1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
40 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
41 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
42 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.FSUtils;
45
46 import com.google.protobuf.HBaseZeroCopyByteString;
47
48
49
50
51
52
53
54
55
56
57 @InterfaceAudience.Private
58 public class SnapshotManifestV1 {
59 private static final Log LOG = LogFactory.getLog(SnapshotManifestV1.class);
60
61 public static final int DESCRIPTOR_VERSION = 0;
62
63 private SnapshotManifestV1() {
64 }
65
66 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
67 HRegionFileSystem, Path> {
68 private final Configuration conf;
69 private final Path snapshotDir;
70 private final FileSystem fs;
71
72 public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
73 this.snapshotDir = snapshotDir;
74 this.conf = conf;
75 this.fs = fs;
76 }
77
78 public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
79 HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
80 fs, snapshotDir, regionInfo);
81 return snapshotRegionFs;
82 }
83
84 public void regionClose(final HRegionFileSystem region) {
85 }
86
87 public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
88 Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
89 return familyDir;
90 }
91
92 public void familyClose(final HRegionFileSystem region, final Path family) {
93 }
94
95 public void storeFile(final HRegionFileSystem region, final Path familyDir,
96 final StoreFileInfo storeFile) throws IOException {
97 Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
98 boolean success = true;
99 if (storeFile.isReference()) {
100
101 storeFile.getReference().write(fs, referenceFile);
102 } else {
103
104
105
106
107 success = fs.createNewFile(referenceFile);
108 }
109 if (!success) {
110 throw new IOException("Failed to create reference file:" + referenceFile);
111 }
112 }
113 }
114
115 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
116 final Executor executor,final FileSystem fs, final Path snapshotDir,
117 final SnapshotDescription desc) throws IOException {
118 FileStatus[] regions = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
119 if (regions == null) {
120 LOG.info("No regions under directory:" + snapshotDir);
121 return null;
122 }
123
124 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
125 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
126 for (final FileStatus region: regions) {
127 completionService.submit(new Callable<SnapshotRegionManifest>() {
128 @Override
129 public SnapshotRegionManifest call() throws IOException {
130 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
131 return buildManifestFromDisk(conf, fs, snapshotDir, hri);
132 }
133 });
134 }
135
136 ArrayList<SnapshotRegionManifest> regionsManifest =
137 new ArrayList<SnapshotRegionManifest>(regions.length);
138 try {
139 for (int i = 0; i < regions.length; ++i) {
140 regionsManifest.add(completionService.take().get());
141 }
142 } catch (InterruptedException e) {
143 throw new InterruptedIOException(e.getMessage());
144 } catch (ExecutionException e) {
145 IOException ex = new IOException();
146 ex.initCause(e.getCause());
147 throw ex;
148 }
149 return regionsManifest;
150 }
151
152 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
153 final SnapshotRegionManifest manifest) throws IOException {
154 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
155 fs.delete(new Path(snapshotDir, regionName), true);
156 }
157
158 static SnapshotRegionManifest buildManifestFromDisk (final Configuration conf,
159 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
160 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
161 tableDir, regionInfo, true);
162 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
163
164
165 LOG.debug("Storing region-info for snapshot.");
166 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
167
168
169 LOG.debug("Creating references for hfiles");
170
171
172
173
174
175
176 Collection<String> familyNames = regionFs.getFamilies();
177 if (familyNames != null) {
178 for (String familyName: familyNames) {
179 Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
180 if (storeFiles == null) {
181 LOG.debug("No files under family: " + familyName);
182 continue;
183 }
184
185
186 SnapshotRegionManifest.FamilyFiles.Builder family =
187 SnapshotRegionManifest.FamilyFiles.newBuilder();
188 family.setFamilyName(HBaseZeroCopyByteString.wrap(Bytes.toBytes(familyName)));
189
190 if (LOG.isDebugEnabled()) {
191 LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
192 }
193
194
195 int i = 0;
196 int sz = storeFiles.size();
197 for (StoreFileInfo storeFile: storeFiles) {
198
199 LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
200 SnapshotRegionManifest.StoreFile.Builder sfManifest =
201 SnapshotRegionManifest.StoreFile.newBuilder();
202 sfManifest.setName(storeFile.getPath().getName());
203 family.addStoreFiles(sfManifest.build());
204 }
205 manifest.addFamilyFiles(family.build());
206 }
207 }
208 return manifest.build();
209 }
210 }