1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28 import java.util.concurrent.atomic.AtomicInteger;
29 import java.util.concurrent.atomic.AtomicLong;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.hbase.classification.InterfaceAudience;
38 import org.apache.hadoop.hbase.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configured;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.util.StringUtils;
43 import org.apache.hadoop.util.Tool;
44 import org.apache.hadoop.util.ToolRunner;
45
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.hbase.HBaseConfiguration;
48 import org.apache.hadoop.hbase.HTableDescriptor;
49 import org.apache.hadoop.hbase.io.HFileLink;
50 import org.apache.hadoop.hbase.io.HLogLink;
51 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
52 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
53 import org.apache.hadoop.hbase.util.FSUtils;
54
55
56
57
58
59
60
61
62
63
64 @InterfaceAudience.Public
65 @InterfaceStability.Evolving
66 public final class SnapshotInfo extends Configured implements Tool {
67 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
68
69
70
71
72
73
74
75
76
77 public static class SnapshotStats {
78
79 static class FileInfo {
80 private final boolean corrupted;
81 private final boolean inArchive;
82 private final long size;
83
84 FileInfo(final boolean inArchive, final long size, final boolean corrupted) {
85 this.corrupted = corrupted;
86 this.inArchive = inArchive;
87 this.size = size;
88 }
89
90
91 public boolean inArchive() {
92 return this.inArchive;
93 }
94
95
96 public boolean isCorrupted() {
97 return this.corrupted;
98 }
99
100
101 public boolean isMissing() {
102 return this.size < 0;
103 }
104
105
106 public long getSize() {
107 return this.size;
108 }
109
110 String getStateToString() {
111 if (isCorrupted()) return "CORRUPTED";
112 if (isMissing()) return "NOT FOUND";
113 if (inArchive()) return "archive";
114 return null;
115 }
116 }
117
118 private AtomicInteger hfileArchiveCount = new AtomicInteger();
119 private AtomicInteger hfilesCorrupted = new AtomicInteger();
120 private AtomicInteger hfilesMissing = new AtomicInteger();
121 private AtomicInteger hfilesCount = new AtomicInteger();
122 private AtomicInteger logsMissing = new AtomicInteger();
123 private AtomicInteger logsCount = new AtomicInteger();
124 private AtomicLong hfileArchiveSize = new AtomicLong();
125 private AtomicLong hfileSize = new AtomicLong();
126 private AtomicLong logSize = new AtomicLong();
127
128 private final SnapshotDescription snapshot;
129 private final TableName snapshotTable;
130 private final Configuration conf;
131 private final FileSystem fs;
132
133 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
134 {
135 this.snapshot = snapshot;
136 this.snapshotTable = TableName.valueOf(snapshot.getTable());
137 this.conf = conf;
138 this.fs = fs;
139 }
140
141
142 public SnapshotDescription getSnapshotDescription() {
143 return this.snapshot;
144 }
145
146
147 public boolean isSnapshotCorrupted() {
148 return hfilesMissing.get() > 0 ||
149 logsMissing.get() > 0 ||
150 hfilesCorrupted.get() > 0;
151 }
152
153
154 public int getStoreFilesCount() {
155 return hfilesCount.get() + hfileArchiveCount.get();
156 }
157
158
159 public int getArchivedStoreFilesCount() {
160 return hfileArchiveCount.get();
161 }
162
163
164 public int getLogsCount() {
165 return logsCount.get();
166 }
167
168
169 public int getMissingStoreFilesCount() {
170 return hfilesMissing.get();
171 }
172
173
174 public int getCorruptedStoreFilesCount() {
175 return hfilesCorrupted.get();
176 }
177
178
179 public int getMissingLogsCount() {
180 return logsMissing.get();
181 }
182
183
184 public long getStoreFilesSize() {
185 return hfileSize.get() + hfileArchiveSize.get();
186 }
187
188
189 public long getSharedStoreFilesSize() {
190 return hfileSize.get();
191 }
192
193
194 public long getArchivedStoreFileSize() {
195 return hfileArchiveSize.get();
196 }
197
198
199 public float getSharedStoreFilePercentage() {
200 return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
201 }
202
203
204 public long getLogsSize() {
205 return logSize.get();
206 }
207
208
209
210
211
212
213
214
215 FileInfo addStoreFile(final HRegionInfo region, final String family,
216 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
217 HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
218 family, storeFile.getName());
219 boolean isCorrupted = false;
220 boolean inArchive = false;
221 long size = -1;
222 try {
223 if ((inArchive = fs.exists(link.getArchivePath()))) {
224 size = fs.getFileStatus(link.getArchivePath()).getLen();
225 hfileArchiveSize.addAndGet(size);
226 hfileArchiveCount.incrementAndGet();
227 } else {
228 size = link.getFileStatus(fs).getLen();
229 hfileSize.addAndGet(size);
230 hfilesCount.incrementAndGet();
231 }
232 isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
233 if (isCorrupted) hfilesCorrupted.incrementAndGet();
234 } catch (FileNotFoundException e) {
235 hfilesMissing.incrementAndGet();
236 }
237 return new FileInfo(inArchive, size, isCorrupted);
238 }
239
240
241
242
243
244
245
246 FileInfo addLogFile(final String server, final String logfile) throws IOException {
247 HLogLink logLink = new HLogLink(conf, server, logfile);
248 long size = -1;
249 try {
250 size = logLink.getFileStatus(fs).getLen();
251 logSize.addAndGet(size);
252 logsCount.incrementAndGet();
253 } catch (FileNotFoundException e) {
254 logsMissing.incrementAndGet();
255 }
256 return new FileInfo(false, size, false);
257 }
258 }
259
260 private boolean printSizeInBytes = false;
261 private FileSystem fs;
262 private Path rootDir;
263
264 private SnapshotManifest snapshotManifest;
265
266 @Override
267 public int run(String[] args) throws IOException, InterruptedException {
268 final Configuration conf = getConf();
269 boolean listSnapshots = false;
270 String snapshotName = null;
271 boolean showSchema = false;
272 boolean showFiles = false;
273 boolean showStats = false;
274
275
276 for (int i = 0; i < args.length; i++) {
277 String cmd = args[i];
278 try {
279 if (cmd.equals("-snapshot")) {
280 snapshotName = args[++i];
281 } else if (cmd.equals("-files")) {
282 showFiles = true;
283 showStats = true;
284 } else if (cmd.equals("-stats")) {
285 showStats = true;
286 } else if (cmd.equals("-schema")) {
287 showSchema = true;
288 } else if (cmd.equals("-remote-dir")) {
289 Path sourceDir = new Path(args[++i]);
290 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
291 FSUtils.setFsDefault(conf, new Path(defaultFs));
292 FSUtils.setRootDir(conf, sourceDir);
293 } else if (cmd.equals("-list-snapshots")) {
294 listSnapshots = true;
295 } else if (cmd.equals("-size-in-bytes")) {
296 printSizeInBytes = true;
297 } else if (cmd.equals("-h") || cmd.equals("--help")) {
298 printUsageAndExit();
299 } else {
300 System.err.println("UNEXPECTED: " + cmd);
301 printUsageAndExit();
302 }
303 } catch (Exception e) {
304 printUsageAndExit();
305 }
306 }
307
308
309 if (listSnapshots) {
310 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
311 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
312 for (SnapshotDescription desc: getSnapshotList(conf)) {
313 System.out.printf("%-20s | %20s | %s%n",
314 desc.getName(),
315 df.format(new Date(desc.getCreationTime())),
316 desc.getTable());
317 }
318 return 0;
319 }
320
321 if (snapshotName == null) {
322 System.err.println("Missing snapshot name!");
323 printUsageAndExit();
324 return 1;
325 }
326
327 rootDir = FSUtils.getRootDir(conf);
328 fs = FileSystem.get(rootDir.toUri(), conf);
329 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
330
331
332 if (!loadSnapshotInfo(snapshotName)) {
333 System.err.println("Snapshot '" + snapshotName + "' not found!");
334 return 1;
335 }
336
337 printInfo();
338 if (showSchema) printSchema();
339 printFiles(showFiles, showStats);
340
341 return 0;
342 }
343
344
345
346
347
348
349 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
350 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
351 if (!fs.exists(snapshotDir)) {
352 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
353 return false;
354 }
355
356 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
357 snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
358 return true;
359 }
360
361
362
363
364 private void printInfo() {
365 SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
366 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
367 System.out.println("Snapshot Info");
368 System.out.println("----------------------------------------");
369 System.out.println(" Name: " + snapshotDesc.getName());
370 System.out.println(" Type: " + snapshotDesc.getType());
371 System.out.println(" Table: " + snapshotDesc.getTable());
372 System.out.println(" Format: " + snapshotDesc.getVersion());
373 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
374 System.out.println();
375 }
376
377
378
379
380 private void printSchema() {
381 System.out.println("Table Descriptor");
382 System.out.println("----------------------------------------");
383 System.out.println(snapshotManifest.getTableDescriptor().toString());
384 System.out.println();
385 }
386
387
388
389
390
391 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
392 if (showFiles) {
393 System.out.println("Snapshot Files");
394 System.out.println("----------------------------------------");
395 }
396
397
398 final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
399 final String table = snapshotDesc.getTable();
400 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
401 SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
402 new SnapshotReferenceUtil.SnapshotVisitor() {
403 @Override
404 public void storeFile(final HRegionInfo regionInfo, final String family,
405 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
406 if (storeFile.hasReference()) return;
407
408 SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
409 if (showFiles) {
410 String state = info.getStateToString();
411 System.out.printf("%8s %s/%s/%s/%s %s%n",
412 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
413 table, regionInfo.getEncodedName(), family, storeFile.getName(),
414 state == null ? "" : "(" + state + ")");
415 }
416 }
417
418 @Override
419 public void logFile (final String server, final String logfile)
420 throws IOException {
421 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
422
423 if (showFiles) {
424 String state = info.getStateToString();
425 System.out.printf("%8s log %s on server %s (%s)%n",
426 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
427 logfile, server,
428 state == null ? "" : "(" + state + ")");
429 }
430 }
431 });
432
433
434 System.out.println();
435 if (stats.isSnapshotCorrupted()) {
436 System.out.println("**************************************************************");
437 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
438 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
439 System.out.printf(" %d hfile(s) corrupted.%n",
440 stats.getCorruptedStoreFilesCount());
441 System.out.println("**************************************************************");
442 }
443
444 if (showStats) {
445 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
446 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
447 fileSizeToString(stats.getStoreFilesSize()),
448 stats.getSharedStoreFilePercentage(),
449 fileSizeToString(stats.getSharedStoreFilesSize())
450 );
451 System.out.printf("%d Logs, total size %s%n",
452 stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
453 System.out.println();
454 }
455 }
456
457 private String fileSizeToString(long size) {
458 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
459 }
460
461 private void printUsageAndExit() {
462 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
463 System.err.println(" where [options] are:");
464 System.err.println(" -h|-help Show this help and exit.");
465 System.err.println(" -remote-dir Root directory that contains the snapshots.");
466 System.err.println(" -list-snapshots List all the available snapshots and exit.");
467 System.err.println(" -size-in-bytes Print the size of the files in bytes.");
468 System.err.println(" -snapshot NAME Snapshot to examine.");
469 System.err.println(" -files Files and logs list.");
470 System.err.println(" -stats Files and logs stats.");
471 System.err.println(" -schema Describe the snapshotted table.");
472 System.err.println();
473 System.err.println("Examples:");
474 System.err.println(" hbase " + getClass() + " \\");
475 System.err.println(" -snapshot MySnapshot -files");
476 System.exit(1);
477 }
478
479
480
481
482
483
484
485 public static SnapshotStats getSnapshotStats(final Configuration conf,
486 final SnapshotDescription snapshot) throws IOException {
487 Path rootDir = FSUtils.getRootDir(conf);
488 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
489 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
490 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
491 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
492 SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
493 new SnapshotReferenceUtil.SnapshotVisitor() {
494 @Override
495 public void storeFile(final HRegionInfo regionInfo, final String family,
496 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
497 if (!storeFile.hasReference()) {
498 stats.addStoreFile(regionInfo, family, storeFile);
499 }
500 }
501
502 @Override
503 public void logFile (final String server, final String logfile) throws IOException {
504 stats.addLogFile(server, logfile);
505 }
506 });
507 return stats;
508 }
509
510
511
512
513
514
515 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
516 throws IOException {
517 Path rootDir = FSUtils.getRootDir(conf);
518 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
519 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
520 FileStatus[] snapshots = fs.listStatus(snapshotDir,
521 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
522 List<SnapshotDescription> snapshotLists =
523 new ArrayList<SnapshotDescription>(snapshots.length);
524 for (FileStatus snapshotDirStat: snapshots) {
525 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
526 }
527 return snapshotLists;
528 }
529
530
531
532
533
534
535
536
537 static int innerMain(final String [] args) throws Exception {
538 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
539 }
540
541 public static void main(String[] args) throws Exception {
542 System.exit(innerMain(args));
543 }
544 }