1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.NavigableMap;
27 import java.util.Set;
28 import java.util.concurrent.locks.Lock;
29 import java.util.concurrent.locks.ReentrantLock;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.fs.PathFilter;
39 import org.apache.hadoop.hbase.ClusterId;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
46 import org.apache.hadoop.hbase.RemoteExceptionHandler;
47 import org.apache.hadoop.hbase.Server;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.backup.HFileArchiver;
50 import org.apache.hadoop.hbase.catalog.MetaReader;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.exceptions.DeserializationException;
53 import org.apache.hadoop.hbase.fs.HFileSystem;
54 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
55 import org.apache.hadoop.hbase.regionserver.HRegion;
56 import org.apache.hadoop.hbase.regionserver.wal.HLog;
57 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
60 import org.apache.hadoop.hbase.util.FSTableDescriptors;
61 import org.apache.hadoop.hbase.util.FSUtils;
62 import org.apache.zookeeper.KeeperException;
63
64
65
66
67
68
69 @InterfaceAudience.Private
70 public class MasterFileSystem {
71 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
72
73 Configuration conf;
74
75 Server master;
76
77 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
78
79 private ClusterId clusterId;
80
81 private final FileSystem fs;
82
83 private volatile boolean fsOk = true;
84
85 private final Path oldLogDir;
86
87 private final Path rootdir;
88
89 private final Path tempdir;
90
91 final Lock splitLogLock = new ReentrantLock();
92 final boolean distributedLogReplay;
93 final SplitLogManager splitLogManager;
94 private final MasterServices services;
95
96 final static PathFilter META_FILTER = new PathFilter() {
97 @Override
98 public boolean accept(Path p) {
99 return HLogUtil.isMetaFile(p);
100 }
101 };
102
103 final static PathFilter NON_META_FILTER = new PathFilter() {
104 @Override
105 public boolean accept(Path p) {
106 return !HLogUtil.isMetaFile(p);
107 }
108 };
109
110 public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
111 throws IOException {
112 this.conf = master.getConfiguration();
113 this.master = master;
114 this.services = services;
115
116
117
118
119 this.rootdir = FSUtils.getRootDir(conf);
120 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
121
122
123 this.fs = this.rootdir.getFileSystem(conf);
124 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
125
126 fs.setConf(conf);
127
128
129 this.oldLogDir = createInitialFileSystemLayout();
130 HFileSystem.addLocationsOrderInterceptor(conf);
131 try {
132 this.splitLogManager = new SplitLogManager(master.getZooKeeper(), master.getConfiguration(),
133 master, services,
134 master.getServerName(), masterRecovery);
135 } catch (KeeperException e) {
136 throw new IOException(e);
137 }
138 this.distributedLogReplay = (this.splitLogManager.getRecoveryMode() == RecoveryMode.LOG_REPLAY);
139 }
140
141
142
143
144
145
146
147
148
149
150
151 private Path createInitialFileSystemLayout() throws IOException {
152
153 checkRootDir(this.rootdir, conf, this.fs);
154
155
156 checkTempDir(this.tempdir, conf, this.fs);
157
158 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
159
160
161 if(!this.fs.exists(oldLogDir)) {
162 this.fs.mkdirs(oldLogDir);
163 }
164
165 return oldLogDir;
166 }
167
168 public FileSystem getFileSystem() {
169 return this.fs;
170 }
171
172
173
174
175
176 public Path getOldLogDir() {
177 return this.oldLogDir;
178 }
179
180
181
182
183
184
185 public boolean checkFileSystem() {
186 if (this.fsOk) {
187 try {
188 FSUtils.checkFileSystemAvailable(this.fs);
189 FSUtils.checkDfsSafeMode(this.conf);
190 } catch (IOException e) {
191 master.abort("Shutting down HBase cluster: file system not available", e);
192 this.fsOk = false;
193 }
194 }
195 return this.fsOk;
196 }
197
198
199
200
201 public Path getRootDir() {
202 return this.rootdir;
203 }
204
205
206
207
208 public Path getTempDir() {
209 return this.tempdir;
210 }
211
212
213
214
215 public ClusterId getClusterId() {
216 return clusterId;
217 }
218
219
220
221
222
223 Set<ServerName> getFailedServersFromLogFolders() {
224 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
225 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
226
227 Set<ServerName> serverNames = new HashSet<ServerName>();
228 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
229
230 do {
231 if (master.isStopped()) {
232 LOG.warn("Master stopped while trying to get failed servers.");
233 break;
234 }
235 try {
236 if (!this.fs.exists(logsDirPath)) return serverNames;
237 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
238
239
240 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
241 .keySet();
242
243 if (logFolders == null || logFolders.length == 0) {
244 LOG.debug("No log files to split, proceeding...");
245 return serverNames;
246 }
247 for (FileStatus status : logFolders) {
248 String sn = status.getPath().getName();
249
250 if (sn.endsWith(HLog.SPLITTING_EXT)) {
251 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
252 }
253 ServerName serverName = ServerName.parseServerName(sn);
254 if (!onlineServers.contains(serverName)) {
255 LOG.info("Log folder " + status.getPath() + " doesn't belong "
256 + "to a known region server, splitting");
257 serverNames.add(serverName);
258 } else {
259 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
260 }
261 }
262 retrySplitting = false;
263 } catch (IOException ioe) {
264 LOG.warn("Failed getting failed servers to be recovered.", ioe);
265 if (!checkFileSystem()) {
266 LOG.warn("Bad Filesystem, exiting");
267 Runtime.getRuntime().halt(1);
268 }
269 try {
270 if (retrySplitting) {
271 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
272 }
273 } catch (InterruptedException e) {
274 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
275 Thread.currentThread().interrupt();
276 retrySplitting = false;
277 Runtime.getRuntime().halt(1);
278 }
279 }
280 } while (retrySplitting);
281
282 return serverNames;
283 }
284
285 public void splitLog(final ServerName serverName) throws IOException {
286 Set<ServerName> serverNames = new HashSet<ServerName>();
287 serverNames.add(serverName);
288 splitLog(serverNames);
289 }
290
291
292
293
294
295
296 public void splitMetaLog(final ServerName serverName) throws IOException {
297 Set<ServerName> serverNames = new HashSet<ServerName>();
298 serverNames.add(serverName);
299 splitMetaLog(serverNames);
300 }
301
302
303
304
305
306
307 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
308 splitLog(serverNames, META_FILTER);
309 }
310
311 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
312 List<Path> logDirs = new ArrayList<Path>();
313 boolean needReleaseLock = false;
314 if (!this.services.isInitialized()) {
315
316 this.splitLogLock.lock();
317 needReleaseLock = true;
318 }
319 try {
320 for (ServerName serverName : serverNames) {
321 Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
322 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
323
324 if (fs.exists(logDir)) {
325 if (!this.fs.rename(logDir, splitDir)) {
326 throw new IOException("Failed fs.rename for log split: " + logDir);
327 }
328 logDir = splitDir;
329 LOG.debug("Renamed region directory: " + splitDir);
330 } else if (!fs.exists(splitDir)) {
331 LOG.info("Log dir for server " + serverName + " does not exist");
332 continue;
333 }
334 logDirs.add(splitDir);
335 }
336 } finally {
337 if (needReleaseLock) {
338 this.splitLogLock.unlock();
339 }
340 }
341 return logDirs;
342 }
343
344
345
346
347
348
349
350 public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
351 if (!this.distributedLogReplay) {
352 return;
353 }
354
355 for (ServerName serverName : serverNames) {
356 NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
357 if (regions == null) {
358 continue;
359 }
360 try {
361 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
362 } catch (KeeperException e) {
363 throw new IOException(e);
364 }
365 }
366 }
367
368
369
370
371
372
373
374 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
375 if (!this.distributedLogReplay) {
376 return;
377 }
378
379 if (regions == null || regions.isEmpty()) {
380 return;
381 }
382 try {
383 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
384 } catch (KeeperException e) {
385 throw new IOException(e);
386 }
387 }
388
389 public void splitLog(final Set<ServerName> serverNames) throws IOException {
390 splitLog(serverNames, NON_META_FILTER);
391 }
392
393
394
395
396
397
398 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
399 throws KeeperException {
400 this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
401 }
402
403
404
405
406
407
408
409
410 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
411 long splitTime = 0, splitLogSize = 0;
412 List<Path> logDirs = getLogDirs(serverNames);
413
414 splitLogManager.handleDeadWorkers(serverNames);
415 splitTime = EnvironmentEdgeManager.currentTimeMillis();
416 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
417 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
418
419 if (this.metricsMasterFilesystem != null) {
420 if (filter == META_FILTER) {
421 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
422 } else {
423 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
424 }
425 }
426 }
427
428
429
430
431
432
433
434
435
436
437 @SuppressWarnings("deprecation")
438 private Path checkRootDir(final Path rd, final Configuration c,
439 final FileSystem fs)
440 throws IOException {
441
442 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
443
444 try {
445 if (!fs.exists(rd)) {
446 fs.mkdirs(rd);
447
448
449
450
451
452
453
454 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
455 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
456 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
457 } else {
458 if (!fs.isDirectory(rd)) {
459 throw new IllegalArgumentException(rd.toString() + " is not a directory");
460 }
461
462 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
463 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
464 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
465 }
466 } catch (DeserializationException de) {
467 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
468 IOException ioe = new IOException();
469 ioe.initCause(de);
470 throw ioe;
471 } catch (IllegalArgumentException iae) {
472 LOG.fatal("Please fix invalid configuration for "
473 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
474 throw iae;
475 }
476
477 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
478 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
479 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
480 }
481 clusterId = FSUtils.getClusterId(fs, rd);
482
483
484 if (!FSUtils.metaRegionExists(fs, rd)) {
485 bootstrap(rd, c);
486 } else {
487
488 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
489 .migrateFSTableDescriptorsIfNecessary(fs, rd);
490 }
491
492
493
494
495
496 FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
497 fsd.createTableDescriptor(
498 new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
499
500 return rd;
501 }
502
503
504
505
506
507 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
508 throws IOException {
509
510 if (fs.exists(tmpdir)) {
511
512
513 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
514 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
515 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
516 }
517 }
518 if (!fs.delete(tmpdir, true)) {
519 throw new IOException("Unable to clean the temp directory: " + tmpdir);
520 }
521 }
522
523
524 if (!fs.mkdirs(tmpdir)) {
525 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
526 }
527 }
528
529 private static void bootstrap(final Path rd, final Configuration c)
530 throws IOException {
531 LOG.info("BOOTSTRAP: creating hbase:meta region");
532 try {
533
534
535
536
537 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
538 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
539 setInfoFamilyCachingForMeta(metaDescriptor, false);
540 HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
541 setInfoFamilyCachingForMeta(metaDescriptor, true);
542 HRegion.closeHRegion(meta);
543 } catch (IOException e) {
544 e = RemoteExceptionHandler.checkIOException(e);
545 LOG.error("bootstrap", e);
546 throw e;
547 }
548 }
549
550
551
552
553 public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
554 final boolean b) {
555 for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
556 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
557 hcd.setBlockCacheEnabled(b);
558 hcd.setInMemory(b);
559 }
560 }
561 }
562
563 public void deleteRegion(HRegionInfo region) throws IOException {
564 HFileArchiver.archiveRegion(conf, fs, region);
565 }
566
567 public void deleteTable(TableName tableName) throws IOException {
568 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
569 }
570
571
572
573
574
575
576
577 public Path moveTableToTemp(TableName tableName) throws IOException {
578 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
579 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
580
581
582 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
583 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
584 }
585
586 if (!fs.rename(srcPath, tempPath)) {
587 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
588 }
589
590 return tempPath;
591 }
592
593 public void updateRegionInfo(HRegionInfo region) {
594
595
596
597 }
598
599 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
600 throws IOException {
601
602 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
603 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
604
605
606 Path familyDir = new Path(tableDir,
607 new Path(region.getEncodedName(), Bytes.toString(familyName)));
608 if (fs.delete(familyDir, true) == false) {
609 throw new IOException("Could not delete family "
610 + Bytes.toString(familyName) + " from FileSystem for region "
611 + region.getRegionNameAsString() + "(" + region.getEncodedName()
612 + ")");
613 }
614 }
615
616 public void stop() {
617 if (splitLogManager != null) {
618 this.splitLogManager.stop();
619 }
620 }
621
622
623
624
625
626
627
628
629 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
630 throws IOException {
631 LOG.info("DeleteColumn. Table = " + tableName
632 + " family = " + Bytes.toString(familyName));
633 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
634 htd.removeFamily(familyName);
635 this.services.getTableDescriptors().add(htd);
636 return htd;
637 }
638
639
640
641
642
643
644
645
646 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
647 throws IOException {
648 LOG.info("AddModifyColumn. Table = " + tableName
649 + " HCD = " + hcd.toString());
650
651 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
652 byte [] familyName = hcd.getName();
653 if(!htd.hasFamily(familyName)) {
654 throw new InvalidFamilyOperationException("Family '" +
655 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
656 }
657 htd.addFamily(hcd);
658 this.services.getTableDescriptors().add(htd);
659 return htd;
660 }
661
662
663
664
665
666
667
668
669 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
670 throws IOException {
671 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
672 hcd.toString());
673 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
674 if (htd == null) {
675 throw new InvalidFamilyOperationException("Family '" +
676 hcd.getNameAsString() + "' cannot be modified as HTD is null");
677 }
678 htd.addFamily(hcd);
679 this.services.getTableDescriptors().add(htd);
680 return htd;
681 }
682
683 private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
684 throws IOException {
685 if (!this.master.isStopped()) {
686 try {
687 this.master.getCatalogTracker().waitForMeta();
688 return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
689 } catch (InterruptedException e) {
690 throw (InterruptedIOException)new InterruptedIOException().initCause(e);
691 }
692 }
693 return null;
694 }
695
696
697
698
699
700
701
702 public void setLogRecoveryMode() throws IOException {
703 try {
704 this.splitLogManager.setRecoveryMode(false);
705 } catch (KeeperException e) {
706 throw new IOException(e);
707 }
708 }
709
710 public RecoveryMode getLogRecoveryMode() {
711 return this.splitLogManager.getRecoveryMode();
712 }
713 }