View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.DataInputStream;
23  import java.io.EOFException;
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.io.InterruptedIOException;
28  import java.lang.reflect.InvocationTargetException;
29  import java.lang.reflect.Method;
30  import java.net.InetSocketAddress;
31  import java.net.URI;
32  import java.net.URISyntaxException;
33  import java.util.ArrayList;
34  import java.util.Collections;
35  import java.util.HashMap;
36  import java.util.LinkedList;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.concurrent.ArrayBlockingQueue;
40  import java.util.concurrent.ConcurrentHashMap;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.TimeUnit;
43  import java.util.regex.Pattern;
44  
45  import org.apache.commons.logging.Log;
46  import org.apache.commons.logging.LogFactory;
47  import org.apache.hadoop.hbase.classification.InterfaceAudience;
48  import org.apache.hadoop.conf.Configuration;
49  import org.apache.hadoop.fs.BlockLocation;
50  import org.apache.hadoop.fs.FSDataInputStream;
51  import org.apache.hadoop.fs.FSDataOutputStream;
52  import org.apache.hadoop.fs.FileStatus;
53  import org.apache.hadoop.fs.FileSystem;
54  import org.apache.hadoop.fs.Path;
55  import org.apache.hadoop.fs.PathFilter;
56  import org.apache.hadoop.fs.permission.FsAction;
57  import org.apache.hadoop.fs.permission.FsPermission;
58  import org.apache.hadoop.hbase.ClusterId;
59  import org.apache.hadoop.hbase.HColumnDescriptor;
60  import org.apache.hadoop.hbase.HConstants;
61  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
62  import org.apache.hadoop.hbase.HRegionInfo;
63  import org.apache.hadoop.hbase.RemoteExceptionHandler;
64  import org.apache.hadoop.hbase.TableName;
65  import org.apache.hadoop.hbase.exceptions.DeserializationException;
66  import org.apache.hadoop.hbase.fs.HFileSystem;
67  import org.apache.hadoop.hbase.master.HMaster;
68  import org.apache.hadoop.hbase.master.RegionPlacementMaintainer;
69  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
70  import org.apache.hadoop.hbase.security.AccessDeniedException;
71  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72  import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
73  import org.apache.hadoop.hbase.regionserver.HRegion;
74  import org.apache.hadoop.hdfs.DistributedFileSystem;
75  import org.apache.hadoop.hdfs.protocol.FSConstants;
76  import org.apache.hadoop.io.IOUtils;
77  import org.apache.hadoop.io.SequenceFile;
78  import org.apache.hadoop.security.UserGroupInformation;
79  import org.apache.hadoop.util.Progressable;
80  import org.apache.hadoop.util.ReflectionUtils;
81  import org.apache.hadoop.util.StringUtils;
82  
83  import com.google.common.primitives.Ints;
84  import com.google.protobuf.InvalidProtocolBufferException;
85  
86  /**
87   * Utility methods for interacting with the underlying file system.
88   */
89  @InterfaceAudience.Private
90  public abstract class FSUtils {
91    private static final Log LOG = LogFactory.getLog(FSUtils.class);
92  
93    /** Full access permissions (starting point for a umask) */
94    public static final String FULL_RWX_PERMISSIONS = "777";
95    private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";
96    private static final int DEFAULT_THREAD_POOLSIZE = 2;
97  
98    /** Set to true on Windows platforms */
99    public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
100 
101   protected FSUtils() {
102     super();
103   }
104 
105   /**
106    * Compare of path component. Does not consider schema; i.e. if schemas different but <code>path
107    * <code> starts with <code>rootPath<code>, then the function returns true
108    * @param rootPath
109    * @param path
110    * @return True if <code>path</code> starts with <code>rootPath</code>
111    */
112   public static boolean isStartingWithPath(final Path rootPath, final String path) {
113     String uriRootPath = rootPath.toUri().getPath();
114     String tailUriPath = (new Path(path)).toUri().getPath();
115     return tailUriPath.startsWith(uriRootPath);
116   }
117 
118   /**
119    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
120    * '/a/b/c' part. Does not consider schema; i.e. if schemas different but path or subpath matches,
121    * the two will equate.
122    * @param pathToSearch Path we will be trying to match.
123    * @param pathTail
124    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
125    */
126   public static boolean isMatchingTail(final Path pathToSearch, String pathTail) {
127     return isMatchingTail(pathToSearch, new Path(pathTail));
128   }
129 
130   /**
131    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
132    * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true.  Does not consider
133    * schema; i.e. if schemas different but path or subpath matches, the two will equate.
134    * @param pathToSearch Path we will be trying to match.
135    * @param pathTail
136    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
137    */
138   public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
139     if (pathToSearch.depth() != pathTail.depth()) return false;
140     Path tailPath = pathTail;
141     String tailName;
142     Path toSearch = pathToSearch;
143     String toSearchName;
144     boolean result = false;
145     do {
146       tailName = tailPath.getName();
147       if (tailName == null || tailName.length() <= 0) {
148         result = true;
149         break;
150       }
151       toSearchName = toSearch.getName();
152       if (toSearchName == null || toSearchName.length() <= 0) break;
153       // Move up a parent on each path for next go around.  Path doesn't let us go off the end.
154       tailPath = tailPath.getParent();
155       toSearch = toSearch.getParent();
156     } while(tailName.equals(toSearchName));
157     return result;
158   }
159 
160   public static FSUtils getInstance(FileSystem fs, Configuration conf) {
161     String scheme = fs.getUri().getScheme();
162     if (scheme == null) {
163       LOG.warn("Could not find scheme for uri " +
164           fs.getUri() + ", default to hdfs");
165       scheme = "hdfs";
166     }
167     Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
168         scheme + ".impl", FSHDFSUtils.class); // Default to HDFS impl
169     FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
170     return fsUtils;
171   }
172 
173   /**
174    * Delete if exists.
175    * @param fs filesystem object
176    * @param dir directory to delete
177    * @return True if deleted <code>dir</code>
178    * @throws IOException e
179    */
180   public static boolean deleteDirectory(final FileSystem fs, final Path dir)
181   throws IOException {
182     return fs.exists(dir) && fs.delete(dir, true);
183   }
184 
185   /**
186    * Delete the region directory if exists.
187    * @param conf
188    * @param hri
189    * @return True if deleted the region directory.
190    * @throws IOException
191    */
192   public static boolean deleteRegionDir(final Configuration conf, final HRegionInfo hri)
193   throws IOException {
194     Path rootDir = getRootDir(conf);
195     FileSystem fs = rootDir.getFileSystem(conf);
196     return deleteDirectory(fs,
197       new Path(getTableDir(rootDir, hri.getTable()), hri.getEncodedName()));
198   }
199 
200   /**
201    * Return the number of bytes that large input files should be optimally
202    * be split into to minimize i/o time.
203    *
204    * use reflection to search for getDefaultBlockSize(Path f)
205    * if the method doesn't exist, fall back to using getDefaultBlockSize()
206    *
207    * @param fs filesystem object
208    * @return the default block size for the path's filesystem
209    * @throws IOException e
210    */
211   public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
212     Method m = null;
213     Class<? extends FileSystem> cls = fs.getClass();
214     try {
215       m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
216     } catch (NoSuchMethodException e) {
217       LOG.info("FileSystem doesn't support getDefaultBlockSize");
218     } catch (SecurityException e) {
219       LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
220       m = null; // could happen on setAccessible()
221     }
222     if (m == null) {
223       return fs.getDefaultBlockSize();
224     } else {
225       try {
226         Object ret = m.invoke(fs, path);
227         return ((Long)ret).longValue();
228       } catch (Exception e) {
229         throw new IOException(e);
230       }
231     }
232   }
233 
234   /*
235    * Get the default replication.
236    *
237    * use reflection to search for getDefaultReplication(Path f)
238    * if the method doesn't exist, fall back to using getDefaultReplication()
239    *
240    * @param fs filesystem object
241    * @param f path of file
242    * @return default replication for the path's filesystem
243    * @throws IOException e
244    */
245   public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
246     Method m = null;
247     Class<? extends FileSystem> cls = fs.getClass();
248     try {
249       m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
250     } catch (NoSuchMethodException e) {
251       LOG.info("FileSystem doesn't support getDefaultReplication");
252     } catch (SecurityException e) {
253       LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
254       m = null; // could happen on setAccessible()
255     }
256     if (m == null) {
257       return fs.getDefaultReplication();
258     } else {
259       try {
260         Object ret = m.invoke(fs, path);
261         return ((Number)ret).shortValue();
262       } catch (Exception e) {
263         throw new IOException(e);
264       }
265     }
266   }
267 
268   /**
269    * Returns the default buffer size to use during writes.
270    *
271    * The size of the buffer should probably be a multiple of hardware
272    * page size (4096 on Intel x86), and it determines how much data is
273    * buffered during read and write operations.
274    *
275    * @param fs filesystem object
276    * @return default buffer size to use during writes
277    */
278   public static int getDefaultBufferSize(final FileSystem fs) {
279     return fs.getConf().getInt("io.file.buffer.size", 4096);
280   }
281 
282   /**
283    * Create the specified file on the filesystem. By default, this will:
284    * <ol>
285    * <li>overwrite the file if it exists</li>
286    * <li>apply the umask in the configuration (if it is enabled)</li>
287    * <li>use the fs configured buffer size (or 4096 if not set)</li>
288    * <li>use the default replication</li>
289    * <li>use the default block size</li>
290    * <li>not track progress</li>
291    * </ol>
292    *
293    * @param fs {@link FileSystem} on which to write the file
294    * @param path {@link Path} to the file to write
295    * @param perm permissions
296    * @param favoredNodes
297    * @return output stream to the created file
298    * @throws IOException if the file cannot be created
299    */
300   public static FSDataOutputStream create(FileSystem fs, Path path,
301       FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {
302     if (fs instanceof HFileSystem) {
303       FileSystem backingFs = ((HFileSystem)fs).getBackingFs();
304       if (backingFs instanceof DistributedFileSystem) {
305         // Try to use the favoredNodes version via reflection to allow backwards-
306         // compatibility.
307         try {
308           return (FSDataOutputStream) (DistributedFileSystem.class
309               .getDeclaredMethod("create", Path.class, FsPermission.class,
310                   boolean.class, int.class, short.class, long.class,
311                   Progressable.class, InetSocketAddress[].class)
312                   .invoke(backingFs, path, perm, true,
313                       getDefaultBufferSize(backingFs),
314                       getDefaultReplication(backingFs, path),
315                       getDefaultBlockSize(backingFs, path),
316                       null, favoredNodes));
317         } catch (InvocationTargetException ite) {
318           // Function was properly called, but threw it's own exception.
319           throw new IOException(ite.getCause());
320         } catch (NoSuchMethodException e) {
321           LOG.debug("DFS Client does not support most favored nodes create; using default create");
322           if (LOG.isTraceEnabled()) LOG.trace("Ignoring; use default create", e);
323         } catch (IllegalArgumentException e) {
324           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
325         } catch (SecurityException e) {
326           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
327         } catch (IllegalAccessException e) {
328           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
329         }
330       }
331     }
332     return create(fs, path, perm, true);
333   }
334 
335   /**
336    * Create the specified file on the filesystem. By default, this will:
337    * <ol>
338    * <li>apply the umask in the configuration (if it is enabled)</li>
339    * <li>use the fs configured buffer size (or 4096 if not set)</li>
340    * <li>use the default replication</li>
341    * <li>use the default block size</li>
342    * <li>not track progress</li>
343    * </ol>
344    *
345    * @param fs {@link FileSystem} on which to write the file
346    * @param path {@link Path} to the file to write
347    * @param perm
348    * @param overwrite Whether or not the created file should be overwritten.
349    * @return output stream to the created file
350    * @throws IOException if the file cannot be created
351    */
352   public static FSDataOutputStream create(FileSystem fs, Path path,
353       FsPermission perm, boolean overwrite) throws IOException {
354     if (LOG.isTraceEnabled()) {
355       LOG.trace("Creating file=" + path + " with permission=" + perm + ", overwrite=" + overwrite);
356     }
357     return fs.create(path, perm, overwrite, getDefaultBufferSize(fs),
358         getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null);
359   }
360 
361   /**
362    * Get the file permissions specified in the configuration, if they are
363    * enabled.
364    *
365    * @param fs filesystem that the file will be created on.
366    * @param conf configuration to read for determining if permissions are
367    *          enabled and which to use
368    * @param permssionConfKey property key in the configuration to use when
369    *          finding the permission
370    * @return the permission to use when creating a new file on the fs. If
371    *         special permissions are not specified in the configuration, then
372    *         the default permissions on the the fs will be returned.
373    */
374   public static FsPermission getFilePermissions(final FileSystem fs,
375       final Configuration conf, final String permssionConfKey) {
376     boolean enablePermissions = conf.getBoolean(
377         HConstants.ENABLE_DATA_FILE_UMASK, false);
378 
379     if (enablePermissions) {
380       try {
381         FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
382         // make sure that we have a mask, if not, go default.
383         String mask = conf.get(permssionConfKey);
384         if (mask == null)
385           return getFileDefault();
386         // appy the umask
387         FsPermission umask = new FsPermission(mask);
388         return perm.applyUMask(umask);
389       } catch (IllegalArgumentException e) {
390         LOG.warn(
391             "Incorrect umask attempted to be created: "
392                 + conf.get(permssionConfKey)
393                 + ", using default file permissions.", e);
394         return getFileDefault();
395       }
396     }
397     return getFileDefault();
398   }
399 
400   /**
401    * Get the default permission for file.
402    * This is the same method as FsPermission.getFileDefault() in Hadoop 2.
403    * We provide the method here to support compatibility with Hadoop 1.
404    * See HBASE-11061.  Would be better to do this as Interface in hadoop-compat
405    * w/ hadoop1 and hadoop2 implementations but punting on this since small
406    * risk this will change in 0.96/0.98 timeframe (only committed to these
407    * branches).
408    */
409   public static FsPermission getFileDefault() {
410     return new FsPermission((short)00666);
411   }
412 
413   /**
414    * Checks to see if the specified file system is available
415    *
416    * @param fs filesystem
417    * @throws IOException e
418    */
419   public static void checkFileSystemAvailable(final FileSystem fs)
420   throws IOException {
421     if (!(fs instanceof DistributedFileSystem)) {
422       return;
423     }
424     IOException exception = null;
425     DistributedFileSystem dfs = (DistributedFileSystem) fs;
426     try {
427       if (dfs.exists(new Path("/"))) {
428         return;
429       }
430     } catch (IOException e) {
431       exception = RemoteExceptionHandler.checkIOException(e);
432     }
433     try {
434       fs.close();
435     } catch (Exception e) {
436       LOG.error("file system close failed: ", e);
437     }
438     IOException io = new IOException("File system is not available");
439     io.initCause(exception);
440     throw io;
441   }
442 
443   /**
444    * We use reflection because {@link DistributedFileSystem#setSafeMode(
445    * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1
446    *
447    * @param dfs
448    * @return whether we're in safe mode
449    * @throws IOException
450    */
451   private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
452     boolean inSafeMode = false;
453     try {
454       Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
455           org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
456       inSafeMode = (Boolean) m.invoke(dfs,
457         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
458     } catch (Exception e) {
459       if (e instanceof IOException) throw (IOException) e;
460 
461       // Check whether dfs is on safemode.
462       inSafeMode = dfs.setSafeMode(
463         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
464     }
465     return inSafeMode;
466   }
467 
468   /**
469    * Check whether dfs is in safemode.
470    * @param conf
471    * @throws IOException
472    */
473   public static void checkDfsSafeMode(final Configuration conf)
474   throws IOException {
475     boolean isInSafeMode = false;
476     FileSystem fs = FileSystem.get(conf);
477     if (fs instanceof DistributedFileSystem) {
478       DistributedFileSystem dfs = (DistributedFileSystem)fs;
479       isInSafeMode = isInSafeMode(dfs);
480     }
481     if (isInSafeMode) {
482       throw new IOException("File system is in safemode, it can't be written now");
483     }
484   }
485 
486   /**
487    * Verifies current version of file system
488    *
489    * @param fs filesystem object
490    * @param rootdir root hbase directory
491    * @return null if no version file exists, version string otherwise.
492    * @throws IOException e
493    * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
494    */
495   public static String getVersion(FileSystem fs, Path rootdir)
496   throws IOException, DeserializationException {
497     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
498     FileStatus[] status = null;
499     try {
500       // hadoop 2.0 throws FNFE if directory does not exist.
501       // hadoop 1.0 returns null if directory does not exist.
502       status = fs.listStatus(versionFile);
503     } catch (FileNotFoundException fnfe) {
504       return null;
505     }
506     if (status == null || status.length == 0) return null;
507     String version = null;
508     byte [] content = new byte [(int)status[0].getLen()];
509     FSDataInputStream s = fs.open(versionFile);
510     try {
511       IOUtils.readFully(s, content, 0, content.length);
512       if (ProtobufUtil.isPBMagicPrefix(content)) {
513         version = parseVersionFrom(content);
514       } else {
515         // Presume it pre-pb format.
516         InputStream is = new ByteArrayInputStream(content);
517         DataInputStream dis = new DataInputStream(is);
518         try {
519           version = dis.readUTF();
520         } finally {
521           dis.close();
522         }
523       }
524     } catch (EOFException eof) {
525       LOG.warn("Version file was empty, odd, will try to set it.");
526     } finally {
527       s.close();
528     }
529     return version;
530   }
531 
532   /**
533    * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.
534    * @param bytes The byte content of the hbase.version file.
535    * @return The version found in the file as a String.
536    * @throws DeserializationException
537    */
538   static String parseVersionFrom(final byte [] bytes)
539   throws DeserializationException {
540     ProtobufUtil.expectPBMagicPrefix(bytes);
541     int pblen = ProtobufUtil.lengthOfPBMagic();
542     FSProtos.HBaseVersionFileContent.Builder builder =
543       FSProtos.HBaseVersionFileContent.newBuilder();
544     FSProtos.HBaseVersionFileContent fileContent;
545     try {
546       fileContent = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
547       return fileContent.getVersion();
548     } catch (InvalidProtocolBufferException e) {
549       // Convert
550       throw new DeserializationException(e);
551     }
552   }
553 
554   /**
555    * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.
556    * @param version Version to persist
557    * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a prefix.
558    */
559   static byte [] toVersionByteArray(final String version) {
560     FSProtos.HBaseVersionFileContent.Builder builder =
561       FSProtos.HBaseVersionFileContent.newBuilder();
562     return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
563   }
564 
565   /**
566    * Verifies current version of file system
567    *
568    * @param fs file system
569    * @param rootdir root directory of HBase installation
570    * @param message if true, issues a message on System.out
571    *
572    * @throws IOException e
573    * @throws DeserializationException
574    */
575   public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
576   throws IOException, DeserializationException {
577     checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
578   }
579 
580   /**
581    * Verifies current version of file system
582    *
583    * @param fs file system
584    * @param rootdir root directory of HBase installation
585    * @param message if true, issues a message on System.out
586    * @param wait wait interval
587    * @param retries number of times to retry
588    *
589    * @throws IOException e
590    * @throws DeserializationException
591    */
592   public static void checkVersion(FileSystem fs, Path rootdir,
593       boolean message, int wait, int retries)
594   throws IOException, DeserializationException {
595     String version = getVersion(fs, rootdir);
596     if (version == null) {
597       if (!metaRegionExists(fs, rootdir)) {
598         // rootDir is empty (no version file and no root region)
599         // just create new version file (HBASE-1195)
600         setVersion(fs, rootdir, wait, retries);
601         return;
602       }
603     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return;
604 
605     // version is deprecated require migration
606     // Output on stdout so user sees it in terminal.
607     String msg = "HBase file layout needs to be upgraded."
608       + " You have version " + version
609       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
610       + ". Consult http://hbase.apache.org/book.html for further information about upgrading HBase."
611       + " Is your hbase.rootdir valid? If so, you may need to run "
612       + "'hbase hbck -fixVersionFile'.";
613     if (message) {
614       System.out.println("WARNING! " + msg);
615     }
616     throw new FileSystemVersionException(msg);
617   }
618 
619   /**
620    * Sets version of file system
621    *
622    * @param fs filesystem object
623    * @param rootdir hbase root
624    * @throws IOException e
625    */
626   public static void setVersion(FileSystem fs, Path rootdir)
627   throws IOException {
628     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
629       HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
630   }
631 
632   /**
633    * Sets version of file system
634    *
635    * @param fs filesystem object
636    * @param rootdir hbase root
637    * @param wait time to wait for retry
638    * @param retries number of times to retry before failing
639    * @throws IOException e
640    */
641   public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
642   throws IOException {
643     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
644   }
645 
646 
647   /**
648    * Sets version of file system
649    *
650    * @param fs filesystem object
651    * @param rootdir hbase root directory
652    * @param version version to set
653    * @param wait time to wait for retry
654    * @param retries number of times to retry before throwing an IOException
655    * @throws IOException e
656    */
657   public static void setVersion(FileSystem fs, Path rootdir, String version,
658       int wait, int retries) throws IOException {
659     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
660     Path tempVersionFile = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY + Path.SEPARATOR +
661       HConstants.VERSION_FILE_NAME);
662     while (true) {
663       try {
664         // Write the version to a temporary file
665         FSDataOutputStream s = fs.create(tempVersionFile);
666         try {
667           s.write(toVersionByteArray(version));
668           s.close();
669           s = null;
670           // Move the temp version file to its normal location. Returns false
671           // if the rename failed. Throw an IOE in that case.
672           if (!fs.rename(tempVersionFile, versionFile)) {
673             throw new IOException("Unable to move temp version file to " + versionFile);
674           }
675         } finally {
676           // Cleaning up the temporary if the rename failed would be trying
677           // too hard. We'll unconditionally create it again the next time
678           // through anyway, files are overwritten by default by create().
679 
680           // Attempt to close the stream on the way out if it is still open.
681           try {
682             if (s != null) s.close();
683           } catch (IOException ignore) { }
684         }
685         LOG.info("Created version file at " + rootdir.toString() + " with version=" + version);
686         return;
687       } catch (IOException e) {
688         if (retries > 0) {
689           LOG.debug("Unable to create version file at " + rootdir.toString() + ", retrying", e);
690           fs.delete(versionFile, false);
691           try {
692             if (wait > 0) {
693               Thread.sleep(wait);
694             }
695           } catch (InterruptedException ex) {
696             // ignore
697           }
698           retries--;
699         } else {
700           throw e;
701         }
702       }
703     }
704   }
705 
706   /**
707    * Checks that a cluster ID file exists in the HBase root directory
708    * @param fs the root directory FileSystem
709    * @param rootdir the HBase root directory in HDFS
710    * @param wait how long to wait between retries
711    * @return <code>true</code> if the file exists, otherwise <code>false</code>
712    * @throws IOException if checking the FileSystem fails
713    */
714   public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
715       int wait) throws IOException {
716     while (true) {
717       try {
718         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
719         return fs.exists(filePath);
720       } catch (IOException ioe) {
721         if (wait > 0) {
722           LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
723               ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
724           try {
725             Thread.sleep(wait);
726           } catch (InterruptedException ie) {
727             throw (InterruptedIOException)new InterruptedIOException().initCause(ie);
728           }
729         } else {
730           throw ioe;
731         }
732       }
733     }
734   }
735 
736   /**
737    * Returns the value of the unique cluster ID stored for this HBase instance.
738    * @param fs the root directory FileSystem
739    * @param rootdir the path to the HBase root directory
740    * @return the unique cluster identifier
741    * @throws IOException if reading the cluster ID file fails
742    */
743   public static ClusterId getClusterId(FileSystem fs, Path rootdir)
744   throws IOException {
745     Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
746     ClusterId clusterId = null;
747     FileStatus status = fs.exists(idPath)? fs.getFileStatus(idPath):  null;
748     if (status != null) {
749       int len = Ints.checkedCast(status.getLen());
750       byte [] content = new byte[len];
751       FSDataInputStream in = fs.open(idPath);
752       try {
753         in.readFully(content);
754       } catch (EOFException eof) {
755         LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
756       } finally{
757         in.close();
758       }
759       try {
760         clusterId = ClusterId.parseFrom(content);
761       } catch (DeserializationException e) {
762         throw new IOException("content=" + Bytes.toString(content), e);
763       }
764       // If not pb'd, make it so.
765       if (!ProtobufUtil.isPBMagicPrefix(content)) {
766         String cid = null;
767         in = fs.open(idPath);
768         try {
769           cid = in.readUTF();
770           clusterId = new ClusterId(cid);
771         } catch (EOFException eof) {
772           LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
773         } finally {
774           in.close();
775         }
776         rewriteAsPb(fs, rootdir, idPath, clusterId);
777       }
778       return clusterId;
779     } else {
780       LOG.warn("Cluster ID file does not exist at " + idPath.toString());
781     }
782     return clusterId;
783   }
784 
785   /**
786    * @param cid
787    * @throws IOException
788    */
789   private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
790       final ClusterId cid)
791   throws IOException {
792     // Rewrite the file as pb.  Move aside the old one first, write new
793     // then delete the moved-aside file.
794     Path movedAsideName = new Path(p + "." + System.currentTimeMillis());
795     if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
796     setClusterId(fs, rootdir, cid, 100);
797     if (!fs.delete(movedAsideName, false)) {
798       throw new IOException("Failed delete of " + movedAsideName);
799     }
800     LOG.debug("Rewrote the hbase.id file as pb");
801   }
802 
803   /**
804    * Writes a new unique identifier for this cluster to the "hbase.id" file
805    * in the HBase root directory
806    * @param fs the root directory FileSystem
807    * @param rootdir the path to the HBase root directory
808    * @param clusterId the unique identifier to store
809    * @param wait how long (in milliseconds) to wait between retries
810    * @throws IOException if writing to the FileSystem fails and no wait value
811    */
812   public static void setClusterId(FileSystem fs, Path rootdir, ClusterId clusterId,
813       int wait) throws IOException {
814     while (true) {
815       try {
816         Path idFile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
817         Path tempIdFile = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY +
818           Path.SEPARATOR + HConstants.CLUSTER_ID_FILE_NAME);
819         // Write the id file to a temporary location
820         FSDataOutputStream s = fs.create(tempIdFile);
821         try {
822           s.write(clusterId.toByteArray());
823           s.close();
824           s = null;
825           // Move the temporary file to its normal location. Throw an IOE if
826           // the rename failed
827           if (!fs.rename(tempIdFile, idFile)) {
828             throw new IOException("Unable to move temp version file to " + idFile);
829           }
830         } finally {
831           // Attempt to close the stream if still open on the way out
832           try {
833             if (s != null) s.close();
834           } catch (IOException ignore) { }
835         }
836         if (LOG.isDebugEnabled()) {
837           LOG.debug("Created cluster ID file at " + idFile.toString() + " with ID: " + clusterId);
838         }
839         return;
840       } catch (IOException ioe) {
841         if (wait > 0) {
842           LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
843               ", retrying in " + wait + "msec: " + StringUtils.stringifyException(ioe));
844           try {
845             Thread.sleep(wait);
846           } catch (InterruptedException ie) {
847             Thread.currentThread().interrupt();
848             break;
849           }
850         } else {
851           throw ioe;
852         }
853       }
854     }
855   }
856 
857   /**
858    * Verifies root directory path is a valid URI with a scheme
859    *
860    * @param root root directory path
861    * @return Passed <code>root</code> argument.
862    * @throws IOException if not a valid URI with a scheme
863    */
864   public static Path validateRootPath(Path root) throws IOException {
865     try {
866       URI rootURI = new URI(root.toString());
867       String scheme = rootURI.getScheme();
868       if (scheme == null) {
869         throw new IOException("Root directory does not have a scheme");
870       }
871       return root;
872     } catch (URISyntaxException e) {
873       IOException io = new IOException("Root directory path is not a valid " +
874         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
875       io.initCause(e);
876       throw io;
877     }
878   }
879 
880   /**
881    * Checks for the presence of the root path (using the provided conf object) in the given path. If
882    * it exists, this method removes it and returns the String representation of remaining relative path.
883    * @param path
884    * @param conf
885    * @return String representation of the remaining relative path
886    * @throws IOException
887    */
888   public static String removeRootPath(Path path, final Configuration conf) throws IOException {
889     Path root = FSUtils.getRootDir(conf);
890     String pathStr = path.toString();
891     // check that the path is absolute... it has the root path in it.
892     if (!pathStr.startsWith(root.toString())) return pathStr;
893     // if not, return as it is.
894     return pathStr.substring(root.toString().length() + 1);// remove the "/" too.
895   }
896 
897   /**
898    * If DFS, check safe mode and if so, wait until we clear it.
899    * @param conf configuration
900    * @param wait Sleep between retries
901    * @throws IOException e
902    */
903   public static void waitOnSafeMode(final Configuration conf,
904     final long wait)
905   throws IOException {
906     FileSystem fs = FileSystem.get(conf);
907     if (!(fs instanceof DistributedFileSystem)) return;
908     DistributedFileSystem dfs = (DistributedFileSystem)fs;
909     // Make sure dfs is not in safe mode
910     while (isInSafeMode(dfs)) {
911       LOG.info("Waiting for dfs to exit safe mode...");
912       try {
913         Thread.sleep(wait);
914       } catch (InterruptedException e) {
915         //continue
916       }
917     }
918   }
919 
920   /**
921    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
922    * method returns the 'path' component of a Path's URI: e.g. If a Path is
923    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
924    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
925    * This method is useful if you want to print out a Path without qualifying
926    * Filesystem instance.
927    * @param p Filesystem Path whose 'path' component we are to return.
928    * @return Path portion of the Filesystem
929    */
930   public static String getPath(Path p) {
931     return p.toUri().getPath();
932   }
933 
934   /**
935    * @param c configuration
936    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
937    * configuration as a qualified Path.
938    * @throws IOException e
939    */
940   public static Path getRootDir(final Configuration c) throws IOException {
941     Path p = new Path(c.get(HConstants.HBASE_DIR));
942     FileSystem fs = p.getFileSystem(c);
943     return p.makeQualified(fs);
944   }
945 
946   public static void setRootDir(final Configuration c, final Path root) throws IOException {
947     c.set(HConstants.HBASE_DIR, root.toString());
948   }
949 
950   public static void setFsDefault(final Configuration c, final Path root) throws IOException {
951     c.set("fs.defaultFS", root.toString());    // for hadoop 0.21+
952     c.set("fs.default.name", root.toString()); // for hadoop 0.20
953   }
954 
955   /**
956    * Checks if meta region exists
957    *
958    * @param fs file system
959    * @param rootdir root directory of HBase installation
960    * @return true if exists
961    * @throws IOException e
962    */
963   @SuppressWarnings("deprecation")
964   public static boolean metaRegionExists(FileSystem fs, Path rootdir)
965   throws IOException {
966     Path metaRegionDir =
967       HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO);
968     return fs.exists(metaRegionDir);
969   }
970 
971   /**
972    * Compute HDFS blocks distribution of a given file, or a portion of the file
973    * @param fs file system
974    * @param status file status of the file
975    * @param start start position of the portion
976    * @param length length of the portion
977    * @return The HDFS blocks distribution
978    */
979   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
980     final FileSystem fs, FileStatus status, long start, long length)
981     throws IOException {
982     HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
983     BlockLocation [] blockLocations =
984       fs.getFileBlockLocations(status, start, length);
985     for(BlockLocation bl : blockLocations) {
986       String [] hosts = bl.getHosts();
987       long len = bl.getLength();
988       blocksDistribution.addHostsAndBlockWeight(hosts, len);
989     }
990 
991     return blocksDistribution;
992   }
993 
994 
995 
996   /**
997    * Runs through the hbase rootdir and checks all stores have only
998    * one file in them -- that is, they've been major compacted.  Looks
999    * at root and meta tables too.
1000    * @param fs filesystem
1001    * @param hbaseRootDir hbase root directory
1002    * @return True if this hbase install is major compacted.
1003    * @throws IOException e
1004    */
1005   public static boolean isMajorCompacted(final FileSystem fs,
1006       final Path hbaseRootDir)
1007   throws IOException {
1008     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1009     PathFilter regionFilter = new RegionDirFilter(fs);
1010     PathFilter familyFilter = new FamilyDirFilter(fs);
1011     for (Path d : tableDirs) {
1012       FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
1013       for (FileStatus regionDir : regionDirs) {
1014         Path dd = regionDir.getPath();
1015         // Else its a region name.  Now look in region for families.
1016         FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1017         for (FileStatus familyDir : familyDirs) {
1018           Path family = familyDir.getPath();
1019           // Now in family make sure only one file.
1020           FileStatus[] familyStatus = fs.listStatus(family);
1021           if (familyStatus.length > 1) {
1022             LOG.debug(family.toString() + " has " + familyStatus.length +
1023                 " files.");
1024             return false;
1025           }
1026         }
1027       }
1028     }
1029     return true;
1030   }
1031 
1032   // TODO move this method OUT of FSUtils. No dependencies to HMaster
1033   /**
1034    * Returns the total overall fragmentation percentage. Includes hbase:meta and
1035    * -ROOT- as well.
1036    *
1037    * @param master  The master defining the HBase root and file system.
1038    * @return A map for each table and its percentage.
1039    * @throws IOException When scanning the directory fails.
1040    */
1041   public static int getTotalTableFragmentation(final HMaster master)
1042   throws IOException {
1043     Map<String, Integer> map = getTableFragmentation(master);
1044     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
1045   }
1046 
1047   /**
1048    * Runs through the HBase rootdir and checks how many stores for each table
1049    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
1050    * percentage across all tables is stored under the special key "-TOTAL-".
1051    *
1052    * @param master  The master defining the HBase root and file system.
1053    * @return A map for each table and its percentage.
1054    *
1055    * @throws IOException When scanning the directory fails.
1056    */
1057   public static Map<String, Integer> getTableFragmentation(
1058     final HMaster master)
1059   throws IOException {
1060     Path path = getRootDir(master.getConfiguration());
1061     // since HMaster.getFileSystem() is package private
1062     FileSystem fs = path.getFileSystem(master.getConfiguration());
1063     return getTableFragmentation(fs, path);
1064   }
1065 
1066   /**
1067    * Runs through the HBase rootdir and checks how many stores for each table
1068    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
1069    * percentage across all tables is stored under the special key "-TOTAL-".
1070    *
1071    * @param fs  The file system to use.
1072    * @param hbaseRootDir  The root directory to scan.
1073    * @return A map for each table and its percentage.
1074    * @throws IOException When scanning the directory fails.
1075    */
1076   public static Map<String, Integer> getTableFragmentation(
1077     final FileSystem fs, final Path hbaseRootDir)
1078   throws IOException {
1079     Map<String, Integer> frags = new HashMap<String, Integer>();
1080     int cfCountTotal = 0;
1081     int cfFragTotal = 0;
1082     PathFilter regionFilter = new RegionDirFilter(fs);
1083     PathFilter familyFilter = new FamilyDirFilter(fs);
1084     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1085     for (Path d : tableDirs) {
1086       int cfCount = 0;
1087       int cfFrag = 0;
1088       FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
1089       for (FileStatus regionDir : regionDirs) {
1090         Path dd = regionDir.getPath();
1091         // else its a region name, now look in region for families
1092         FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1093         for (FileStatus familyDir : familyDirs) {
1094           cfCount++;
1095           cfCountTotal++;
1096           Path family = familyDir.getPath();
1097           // now in family make sure only one file
1098           FileStatus[] familyStatus = fs.listStatus(family);
1099           if (familyStatus.length > 1) {
1100             cfFrag++;
1101             cfFragTotal++;
1102           }
1103         }
1104       }
1105       // compute percentage per table and store in result list
1106       frags.put(FSUtils.getTableName(d).getNameAsString(),
1107           Math.round((float) cfFrag / cfCount * 100));
1108     }
1109     // set overall percentage for all tables
1110     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
1111     return frags;
1112   }
1113 
1114   /**
1115    * Returns the {@link org.apache.hadoop.fs.Path} object representing the table directory under
1116    * path rootdir
1117    *
1118    * @param rootdir qualified path of HBase root directory
1119    * @param tableName name of table
1120    * @return {@link org.apache.hadoop.fs.Path} for table
1121    */
1122   public static Path getTableDir(Path rootdir, final TableName tableName) {
1123     return new Path(getNamespaceDir(rootdir, tableName.getNamespaceAsString()),
1124         tableName.getQualifierAsString());
1125   }
1126 
1127   /**
1128    * Returns the {@link org.apache.hadoop.hbase.TableName} object representing
1129    * the table directory under
1130    * path rootdir
1131    *
1132    * @param tablePath path of table
1133    * @return {@link org.apache.hadoop.fs.Path} for table
1134    */
1135   public static TableName getTableName(Path tablePath) {
1136     return TableName.valueOf(tablePath.getParent().getName(), tablePath.getName());
1137   }
1138 
1139   /**
1140    * Returns the {@link org.apache.hadoop.fs.Path} object representing
1141    * the namespace directory under path rootdir
1142    *
1143    * @param rootdir qualified path of HBase root directory
1144    * @param namespace namespace name
1145    * @return {@link org.apache.hadoop.fs.Path} for table
1146    */
1147   public static Path getNamespaceDir(Path rootdir, final String namespace) {
1148     return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
1149         new Path(namespace)));
1150   }
1151 
1152   /**
1153    * A {@link PathFilter} that returns only regular files.
1154    */
1155   static class FileFilter implements PathFilter {
1156     private final FileSystem fs;
1157 
1158     public FileFilter(final FileSystem fs) {
1159       this.fs = fs;
1160     }
1161 
1162     @Override
1163     public boolean accept(Path p) {
1164       try {
1165         return fs.isFile(p);
1166       } catch (IOException e) {
1167         LOG.debug("unable to verify if path=" + p + " is a regular file", e);
1168         return false;
1169       }
1170     }
1171   }
1172 
1173   /**
1174    * Directory filter that doesn't include any of the directories in the specified blacklist
1175    */
1176   public static class BlackListDirFilter implements PathFilter {
1177     private final FileSystem fs;
1178     private List<String> blacklist;
1179 
1180     /**
1181      * Create a filter on the give filesystem with the specified blacklist
1182      * @param fs filesystem to filter
1183      * @param directoryNameBlackList list of the names of the directories to filter. If
1184      *          <tt>null</tt>, all directories are returned
1185      */
1186     @SuppressWarnings("unchecked")
1187     public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {
1188       this.fs = fs;
1189       blacklist =
1190         (List<String>) (directoryNameBlackList == null ? Collections.emptyList()
1191           : directoryNameBlackList);
1192     }
1193 
1194     @Override
1195     public boolean accept(Path p) {
1196       boolean isValid = false;
1197       try {
1198         if (isValidName(p.getName())) {
1199           isValid = fs.getFileStatus(p).isDir();
1200         } else {
1201           isValid = false;
1202         }
1203       } catch (IOException e) {
1204         LOG.warn("An error occurred while verifying if [" + p.toString()
1205             + "] is a valid directory. Returning 'not valid' and continuing.", e);
1206       }
1207       return isValid;
1208     }
1209 
1210     protected boolean isValidName(final String name) {
1211       return !blacklist.contains(name);
1212     }
1213   }
1214 
1215   /**
1216    * A {@link PathFilter} that only allows directories.
1217    */
1218   public static class DirFilter extends BlackListDirFilter {
1219 
1220     public DirFilter(FileSystem fs) {
1221       super(fs, null);
1222     }
1223   }
1224 
1225   /**
1226    * A {@link PathFilter} that returns usertable directories. To get all directories use the
1227    * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
1228    */
1229   public static class UserTableDirFilter extends BlackListDirFilter {
1230     public UserTableDirFilter(FileSystem fs) {
1231       super(fs, HConstants.HBASE_NON_TABLE_DIRS);
1232     }
1233 
1234     protected boolean isValidName(final String name) {
1235       if (!super.isValidName(name))
1236         return false;
1237 
1238       try {
1239         TableName.isLegalTableQualifierName(Bytes.toBytes(name));
1240       } catch (IllegalArgumentException e) {
1241         LOG.info("INVALID NAME " + name);
1242         return false;
1243       }
1244       return true;
1245     }
1246   }
1247 
1248   /**
1249    * Heuristic to determine whether is safe or not to open a file for append
1250    * Looks both for dfs.support.append and use reflection to search
1251    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
1252    * @param conf
1253    * @return True if append support
1254    */
1255   public static boolean isAppendSupported(final Configuration conf) {
1256     boolean append = conf.getBoolean("dfs.support.append", false);
1257     if (append) {
1258       try {
1259         // TODO: The implementation that comes back when we do a createWriter
1260         // may not be using SequenceFile so the below is not a definitive test.
1261         // Will do for now (hdfs-200).
1262         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
1263         append = true;
1264       } catch (SecurityException e) {
1265       } catch (NoSuchMethodException e) {
1266         append = false;
1267       }
1268     }
1269     if (!append) {
1270       // Look for the 0.21, 0.22, new-style append evidence.
1271       try {
1272         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1273         append = true;
1274       } catch (NoSuchMethodException e) {
1275         append = false;
1276       }
1277     }
1278     return append;
1279   }
1280 
1281   /**
1282    * @param conf
1283    * @return True if this filesystem whose scheme is 'hdfs'.
1284    * @throws IOException
1285    */
1286   public static boolean isHDFS(final Configuration conf) throws IOException {
1287     FileSystem fs = FileSystem.get(conf);
1288     String scheme = fs.getUri().getScheme();
1289     return scheme.equalsIgnoreCase("hdfs");
1290   }
1291 
1292   /**
1293    * Recover file lease. Used when a file might be suspect
1294    * to be had been left open by another process.
1295    * @param fs FileSystem handle
1296    * @param p Path of file to recover lease
1297    * @param conf Configuration handle
1298    * @throws IOException
1299    */
1300   public abstract void recoverFileLease(final FileSystem fs, final Path p,
1301       Configuration conf, CancelableProgressable reporter) throws IOException;
1302 
1303   public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1304       throws IOException {
1305     List<Path> tableDirs = new LinkedList<Path>();
1306 
1307     for(FileStatus status :
1308         fs.globStatus(new Path(rootdir,
1309             new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) {
1310       tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
1311     }
1312     return tableDirs;
1313   }
1314 
1315   /**
1316    * @param fs
1317    * @param rootdir
1318    * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as
1319    * .logs, .oldlogs, .corrupt folders.
1320    * @throws IOException
1321    */
1322   public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
1323       throws IOException {
1324     // presumes any directory under hbase.rootdir is a table
1325     FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
1326     List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1327     for (FileStatus dir: dirs) {
1328       tabledirs.add(dir.getPath());
1329     }
1330     return tabledirs;
1331   }
1332 
1333   /**
1334    * Checks if the given path is the one with 'recovered.edits' dir.
1335    * @param path
1336    * @return True if we recovered edits
1337    */
1338   public static boolean isRecoveredEdits(Path path) {
1339     return path.toString().contains(HConstants.RECOVERED_EDITS_DIR);
1340   }
1341 
1342   /**
1343    * Filter for all dirs that don't start with '.'
1344    */
1345   public static class RegionDirFilter implements PathFilter {
1346     // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
1347     final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1348     final FileSystem fs;
1349 
1350     public RegionDirFilter(FileSystem fs) {
1351       this.fs = fs;
1352     }
1353 
1354     @Override
1355     public boolean accept(Path rd) {
1356       if (!regionDirPattern.matcher(rd.getName()).matches()) {
1357         return false;
1358       }
1359 
1360       try {
1361         return fs.getFileStatus(rd).isDir();
1362       } catch (IOException ioe) {
1363         // Maybe the file was moved or the fs was disconnected.
1364         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1365         return false;
1366       }
1367     }
1368   }
1369 
1370   /**
1371    * Given a particular table dir, return all the regiondirs inside it, excluding files such as
1372    * .tableinfo
1373    * @param fs A file system for the Path
1374    * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
1375    * @return List of paths to valid region directories in table dir.
1376    * @throws IOException
1377    */
1378   public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1379     // assumes we are in a table dir.
1380     FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1381     List<Path> regionDirs = new ArrayList<Path>(rds.length);
1382     for (FileStatus rdfs: rds) {
1383       Path rdPath = rdfs.getPath();
1384       regionDirs.add(rdPath);
1385     }
1386     return regionDirs;
1387   }
1388 
1389   /**
1390    * Filter for all dirs that are legal column family names.  This is generally used for colfam
1391    * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
1392    */
1393   public static class FamilyDirFilter implements PathFilter {
1394     final FileSystem fs;
1395 
1396     public FamilyDirFilter(FileSystem fs) {
1397       this.fs = fs;
1398     }
1399 
1400     @Override
1401     public boolean accept(Path rd) {
1402       try {
1403         // throws IAE if invalid
1404         HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1405       } catch (IllegalArgumentException iae) {
1406         // path name is an invalid family name and thus is excluded.
1407         return false;
1408       }
1409 
1410       try {
1411         return fs.getFileStatus(rd).isDir();
1412       } catch (IOException ioe) {
1413         // Maybe the file was moved or the fs was disconnected.
1414         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1415         return false;
1416       }
1417     }
1418   }
1419 
1420   /**
1421    * Given a particular region dir, return all the familydirs inside it
1422    *
1423    * @param fs A file system for the Path
1424    * @param regionDir Path to a specific region directory
1425    * @return List of paths to valid family directories in region dir.
1426    * @throws IOException
1427    */
1428   public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1429     // assumes we are in a region dir.
1430     FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1431     List<Path> familyDirs = new ArrayList<Path>(fds.length);
1432     for (FileStatus fdfs: fds) {
1433       Path fdPath = fdfs.getPath();
1434       familyDirs.add(fdPath);
1435     }
1436     return familyDirs;
1437   }
1438 
1439   public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir) throws IOException {
1440     FileStatus[] fds = fs.listStatus(familyDir, new ReferenceFileFilter(fs));
1441     List<Path> referenceFiles = new ArrayList<Path>(fds.length);
1442     for (FileStatus fdfs: fds) {
1443       Path fdPath = fdfs.getPath();
1444       referenceFiles.add(fdPath);
1445     }
1446     return referenceFiles;
1447   }
1448 
1449   /**
1450    * Filter for HFiles that excludes reference files.
1451    */
1452   public static class HFileFilter implements PathFilter {
1453     final FileSystem fs;
1454 
1455     public HFileFilter(FileSystem fs) {
1456       this.fs = fs;
1457     }
1458 
1459     @Override
1460     public boolean accept(Path rd) {
1461       try {
1462         // only files
1463         return !fs.getFileStatus(rd).isDir() && StoreFileInfo.isHFile(rd);
1464       } catch (IOException ioe) {
1465         // Maybe the file was moved or the fs was disconnected.
1466         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1467         return false;
1468       }
1469     }
1470   }
1471 
1472   public static class ReferenceFileFilter implements PathFilter {
1473 
1474     private final FileSystem fs;
1475 
1476     public ReferenceFileFilter(FileSystem fs) {
1477       this.fs = fs;
1478     }
1479 
1480     @Override
1481     public boolean accept(Path rd) {
1482       try {
1483         // only files can be references.
1484         return !fs.getFileStatus(rd).isDir() && StoreFileInfo.isReference(rd);
1485       } catch (IOException ioe) {
1486         // Maybe the file was moved or the fs was disconnected.
1487         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1488         return false;
1489       }
1490     }
1491   }
1492 
1493 
1494   /**
1495    * @param conf
1496    * @return Returns the filesystem of the hbase rootdir.
1497    * @throws IOException
1498    */
1499   public static FileSystem getCurrentFileSystem(Configuration conf)
1500   throws IOException {
1501     return getRootDir(conf).getFileSystem(conf);
1502   }
1503 
1504 
1505   /**
1506    * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
1507    * table StoreFile names to the full Path.
1508    * <br>
1509    * Example...<br>
1510    * Key = 3944417774205889744  <br>
1511    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1512    *
1513    * @param map map to add values.  If null, this method will create and populate one to return
1514    * @param fs  The file system to use.
1515    * @param hbaseRootDir  The root directory to scan.
1516    * @param tableName name of the table to scan.
1517    * @return Map keyed by StoreFile name with a value of the full Path.
1518    * @throws IOException When scanning the directory fails.
1519    */
1520   public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1521   final FileSystem fs, final Path hbaseRootDir, TableName tableName)
1522   throws IOException {
1523     if (map == null) {
1524       map = new HashMap<String, Path>();
1525     }
1526 
1527     // only include the directory paths to tables
1528     Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
1529     // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
1530     // should be regions.
1531     PathFilter familyFilter = new FamilyDirFilter(fs);
1532     FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs));
1533     for (FileStatus regionDir : regionDirs) {
1534       Path dd = regionDir.getPath();
1535       // else its a region name, now look in region for families
1536       FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1537       for (FileStatus familyDir : familyDirs) {
1538         Path family = familyDir.getPath();
1539         // now in family, iterate over the StoreFiles and
1540         // put in map
1541         FileStatus[] familyStatus = fs.listStatus(family);
1542         for (FileStatus sfStatus : familyStatus) {
1543           Path sf = sfStatus.getPath();
1544           map.put( sf.getName(), sf);
1545         }
1546       }
1547     }
1548     return map;
1549   }
1550 
1551   public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
1552     int result = 0;
1553     try {
1554       for (Path familyDir:getFamilyDirs(fs, p)){
1555         result += getReferenceFilePaths(fs, familyDir).size();
1556       }
1557     } catch (IOException e) {
1558       LOG.warn("Error Counting reference files.", e);
1559     }
1560     return result;
1561   }
1562 
1563 
1564   /**
1565    * Runs through the HBase rootdir and creates a reverse lookup map for
1566    * table StoreFile names to the full Path.
1567    * <br>
1568    * Example...<br>
1569    * Key = 3944417774205889744  <br>
1570    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1571    *
1572    * @param fs  The file system to use.
1573    * @param hbaseRootDir  The root directory to scan.
1574    * @return Map keyed by StoreFile name with a value of the full Path.
1575    * @throws IOException When scanning the directory fails.
1576    */
1577   public static Map<String, Path> getTableStoreFilePathMap(
1578     final FileSystem fs, final Path hbaseRootDir)
1579   throws IOException {
1580     Map<String, Path> map = new HashMap<String, Path>();
1581 
1582     // if this method looks similar to 'getTableFragmentation' that is because
1583     // it was borrowed from it.
1584 
1585     // only include the directory paths to tables
1586     for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
1587       getTableStoreFilePathMap(map, fs, hbaseRootDir,
1588           FSUtils.getTableName(tableDir));
1589     }
1590     return map;
1591   }
1592 
1593   /**
1594    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1595    * This accommodates differences between hadoop versions, where hadoop 1
1596    * does not throw a FileNotFoundException, and return an empty FileStatus[]
1597    * while Hadoop 2 will throw FileNotFoundException.
1598    *
1599    * @param fs file system
1600    * @param dir directory
1601    * @param filter path filter
1602    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1603    */
1604   public static FileStatus [] listStatus(final FileSystem fs,
1605       final Path dir, final PathFilter filter) throws IOException {
1606     FileStatus [] status = null;
1607     try {
1608       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1609     } catch (FileNotFoundException fnfe) {
1610       // if directory doesn't exist, return null
1611       if (LOG.isTraceEnabled()) {
1612         LOG.trace(dir + " doesn't exist");
1613       }
1614     }
1615     if (status == null || status.length < 1) return null;
1616     return status;
1617   }
1618 
1619   /**
1620    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1621    * This would accommodates differences between hadoop versions
1622    *
1623    * @param fs file system
1624    * @param dir directory
1625    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1626    */
1627   public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1628     return listStatus(fs, dir, null);
1629   }
1630 
1631   /**
1632    * Calls fs.delete() and returns the value returned by the fs.delete()
1633    *
1634    * @param fs
1635    * @param path
1636    * @param recursive
1637    * @return the value returned by the fs.delete()
1638    * @throws IOException
1639    */
1640   public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1641       throws IOException {
1642     return fs.delete(path, recursive);
1643   }
1644 
1645   /**
1646    * Calls fs.exists(). Checks if the specified path exists
1647    *
1648    * @param fs
1649    * @param path
1650    * @return the value returned by fs.exists()
1651    * @throws IOException
1652    */
1653   public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1654     return fs.exists(path);
1655   }
1656 
1657   /**
1658    * Throw an exception if an action is not permitted by a user on a file.
1659    *
1660    * @param ugi
1661    *          the user
1662    * @param file
1663    *          the file
1664    * @param action
1665    *          the action
1666    */
1667   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
1668       FsAction action) throws AccessDeniedException {
1669     if (ugi.getShortUserName().equals(file.getOwner())) {
1670       if (file.getPermission().getUserAction().implies(action)) {
1671         return;
1672       }
1673     } else if (contains(ugi.getGroupNames(), file.getGroup())) {
1674       if (file.getPermission().getGroupAction().implies(action)) {
1675         return;
1676       }
1677     } else if (file.getPermission().getOtherAction().implies(action)) {
1678       return;
1679     }
1680     throw new AccessDeniedException("Permission denied:" + " action=" + action
1681         + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
1682   }
1683 
1684   private static boolean contains(String[] groups, String user) {
1685     for (String group : groups) {
1686       if (group.equals(user)) {
1687         return true;
1688       }
1689     }
1690     return false;
1691   }
1692 
1693   /**
1694    * Log the current state of the filesystem from a certain root directory
1695    * @param fs filesystem to investigate
1696    * @param root root file/directory to start logging from
1697    * @param LOG log to output information
1698    * @throws IOException if an unexpected exception occurs
1699    */
1700   public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1701       throws IOException {
1702     LOG.debug("Current file system:");
1703     logFSTree(LOG, fs, root, "|-");
1704   }
1705 
1706   /**
1707    * Recursive helper to log the state of the FS
1708    *
1709    * @see #logFileSystemState(FileSystem, Path, Log)
1710    */
1711   private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1712       throws IOException {
1713     FileStatus[] files = FSUtils.listStatus(fs, root, null);
1714     if (files == null) return;
1715 
1716     for (FileStatus file : files) {
1717       if (file.isDir()) {
1718         LOG.debug(prefix + file.getPath().getName() + "/");
1719         logFSTree(LOG, fs, file.getPath(), prefix + "---");
1720       } else {
1721         LOG.debug(prefix + file.getPath().getName());
1722       }
1723     }
1724   }
1725 
1726   public static boolean renameAndSetModifyTime(final FileSystem fs, final Path src, final Path dest)
1727       throws IOException {
1728     // set the modify time for TimeToLive Cleaner
1729     fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
1730     return fs.rename(src, dest);
1731   }
1732 
1733   /**
1734    * This function is to scan the root path of the file system to get the
1735    * degree of locality for each region on each of the servers having at least
1736    * one block of that region.
1737    * This is used by the tool {@link RegionPlacementMaintainer}
1738    *
1739    * @param conf
1740    *          the configuration to use
1741    * @return the mapping from region encoded name to a map of server names to
1742    *           locality fraction
1743    * @throws IOException
1744    *           in case of file system errors or interrupts
1745    */
1746   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1747       final Configuration conf) throws IOException {
1748     return getRegionDegreeLocalityMappingFromFS(
1749         conf, null,
1750         conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));
1751 
1752   }
1753 
1754   /**
1755    * This function is to scan the root path of the file system to get the
1756    * degree of locality for each region on each of the servers having at least
1757    * one block of that region.
1758    *
1759    * @param conf
1760    *          the configuration to use
1761    * @param desiredTable
1762    *          the table you wish to scan locality for
1763    * @param threadPoolSize
1764    *          the thread pool size to use
1765    * @return the mapping from region encoded name to a map of server names to
1766    *           locality fraction
1767    * @throws IOException
1768    *           in case of file system errors or interrupts
1769    */
1770   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1771       final Configuration conf, final String desiredTable, int threadPoolSize)
1772       throws IOException {
1773     Map<String, Map<String, Float>> regionDegreeLocalityMapping =
1774         new ConcurrentHashMap<String, Map<String, Float>>();
1775     getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, null,
1776         regionDegreeLocalityMapping);
1777     return regionDegreeLocalityMapping;
1778   }
1779 
1780   /**
1781    * This function is to scan the root path of the file system to get either the
1782    * mapping between the region name and its best locality region server or the
1783    * degree of locality of each region on each of the servers having at least
1784    * one block of that region. The output map parameters are both optional.
1785    *
1786    * @param conf
1787    *          the configuration to use
1788    * @param desiredTable
1789    *          the table you wish to scan locality for
1790    * @param threadPoolSize
1791    *          the thread pool size to use
1792    * @param regionToBestLocalityRSMapping
1793    *          the map into which to put the best locality mapping or null
1794    * @param regionDegreeLocalityMapping
1795    *          the map into which to put the locality degree mapping or null,
1796    *          must be a thread-safe implementation
1797    * @throws IOException
1798    *           in case of file system errors or interrupts
1799    */
1800   private static void getRegionLocalityMappingFromFS(
1801       final Configuration conf, final String desiredTable,
1802       int threadPoolSize,
1803       Map<String, String> regionToBestLocalityRSMapping,
1804       Map<String, Map<String, Float>> regionDegreeLocalityMapping)
1805       throws IOException {
1806     FileSystem fs =  FileSystem.get(conf);
1807     Path rootPath = FSUtils.getRootDir(conf);
1808     long startTime = EnvironmentEdgeManager.currentTimeMillis();
1809     Path queryPath;
1810     // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
1811     if (null == desiredTable) {
1812       queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
1813     } else {
1814       queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
1815     }
1816 
1817     // reject all paths that are not appropriate
1818     PathFilter pathFilter = new PathFilter() {
1819       @Override
1820       public boolean accept(Path path) {
1821         // this is the region name; it may get some noise data
1822         if (null == path) {
1823           return false;
1824         }
1825 
1826         // no parent?
1827         Path parent = path.getParent();
1828         if (null == parent) {
1829           return false;
1830         }
1831 
1832         String regionName = path.getName();
1833         if (null == regionName) {
1834           return false;
1835         }
1836 
1837         if (!regionName.toLowerCase().matches("[0-9a-f]+")) {
1838           return false;
1839         }
1840         return true;
1841       }
1842     };
1843 
1844     FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
1845 
1846     if (null == statusList) {
1847       return;
1848     } else {
1849       LOG.debug("Query Path: " + queryPath + " ; # list of files: " +
1850           statusList.length);
1851     }
1852 
1853     // lower the number of threads in case we have very few expected regions
1854     threadPoolSize = Math.min(threadPoolSize, statusList.length);
1855 
1856     // run in multiple threads
1857     ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize,
1858         threadPoolSize, 60, TimeUnit.SECONDS,
1859         new ArrayBlockingQueue<Runnable>(statusList.length));
1860     try {
1861       // ignore all file status items that are not of interest
1862       for (FileStatus regionStatus : statusList) {
1863         if (null == regionStatus) {
1864           continue;
1865         }
1866 
1867         if (!regionStatus.isDir()) {
1868           continue;
1869         }
1870 
1871         Path regionPath = regionStatus.getPath();
1872         if (null == regionPath) {
1873           continue;
1874         }
1875 
1876         tpe.execute(new FSRegionScanner(fs, regionPath,
1877             regionToBestLocalityRSMapping, regionDegreeLocalityMapping));
1878       }
1879     } finally {
1880       tpe.shutdown();
1881       int threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
1882           60 * 1000);
1883       try {
1884         // here we wait until TPE terminates, which is either naturally or by
1885         // exceptions in the execution of the threads
1886         while (!tpe.awaitTermination(threadWakeFrequency,
1887             TimeUnit.MILLISECONDS)) {
1888           // printing out rough estimate, so as to not introduce
1889           // AtomicInteger
1890           LOG.info("Locality checking is underway: { Scanned Regions : "
1891               + tpe.getCompletedTaskCount() + "/"
1892               + tpe.getTaskCount() + " }");
1893         }
1894       } catch (InterruptedException e) {
1895         throw (InterruptedIOException)new InterruptedIOException().initCause(e);
1896       }
1897     }
1898 
1899     long overhead = EnvironmentEdgeManager.currentTimeMillis() - startTime;
1900     String overheadMsg = "Scan DFS for locality info takes " + overhead + " ms";
1901 
1902     LOG.info(overheadMsg);
1903   }
1904 
1905   /**
1906    * Do our short circuit read setup.
1907    * Checks buffer size to use and whether to do checksumming in hbase or hdfs.
1908    * @param conf
1909    */
1910   public static void setupShortCircuitRead(final Configuration conf) {
1911     // Check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.
1912     boolean shortCircuitSkipChecksum =
1913       conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);
1914     boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1915     if (shortCircuitSkipChecksum) {
1916       LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " +
1917         "be set to true." + (useHBaseChecksum ? " HBase checksum doesn't require " +
1918         "it, see https://issues.apache.org/jira/browse/HBASE-6868." : ""));
1919       assert !shortCircuitSkipChecksum; //this will fail if assertions are on
1920     }
1921     checkShortCircuitReadBufferSize(conf);
1922   }
1923 
1924   /**
1925    * Check if short circuit read buffer size is set and if not, set it to hbase value.
1926    * @param conf
1927    */
1928   public static void checkShortCircuitReadBufferSize(final Configuration conf) {
1929     final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;
1930     final int notSet = -1;
1931     // DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY is only defined in h2
1932     final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";
1933     int size = conf.getInt(dfsKey, notSet);
1934     // If a size is set, return -- we will use it.
1935     if (size != notSet) return;
1936     // But short circuit buffer size is normally not set.  Put in place the hbase wanted size.
1937     int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);
1938     conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));
1939   }
1940 }