View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.NavigableSet;
25  import java.util.TreeSet;
26  import java.util.UUID;
27  import java.util.concurrent.atomic.AtomicLong;
28  import java.util.regex.Matcher;
29  import java.util.regex.Pattern;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
45  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  
48  import com.google.protobuf.TextFormat;
49  
50  @InterfaceAudience.Private
51  public class HLogUtil {
52    static final Log LOG = LogFactory.getLog(HLogUtil.class);
53  
54    /**
55     * Pattern used to validate a HLog file name
56     */
57    private static final Pattern pattern =
58        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
59  
60    /**
61     * @param filename
62     *          name of the file to validate
63     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
64     *         otherwise
65     */
66    public static boolean validateHLogFilename(String filename) {
67      return pattern.matcher(filename).matches();
68    }
69  
70    /**
71     * Construct the HLog directory name
72     *
73     * @param serverName
74     *          Server name formatted as described in {@link ServerName}
75     * @return the relative HLog directory name, e.g.
76     *         <code>.logs/1.example.org,60030,12345</code> if
77     *         <code>serverName</code> passed is
78     *         <code>1.example.org,60030,12345</code>
79     */
80    public static String getHLogDirectoryName(final String serverName) {
81      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
82      dirName.append("/");
83      dirName.append(serverName);
84      return dirName.toString();
85    }
86  
87    /**
88     * @param regiondir
89     *          This regions directory in the filesystem.
90     * @return The directory that holds recovered edits files for the region
91     *         <code>regiondir</code>
92     */
93    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
94      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
95    }
96  
97    /**
98     * Move aside a bad edits file.
99     *
100    * @param fs
101    * @param edits
102    *          Edits file to move aside.
103    * @return The name of the moved aside file.
104    * @throws IOException
105    */
106   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
107       throws IOException {
108     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
109         + System.currentTimeMillis());
110     if (!fs.rename(edits, moveAsideName)) {
111       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
112     }
113     return moveAsideName;
114   }
115 
116   /**
117    * @param path
118    *          - the path to analyze. Expected format, if it's in hlog directory:
119    *          / [base directory for hbase] / hbase / .logs / ServerName /
120    *          logfile
121    * @return null if it's not a log file. Returns the ServerName of the region
122    *         server that created this log file otherwise.
123    */
124   public static ServerName getServerNameFromHLogDirectoryName(
125       Configuration conf, String path) throws IOException {
126     if (path == null
127         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
128       return null;
129     }
130 
131     if (conf == null) {
132       throw new IllegalArgumentException("parameter conf must be set");
133     }
134 
135     final String rootDir = conf.get(HConstants.HBASE_DIR);
136     if (rootDir == null || rootDir.isEmpty()) {
137       throw new IllegalArgumentException(HConstants.HBASE_DIR
138           + " key not found in conf.");
139     }
140 
141     final StringBuilder startPathSB = new StringBuilder(rootDir);
142     if (!rootDir.endsWith("/"))
143       startPathSB.append('/');
144     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
145     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
146       startPathSB.append('/');
147     final String startPath = startPathSB.toString();
148 
149     String fullPath;
150     try {
151       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
152     } catch (IllegalArgumentException e) {
153       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
154       return null;
155     }
156 
157     if (!fullPath.startsWith(startPath)) {
158       return null;
159     }
160 
161     final String serverNameAndFile = fullPath.substring(startPath.length());
162 
163     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
164       // Either it's a file (not a directory) or it's not a ServerName format
165       return null;
166     }
167 
168     Path p = new Path(path);
169     return getServerNameFromHLogDirectoryName(p);
170   }
171 
172   /**
173    * This function returns region server name from a log file name which is in either format:
174    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
175    * node>/hbase/.logs/<server name>/...
176    * @param logFile
177    * @return null if the passed in logFile isn't a valid HLog file path
178    */
179   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
180     Path logDir = logFile.getParent();
181     String logDirName = logDir.getName();
182     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
183       logDir = logFile;
184       logDirName = logDir.getName();
185     }
186     ServerName serverName = null;
187     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
188       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
189     }
190     try {
191       serverName = ServerName.parseServerName(logDirName);
192     } catch (IllegalArgumentException ex) {
193       serverName = null;
194       LOG.warn("Invalid log file path=" + logFile, ex);
195     }
196     if (serverName != null && serverName.getStartcode() < 0) {
197       LOG.warn("Invalid log file path=" + logFile);
198       return null;
199     }
200     return serverName;
201   }
202 
203   /**
204    * Returns sorted set of edit files made by wal-log splitter, excluding files
205    * with '.temp' suffix.
206    *
207    * @param fs
208    * @param regiondir
209    * @return Files in passed <code>regiondir</code> as a sorted set.
210    * @throws IOException
211    */
212   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
213       final Path regiondir) throws IOException {
214     NavigableSet<Path> filesSorted = new TreeSet<Path>();
215     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
216     if (!fs.exists(editsdir))
217       return filesSorted;
218     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
219       @Override
220       public boolean accept(Path p) {
221         boolean result = false;
222         try {
223           // Return files and only files that match the editfile names pattern.
224           // There can be other files in this directory other than edit files.
225           // In particular, on error, we'll move aside the bad edit file giving
226           // it a timestamp suffix. See moveAsideBadEditsFile.
227           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
228           result = fs.isFile(p) && m.matches();
229           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
230           // because it means splithlog thread is writting this file.
231           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
232             result = false;
233           }
234         } catch (IOException e) {
235           LOG.warn("Failed isFile check on " + p);
236         }
237         return result;
238       }
239     });
240     if (files == null)
241       return filesSorted;
242     for (FileStatus status : files) {
243       filesSorted.add(status.getPath());
244     }
245     return filesSorted;
246   }
247 
248   public static boolean isMetaFile(Path p) {
249     return isMetaFile(p.getName());
250   }
251 
252   public static boolean isMetaFile(String p) {
253     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
254       return true;
255     }
256     return false;
257   }
258 
259   /**
260    * Write the marker that a compaction has succeeded and is about to be committed.
261    * This provides info to the HMaster to allow it to recover the compaction if
262    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
263    * the compaction from finishing if this regionserver has already lost its lease on the log.
264    * @param sequenceId Used by HLog to get sequence Id for the waledit.
265    */
266   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
267       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
268     WALEdit e = WALEdit.createCompaction(info, c);
269     long now = EnvironmentEdgeManager.currentTimeMillis();
270     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
271     long txid = log.appendNoSync(info, tn, e, new ArrayList<UUID>(), now, htd, sequenceId,
272         false, HConstants.NO_NONCE, HConstants.NO_NONCE);
273     log.sync(txid);
274 
275     if (LOG.isTraceEnabled()) {
276       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
277     }
278   }
279 }