View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.List;
26  import java.util.UUID;
27  import java.util.concurrent.atomic.AtomicLong;
28  import java.util.regex.Pattern;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FSDataInputStream;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
42  import org.apache.hadoop.io.Writable;
43  
44  import com.google.common.annotations.VisibleForTesting;
45  
46  
47  @InterfaceAudience.Private
48  // TODO: Rename interface to WAL
49  public interface HLog {
50    Log LOG = LogFactory.getLog(HLog.class);
51  
52    /** File Extension used while splitting an HLog into regions (HBASE-2312) */
53    // TODO: this seems like an implementation detail that does not belong here.
54    String SPLITTING_EXT = "-splitting";
55    boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
56    /** The hbase:meta region's HLog filename extension */
57    String META_HLOG_FILE_EXTN = ".meta";
58  
59    /**
60     * Configuration name of HLog Trailer's warning size. If a waltrailer's size is greater than the
61     * configured size, a warning is logged. This is used with Protobuf reader/writer.
62     */
63    // TODO: Implementation detail.  Why in here?
64    String WAL_TRAILER_WARN_SIZE = "hbase.regionserver.waltrailer.warn.size";
65    int DEFAULT_WAL_TRAILER_WARN_SIZE = 1024 * 1024; // 1MB
66  
67    // TODO: Implemenation detail.  Why in here?
68    Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");
69    String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
70  
71    interface Reader {
72  
73      /**
74       * @param fs File system.
75       * @param path Path.
76       * @param c Configuration.
77       * @param s Input stream that may have been pre-opened by the caller; may be null.
78       */
79      void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
80  
81      void close() throws IOException;
82  
83      Entry next() throws IOException;
84  
85      Entry next(Entry reuse) throws IOException;
86  
87      void seek(long pos) throws IOException;
88  
89      long getPosition() throws IOException;
90      void reset() throws IOException;
91  
92      /**
93       * @return the WALTrailer of the current HLog. It may be null in case of legacy or corrupt WAL
94       *         files.
95       */
96      // TODO: What we need a trailer on WAL for?
97      WALTrailer getWALTrailer();
98    }
99  
100   interface Writer {
101     void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
102 
103     void close() throws IOException;
104 
105     void sync() throws IOException;
106 
107     void append(Entry entry) throws IOException;
108 
109     long getLength() throws IOException;
110 
111     /**
112      * Sets HLog's WALTrailer. This trailer is appended at the end of WAL on closing.
113      * @param walTrailer trailer to append to WAL.
114      */
115     void setWALTrailer(WALTrailer walTrailer);
116   }
117 
118   /**
119    * Utility class that lets us keep track of the edit with it's key.
120    * Only used when splitting logs.
121    */
122   // TODO: Remove this Writable.
123   @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
124   class Entry implements Writable {
125     private WALEdit edit;
126     private HLogKey key;
127 
128     public Entry() {
129       edit = new WALEdit();
130       key = new HLogKey();
131     }
132 
133     /**
134      * Constructor for both params
135      *
136      * @param edit log's edit
137      * @param key log's key
138      */
139     public Entry(HLogKey key, WALEdit edit) {
140       super();
141       this.key = key;
142       this.edit = edit;
143     }
144 
145     /**
146      * Gets the edit
147      *
148      * @return edit
149      */
150     public WALEdit getEdit() {
151       return edit;
152     }
153 
154     /**
155      * Gets the key
156      *
157      * @return key
158      */
159     public HLogKey getKey() {
160       return key;
161     }
162 
163     /**
164      * Set compression context for this entry.
165      *
166      * @param compressionContext
167      *          Compression context
168      */
169     public void setCompressionContext(CompressionContext compressionContext) {
170       edit.setCompressionContext(compressionContext);
171       key.setCompressionContext(compressionContext);
172     }
173 
174     @Override
175     public String toString() {
176       return this.key + "=" + this.edit;
177     }
178 
179     @Override
180     @SuppressWarnings("deprecation")
181     public void write(DataOutput dataOutput) throws IOException {
182       this.key.write(dataOutput);
183       this.edit.write(dataOutput);
184     }
185 
186     @Override
187     public void readFields(DataInput dataInput) throws IOException {
188       this.key.readFields(dataInput);
189       this.edit.readFields(dataInput);
190     }
191   }
192 
193   /**
194    * registers WALActionsListener
195    *
196    * @param listener
197    */
198   void registerWALActionsListener(final WALActionsListener listener);
199 
200   /**
201    * unregisters WALActionsListener
202    *
203    * @param listener
204    */
205   boolean unregisterWALActionsListener(final WALActionsListener listener);
206 
207   /**
208    * @return Current state of the monotonically increasing file id.
209    */
210   // TODO: Remove.  Implementation detail.
211   long getFilenum();
212 
213   /**
214    * @return the number of HLog files
215    */
216   int getNumLogFiles();
217 
218   /**
219    * @return the size of HLog files
220    */
221   long getLogFileSize();
222 
223   // TODO: Log rolling should not be in this interface.
224   /**
225    * Roll the log writer. That is, start writing log messages to a new file.
226    *
227    * <p>
228    * The implementation is synchronized in order to make sure there's one rollWriter
229    * running at any given time.
230    *
231    * @return If lots of logs, flush the returned regions so next time through we
232    *         can clean logs. Returns null if nothing to flush. Names are actual
233    *         region names as returned by {@link HRegionInfo#getEncodedName()}
234    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
235    * @throws IOException
236    */
237   byte[][] rollWriter() throws FailedLogCloseException, IOException;
238 
239   /**
240    * Roll the log writer. That is, start writing log messages to a new file.
241    *
242    * <p>
243    * The implementation is synchronized in order to make sure there's one rollWriter
244    * running at any given time.
245    *
246    * @param force
247    *          If true, force creation of a new writer even if no entries have
248    *          been written to the current writer
249    * @return If lots of logs, flush the returned regions so next time through we
250    *         can clean logs. Returns null if nothing to flush. Names are actual
251    *         region names as returned by {@link HRegionInfo#getEncodedName()}
252    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
253    * @throws IOException
254    */
255   byte[][] rollWriter(boolean force) throws FailedLogCloseException,
256       IOException;
257 
258   /**
259    * Shut down the log.
260    *
261    * @throws IOException
262    */
263   void close() throws IOException;
264 
265   /**
266    * Shut down the log and delete the log directory
267    *
268    * @throws IOException
269    */
270   void closeAndDelete() throws IOException;
271 
272   /**
273    * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor),
274    * except it causes a sync on the log
275    * @param sequenceId of the region.
276    */
277   @VisibleForTesting
278   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
279       final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException;
280 
281   /**
282    * For notification post append to the writer.
283    * @param entries
284    */
285   void postAppend(final List<Entry> entries);
286 
287   /**
288    * For notification post writer sync.
289    */
290   void postSync();
291 
292   /**
293    * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and
294    * log-sequence-id. The HLog is not flushed after this transaction is written to the log.
295    * @param info
296    * @param tableName
297    * @param edits
298    * @param clusterIds The clusters that have consumed the change (for replication)
299    * @param now
300    * @param htd
301    * @param sequenceId of the region
302    * @return txid of this transaction
303    * @throws IOException
304    */
305   @Deprecated
306   long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
307       List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId,
308       boolean isInMemstore, long nonceGroup, long nonce) throws IOException;
309 
310   // TODO: Do we need all these versions of sync?
311   void hsync() throws IOException;
312 
313   void hflush() throws IOException;
314 
315   void sync() throws IOException;
316 
317   void sync(long txid) throws IOException;
318 
319   /**
320    * WAL keeps track of the sequence numbers that were not yet flushed from memstores
321    * in order to be able to do cleanup. This method tells WAL that some region is about
322    * to flush memstore.
323    *
324    * We stash the oldest seqNum for the region, and let the the next edit inserted in this
325    * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor,
326    * AtomicLong)} as new oldest seqnum.
327    * In case of flush being aborted, we put the stashed value back; in case of flush succeeding,
328    * the seqNum of that first edit after start becomes the valid oldest seqNum for this region.
329    *
330    * @return true if the flush can proceed, false in case wal is closing (ususally, when server is
331    * closing) and flush couldn't be started.
332    */
333   boolean startCacheFlush(final byte[] encodedRegionName);
334 
335   /**
336    * Complete the cache flush.
337    * @param encodedRegionName Encoded region name.
338    */
339   void completeCacheFlush(final byte[] encodedRegionName);
340 
341   /**
342    * Abort a cache flush. Call if the flush fails. Note that the only recovery
343    * for an aborted flush currently is a restart of the regionserver so the
344    * snapshot content dropped by the failure gets restored to the memstore.v
345    * @param encodedRegionName Encoded region name.
346    */
347   void abortCacheFlush(byte[] encodedRegionName);
348 
349   /**
350    * @return Coprocessor host.
351    */
352   WALCoprocessorHost getCoprocessorHost();
353 
354   /**
355    * Get LowReplication-Roller status
356    *
357    * @return lowReplicationRollEnabled
358    */
359   // TODO: This is implementation detail?
360   boolean isLowReplicationRollEnabled();
361 
362   /** Gets the earliest sequence number in the memstore for this particular region.
363    * This can serve as best-effort "recent" WAL number for this region.
364    * @param encodedRegionName The region to get the number for.
365    * @return The number if present, HConstants.NO_SEQNUM if absent.
366    */
367   long getEarliestMemstoreSeqNum(byte[] encodedRegionName);
368 }