View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.ResultScanner;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.regionserver.InternalScanner;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.FSTableDescriptors;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.hdfs.MiniDFSCluster;
48  
49  /**
50   * Abstract HBase test class.  Initializes a few things that can come in handly
51   * like an HBaseConfiguration and filesystem.
52   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
53   */
54  public abstract class HBaseTestCase extends TestCase {
55    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56  
57    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
58    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
59    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
60  
61    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
62  
63    private boolean localfs = false;
64    protected static Path testDir = null;
65    protected FileSystem fs = null;
66    protected HRegion meta = null;
67    protected static final char FIRST_CHAR = 'a';
68    protected static final char LAST_CHAR = 'z';
69    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
70    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
71    protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
72    protected static final int MAXVERSIONS = 3;
73  
74    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
75  
76    public volatile Configuration conf = HBaseConfiguration.create();
77    public final FSTableDescriptors fsTableDescriptors;
78    {
79      try {
80        fsTableDescriptors = new FSTableDescriptors(conf);
81      } catch (IOException e) {
82        throw new RuntimeException("Failed to init descriptors", e);
83      }
84    }
85  
86    /** constructor */
87    public HBaseTestCase() {
88      super();
89    }
90  
91    /**
92     * @param name
93     */
94    public HBaseTestCase(String name) {
95      super(name);
96    }
97  
98    /**
99     * Note that this method must be called after the mini hdfs cluster has
100    * started or we end up with a local file system.
101    */
102   @Override
103   protected void setUp() throws Exception {
104     super.setUp();
105     localfs =
106       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
107 
108     if (fs == null) {
109       this.fs = FileSystem.get(conf);
110     }
111     try {
112       if (localfs) {
113         this.testDir = getUnitTestdir(getName());
114         if (fs.exists(testDir)) {
115           fs.delete(testDir, true);
116         }
117       } else {
118         this.testDir = FSUtils.getRootDir(conf);
119       }
120     } catch (Exception e) {
121       LOG.fatal("error during setup", e);
122       throw e;
123     }
124   }
125 
126   @Override
127   protected void tearDown() throws Exception {
128     try {
129       if (localfs) {
130         if (this.fs.exists(testDir)) {
131           this.fs.delete(testDir, true);
132         }
133       }
134     } catch (Exception e) {
135       LOG.fatal("error during tear down", e);
136     }
137     super.tearDown();
138   }
139 
140   /**
141    * @see HBaseTestingUtility#getBaseTestDir
142    * @param testName
143    * @return directory to use for this test
144    */
145     protected Path getUnitTestdir(String testName) {
146       return testUtil.getDataTestDir(testName);
147     }
148 
149   /**
150    * You must call close on the returned region and then close on the log file
151    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()}
152    * and on it call close.
153    * @param desc
154    * @param startKey
155    * @param endKey
156    * @return An {@link HRegion}
157    * @throws IOException
158    */
159   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
160       byte [] endKey)
161   throws IOException {
162     return createNewHRegion(desc, startKey, endKey, this.conf);
163   }
164 
165   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
166       byte [] endKey, Configuration conf)
167   throws IOException {
168     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
169     return HRegion.createHRegion(hri, testDir, conf, desc);
170   }
171 
172   protected HRegion openClosedRegion(final HRegion closedRegion)
173   throws IOException {
174     return HRegion.openHRegion(closedRegion, null);
175   }
176 
177   /**
178    * Create a table of name <code>name</code> with {@link COLUMNS} for
179    * families.
180    * @param name Name to give table.
181    * @return Column descriptor.
182    */
183   protected HTableDescriptor createTableDescriptor(final String name) {
184     return createTableDescriptor(name, MAXVERSIONS);
185   }
186 
187   /**
188    * Create a table of name <code>name</code> with {@link COLUMNS} for
189    * families.
190    * @param name Name to give table.
191    * @param versions How many versions to allow per column.
192    * @return Column descriptor.
193    */
194   protected HTableDescriptor createTableDescriptor(final String name,
195       final int versions) {
196     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
197         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
198   }
199 
200   /**
201    * Create a table of name <code>name</code> with {@link COLUMNS} for
202    * families.
203    * @param name Name to give table.
204    * @param versions How many versions to allow per column.
205    * @return Column descriptor.
206    */
207   protected HTableDescriptor createTableDescriptor(final String name,
208       final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
209     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
210     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
211       htd.addFamily(new HColumnDescriptor(cfName)
212           .setMinVersions(minVersions)
213           .setMaxVersions(versions)
214           .setKeepDeletedCells(keepDeleted)
215           .setBlockCacheEnabled(false)
216           .setTimeToLive(ttl)
217       );
218     }
219     return htd;
220   }
221 
222   /**
223    * Add content to region <code>r</code> on the passed column
224    * <code>column</code>.
225    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
226    * @param r
227    * @param columnFamily
228    * @param column
229    * @throws IOException
230    * @return count of what we added.
231    */
232   public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
233   throws IOException {
234     byte [] startKey = r.getRegionInfo().getStartKey();
235     byte [] endKey = r.getRegionInfo().getEndKey();
236     byte [] startKeyBytes = startKey;
237     if (startKeyBytes == null || startKeyBytes.length == 0) {
238       startKeyBytes = START_KEY_BYTES;
239     }
240     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
241       startKeyBytes, endKey, -1);
242   }
243 
244   public static long addContent(final HRegion r, final byte [] columnFamily)
245   throws IOException {
246     return addContent(r, columnFamily, null);
247   }
248 
249   /**
250    * Add content to region <code>r</code> on the passed column
251    * <code>column</code>.
252    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
253    * @param updater  An instance of {@link Incommon}.
254    * @param columnFamily
255    * @param writeToWAL
256    * @throws IOException
257    * @return count of what we added.
258    */
259   public static long addContent(final Incommon updater,
260       final String columnFamily) throws IOException {
261     return addContent(updater, columnFamily, START_KEY_BYTES, null);
262   }
263 
264   public static long addContent(final Incommon updater, final String family,
265       final String column) throws IOException {
266     return addContent(updater, family, column, START_KEY_BYTES, null);
267   }
268 
269   /**
270    * Add content to region <code>r</code> on the passed column
271    * <code>column</code>.
272    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
273    * @param updater  An instance of {@link Incommon}.
274    * @param columnFamily
275    * @param startKeyBytes Where to start the rows inserted
276    * @param endKey Where to stop inserting rows.
277    * @param writeToWAL
278    * @return count of what we added.
279    * @throws IOException
280    */
281   public static long addContent(final Incommon updater, final String columnFamily,
282       final byte [] startKeyBytes, final byte [] endKey)
283   throws IOException {
284     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
285   }
286 
287   public static long addContent(final Incommon updater, final String family, String column,
288       final byte [] startKeyBytes, final byte [] endKey) throws IOException {
289     return addContent(updater, family, column, startKeyBytes, endKey, -1);
290   }
291 
292   /**
293    * Add content to region <code>r</code> on the passed column
294    * <code>column</code>.
295    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
296    * @param updater  An instance of {@link Incommon}.
297    * @param column
298    * @param startKeyBytes Where to start the rows inserted
299    * @param endKey Where to stop inserting rows.
300    * @param ts Timestamp to write the content with.
301    * @param writeToWAL
302    * @return count of what we added.
303    * @throws IOException
304    */
305   public static long addContent(final Incommon updater,
306                                    final String columnFamily, 
307                                    final String column,
308       final byte [] startKeyBytes, final byte [] endKey, final long ts)
309   throws IOException {
310     long count = 0;
311     // Add rows of three characters.  The first character starts with the
312     // 'a' character and runs up to 'z'.  Per first character, we run the
313     // second character over same range.  And same for the third so rows
314     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
315     char secondCharStart = (char)startKeyBytes[1];
316     char thirdCharStart = (char)startKeyBytes[2];
317     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
318       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
319         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
320           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
321           if (endKey != null && endKey.length > 0
322               && Bytes.compareTo(endKey, t) <= 0) {
323             break EXIT;
324           }
325           try {
326             Put put;
327             if(ts != -1) {
328               put = new Put(t, ts);
329             } else {
330               put = new Put(t);
331             }
332             try {
333               StringBuilder sb = new StringBuilder();
334               if (column != null && column.contains(":")) {
335                 sb.append(column);
336               } else {
337                 if (columnFamily != null) {
338                   sb.append(columnFamily);
339                   if (!columnFamily.endsWith(":")) {
340                     sb.append(":");
341                   }
342                   if (column != null) {
343                     sb.append(column);
344                   }
345                 }
346               }
347               byte[][] split =
348                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
349               if(split.length == 1) {
350                 put.add(split[0], new byte[0], t);
351               } else {
352                 put.add(split[0], split[1], t);
353               }
354               put.setDurability(Durability.SKIP_WAL);
355               updater.put(put);
356               count++;
357             } catch (RuntimeException ex) {
358               ex.printStackTrace();
359               throw ex;
360             } catch (IOException ex) {
361               ex.printStackTrace();
362               throw ex;
363             }
364           } catch (RuntimeException ex) {
365             ex.printStackTrace();
366             throw ex;
367           } catch (IOException ex) {
368             ex.printStackTrace();
369             throw ex;
370           }
371         }
372         // Set start character back to FIRST_CHAR after we've done first loop.
373         thirdCharStart = FIRST_CHAR;
374       }
375       secondCharStart = FIRST_CHAR;
376     }
377     return count;
378   }
379 
380   /**
381    * Implementors can flushcache.
382    */
383   public interface FlushCache {
384     /**
385      * @throws IOException
386      */
387     void flushcache() throws IOException;
388   }
389 
390   /**
391    * Interface used by tests so can do common operations against an HTable
392    * or an HRegion.
393    *
394    * TOOD: Come up w/ a better name for this interface.
395    */
396   public interface Incommon {
397     /**
398      *
399      * @param delete
400      * @param writeToWAL
401      * @throws IOException
402      */
403     void delete(Delete delete, boolean writeToWAL)
404     throws IOException;
405 
406     /**
407      * @param put
408      * @throws IOException
409      */
410     void put(Put put) throws IOException;
411 
412     Result get(Get get) throws IOException;
413 
414     /**
415      * @param family
416      * @param qualifiers
417      * @param firstRow
418      * @param ts
419      * @return scanner for specified columns, first row and timestamp
420      * @throws IOException
421      */
422     ScannerIncommon getScanner(
423       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
424     )
425     throws IOException;
426   }
427 
428   /**
429    * A class that makes a {@link Incommon} out of a {@link HRegion}
430    */
431   public static class HRegionIncommon implements Incommon, FlushCache {
432     final HRegion region;
433 
434     /**
435      * @param HRegion
436      */
437     public HRegionIncommon(final HRegion HRegion) {
438       this.region = HRegion;
439     }
440 
441     public void put(Put put) throws IOException {
442       region.put(put);
443     }
444 
445     public void delete(Delete delete,  boolean writeToWAL)
446     throws IOException {
447       this.region.delete(delete);
448     }
449 
450     public Result get(Get get) throws IOException {
451       return region.get(get);
452     }
453 
454     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
455         byte [] firstRow, long ts)
456       throws IOException {
457         Scan scan = new Scan(firstRow);
458         if(qualifiers == null || qualifiers.length == 0) {
459           scan.addFamily(family);
460         } else {
461           for(int i=0; i<qualifiers.length; i++){
462             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
463           }
464         }
465         scan.setTimeRange(0, ts);
466         return new
467           InternalScannerIncommon(region.getScanner(scan));
468       }
469 
470     public void flushcache() throws IOException {
471       this.region.flushcache();
472     }
473   }
474 
475   /**
476    * A class that makes a {@link Incommon} out of a {@link HTable}
477    */
478   public static class HTableIncommon implements Incommon {
479     final HTable table;
480 
481     /**
482      * @param table
483      */
484     public HTableIncommon(final HTable table) {
485       super();
486       this.table = table;
487     }
488 
489     public void put(Put put) throws IOException {
490       table.put(put);
491     }
492 
493 
494     public void delete(Delete delete, boolean writeToWAL)
495     throws IOException {
496       this.table.delete(delete);
497     }
498 
499     public Result get(Get get) throws IOException {
500       return table.get(get);
501     }
502 
503     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
504         byte [] firstRow, long ts)
505       throws IOException {
506       Scan scan = new Scan(firstRow);
507       if(qualifiers == null || qualifiers.length == 0) {
508         scan.addFamily(family);
509       } else {
510         for(int i=0; i<qualifiers.length; i++){
511           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
512         }
513       }
514       scan.setTimeRange(0, ts);
515       return new
516         ClientScannerIncommon(table.getScanner(scan));
517     }
518   }
519 
520   public interface ScannerIncommon
521   extends Iterable<Result> {
522     boolean next(List<Cell> values)
523     throws IOException;
524 
525     void close() throws IOException;
526   }
527 
528   public static class ClientScannerIncommon implements ScannerIncommon {
529     ResultScanner scanner;
530     public ClientScannerIncommon(ResultScanner scanner) {
531       this.scanner = scanner;
532     }
533 
534     @Override
535     public boolean next(List<Cell> values)
536     throws IOException {
537       Result results = scanner.next();
538       if (results == null) {
539         return false;
540       }
541       values.clear();
542       values.addAll(results.listCells());
543       return true;
544     }
545 
546     public void close() throws IOException {
547       scanner.close();
548     }
549 
550     public Iterator<Result> iterator() {
551       return scanner.iterator();
552     }
553   }
554 
555   public static class InternalScannerIncommon implements ScannerIncommon {
556     InternalScanner scanner;
557 
558     public InternalScannerIncommon(InternalScanner scanner) {
559       this.scanner = scanner;
560     }
561 
562     @Override
563     public boolean next(List<Cell> results)
564     throws IOException {
565       return scanner.next(results);
566     }
567 
568     @Override
569     public void close() throws IOException {
570       scanner.close();
571     }
572 
573     @Override
574     public Iterator<Result> iterator() {
575       throw new UnsupportedOperationException();
576     }
577   }
578 
579   protected void assertResultEquals(final HRegion region, final byte [] row,
580       final byte [] family, final byte [] qualifier, final long timestamp,
581       final byte [] value)
582     throws IOException {
583       Get get = new Get(row);
584       get.setTimeStamp(timestamp);
585       Result res = region.get(get);
586       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
587         res.getMap();
588       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
589 
590       if (value == null) {
591         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
592             " at timestamp " + timestamp, null, res_value);
593       } else {
594         if (res_value == null) {
595           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
596               " at timestamp " + timestamp + "\" was expected to be \"" +
597               Bytes.toStringBinary(value) + " but was null");
598         }
599         if (res_value != null) {
600           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601               " at timestamp " +
602               timestamp, value, new String(res_value));
603         }
604       }
605     }
606 
607   /**
608    * Common method to close down a MiniDFSCluster and the associated file system
609    *
610    * @param cluster
611    */
612   public static void shutdownDfs(MiniDFSCluster cluster) {
613     if (cluster != null) {
614       LOG.info("Shutting down Mini DFS ");
615       try {
616         cluster.shutdown();
617       } catch (Exception e) {
618         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
619         // here because of an InterruptedException. Don't let exceptions in
620         // here be cause of test failure.
621       }
622       try {
623         FileSystem fs = cluster.getFileSystem();
624         if (fs != null) {
625           LOG.info("Shutting down FileSystem");
626           fs.close();
627         }
628         FileSystem.closeAll();
629       } catch (IOException e) {
630         LOG.error("error closing file system", e);
631       }
632     }
633   }
634 
635   /**
636    * You must call {@link #closeRootAndMeta()} when done after calling this
637    * method. It does cleanup.
638    * @throws IOException
639    */
640   protected void createMetaRegion() throws IOException {
641     FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
642     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
643       fsTableDescriptors.get(TableName.META_TABLE_NAME));
644   }
645 
646   protected void closeRootAndMeta() throws IOException {
647     HRegion.closeHRegion(meta);
648   }
649 
650   public static void assertByteEquals(byte[] expected,
651                                byte[] actual) {
652     if (Bytes.compareTo(expected, actual) != 0) {
653       throw new AssertionFailedError("expected:<" +
654       Bytes.toString(expected) + "> but was:<" +
655       Bytes.toString(actual) + ">");
656     }
657   }
658 
659   public static void assertEquals(byte[] expected,
660                                byte[] actual) {
661     if (Bytes.compareTo(expected, actual) != 0) {
662       throw new AssertionFailedError("expected:<" +
663       Bytes.toStringBinary(expected) + "> but was:<" +
664       Bytes.toStringBinary(actual) + ">");
665     }
666   }
667 
668 }