View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  
45  import java.io.IOException;
46  import java.io.InterruptedIOException;
47  import java.security.PrivilegedExceptionAction;
48  import java.util.ArrayList;
49  import java.util.Arrays;
50  import java.util.Collection;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.NavigableMap;
54  import java.util.TreeMap;
55  import java.util.UUID;
56  import java.util.concurrent.atomic.AtomicBoolean;
57  import java.util.concurrent.atomic.AtomicInteger;
58  import java.util.concurrent.atomic.AtomicLong;
59  import java.util.concurrent.atomic.AtomicReference;
60  
61  import org.apache.commons.logging.Log;
62  import org.apache.commons.logging.LogFactory;
63  import org.apache.hadoop.conf.Configuration;
64  import org.apache.hadoop.fs.FSDataOutputStream;
65  import org.apache.hadoop.fs.FileStatus;
66  import org.apache.hadoop.fs.FileSystem;
67  import org.apache.hadoop.fs.Path;
68  import org.apache.hadoop.hbase.Cell;
69  import org.apache.hadoop.hbase.CellComparator;
70  import org.apache.hadoop.hbase.CellUtil;
71  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72  import org.apache.hadoop.hbase.DroppedSnapshotException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseTestCase;
75  import org.apache.hadoop.hbase.HBaseTestingUtility;
76  import org.apache.hadoop.hbase.HColumnDescriptor;
77  import org.apache.hadoop.hbase.HConstants;
78  import org.apache.hadoop.hbase.RegionTooBusyException;
79  import org.apache.hadoop.hbase.Tag;
80  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
81  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
82  import org.apache.hadoop.hbase.HRegionInfo;
83  import org.apache.hadoop.hbase.HTableDescriptor;
84  import org.apache.hadoop.hbase.KeyValue;
85  import org.apache.hadoop.hbase.testclassification.MediumTests;
86  import org.apache.hadoop.hbase.MiniHBaseCluster;
87  import org.apache.hadoop.hbase.MultithreadedTestUtil;
88  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
89  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
90  import org.apache.hadoop.hbase.NotServingRegionException;
91  import org.apache.hadoop.hbase.TableName;
92  import org.apache.hadoop.hbase.TagType;
93  import org.apache.hadoop.hbase.Waiter;
94  import org.apache.hadoop.hbase.client.Append;
95  import org.apache.hadoop.hbase.client.Delete;
96  import org.apache.hadoop.hbase.client.Durability;
97  import org.apache.hadoop.hbase.client.Get;
98  import org.apache.hadoop.hbase.client.HTable;
99  import org.apache.hadoop.hbase.client.Increment;
100 import org.apache.hadoop.hbase.client.Put;
101 import org.apache.hadoop.hbase.client.Result;
102 import org.apache.hadoop.hbase.client.Scan;
103 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
104 import org.apache.hadoop.hbase.filter.BinaryComparator;
105 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
106 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
107 import org.apache.hadoop.hbase.filter.Filter;
108 import org.apache.hadoop.hbase.filter.FilterBase;
109 import org.apache.hadoop.hbase.filter.FilterList;
110 import org.apache.hadoop.hbase.filter.NullComparator;
111 import org.apache.hadoop.hbase.filter.PrefixFilter;
112 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
113 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
114 import org.apache.hadoop.hbase.io.hfile.HFile;
115 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
116 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
117 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
118 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
119 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
120 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
121 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
122 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
123 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
124 import org.apache.hadoop.hbase.regionserver.wal.HLog;
125 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
126 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
128 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
129 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
130 import org.apache.hadoop.hbase.security.User;
131 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
132 import org.apache.hadoop.hbase.util.Bytes;
133 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
134 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
135 import org.apache.hadoop.hbase.util.FSUtils;
136 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
137 import org.apache.hadoop.hbase.util.PairOfSameType;
138 import org.apache.hadoop.hbase.util.Threads;
139 import org.junit.After;
140 import org.junit.Assert;
141 import org.junit.Before;
142 import org.junit.Rule;
143 import org.junit.Test;
144 import org.junit.experimental.categories.Category;
145 import org.junit.rules.TestName;
146 import org.mockito.Mockito;
147 
148 import com.google.common.collect.ImmutableList;
149 import com.google.common.collect.Lists;
150 import com.google.protobuf.ByteString;
151 
152 /**
153  * Basic stand-alone testing of HRegion.  No clusters!
154  *
155  * A lot of the meta information for an HRegion now lives inside other HRegions
156  * or in the HBaseMaster, so only basic testing is possible.
157  */
158 @Category(MediumTests.class)
159 @SuppressWarnings("deprecation")
160 public class TestHRegion {
161   // Do not spin up clusters in here. If you need to spin up a cluster, do it
162   // over in TestHRegionOnCluster.
163   static final Log LOG = LogFactory.getLog(TestHRegion.class);
164   @Rule public TestName name = new TestName();
165 
166   private static final String COLUMN_FAMILY = "MyCF";
167   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
168 
169   HRegion region = null;
170   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
171   private static HBaseTestingUtility TEST_UTIL;
172   public static Configuration CONF ;
173   private String dir;
174   private static FileSystem FILESYSTEM;
175   private final int MAX_VERSIONS = 2;
176 
177   // Test names
178   protected byte[] tableName;
179   protected String method;
180   protected final byte[] qual1 = Bytes.toBytes("qual1");
181   protected final byte[] qual2 = Bytes.toBytes("qual2");
182   protected final byte[] qual3 = Bytes.toBytes("qual3");
183   protected final byte[] value1 = Bytes.toBytes("value1");
184   protected final byte[] value2 = Bytes.toBytes("value2");
185   protected final byte[] row = Bytes.toBytes("rowA");
186   protected final byte[] row2 = Bytes.toBytes("rowB");
187 
188   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
189       .getInstance(MetricsAssertHelper.class);
190 
191   @Before
192   public void setup() throws IOException {
193     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
194     FILESYSTEM = TEST_UTIL.getTestFileSystem();
195     CONF = TEST_UTIL.getConfiguration();
196     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
197     method = name.getMethodName();
198     tableName = Bytes.toBytes(name.getMethodName());
199   }
200 
201   @After
202   public void tearDown() throws Exception {
203     EnvironmentEdgeManagerTestHelper.reset();
204     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
205     TEST_UTIL.cleanupTestDir();
206   }
207 
208   String getName() {
209     return name.getMethodName();
210   }
211 
212   /**
213    * Test for Bug 2 of HBASE-10466.
214    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
215    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
216    * flush is skipped and only the second flush takes place. However, two flushes are required in
217    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
218    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
219    * flushes for region close."
220    * @throws IOException
221    */
222   @Test (timeout=60000)
223   public void testCloseCarryingSnapshot() throws IOException {
224     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
225     Store store = region.getStore(COLUMN_FAMILY_BYTES);
226     // Get some random bytes.
227     byte [] value = Bytes.toBytes(name.getMethodName());
228     // Make a random put against our cf.
229     Put put = new Put(value);
230     put.add(COLUMN_FAMILY_BYTES, null, value);
231     // First put something in current memstore, which will be in snapshot after flusher.prepare()
232     region.put(put);
233     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
234     storeFlushCtx.prepare();
235     // Second put something in current memstore
236     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
237     region.put(put);
238     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
239     region.close();
240     assertEquals(0, region.getMemstoreSize().get());
241     HRegion.closeHRegion(region);
242   }
243 
244   /*
245    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
246    * See HBASE-10845
247    */
248   @Test (timeout=60000)
249   public void testMemstoreSnapshotSize() throws IOException {
250     class MyFaultyHLog extends FaultyHLog {
251       StoreFlushContext storeFlushCtx;
252       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
253           throws IOException {
254         super(fs, rootDir, logName, conf);
255       }
256       
257       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
258         this.storeFlushCtx = storeFlushCtx;
259       }
260 
261       @Override
262       public void sync(long txid) throws IOException {
263         storeFlushCtx.prepare();
264         super.sync(txid);
265       }
266     }
267     
268     FileSystem fs = FileSystem.get(CONF);
269     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
270     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
271     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
272       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
273     
274     Store store = region.getStore(COLUMN_FAMILY_BYTES);
275     // Get some random bytes.
276     byte [] value = Bytes.toBytes(name.getMethodName());
277     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
278     
279     Put put = new Put(value);
280     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
281     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
282 
283     boolean threwIOE = false;
284     try {
285       region.put(put);
286     } catch (IOException ioe) {
287       threwIOE = true;
288     } finally {
289       assertTrue("The regionserver should have thrown an exception", threwIOE);
290     }
291     long sz = store.getFlushableSize();
292     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
293     HRegion.closeHRegion(region);
294   }
295   
296   /**
297    * Test we do not lose data if we fail a flush and then close.
298    * Part of HBase-10466.  Tests the following from the issue description:
299    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
300    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
301    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
302    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
303    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
304    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
305    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
306    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
307    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
308    * if memstoreSize is not larger than 0."
309    * @throws Exception
310    */
311   @Test (timeout=60000)
312   public void testFlushSizeAccounting() throws Exception {
313     final Configuration conf = HBaseConfiguration.create(CONF);
314     // Only retry once.
315     conf.setInt("hbase.hstore.flush.retries.number", 1);
316     final User user =
317       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
318     // Inject our faulty LocalFileSystem
319     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
320     user.runAs(new PrivilegedExceptionAction<Object>() {
321       @Override
322       public Object run() throws Exception {
323         // Make sure it worked (above is sensitive to caching details in hadoop core)
324         FileSystem fs = FileSystem.get(conf);
325         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
326         FaultyFileSystem ffs = (FaultyFileSystem)fs;
327         HRegion region = null;
328         try {
329           // Initialize region
330           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
331           long size = region.getMemstoreSize().get();
332           Assert.assertEquals(0, size);
333           // Put one item into memstore.  Measure the size of one item in memstore.
334           Put p1 = new Put(row);
335           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
336           region.put(p1);
337           final long sizeOfOnePut = region.getMemstoreSize().get();
338           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
339           try {
340             LOG.info("Flushing");
341             region.flushcache();
342             Assert.fail("Didn't bubble up IOE!");
343           } catch (DroppedSnapshotException dse) {
344             // What we are expecting
345           }
346           // Make it so all writes succeed from here on out
347           ffs.fault.set(false);
348           // Check sizes.  Should still be the one entry.
349           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
350           // Now add two entries so that on this next flush that fails, we can see if we
351           // subtract the right amount, the snapshot size only.
352           Put p2 = new Put(row);
353           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
354           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
355           region.put(p2);
356           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
357           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
358           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
359           // it
360           region.flushcache();
361           // Make sure our memory accounting is right.
362           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
363         } finally {
364           HRegion.closeHRegion(region);
365         }
366         return null;
367       }
368     });
369     FileSystem.closeAllForUGI(user.getUGI());
370   }
371 
372   @Test
373   public void testCompactionAffectedByScanners() throws Exception {
374     byte[] family = Bytes.toBytes("family");
375     this.region = initHRegion(tableName, method, CONF, family);
376 
377     Put put = new Put(Bytes.toBytes("r1"));
378     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
379     region.put(put);
380     region.flushcache();
381 
382     Scan scan = new Scan();
383     scan.setMaxVersions(3);
384     // open the first scanner
385     RegionScanner scanner1 = region.getScanner(scan);
386 
387     Delete delete = new Delete(Bytes.toBytes("r1"));
388     region.delete(delete);
389     region.flushcache();
390 
391     // open the second scanner
392     RegionScanner scanner2 = region.getScanner(scan);
393 
394     List<Cell> results = new ArrayList<Cell>();
395 
396     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
397 
398     // make a major compaction
399     region.compactStores(true);
400 
401     // open the third scanner
402     RegionScanner scanner3 = region.getScanner(scan);
403 
404     // get data from scanner 1, 2, 3 after major compaction
405     scanner1.next(results);
406     System.out.println(results);
407     assertEquals(1, results.size());
408 
409     results.clear();
410     scanner2.next(results);
411     System.out.println(results);
412     assertEquals(0, results.size());
413 
414     results.clear();
415     scanner3.next(results);
416     System.out.println(results);
417     assertEquals(0, results.size());
418   }
419 
420   @Test
421   public void testToShowNPEOnRegionScannerReseek() throws Exception {
422     byte[] family = Bytes.toBytes("family");
423     this.region = initHRegion(tableName, method, CONF, family);
424 
425     Put put = new Put(Bytes.toBytes("r1"));
426     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
427     region.put(put);
428     put = new Put(Bytes.toBytes("r2"));
429     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
430     region.put(put);
431     region.flushcache();
432 
433     Scan scan = new Scan();
434     scan.setMaxVersions(3);
435     // open the first scanner
436     RegionScanner scanner1 = region.getScanner(scan);
437 
438     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
439 
440     region.compactStores(true);
441 
442     scanner1.reseek(Bytes.toBytes("r2"));
443     List<Cell> results = new ArrayList<Cell>();
444     scanner1.next(results);
445     Cell keyValue = results.get(0);
446     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
447     scanner1.close();
448   }
449 
450   @Test
451   public void testSkipRecoveredEditsReplay() throws Exception {
452     String method = "testSkipRecoveredEditsReplay";
453     TableName tableName = TableName.valueOf(method);
454     byte[] family = Bytes.toBytes("family");
455     this.region = initHRegion(tableName, method, CONF, family);
456     try {
457       Path regiondir = region.getRegionFileSystem().getRegionDir();
458       FileSystem fs = region.getRegionFileSystem().getFileSystem();
459       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
460 
461       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
462 
463       long maxSeqId = 1050;
464       long minSeqId = 1000;
465 
466       for (long i = minSeqId; i <= maxSeqId; i += 10) {
467         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
468         fs.create(recoveredEdits);
469         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
470 
471         long time = System.nanoTime();
472         WALEdit edit = new WALEdit();
473         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
474             .toBytes(i)));
475         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
476             HConstants.DEFAULT_CLUSTER_ID), edit));
477 
478         writer.close();
479       }
480       MonitoredTask status = TaskMonitor.get().createStatus(method);
481       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
482       for (Store store : region.getStores().values()) {
483         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
484       }
485       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
486       assertEquals(maxSeqId, seqId);
487       Get get = new Get(row);
488       Result result = region.get(get);
489       for (long i = minSeqId; i <= maxSeqId; i += 10) {
490         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
491         assertEquals(1, kvs.size());
492         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
493       }
494     } finally {
495       HRegion.closeHRegion(this.region);
496       this.region = null;
497     }
498   }
499 
500   @Test
501   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
502     String method = "testSkipRecoveredEditsReplaySomeIgnored";
503     TableName tableName = TableName.valueOf(method);
504     byte[] family = Bytes.toBytes("family");
505     this.region = initHRegion(tableName, method, CONF, family);
506     try {
507       Path regiondir = region.getRegionFileSystem().getRegionDir();
508       FileSystem fs = region.getRegionFileSystem().getFileSystem();
509       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
510 
511       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
512 
513       long maxSeqId = 1050;
514       long minSeqId = 1000;
515 
516       for (long i = minSeqId; i <= maxSeqId; i += 10) {
517         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
518         fs.create(recoveredEdits);
519         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
520 
521         long time = System.nanoTime();
522         WALEdit edit = new WALEdit();
523         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
524             .toBytes(i)));
525         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
526             HConstants.DEFAULT_CLUSTER_ID), edit));
527 
528         writer.close();
529       }
530       long recoverSeqId = 1030;
531       MonitoredTask status = TaskMonitor.get().createStatus(method);
532       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
533       for (Store store : region.getStores().values()) {
534         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
535       }
536       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
537       assertEquals(maxSeqId, seqId);
538       Get get = new Get(row);
539       Result result = region.get(get);
540       for (long i = minSeqId; i <= maxSeqId; i += 10) {
541         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
542         if (i < recoverSeqId) {
543           assertEquals(0, kvs.size());
544         } else {
545           assertEquals(1, kvs.size());
546           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
547         }
548       }
549     } finally {
550       HRegion.closeHRegion(this.region);
551       this.region = null;
552     }
553   }
554 
555   @Test
556   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
557     byte[] family = Bytes.toBytes("family");
558     this.region = initHRegion(tableName, method, CONF, family);
559     try {
560       Path regiondir = region.getRegionFileSystem().getRegionDir();
561       FileSystem fs = region.getRegionFileSystem().getFileSystem();
562 
563       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
564       for (int i = 1000; i < 1050; i += 10) {
565         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
566         FSDataOutputStream dos = fs.create(recoveredEdits);
567         dos.writeInt(i);
568         dos.close();
569       }
570       long minSeqId = 2000;
571       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
572       FSDataOutputStream dos = fs.create(recoveredEdits);
573       dos.close();
574 
575       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
576       for (Store store : region.getStores().values()) {
577         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
578       }
579       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
580       assertEquals(minSeqId, seqId);
581     } finally {
582       HRegion.closeHRegion(this.region);
583       this.region = null;
584     }
585   }
586 
587   @Test
588   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
589     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
590     TableName tableName = TableName.valueOf(method);
591     byte[] family = Bytes.toBytes("family");
592     this.region = initHRegion(tableName, method, CONF, family);
593     try {
594       Path regiondir = region.getRegionFileSystem().getRegionDir();
595       FileSystem fs = region.getRegionFileSystem().getFileSystem();
596       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
597 
598       assertEquals(0, region.getStoreFileList(
599         region.getStores().keySet().toArray(new byte[0][])).size());
600 
601       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
602 
603       long maxSeqId = 1050;
604       long minSeqId = 1000;
605 
606       for (long i = minSeqId; i <= maxSeqId; i += 10) {
607         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
608         fs.create(recoveredEdits);
609         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
610 
611         long time = System.nanoTime();
612         WALEdit edit = null;
613         if (i == maxSeqId) {
614           edit = WALEdit.createCompaction(region.getRegionInfo(),
615           CompactionDescriptor.newBuilder()
616           .setTableName(ByteString.copyFrom(tableName.getName()))
617           .setFamilyName(ByteString.copyFrom(regionName))
618           .setEncodedRegionName(ByteString.copyFrom(regionName))
619           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
620           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
621           .build());
622         } else {
623           edit = new WALEdit();
624           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
625             .toBytes(i)));
626         }
627         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
628             HConstants.DEFAULT_CLUSTER_ID), edit));
629         writer.close();
630       }
631 
632       long recoverSeqId = 1030;
633       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
634       MonitoredTask status = TaskMonitor.get().createStatus(method);
635       for (Store store : region.getStores().values()) {
636         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
637       }
638       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
639       assertEquals(maxSeqId, seqId);
640 
641       // assert that the files are flushed
642       assertEquals(1, region.getStoreFileList(
643         region.getStores().keySet().toArray(new byte[0][])).size());
644 
645     } finally {
646       HRegion.closeHRegion(this.region);
647       this.region = null;
648     }  }
649 
650   @Test
651   public void testRecoveredEditsReplayCompaction() throws Exception {
652     String method = name.getMethodName();
653     TableName tableName = TableName.valueOf(method);
654     byte[] family = Bytes.toBytes("family");
655     this.region = initHRegion(tableName, method, CONF, family);
656     try {
657       Path regiondir = region.getRegionFileSystem().getRegionDir();
658       FileSystem fs = region.getRegionFileSystem().getFileSystem();
659       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
660 
661       long maxSeqId = 3;
662       long minSeqId = 0;
663 
664       for (long i = minSeqId; i < maxSeqId; i++) {
665         Put put = new Put(Bytes.toBytes(i));
666         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
667         region.put(put);
668         region.flushcache();
669       }
670 
671       // this will create a region with 3 files
672       assertEquals(3, region.getStore(family).getStorefilesCount());
673       List<Path> storeFiles = new ArrayList<Path>(3);
674       for (StoreFile sf : region.getStore(family).getStorefiles()) {
675         storeFiles.add(sf.getPath());
676       }
677 
678       // disable compaction completion
679       CONF.setBoolean("hbase.hstore.compaction.complete", false);
680       region.compactStores();
681 
682       // ensure that nothing changed
683       assertEquals(3, region.getStore(family).getStorefilesCount());
684 
685       // now find the compacted file, and manually add it to the recovered edits
686       Path tmpDir = region.getRegionFileSystem().getTempDir();
687       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
688       String errorMsg = "Expected to find 1 file in the region temp directory "
689           + "from the compaction, could not find any";
690       assertNotNull(errorMsg, files);
691       assertEquals(errorMsg, 1, files.length);
692       // move the file inside region dir
693       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
694           files[0].getPath());
695 
696       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
697           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
698           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
699 
700       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
701           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
702 
703       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
704 
705       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
706       fs.create(recoveredEdits);
707       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
708 
709       long time = System.nanoTime();
710 
711       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
712           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
713           compactionDescriptor)));
714       writer.close();
715 
716       // close the region now, and reopen again
717       region.getTableDesc();
718       region.getRegionInfo();
719       region.close();
720       region = HRegion.openHRegion(region, null);
721 
722       // now check whether we have only one store file, the compacted one
723       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
724       for (StoreFile sf : sfs) {
725         LOG.info(sf.getPath());
726       }
727       assertEquals(1, region.getStore(family).getStorefilesCount());
728       files = FSUtils.listStatus(fs, tmpDir);
729       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
730 
731       for (long i = minSeqId; i < maxSeqId; i++) {
732         Get get = new Get(Bytes.toBytes(i));
733         Result result = region.get(get);
734         byte[] value = result.getValue(family, Bytes.toBytes(i));
735         assertArrayEquals(Bytes.toBytes(i), value);
736       }
737     } finally {
738       HRegion.closeHRegion(this.region);
739       this.region = null;
740     }
741   }
742 
743   @Test
744   public void testGetWhileRegionClose() throws IOException {
745     TableName tableName = TableName.valueOf(name.getMethodName());
746     Configuration hc = initSplit();
747     int numRows = 100;
748     byte[][] families = { fam1, fam2, fam3 };
749 
750     // Setting up region
751     String method = name.getMethodName();
752     this.region = initHRegion(tableName, method, hc, families);
753     try {
754       // Put data in region
755       final int startRow = 100;
756       putData(startRow, numRows, qual1, families);
757       putData(startRow, numRows, qual2, families);
758       putData(startRow, numRows, qual3, families);
759       final AtomicBoolean done = new AtomicBoolean(false);
760       final AtomicInteger gets = new AtomicInteger(0);
761       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
762       try {
763         // Set ten threads running concurrently getting from the region.
764         for (int i = 0; i < threads.length / 2; i++) {
765           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
766           threads[i].setDaemon(true);
767           threads[i].start();
768         }
769         // Artificially make the condition by setting closing flag explicitly.
770         // I can't make the issue happen with a call to region.close().
771         this.region.closing.set(true);
772         for (int i = threads.length / 2; i < threads.length; i++) {
773           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
774           threads[i].setDaemon(true);
775           threads[i].start();
776         }
777       } finally {
778         if (this.region != null) {
779           HRegion.closeHRegion(this.region);
780         }
781       }
782       done.set(true);
783       for (GetTillDoneOrException t : threads) {
784         try {
785           t.join();
786         } catch (InterruptedException e) {
787           e.printStackTrace();
788         }
789         if (t.e != null) {
790           LOG.info("Exception=" + t.e);
791           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
792         }
793       }
794     } finally {
795       HRegion.closeHRegion(this.region);
796       this.region = null;
797     }
798   }
799 
800   /*
801    * Thread that does get on single row until 'done' flag is flipped. If an
802    * exception causes us to fail, it records it.
803    */
804   class GetTillDoneOrException extends Thread {
805     private final Get g;
806     private final AtomicBoolean done;
807     private final AtomicInteger count;
808     private Exception e;
809 
810     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
811       super("getter." + i);
812       this.g = new Get(r);
813       this.done = d;
814       this.count = c;
815     }
816 
817     @Override
818     public void run() {
819       while (!this.done.get()) {
820         try {
821           assertTrue(region.get(g).size() > 0);
822           this.count.incrementAndGet();
823         } catch (Exception e) {
824           this.e = e;
825           break;
826         }
827       }
828     }
829   }
830 
831   /*
832    * An involved filter test. Has multiple column families and deletes in mix.
833    */
834   @Test
835   public void testWeirdCacheBehaviour() throws Exception {
836     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
837     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
838         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
839     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
840     try {
841       String value = "this is the value";
842       String value2 = "this is some other value";
843       String keyPrefix1 = "prefix1";
844       String keyPrefix2 = "prefix2";
845       String keyPrefix3 = "prefix3";
846       putRows(this.region, 3, value, keyPrefix1);
847       putRows(this.region, 3, value, keyPrefix2);
848       putRows(this.region, 3, value, keyPrefix3);
849       putRows(this.region, 3, value2, keyPrefix1);
850       putRows(this.region, 3, value2, keyPrefix2);
851       putRows(this.region, 3, value2, keyPrefix3);
852       System.out.println("Checking values for key: " + keyPrefix1);
853       assertEquals("Got back incorrect number of rows from scan", 3,
854           getNumberOfRows(keyPrefix1, value2, this.region));
855       System.out.println("Checking values for key: " + keyPrefix2);
856       assertEquals("Got back incorrect number of rows from scan", 3,
857           getNumberOfRows(keyPrefix2, value2, this.region));
858       System.out.println("Checking values for key: " + keyPrefix3);
859       assertEquals("Got back incorrect number of rows from scan", 3,
860           getNumberOfRows(keyPrefix3, value2, this.region));
861       deleteColumns(this.region, value2, keyPrefix1);
862       deleteColumns(this.region, value2, keyPrefix2);
863       deleteColumns(this.region, value2, keyPrefix3);
864       System.out.println("Starting important checks.....");
865       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
866           getNumberOfRows(keyPrefix1, value2, this.region));
867       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
868           getNumberOfRows(keyPrefix2, value2, this.region));
869       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
870           getNumberOfRows(keyPrefix3, value2, this.region));
871     } finally {
872       HRegion.closeHRegion(this.region);
873       this.region = null;
874     }
875   }
876 
877   @Test
878   public void testAppendWithReadOnlyTable() throws Exception {
879     byte[] TABLE = Bytes.toBytes("readOnlyTable");
880     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
881     boolean exceptionCaught = false;
882     Append append = new Append(Bytes.toBytes("somerow"));
883     append.setDurability(Durability.SKIP_WAL);
884     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
885         Bytes.toBytes("somevalue"));
886     try {
887       region.append(append);
888     } catch (IOException e) {
889       exceptionCaught = true;
890     } finally {
891       HRegion.closeHRegion(this.region);
892       this.region = null;
893     }
894     assertTrue(exceptionCaught == true);
895   }
896 
897   @Test
898   public void testIncrWithReadOnlyTable() throws Exception {
899     byte[] TABLE = Bytes.toBytes("readOnlyTable");
900     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
901     boolean exceptionCaught = false;
902     Increment inc = new Increment(Bytes.toBytes("somerow"));
903     inc.setDurability(Durability.SKIP_WAL);
904     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
905     try {
906       region.increment(inc);
907     } catch (IOException e) {
908       exceptionCaught = true;
909     } finally {
910       HRegion.closeHRegion(this.region);
911       this.region = null;
912     }
913     assertTrue(exceptionCaught == true);
914   }
915 
916   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
917     InternalScanner scanner = buildScanner(keyPrefix, value, r);
918     int count = 0;
919     boolean more = false;
920     List<Cell> results = new ArrayList<Cell>();
921     do {
922       more = scanner.next(results);
923       if (results != null && !results.isEmpty())
924         count++;
925       else
926         break;
927       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
928       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
929       r.delete(delete);
930       results.clear();
931     } while (more);
932     assertEquals("Did not perform correct number of deletes", 3, count);
933   }
934 
935   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
936     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
937     int numberOfResults = 0;
938     List<Cell> results = new ArrayList<Cell>();
939     boolean more = false;
940     do {
941       more = resultScanner.next(results);
942       if (results != null && !results.isEmpty())
943         numberOfResults++;
944       else
945         break;
946       for (Cell kv : results) {
947         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
948       }
949       results.clear();
950     } while (more);
951     return numberOfResults;
952   }
953 
954   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
955       throws IOException {
956     // Defaults FilterList.Operator.MUST_PASS_ALL.
957     FilterList allFilters = new FilterList();
958     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
959     // Only return rows where this column value exists in the row.
960     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
961         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
962     filter.setFilterIfMissing(true);
963     allFilters.addFilter(filter);
964     Scan scan = new Scan();
965     scan.addFamily(Bytes.toBytes("trans-blob"));
966     scan.addFamily(Bytes.toBytes("trans-type"));
967     scan.addFamily(Bytes.toBytes("trans-date"));
968     scan.addFamily(Bytes.toBytes("trans-tags"));
969     scan.addFamily(Bytes.toBytes("trans-group"));
970     scan.setFilter(allFilters);
971     return r.getScanner(scan);
972   }
973 
974   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
975     for (int i = 0; i < numRows; i++) {
976       String row = key + "_" + i/* UUID.randomUUID().toString() */;
977       System.out.println(String.format("Saving row: %s, with value %s", row, value));
978       Put put = new Put(Bytes.toBytes(row));
979       put.setDurability(Durability.SKIP_WAL);
980       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
981       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
982       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
983       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
984       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
985       r.put(put);
986     }
987   }
988 
989   @Test
990   public void testFamilyWithAndWithoutColon() throws Exception {
991     byte[] b = Bytes.toBytes(getName());
992     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
993     this.region = initHRegion(b, getName(), CONF, cf);
994     try {
995       Put p = new Put(b);
996       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
997       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
998       boolean exception = false;
999       try {
1000         this.region.put(p);
1001       } catch (NoSuchColumnFamilyException e) {
1002         exception = true;
1003       }
1004       assertTrue(exception);
1005     } finally {
1006       HRegion.closeHRegion(this.region);
1007       this.region = null;
1008     }
1009   }
1010 
1011   @Test
1012   public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1013     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1014     byte[] qual = Bytes.toBytes("qual");
1015     byte[] val = Bytes.toBytes("val");
1016     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1017     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1018     try {
1019       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1020       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1021 
1022       LOG.info("First a batch put with all valid puts");
1023       final Put[] puts = new Put[10];
1024       for (int i = 0; i < 10; i++) {
1025         puts[i] = new Put(Bytes.toBytes("row_" + i));
1026         puts[i].add(cf, qual, val);
1027       }
1028 
1029       OperationStatus[] codes = this.region.batchMutate(puts);
1030       assertEquals(10, codes.length);
1031       for (int i = 0; i < 10; i++) {
1032         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1033       }
1034       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1035 
1036       LOG.info("Next a batch put with one invalid family");
1037       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1038       codes = this.region.batchMutate(puts);
1039       assertEquals(10, codes.length);
1040       for (int i = 0; i < 10; i++) {
1041         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1042             codes[i].getOperationStatusCode());
1043       }
1044 
1045       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1046     } finally {
1047       HRegion.closeHRegion(this.region);
1048       this.region = null;
1049     }
1050   }
1051 
1052   @Test
1053   public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1054     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1055     byte[] qual = Bytes.toBytes("qual");
1056     byte[] val = Bytes.toBytes("val");
1057     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1058     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1059     try {
1060       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1061       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1062 
1063       final Put[] puts = new Put[10];
1064       for (int i = 0; i < 10; i++) {
1065         puts[i] = new Put(Bytes.toBytes("row_" + i));
1066         puts[i].add(cf, qual, val);
1067       }
1068       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1069 
1070       LOG.info("batchPut will have to break into four batches to avoid row locks");
1071       RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1072       RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1073       RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1074 
1075       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1076       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1077       TestThread putter = new TestThread(ctx) {
1078         @Override
1079         public void doWork() throws IOException {
1080           retFromThread.set(region.batchMutate(puts));
1081         }
1082       };
1083       LOG.info("...starting put thread while holding locks");
1084       ctx.addThread(putter);
1085       ctx.startThreads();
1086 
1087       LOG.info("...waiting for put thread to sync 1st time");
1088       waitForCounter(source, "syncTimeNumOps", syncs + 1);
1089 
1090       // Now attempt to close the region from another thread.  Prior to HBASE-12565
1091       // this would cause the in-progress batchMutate operation to to fail with
1092       // exception because it use to release and re-acquire the close-guard lock
1093       // between batches.  Caller then didn't get status indicating which writes succeeded.
1094       // We now expect this thread to block until the batchMutate call finishes.
1095       Thread regionCloseThread = new Thread() {
1096         @Override
1097         public void run() {
1098           try {
1099             HRegion.closeHRegion(region);
1100           } catch (IOException e) {
1101             throw new RuntimeException(e);
1102           }
1103         }
1104       };
1105       regionCloseThread.start();
1106 
1107       LOG.info("...releasing row lock 1, which should let put thread continue");
1108       rowLock1.release();
1109 
1110       LOG.info("...waiting for put thread to sync 2nd time");
1111       waitForCounter(source, "syncTimeNumOps", syncs + 2);
1112 
1113       LOG.info("...releasing row lock 2, which should let put thread continue");
1114       rowLock2.release();
1115 
1116       LOG.info("...waiting for put thread to sync 3rd time");
1117       waitForCounter(source, "syncTimeNumOps", syncs + 3);
1118 
1119       LOG.info("...releasing row lock 3, which should let put thread continue");
1120       rowLock3.release();
1121 
1122       LOG.info("...waiting for put thread to sync 4th time");
1123       waitForCounter(source, "syncTimeNumOps", syncs + 4);
1124 
1125       LOG.info("...joining on put thread");
1126       ctx.stop();
1127       regionCloseThread.join();
1128 
1129       OperationStatus[] codes = retFromThread.get();
1130       for (int i = 0; i < codes.length; i++) {
1131         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1132             codes[i].getOperationStatusCode());
1133       }
1134     } finally {
1135       HRegion.closeHRegion(this.region);
1136       this.region = null;
1137     }
1138   }
1139 
1140   private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1141       throws InterruptedException {
1142     long startWait = System.currentTimeMillis();
1143     long currentCount;
1144     while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1145       Thread.sleep(100);
1146       if (System.currentTimeMillis() - startWait > 10000) {
1147         fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1148           expectedCount, currentCount));
1149       }
1150     }
1151   }
1152 
1153 
1154   @Test
1155   public void testBatchPutWithTsSlop() throws Exception {
1156     byte[] b = Bytes.toBytes(getName());
1157     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1158     byte[] qual = Bytes.toBytes("qual");
1159     byte[] val = Bytes.toBytes("val");
1160 
1161     // add data with a timestamp that is too recent for range. Ensure assert
1162     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1163     this.region = initHRegion(b, getName(), CONF, cf);
1164 
1165     try {
1166       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1167       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1168       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1169 
1170       final Put[] puts = new Put[10];
1171       for (int i = 0; i < 10; i++) {
1172         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1173         puts[i].add(cf, qual, val);
1174       }
1175 
1176       OperationStatus[] codes = this.region.batchMutate(puts);
1177       assertEquals(10, codes.length);
1178       for (int i = 0; i < 10; i++) {
1179         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1180       }
1181       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1182 
1183     } finally {
1184       HRegion.closeHRegion(this.region);
1185       this.region = null;
1186     }
1187 
1188   }
1189 
1190   // ////////////////////////////////////////////////////////////////////////////
1191   // checkAndMutate tests
1192   // ////////////////////////////////////////////////////////////////////////////
1193   @Test
1194   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1195     byte[] row1 = Bytes.toBytes("row1");
1196     byte[] fam1 = Bytes.toBytes("fam1");
1197     byte[] qf1 = Bytes.toBytes("qualifier");
1198     byte[] emptyVal = new byte[] {};
1199     byte[] val1 = Bytes.toBytes("value1");
1200     byte[] val2 = Bytes.toBytes("value2");
1201 
1202     // Setting up region
1203     String method = this.getName();
1204     this.region = initHRegion(tableName, method, CONF, fam1);
1205     try {
1206       // Putting empty data in key
1207       Put put = new Put(row1);
1208       put.add(fam1, qf1, emptyVal);
1209 
1210       // checkAndPut with empty value
1211       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1212           emptyVal), put, true);
1213       assertTrue(res);
1214 
1215       // Putting data in key
1216       put = new Put(row1);
1217       put.add(fam1, qf1, val1);
1218 
1219       // checkAndPut with correct value
1220       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1221           put, true);
1222       assertTrue(res);
1223 
1224       // not empty anymore
1225       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1226           put, true);
1227       assertFalse(res);
1228 
1229       Delete delete = new Delete(row1);
1230       delete.deleteColumn(fam1, qf1);
1231       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1232           delete, true);
1233       assertFalse(res);
1234 
1235       put = new Put(row1);
1236       put.add(fam1, qf1, val2);
1237       // checkAndPut with correct value
1238       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1239           put, true);
1240       assertTrue(res);
1241 
1242       // checkAndDelete with correct value
1243       delete = new Delete(row1);
1244       delete.deleteColumn(fam1, qf1);
1245       delete.deleteColumn(fam1, qf1);
1246       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1247           delete, true);
1248       assertTrue(res);
1249 
1250       delete = new Delete(row1);
1251       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1252           delete, true);
1253       assertTrue(res);
1254 
1255       // checkAndPut looking for a null value
1256       put = new Put(row1);
1257       put.add(fam1, qf1, val1);
1258 
1259       res = region
1260           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1261       assertTrue(res);
1262     } finally {
1263       HRegion.closeHRegion(this.region);
1264       this.region = null;
1265     }
1266   }
1267 
1268   @Test
1269   public void testCheckAndMutate_WithWrongValue() throws IOException {
1270     byte[] row1 = Bytes.toBytes("row1");
1271     byte[] fam1 = Bytes.toBytes("fam1");
1272     byte[] qf1 = Bytes.toBytes("qualifier");
1273     byte[] val1 = Bytes.toBytes("value1");
1274     byte[] val2 = Bytes.toBytes("value2");
1275 
1276     // Setting up region
1277     String method = this.getName();
1278     this.region = initHRegion(tableName, method, CONF, fam1);
1279     try {
1280       // Putting data in key
1281       Put put = new Put(row1);
1282       put.add(fam1, qf1, val1);
1283       region.put(put);
1284 
1285       // checkAndPut with wrong value
1286       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1287           val2), put, true);
1288       assertEquals(false, res);
1289 
1290       // checkAndDelete with wrong value
1291       Delete delete = new Delete(row1);
1292       delete.deleteFamily(fam1);
1293       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1294           put, true);
1295       assertEquals(false, res);
1296     } finally {
1297       HRegion.closeHRegion(this.region);
1298       this.region = null;
1299     }
1300   }
1301 
1302   @Test
1303   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1304     byte[] row1 = Bytes.toBytes("row1");
1305     byte[] fam1 = Bytes.toBytes("fam1");
1306     byte[] qf1 = Bytes.toBytes("qualifier");
1307     byte[] val1 = Bytes.toBytes("value1");
1308 
1309     // Setting up region
1310     String method = this.getName();
1311     this.region = initHRegion(tableName, method, CONF, fam1);
1312     try {
1313       // Putting data in key
1314       Put put = new Put(row1);
1315       put.add(fam1, qf1, val1);
1316       region.put(put);
1317 
1318       // checkAndPut with correct value
1319       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1320           val1), put, true);
1321       assertEquals(true, res);
1322 
1323       // checkAndDelete with correct value
1324       Delete delete = new Delete(row1);
1325       delete.deleteColumn(fam1, qf1);
1326       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1327           delete, true);
1328       assertEquals(true, res);
1329     } finally {
1330       HRegion.closeHRegion(this.region);
1331       this.region = null;
1332     }
1333   }
1334 
1335   @Test
1336   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1337     byte[] row1 = Bytes.toBytes("row1");
1338     byte[] fam1 = Bytes.toBytes("fam1");
1339     byte[] qf1 = Bytes.toBytes("qualifier");
1340     byte[] val1 = Bytes.toBytes("value1");
1341     byte[] val2 = Bytes.toBytes("value2");
1342     byte[] val3 = Bytes.toBytes("value3");
1343     byte[] val4 = Bytes.toBytes("value4");
1344 
1345     // Setting up region
1346     String method = this.getName();
1347     this.region = initHRegion(tableName, method, CONF, fam1);
1348     try {
1349       // Putting val3 in key
1350       Put put = new Put(row1);
1351       put.add(fam1, qf1, val3);
1352       region.put(put);
1353 
1354       // Test CompareOp.LESS: original = val3, compare with val3, fail
1355       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1356           new BinaryComparator(val3), put, true);
1357       assertEquals(false, res);
1358 
1359       // Test CompareOp.LESS: original = val3, compare with val4, fail
1360       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1361           new BinaryComparator(val4), put, true);
1362       assertEquals(false, res);
1363 
1364       // Test CompareOp.LESS: original = val3, compare with val2,
1365       // succeed (now value = val2)
1366       put = new Put(row1);
1367       put.add(fam1, qf1, val2);
1368       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1369           new BinaryComparator(val2), put, true);
1370       assertEquals(true, res);
1371 
1372       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1373       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1374           new BinaryComparator(val3), put, true);
1375       assertEquals(false, res);
1376 
1377       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1378       // succeed (value still = val2)
1379       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1380           new BinaryComparator(val2), put, true);
1381       assertEquals(true, res);
1382 
1383       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1384       // succeed (now value = val3)
1385       put = new Put(row1);
1386       put.add(fam1, qf1, val3);
1387       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1388           new BinaryComparator(val1), put, true);
1389       assertEquals(true, res);
1390 
1391       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1392       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1393           new BinaryComparator(val3), put, true);
1394       assertEquals(false, res);
1395 
1396       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1397       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1398           new BinaryComparator(val2), put, true);
1399       assertEquals(false, res);
1400 
1401       // Test CompareOp.GREATER: original = val3, compare with val4,
1402       // succeed (now value = val2)
1403       put = new Put(row1);
1404       put.add(fam1, qf1, val2);
1405       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1406           new BinaryComparator(val4), put, true);
1407       assertEquals(true, res);
1408 
1409       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1410       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1411           new BinaryComparator(val1), put, true);
1412       assertEquals(false, res);
1413 
1414       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1415       // succeed (value still = val2)
1416       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1417           new BinaryComparator(val2), put, true);
1418       assertEquals(true, res);
1419 
1420       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1421       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1422           new BinaryComparator(val3), put, true);
1423       assertEquals(true, res);
1424     } finally {
1425       HRegion.closeHRegion(this.region);
1426       this.region = null;
1427     }
1428   }
1429 
1430   @Test
1431   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1432     byte[] row1 = Bytes.toBytes("row1");
1433     byte[] fam1 = Bytes.toBytes("fam1");
1434     byte[] fam2 = Bytes.toBytes("fam2");
1435     byte[] qf1 = Bytes.toBytes("qualifier");
1436     byte[] val1 = Bytes.toBytes("value1");
1437     byte[] val2 = Bytes.toBytes("value2");
1438 
1439     byte[][] families = { fam1, fam2 };
1440 
1441     // Setting up region
1442     String method = this.getName();
1443     this.region = initHRegion(tableName, method, CONF, families);
1444     try {
1445       // Putting data in the key to check
1446       Put put = new Put(row1);
1447       put.add(fam1, qf1, val1);
1448       region.put(put);
1449 
1450       // Creating put to add
1451       long ts = System.currentTimeMillis();
1452       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1453       put = new Put(row1);
1454       put.add(kv);
1455 
1456       // checkAndPut with wrong value
1457       HStore store = (HStore) region.getStore(fam1);
1458       store.memstore.kvset.size();
1459 
1460       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1461           val1), put, true);
1462       assertEquals(true, res);
1463       store.memstore.kvset.size();
1464 
1465       Get get = new Get(row1);
1466       get.addColumn(fam2, qf1);
1467       Cell[] actual = region.get(get).rawCells();
1468 
1469       Cell[] expected = { kv };
1470 
1471       assertEquals(expected.length, actual.length);
1472       for (int i = 0; i < actual.length; i++) {
1473         assertEquals(expected[i], actual[i]);
1474       }
1475     } finally {
1476       HRegion.closeHRegion(this.region);
1477       this.region = null;
1478     }
1479   }
1480 
1481   @Test
1482   public void testCheckAndPut_wrongRowInPut() throws IOException {
1483     TableName tableName = TableName.valueOf(name.getMethodName());
1484     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1485     try {
1486       Put put = new Put(row2);
1487       put.add(fam1, qual1, value1);
1488       try {
1489         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1490             new BinaryComparator(value2), put, false);
1491         fail();
1492       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1493         // expected exception.
1494       }
1495     } finally {
1496       HRegion.closeHRegion(this.region);
1497       this.region = null;
1498     }
1499   }
1500 
1501   @Test
1502   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1503     byte[] row1 = Bytes.toBytes("row1");
1504     byte[] fam1 = Bytes.toBytes("fam1");
1505     byte[] fam2 = Bytes.toBytes("fam2");
1506     byte[] qf1 = Bytes.toBytes("qualifier1");
1507     byte[] qf2 = Bytes.toBytes("qualifier2");
1508     byte[] qf3 = Bytes.toBytes("qualifier3");
1509     byte[] val1 = Bytes.toBytes("value1");
1510     byte[] val2 = Bytes.toBytes("value2");
1511     byte[] val3 = Bytes.toBytes("value3");
1512     byte[] emptyVal = new byte[] {};
1513 
1514     byte[][] families = { fam1, fam2 };
1515 
1516     // Setting up region
1517     String method = this.getName();
1518     this.region = initHRegion(tableName, method, CONF, families);
1519     try {
1520       // Put content
1521       Put put = new Put(row1);
1522       put.add(fam1, qf1, val1);
1523       region.put(put);
1524       Threads.sleep(2);
1525 
1526       put = new Put(row1);
1527       put.add(fam1, qf1, val2);
1528       put.add(fam2, qf1, val3);
1529       put.add(fam2, qf2, val2);
1530       put.add(fam2, qf3, val1);
1531       put.add(fam1, qf3, val1);
1532       region.put(put);
1533 
1534       // Multi-column delete
1535       Delete delete = new Delete(row1);
1536       delete.deleteColumn(fam1, qf1);
1537       delete.deleteColumn(fam2, qf1);
1538       delete.deleteColumn(fam1, qf3);
1539       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1540           val2), delete, true);
1541       assertEquals(true, res);
1542 
1543       Get get = new Get(row1);
1544       get.addColumn(fam1, qf1);
1545       get.addColumn(fam1, qf3);
1546       get.addColumn(fam2, qf2);
1547       Result r = region.get(get);
1548       assertEquals(2, r.size());
1549       assertArrayEquals(val1, r.getValue(fam1, qf1));
1550       assertArrayEquals(val2, r.getValue(fam2, qf2));
1551 
1552       // Family delete
1553       delete = new Delete(row1);
1554       delete.deleteFamily(fam2);
1555       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1556           delete, true);
1557       assertEquals(true, res);
1558 
1559       get = new Get(row1);
1560       r = region.get(get);
1561       assertEquals(1, r.size());
1562       assertArrayEquals(val1, r.getValue(fam1, qf1));
1563 
1564       // Row delete
1565       delete = new Delete(row1);
1566       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1567           delete, true);
1568       assertEquals(true, res);
1569       get = new Get(row1);
1570       r = region.get(get);
1571       assertEquals(0, r.size());
1572     } finally {
1573       HRegion.closeHRegion(this.region);
1574       this.region = null;
1575     }
1576   }
1577 
1578   // ////////////////////////////////////////////////////////////////////////////
1579   // Delete tests
1580   // ////////////////////////////////////////////////////////////////////////////
1581   @Test
1582   public void testDelete_multiDeleteColumn() throws IOException {
1583     byte[] row1 = Bytes.toBytes("row1");
1584     byte[] fam1 = Bytes.toBytes("fam1");
1585     byte[] qual = Bytes.toBytes("qualifier");
1586     byte[] value = Bytes.toBytes("value");
1587 
1588     Put put = new Put(row1);
1589     put.add(fam1, qual, 1, value);
1590     put.add(fam1, qual, 2, value);
1591 
1592     String method = this.getName();
1593     this.region = initHRegion(tableName, method, CONF, fam1);
1594     try {
1595       region.put(put);
1596 
1597       // We do support deleting more than 1 'latest' version
1598       Delete delete = new Delete(row1);
1599       delete.deleteColumn(fam1, qual);
1600       delete.deleteColumn(fam1, qual);
1601       region.delete(delete);
1602 
1603       Get get = new Get(row1);
1604       get.addFamily(fam1);
1605       Result r = region.get(get);
1606       assertEquals(0, r.size());
1607     } finally {
1608       HRegion.closeHRegion(this.region);
1609       this.region = null;
1610     }
1611   }
1612 
1613   @Test
1614   public void testDelete_CheckFamily() throws IOException {
1615     byte[] row1 = Bytes.toBytes("row1");
1616     byte[] fam1 = Bytes.toBytes("fam1");
1617     byte[] fam2 = Bytes.toBytes("fam2");
1618     byte[] fam3 = Bytes.toBytes("fam3");
1619     byte[] fam4 = Bytes.toBytes("fam4");
1620 
1621     // Setting up region
1622     String method = this.getName();
1623     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1624     try {
1625       List<Cell> kvs = new ArrayList<Cell>();
1626       kvs.add(new KeyValue(row1, fam4, null, null));
1627 
1628       // testing existing family
1629       byte[] family = fam2;
1630       try {
1631         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1632             Bytes.BYTES_COMPARATOR);
1633         deleteMap.put(family, kvs);
1634         region.delete(deleteMap, Durability.SYNC_WAL);
1635       } catch (Exception e) {
1636         assertTrue("Family " + new String(family) + " does not exist", false);
1637       }
1638 
1639       // testing non existing family
1640       boolean ok = false;
1641       family = fam4;
1642       try {
1643         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1644             Bytes.BYTES_COMPARATOR);
1645         deleteMap.put(family, kvs);
1646         region.delete(deleteMap, Durability.SYNC_WAL);
1647       } catch (Exception e) {
1648         ok = true;
1649       }
1650       assertEquals("Family " + new String(family) + " does exist", true, ok);
1651     } finally {
1652       HRegion.closeHRegion(this.region);
1653       this.region = null;
1654     }
1655   }
1656 
1657   @Test
1658   public void testDelete_mixed() throws IOException, InterruptedException {
1659     byte[] fam = Bytes.toBytes("info");
1660     byte[][] families = { fam };
1661     String method = this.getName();
1662     this.region = initHRegion(tableName, method, CONF, families);
1663     try {
1664       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1665 
1666       byte[] row = Bytes.toBytes("table_name");
1667       // column names
1668       byte[] serverinfo = Bytes.toBytes("serverinfo");
1669       byte[] splitA = Bytes.toBytes("splitA");
1670       byte[] splitB = Bytes.toBytes("splitB");
1671 
1672       // add some data:
1673       Put put = new Put(row);
1674       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1675       region.put(put);
1676 
1677       put = new Put(row);
1678       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1679       region.put(put);
1680 
1681       put = new Put(row);
1682       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1683       region.put(put);
1684 
1685       // ok now delete a split:
1686       Delete delete = new Delete(row);
1687       delete.deleteColumns(fam, splitA);
1688       region.delete(delete);
1689 
1690       // assert some things:
1691       Get get = new Get(row).addColumn(fam, serverinfo);
1692       Result result = region.get(get);
1693       assertEquals(1, result.size());
1694 
1695       get = new Get(row).addColumn(fam, splitA);
1696       result = region.get(get);
1697       assertEquals(0, result.size());
1698 
1699       get = new Get(row).addColumn(fam, splitB);
1700       result = region.get(get);
1701       assertEquals(1, result.size());
1702 
1703       // Assert that after a delete, I can put.
1704       put = new Put(row);
1705       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1706       region.put(put);
1707       get = new Get(row);
1708       result = region.get(get);
1709       assertEquals(3, result.size());
1710 
1711       // Now delete all... then test I can add stuff back
1712       delete = new Delete(row);
1713       region.delete(delete);
1714       assertEquals(0, region.get(get).size());
1715 
1716       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1717       result = region.get(get);
1718       assertEquals(1, result.size());
1719     } finally {
1720       HRegion.closeHRegion(this.region);
1721       this.region = null;
1722     }
1723   }
1724 
1725   @Test
1726   public void testDeleteRowWithFutureTs() throws IOException {
1727     byte[] fam = Bytes.toBytes("info");
1728     byte[][] families = { fam };
1729     String method = this.getName();
1730     this.region = initHRegion(tableName, method, CONF, families);
1731     try {
1732       byte[] row = Bytes.toBytes("table_name");
1733       // column names
1734       byte[] serverinfo = Bytes.toBytes("serverinfo");
1735 
1736       // add data in the far future
1737       Put put = new Put(row);
1738       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1739       region.put(put);
1740 
1741       // now delete something in the present
1742       Delete delete = new Delete(row);
1743       region.delete(delete);
1744 
1745       // make sure we still see our data
1746       Get get = new Get(row).addColumn(fam, serverinfo);
1747       Result result = region.get(get);
1748       assertEquals(1, result.size());
1749 
1750       // delete the future row
1751       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1752       region.delete(delete);
1753 
1754       // make sure it is gone
1755       get = new Get(row).addColumn(fam, serverinfo);
1756       result = region.get(get);
1757       assertEquals(0, result.size());
1758     } finally {
1759       HRegion.closeHRegion(this.region);
1760       this.region = null;
1761     }
1762   }
1763 
1764   /**
1765    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1766    * the actual timestamp
1767    */
1768   @Test
1769   public void testPutWithLatestTS() throws IOException {
1770     byte[] fam = Bytes.toBytes("info");
1771     byte[][] families = { fam };
1772     String method = this.getName();
1773     this.region = initHRegion(tableName, method, CONF, families);
1774     try {
1775       byte[] row = Bytes.toBytes("row1");
1776       // column names
1777       byte[] qual = Bytes.toBytes("qual");
1778 
1779       // add data with LATEST_TIMESTAMP, put without WAL
1780       Put put = new Put(row);
1781       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1782       region.put(put);
1783 
1784       // Make sure it shows up with an actual timestamp
1785       Get get = new Get(row).addColumn(fam, qual);
1786       Result result = region.get(get);
1787       assertEquals(1, result.size());
1788       Cell kv = result.rawCells()[0];
1789       LOG.info("Got: " + kv);
1790       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1791           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1792 
1793       // Check same with WAL enabled (historically these took different
1794       // code paths, so check both)
1795       row = Bytes.toBytes("row2");
1796       put = new Put(row);
1797       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1798       region.put(put);
1799 
1800       // Make sure it shows up with an actual timestamp
1801       get = new Get(row).addColumn(fam, qual);
1802       result = region.get(get);
1803       assertEquals(1, result.size());
1804       kv = result.rawCells()[0];
1805       LOG.info("Got: " + kv);
1806       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1807           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1808     } finally {
1809       HRegion.closeHRegion(this.region);
1810       this.region = null;
1811     }
1812 
1813   }
1814 
1815   /**
1816    * Tests that there is server-side filtering for invalid timestamp upper
1817    * bound. Note that the timestamp lower bound is automatically handled for us
1818    * by the TTL field.
1819    */
1820   @Test
1821   public void testPutWithTsSlop() throws IOException {
1822     byte[] fam = Bytes.toBytes("info");
1823     byte[][] families = { fam };
1824     String method = this.getName();
1825 
1826     // add data with a timestamp that is too recent for range. Ensure assert
1827     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1828     this.region = initHRegion(tableName, method, CONF, families);
1829     boolean caughtExcep = false;
1830     try {
1831       try {
1832         // no TS specified == use latest. should not error
1833         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1834         // TS out of range. should error
1835         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1836             Bytes.toBytes("value")));
1837         fail("Expected IOE for TS out of configured timerange");
1838       } catch (FailedSanityCheckException ioe) {
1839         LOG.debug("Received expected exception", ioe);
1840         caughtExcep = true;
1841       }
1842       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1843     } finally {
1844       HRegion.closeHRegion(this.region);
1845       this.region = null;
1846     }
1847   }
1848 
1849   @Test
1850   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1851     byte[] fam1 = Bytes.toBytes("columnA");
1852     byte[] fam2 = Bytes.toBytes("columnB");
1853     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1854     try {
1855       byte[] rowA = Bytes.toBytes("rowA");
1856       byte[] rowB = Bytes.toBytes("rowB");
1857 
1858       byte[] value = Bytes.toBytes("value");
1859 
1860       Delete delete = new Delete(rowA);
1861       delete.deleteFamily(fam1);
1862 
1863       region.delete(delete);
1864 
1865       // now create data.
1866       Put put = new Put(rowA);
1867       put.add(fam2, null, value);
1868       region.put(put);
1869 
1870       put = new Put(rowB);
1871       put.add(fam1, null, value);
1872       put.add(fam2, null, value);
1873       region.put(put);
1874 
1875       Scan scan = new Scan();
1876       scan.addFamily(fam1).addFamily(fam2);
1877       InternalScanner s = region.getScanner(scan);
1878       List<Cell> results = new ArrayList<Cell>();
1879       s.next(results);
1880       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1881 
1882       results.clear();
1883       s.next(results);
1884       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1885     } finally {
1886       HRegion.closeHRegion(this.region);
1887       this.region = null;
1888     }
1889   }
1890 
1891   @Test
1892   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1893     Delete delete = new Delete(row);
1894     delete.deleteColumns(fam1, qual1);
1895     doTestDelete_AndPostInsert(delete);
1896   }
1897 
1898   @Test
1899   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1900     Delete delete = new Delete(row);
1901     delete.deleteFamily(fam1);
1902     doTestDelete_AndPostInsert(delete);
1903   }
1904 
1905   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1906     TableName tableName = TableName.valueOf(name.getMethodName());
1907     this.region = initHRegion(tableName, getName(), CONF, fam1);
1908     try {
1909       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1910       Put put = new Put(row);
1911       put.add(fam1, qual1, value1);
1912       region.put(put);
1913 
1914       // now delete the value:
1915       region.delete(delete);
1916 
1917       // ok put data:
1918       put = new Put(row);
1919       put.add(fam1, qual1, value2);
1920       region.put(put);
1921 
1922       // ok get:
1923       Get get = new Get(row);
1924       get.addColumn(fam1, qual1);
1925 
1926       Result r = region.get(get);
1927       assertEquals(1, r.size());
1928       assertArrayEquals(value2, r.getValue(fam1, qual1));
1929 
1930       // next:
1931       Scan scan = new Scan(row);
1932       scan.addColumn(fam1, qual1);
1933       InternalScanner s = region.getScanner(scan);
1934 
1935       List<Cell> results = new ArrayList<Cell>();
1936       assertEquals(false, s.next(results));
1937       assertEquals(1, results.size());
1938       Cell kv = results.get(0);
1939 
1940       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1941       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1942       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1943       assertArrayEquals(row, CellUtil.cloneRow(kv));
1944     } finally {
1945       HRegion.closeHRegion(this.region);
1946       this.region = null;
1947     }
1948   }
1949 
1950   @Test
1951   public void testDelete_CheckTimestampUpdated() throws IOException {
1952     TableName tableName = TableName.valueOf(name.getMethodName());
1953     byte[] row1 = Bytes.toBytes("row1");
1954     byte[] col1 = Bytes.toBytes("col1");
1955     byte[] col2 = Bytes.toBytes("col2");
1956     byte[] col3 = Bytes.toBytes("col3");
1957 
1958     // Setting up region
1959     String method = this.getName();
1960     this.region = initHRegion(tableName, method, CONF, fam1);
1961     try {
1962       // Building checkerList
1963       List<Cell> kvs = new ArrayList<Cell>();
1964       kvs.add(new KeyValue(row1, fam1, col1, null));
1965       kvs.add(new KeyValue(row1, fam1, col2, null));
1966       kvs.add(new KeyValue(row1, fam1, col3, null));
1967 
1968       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1969           Bytes.BYTES_COMPARATOR);
1970       deleteMap.put(fam1, kvs);
1971       region.delete(deleteMap, Durability.SYNC_WAL);
1972 
1973       // extract the key values out the memstore:
1974       // This is kinda hacky, but better than nothing...
1975       long now = System.currentTimeMillis();
1976       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1977       assertTrue(firstKv.getTimestamp() <= now);
1978       now = firstKv.getTimestamp();
1979       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1980         assertTrue(kv.getTimestamp() <= now);
1981         now = kv.getTimestamp();
1982       }
1983     } finally {
1984       HRegion.closeHRegion(this.region);
1985       this.region = null;
1986     }
1987   }
1988 
1989   // ////////////////////////////////////////////////////////////////////////////
1990   // Get tests
1991   // ////////////////////////////////////////////////////////////////////////////
1992   @Test
1993   public void testGet_FamilyChecker() throws IOException {
1994     byte[] row1 = Bytes.toBytes("row1");
1995     byte[] fam1 = Bytes.toBytes("fam1");
1996     byte[] fam2 = Bytes.toBytes("False");
1997     byte[] col1 = Bytes.toBytes("col1");
1998 
1999     // Setting up region
2000     String method = this.getName();
2001     this.region = initHRegion(tableName, method, CONF, fam1);
2002     try {
2003       Get get = new Get(row1);
2004       get.addColumn(fam2, col1);
2005 
2006       // Test
2007       try {
2008         region.get(get);
2009       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2010         assertFalse(false);
2011         return;
2012       }
2013       assertFalse(true);
2014     } finally {
2015       HRegion.closeHRegion(this.region);
2016       this.region = null;
2017     }
2018   }
2019 
2020   @Test
2021   public void testGet_Basic() throws IOException {
2022     byte[] row1 = Bytes.toBytes("row1");
2023     byte[] fam1 = Bytes.toBytes("fam1");
2024     byte[] col1 = Bytes.toBytes("col1");
2025     byte[] col2 = Bytes.toBytes("col2");
2026     byte[] col3 = Bytes.toBytes("col3");
2027     byte[] col4 = Bytes.toBytes("col4");
2028     byte[] col5 = Bytes.toBytes("col5");
2029 
2030     // Setting up region
2031     String method = this.getName();
2032     this.region = initHRegion(tableName, method, CONF, fam1);
2033     try {
2034       // Add to memstore
2035       Put put = new Put(row1);
2036       put.add(fam1, col1, null);
2037       put.add(fam1, col2, null);
2038       put.add(fam1, col3, null);
2039       put.add(fam1, col4, null);
2040       put.add(fam1, col5, null);
2041       region.put(put);
2042 
2043       Get get = new Get(row1);
2044       get.addColumn(fam1, col2);
2045       get.addColumn(fam1, col4);
2046       // Expected result
2047       KeyValue kv1 = new KeyValue(row1, fam1, col2);
2048       KeyValue kv2 = new KeyValue(row1, fam1, col4);
2049       KeyValue[] expected = { kv1, kv2 };
2050 
2051       // Test
2052       Result res = region.get(get);
2053       assertEquals(expected.length, res.size());
2054       for (int i = 0; i < res.size(); i++) {
2055         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2056         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2057         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2058       }
2059 
2060       // Test using a filter on a Get
2061       Get g = new Get(row1);
2062       final int count = 2;
2063       g.setFilter(new ColumnCountGetFilter(count));
2064       res = region.get(g);
2065       assertEquals(count, res.size());
2066     } finally {
2067       HRegion.closeHRegion(this.region);
2068       this.region = null;
2069     }
2070   }
2071 
2072   @Test
2073   public void testGet_Empty() throws IOException {
2074     byte[] row = Bytes.toBytes("row");
2075     byte[] fam = Bytes.toBytes("fam");
2076 
2077     String method = this.getName();
2078     this.region = initHRegion(tableName, method, CONF, fam);
2079     try {
2080       Get get = new Get(row);
2081       get.addFamily(fam);
2082       Result r = region.get(get);
2083 
2084       assertTrue(r.isEmpty());
2085     } finally {
2086       HRegion.closeHRegion(this.region);
2087       this.region = null;
2088     }
2089   }
2090 
2091   // ////////////////////////////////////////////////////////////////////////////
2092   // Merge test
2093   // ////////////////////////////////////////////////////////////////////////////
2094   @Test
2095   public void testMerge() throws IOException {
2096     byte[][] families = { fam1, fam2, fam3 };
2097     Configuration hc = initSplit();
2098     // Setting up region
2099     String method = this.getName();
2100     this.region = initHRegion(tableName, method, hc, families);
2101     try {
2102       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2103       region.flushcache();
2104       region.compactStores();
2105       byte[] splitRow = region.checkSplit();
2106       assertNotNull(splitRow);
2107       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2108       HRegion[] subregions = splitRegion(region, splitRow);
2109       try {
2110         // Need to open the regions.
2111         for (int i = 0; i < subregions.length; i++) {
2112           HRegion.openHRegion(subregions[i], null);
2113           subregions[i].compactStores();
2114         }
2115         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2116         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2117         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2118         long startTime = System.currentTimeMillis();
2119         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2120         LOG.info("Merge regions elapsed time: "
2121             + ((System.currentTimeMillis() - startTime) / 1000.0));
2122         FILESYSTEM.delete(oldRegion1, true);
2123         FILESYSTEM.delete(oldRegion2, true);
2124         FILESYSTEM.delete(oldRegionPath, true);
2125         LOG.info("splitAndMerge completed.");
2126       } finally {
2127         for (int i = 0; i < subregions.length; i++) {
2128           try {
2129             HRegion.closeHRegion(subregions[i]);
2130           } catch (IOException e) {
2131             // Ignore.
2132           }
2133         }
2134       }
2135     } finally {
2136       HRegion.closeHRegion(this.region);
2137       this.region = null;
2138     }
2139   }
2140 
2141   /**
2142    * @param parent
2143    *          Region to split.
2144    * @param midkey
2145    *          Key to split around.
2146    * @return The Regions we created.
2147    * @throws IOException
2148    */
2149   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2150     PairOfSameType<HRegion> result = null;
2151     SplitTransaction st = new SplitTransaction(parent, midkey);
2152     // If prepare does not return true, for some reason -- logged inside in
2153     // the prepare call -- we are not ready to split just now. Just return.
2154     if (!st.prepare()) {
2155       parent.clearSplit();
2156       return null;
2157     }
2158     try {
2159       result = st.execute(null, null);
2160     } catch (IOException ioe) {
2161       try {
2162         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2163             + ioe.getMessage());
2164         st.rollback(null, null);
2165         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2166         return null;
2167       } catch (RuntimeException e) {
2168         // If failed rollback, kill this server to avoid having a hole in table.
2169         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2170             + " -- aborting server", e);
2171       }
2172     }
2173     finally {
2174       parent.clearSplit();
2175     }
2176     return new HRegion[] { result.getFirst(), result.getSecond() };
2177   }
2178 
2179   // ////////////////////////////////////////////////////////////////////////////
2180   // Scanner tests
2181   // ////////////////////////////////////////////////////////////////////////////
2182   @Test
2183   public void testGetScanner_WithOkFamilies() throws IOException {
2184     byte[] fam1 = Bytes.toBytes("fam1");
2185     byte[] fam2 = Bytes.toBytes("fam2");
2186 
2187     byte[][] families = { fam1, fam2 };
2188 
2189     // Setting up region
2190     String method = this.getName();
2191     this.region = initHRegion(tableName, method, CONF, families);
2192     try {
2193       Scan scan = new Scan();
2194       scan.addFamily(fam1);
2195       scan.addFamily(fam2);
2196       try {
2197         region.getScanner(scan);
2198       } catch (Exception e) {
2199         assertTrue("Families could not be found in Region", false);
2200       }
2201     } finally {
2202       HRegion.closeHRegion(this.region);
2203       this.region = null;
2204     }
2205   }
2206 
2207   @Test
2208   public void testGetScanner_WithNotOkFamilies() throws IOException {
2209     byte[] fam1 = Bytes.toBytes("fam1");
2210     byte[] fam2 = Bytes.toBytes("fam2");
2211 
2212     byte[][] families = { fam1 };
2213 
2214     // Setting up region
2215     String method = this.getName();
2216     this.region = initHRegion(tableName, method, CONF, families);
2217     try {
2218       Scan scan = new Scan();
2219       scan.addFamily(fam2);
2220       boolean ok = false;
2221       try {
2222         region.getScanner(scan);
2223       } catch (Exception e) {
2224         ok = true;
2225       }
2226       assertTrue("Families could not be found in Region", ok);
2227     } finally {
2228       HRegion.closeHRegion(this.region);
2229       this.region = null;
2230     }
2231   }
2232 
2233   @Test
2234   public void testGetScanner_WithNoFamilies() throws IOException {
2235     byte[] row1 = Bytes.toBytes("row1");
2236     byte[] fam1 = Bytes.toBytes("fam1");
2237     byte[] fam2 = Bytes.toBytes("fam2");
2238     byte[] fam3 = Bytes.toBytes("fam3");
2239     byte[] fam4 = Bytes.toBytes("fam4");
2240 
2241     byte[][] families = { fam1, fam2, fam3, fam4 };
2242 
2243     // Setting up region
2244     String method = this.getName();
2245     this.region = initHRegion(tableName, method, CONF, families);
2246     try {
2247 
2248       // Putting data in Region
2249       Put put = new Put(row1);
2250       put.add(fam1, null, null);
2251       put.add(fam2, null, null);
2252       put.add(fam3, null, null);
2253       put.add(fam4, null, null);
2254       region.put(put);
2255 
2256       Scan scan = null;
2257       HRegion.RegionScannerImpl is = null;
2258 
2259       // Testing to see how many scanners that is produced by getScanner,
2260       // starting
2261       // with known number, 2 - current = 1
2262       scan = new Scan();
2263       scan.addFamily(fam2);
2264       scan.addFamily(fam4);
2265       is = (RegionScannerImpl) region.getScanner(scan);
2266       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2267 
2268       scan = new Scan();
2269       is = (RegionScannerImpl) region.getScanner(scan);
2270       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2271     } finally {
2272       HRegion.closeHRegion(this.region);
2273       this.region = null;
2274     }
2275   }
2276 
2277   /**
2278    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2279    * 
2280    * @throws IOException
2281    */
2282   @Test
2283   public void testGetScanner_WithRegionClosed() throws IOException {
2284     byte[] fam1 = Bytes.toBytes("fam1");
2285     byte[] fam2 = Bytes.toBytes("fam2");
2286 
2287     byte[][] families = { fam1, fam2 };
2288 
2289     // Setting up region
2290     String method = this.getName();
2291     try {
2292       this.region = initHRegion(tableName, method, CONF, families);
2293     } catch (IOException e) {
2294       e.printStackTrace();
2295       fail("Got IOException during initHRegion, " + e.getMessage());
2296     }
2297     try {
2298       region.closed.set(true);
2299       try {
2300         region.getScanner(null);
2301         fail("Expected to get an exception during getScanner on a region that is closed");
2302       } catch (NotServingRegionException e) {
2303         // this is the correct exception that is expected
2304       } catch (IOException e) {
2305         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2306             + e.getMessage());
2307       }
2308     } finally {
2309       HRegion.closeHRegion(this.region);
2310       this.region = null;
2311     }
2312   }
2313 
2314   @Test
2315   public void testRegionScanner_Next() throws IOException {
2316     byte[] row1 = Bytes.toBytes("row1");
2317     byte[] row2 = Bytes.toBytes("row2");
2318     byte[] fam1 = Bytes.toBytes("fam1");
2319     byte[] fam2 = Bytes.toBytes("fam2");
2320     byte[] fam3 = Bytes.toBytes("fam3");
2321     byte[] fam4 = Bytes.toBytes("fam4");
2322 
2323     byte[][] families = { fam1, fam2, fam3, fam4 };
2324     long ts = System.currentTimeMillis();
2325 
2326     // Setting up region
2327     String method = this.getName();
2328     this.region = initHRegion(tableName, method, CONF, families);
2329     try {
2330       // Putting data in Region
2331       Put put = null;
2332       put = new Put(row1);
2333       put.add(fam1, (byte[]) null, ts, null);
2334       put.add(fam2, (byte[]) null, ts, null);
2335       put.add(fam3, (byte[]) null, ts, null);
2336       put.add(fam4, (byte[]) null, ts, null);
2337       region.put(put);
2338 
2339       put = new Put(row2);
2340       put.add(fam1, (byte[]) null, ts, null);
2341       put.add(fam2, (byte[]) null, ts, null);
2342       put.add(fam3, (byte[]) null, ts, null);
2343       put.add(fam4, (byte[]) null, ts, null);
2344       region.put(put);
2345 
2346       Scan scan = new Scan();
2347       scan.addFamily(fam2);
2348       scan.addFamily(fam4);
2349       InternalScanner is = region.getScanner(scan);
2350 
2351       List<Cell> res = null;
2352 
2353       // Result 1
2354       List<Cell> expected1 = new ArrayList<Cell>();
2355       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2356       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2357 
2358       res = new ArrayList<Cell>();
2359       is.next(res);
2360       for (int i = 0; i < res.size(); i++) {
2361         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2362       }
2363 
2364       // Result 2
2365       List<Cell> expected2 = new ArrayList<Cell>();
2366       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2367       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2368 
2369       res = new ArrayList<Cell>();
2370       is.next(res);
2371       for (int i = 0; i < res.size(); i++) {
2372         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2373       }
2374     } finally {
2375       HRegion.closeHRegion(this.region);
2376       this.region = null;
2377     }
2378   }
2379 
2380   @Test
2381   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2382     byte[] row1 = Bytes.toBytes("row1");
2383     byte[] qf1 = Bytes.toBytes("qualifier1");
2384     byte[] qf2 = Bytes.toBytes("qualifier2");
2385     byte[] fam1 = Bytes.toBytes("fam1");
2386     byte[][] families = { fam1 };
2387 
2388     long ts1 = System.currentTimeMillis();
2389     long ts2 = ts1 + 1;
2390     long ts3 = ts1 + 2;
2391 
2392     // Setting up region
2393     String method = this.getName();
2394     this.region = initHRegion(tableName, method, CONF, families);
2395     try {
2396       // Putting data in Region
2397       Put put = null;
2398       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2399       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2400       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2401 
2402       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2403       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2404       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2405 
2406       put = new Put(row1);
2407       put.add(kv13);
2408       put.add(kv12);
2409       put.add(kv11);
2410       put.add(kv23);
2411       put.add(kv22);
2412       put.add(kv21);
2413       region.put(put);
2414 
2415       // Expected
2416       List<Cell> expected = new ArrayList<Cell>();
2417       expected.add(kv13);
2418       expected.add(kv12);
2419 
2420       Scan scan = new Scan(row1);
2421       scan.addColumn(fam1, qf1);
2422       scan.setMaxVersions(MAX_VERSIONS);
2423       List<Cell> actual = new ArrayList<Cell>();
2424       InternalScanner scanner = region.getScanner(scan);
2425 
2426       boolean hasNext = scanner.next(actual);
2427       assertEquals(false, hasNext);
2428 
2429       // Verify result
2430       for (int i = 0; i < expected.size(); i++) {
2431         assertEquals(expected.get(i), actual.get(i));
2432       }
2433     } finally {
2434       HRegion.closeHRegion(this.region);
2435       this.region = null;
2436     }
2437   }
2438 
2439   @Test
2440   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2441     byte[] row1 = Bytes.toBytes("row1");
2442     byte[] qf1 = Bytes.toBytes("qualifier1");
2443     byte[] qf2 = Bytes.toBytes("qualifier2");
2444     byte[] fam1 = Bytes.toBytes("fam1");
2445     byte[][] families = { fam1 };
2446 
2447     long ts1 = 1; // System.currentTimeMillis();
2448     long ts2 = ts1 + 1;
2449     long ts3 = ts1 + 2;
2450 
2451     // Setting up region
2452     String method = this.getName();
2453     this.region = initHRegion(tableName, method, CONF, families);
2454     try {
2455       // Putting data in Region
2456       Put put = null;
2457       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2458       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2459       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2460 
2461       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2462       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2463       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2464 
2465       put = new Put(row1);
2466       put.add(kv13);
2467       put.add(kv12);
2468       put.add(kv11);
2469       put.add(kv23);
2470       put.add(kv22);
2471       put.add(kv21);
2472       region.put(put);
2473       region.flushcache();
2474 
2475       // Expected
2476       List<Cell> expected = new ArrayList<Cell>();
2477       expected.add(kv13);
2478       expected.add(kv12);
2479       expected.add(kv23);
2480       expected.add(kv22);
2481 
2482       Scan scan = new Scan(row1);
2483       scan.addColumn(fam1, qf1);
2484       scan.addColumn(fam1, qf2);
2485       scan.setMaxVersions(MAX_VERSIONS);
2486       List<Cell> actual = new ArrayList<Cell>();
2487       InternalScanner scanner = region.getScanner(scan);
2488 
2489       boolean hasNext = scanner.next(actual);
2490       assertEquals(false, hasNext);
2491 
2492       // Verify result
2493       for (int i = 0; i < expected.size(); i++) {
2494         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2495       }
2496     } finally {
2497       HRegion.closeHRegion(this.region);
2498       this.region = null;
2499     }
2500   }
2501 
2502   @Test
2503   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2504     byte[] row1 = Bytes.toBytes("row1");
2505     byte[] fam1 = Bytes.toBytes("fam1");
2506     byte[][] families = { fam1 };
2507     byte[] qf1 = Bytes.toBytes("qualifier1");
2508     byte[] qf2 = Bytes.toBytes("qualifier2");
2509 
2510     long ts1 = 1;
2511     long ts2 = ts1 + 1;
2512     long ts3 = ts1 + 2;
2513     long ts4 = ts1 + 3;
2514 
2515     // Setting up region
2516     String method = this.getName();
2517     this.region = initHRegion(tableName, method, CONF, families);
2518     try {
2519       // Putting data in Region
2520       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2521       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2522       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2523       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2524 
2525       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2526       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2527       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2528       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2529 
2530       Put put = null;
2531       put = new Put(row1);
2532       put.add(kv14);
2533       put.add(kv24);
2534       region.put(put);
2535       region.flushcache();
2536 
2537       put = new Put(row1);
2538       put.add(kv23);
2539       put.add(kv13);
2540       region.put(put);
2541       region.flushcache();
2542 
2543       put = new Put(row1);
2544       put.add(kv22);
2545       put.add(kv12);
2546       region.put(put);
2547       region.flushcache();
2548 
2549       put = new Put(row1);
2550       put.add(kv21);
2551       put.add(kv11);
2552       region.put(put);
2553 
2554       // Expected
2555       List<Cell> expected = new ArrayList<Cell>();
2556       expected.add(kv14);
2557       expected.add(kv13);
2558       expected.add(kv12);
2559       expected.add(kv24);
2560       expected.add(kv23);
2561       expected.add(kv22);
2562 
2563       Scan scan = new Scan(row1);
2564       scan.addColumn(fam1, qf1);
2565       scan.addColumn(fam1, qf2);
2566       int versions = 3;
2567       scan.setMaxVersions(versions);
2568       List<Cell> actual = new ArrayList<Cell>();
2569       InternalScanner scanner = region.getScanner(scan);
2570 
2571       boolean hasNext = scanner.next(actual);
2572       assertEquals(false, hasNext);
2573 
2574       // Verify result
2575       for (int i = 0; i < expected.size(); i++) {
2576         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2577       }
2578     } finally {
2579       HRegion.closeHRegion(this.region);
2580       this.region = null;
2581     }
2582   }
2583 
2584   @Test
2585   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2586     byte[] row1 = Bytes.toBytes("row1");
2587     byte[] qf1 = Bytes.toBytes("qualifier1");
2588     byte[] qf2 = Bytes.toBytes("qualifier2");
2589     byte[] fam1 = Bytes.toBytes("fam1");
2590     byte[][] families = { fam1 };
2591 
2592     long ts1 = System.currentTimeMillis();
2593     long ts2 = ts1 + 1;
2594     long ts3 = ts1 + 2;
2595 
2596     // Setting up region
2597     String method = this.getName();
2598     this.region = initHRegion(tableName, method, CONF, families);
2599     try {
2600       // Putting data in Region
2601       Put put = null;
2602       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2603       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2604       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2605 
2606       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2607       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2608       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2609 
2610       put = new Put(row1);
2611       put.add(kv13);
2612       put.add(kv12);
2613       put.add(kv11);
2614       put.add(kv23);
2615       put.add(kv22);
2616       put.add(kv21);
2617       region.put(put);
2618 
2619       // Expected
2620       List<Cell> expected = new ArrayList<Cell>();
2621       expected.add(kv13);
2622       expected.add(kv12);
2623       expected.add(kv23);
2624       expected.add(kv22);
2625 
2626       Scan scan = new Scan(row1);
2627       scan.addFamily(fam1);
2628       scan.setMaxVersions(MAX_VERSIONS);
2629       List<Cell> actual = new ArrayList<Cell>();
2630       InternalScanner scanner = region.getScanner(scan);
2631 
2632       boolean hasNext = scanner.next(actual);
2633       assertEquals(false, hasNext);
2634 
2635       // Verify result
2636       for (int i = 0; i < expected.size(); i++) {
2637         assertEquals(expected.get(i), actual.get(i));
2638       }
2639     } finally {
2640       HRegion.closeHRegion(this.region);
2641       this.region = null;
2642     }
2643   }
2644 
2645   @Test
2646   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2647     byte[] row1 = Bytes.toBytes("row1");
2648     byte[] qf1 = Bytes.toBytes("qualifier1");
2649     byte[] qf2 = Bytes.toBytes("qualifier2");
2650     byte[] fam1 = Bytes.toBytes("fam1");
2651 
2652     long ts1 = 1; // System.currentTimeMillis();
2653     long ts2 = ts1 + 1;
2654     long ts3 = ts1 + 2;
2655 
2656     // Setting up region
2657     String method = this.getName();
2658     this.region = initHRegion(tableName, method, CONF, fam1);
2659     try {
2660       // Putting data in Region
2661       Put put = null;
2662       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2663       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2664       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2665 
2666       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2667       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2668       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2669 
2670       put = new Put(row1);
2671       put.add(kv13);
2672       put.add(kv12);
2673       put.add(kv11);
2674       put.add(kv23);
2675       put.add(kv22);
2676       put.add(kv21);
2677       region.put(put);
2678       region.flushcache();
2679 
2680       // Expected
2681       List<Cell> expected = new ArrayList<Cell>();
2682       expected.add(kv13);
2683       expected.add(kv12);
2684       expected.add(kv23);
2685       expected.add(kv22);
2686 
2687       Scan scan = new Scan(row1);
2688       scan.addFamily(fam1);
2689       scan.setMaxVersions(MAX_VERSIONS);
2690       List<Cell> actual = new ArrayList<Cell>();
2691       InternalScanner scanner = region.getScanner(scan);
2692 
2693       boolean hasNext = scanner.next(actual);
2694       assertEquals(false, hasNext);
2695 
2696       // Verify result
2697       for (int i = 0; i < expected.size(); i++) {
2698         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2699       }
2700     } finally {
2701       HRegion.closeHRegion(this.region);
2702       this.region = null;
2703     }
2704   }
2705 
2706   @Test
2707   public void testScanner_StopRow1542() throws IOException {
2708     byte[] family = Bytes.toBytes("testFamily");
2709     this.region = initHRegion(tableName, getName(), CONF, family);
2710     try {
2711       byte[] row1 = Bytes.toBytes("row111");
2712       byte[] row2 = Bytes.toBytes("row222");
2713       byte[] row3 = Bytes.toBytes("row333");
2714       byte[] row4 = Bytes.toBytes("row444");
2715       byte[] row5 = Bytes.toBytes("row555");
2716 
2717       byte[] col1 = Bytes.toBytes("Pub111");
2718       byte[] col2 = Bytes.toBytes("Pub222");
2719 
2720       Put put = new Put(row1);
2721       put.add(family, col1, Bytes.toBytes(10L));
2722       region.put(put);
2723 
2724       put = new Put(row2);
2725       put.add(family, col1, Bytes.toBytes(15L));
2726       region.put(put);
2727 
2728       put = new Put(row3);
2729       put.add(family, col2, Bytes.toBytes(20L));
2730       region.put(put);
2731 
2732       put = new Put(row4);
2733       put.add(family, col2, Bytes.toBytes(30L));
2734       region.put(put);
2735 
2736       put = new Put(row5);
2737       put.add(family, col1, Bytes.toBytes(40L));
2738       region.put(put);
2739 
2740       Scan scan = new Scan(row3, row4);
2741       scan.setMaxVersions();
2742       scan.addColumn(family, col1);
2743       InternalScanner s = region.getScanner(scan);
2744 
2745       List<Cell> results = new ArrayList<Cell>();
2746       assertEquals(false, s.next(results));
2747       assertEquals(0, results.size());
2748     } finally {
2749       HRegion.closeHRegion(this.region);
2750       this.region = null;
2751     }
2752   }
2753 
2754   @Test
2755   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2756     byte[] row1 = Bytes.toBytes("row1");
2757     byte[] fam1 = Bytes.toBytes("fam1");
2758     byte[] qf1 = Bytes.toBytes("qualifier1");
2759     byte[] qf2 = Bytes.toBytes("quateslifier2");
2760 
2761     long ts1 = 1;
2762     long ts2 = ts1 + 1;
2763     long ts3 = ts1 + 2;
2764     long ts4 = ts1 + 3;
2765 
2766     // Setting up region
2767     String method = this.getName();
2768     this.region = initHRegion(tableName, method, CONF, fam1);
2769     try {
2770       // Putting data in Region
2771       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2772       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2773       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2774       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2775 
2776       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2777       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2778       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2779       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2780 
2781       Put put = null;
2782       put = new Put(row1);
2783       put.add(kv14);
2784       put.add(kv24);
2785       region.put(put);
2786       region.flushcache();
2787 
2788       put = new Put(row1);
2789       put.add(kv23);
2790       put.add(kv13);
2791       region.put(put);
2792       region.flushcache();
2793 
2794       put = new Put(row1);
2795       put.add(kv22);
2796       put.add(kv12);
2797       region.put(put);
2798       region.flushcache();
2799 
2800       put = new Put(row1);
2801       put.add(kv21);
2802       put.add(kv11);
2803       region.put(put);
2804 
2805       // Expected
2806       List<KeyValue> expected = new ArrayList<KeyValue>();
2807       expected.add(kv14);
2808       expected.add(kv13);
2809       expected.add(kv12);
2810       expected.add(kv24);
2811       expected.add(kv23);
2812       expected.add(kv22);
2813 
2814       Scan scan = new Scan(row1);
2815       int versions = 3;
2816       scan.setMaxVersions(versions);
2817       List<Cell> actual = new ArrayList<Cell>();
2818       InternalScanner scanner = region.getScanner(scan);
2819 
2820       boolean hasNext = scanner.next(actual);
2821       assertEquals(false, hasNext);
2822 
2823       // Verify result
2824       for (int i = 0; i < expected.size(); i++) {
2825         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2826       }
2827     } finally {
2828       HRegion.closeHRegion(this.region);
2829       this.region = null;
2830     }
2831   }
2832 
2833   /**
2834    * Added for HBASE-5416
2835    * 
2836    * Here we test scan optimization when only subset of CFs are used in filter
2837    * conditions.
2838    */
2839   @Test
2840   public void testScanner_JoinedScanners() throws IOException {
2841     byte[] cf_essential = Bytes.toBytes("essential");
2842     byte[] cf_joined = Bytes.toBytes("joined");
2843     byte[] cf_alpha = Bytes.toBytes("alpha");
2844     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2845     try {
2846       byte[] row1 = Bytes.toBytes("row1");
2847       byte[] row2 = Bytes.toBytes("row2");
2848       byte[] row3 = Bytes.toBytes("row3");
2849 
2850       byte[] col_normal = Bytes.toBytes("d");
2851       byte[] col_alpha = Bytes.toBytes("a");
2852 
2853       byte[] filtered_val = Bytes.toBytes(3);
2854 
2855       Put put = new Put(row1);
2856       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2857       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2858       region.put(put);
2859 
2860       put = new Put(row2);
2861       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2862       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2863       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2864       region.put(put);
2865 
2866       put = new Put(row3);
2867       put.add(cf_essential, col_normal, filtered_val);
2868       put.add(cf_joined, col_normal, filtered_val);
2869       region.put(put);
2870 
2871       // Check two things:
2872       // 1. result list contains expected values
2873       // 2. result list is sorted properly
2874 
2875       Scan scan = new Scan();
2876       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2877           CompareOp.NOT_EQUAL, filtered_val);
2878       scan.setFilter(filter);
2879       scan.setLoadColumnFamiliesOnDemand(true);
2880       InternalScanner s = region.getScanner(scan);
2881 
2882       List<Cell> results = new ArrayList<Cell>();
2883       assertTrue(s.next(results));
2884       assertEquals(results.size(), 1);
2885       results.clear();
2886 
2887       assertTrue(s.next(results));
2888       assertEquals(results.size(), 3);
2889       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2890       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2891       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2892       results.clear();
2893 
2894       assertFalse(s.next(results));
2895       assertEquals(results.size(), 0);
2896     } finally {
2897       HRegion.closeHRegion(this.region);
2898       this.region = null;
2899     }
2900   }
2901 
2902   /**
2903    * HBASE-5416
2904    * 
2905    * Test case when scan limits amount of KVs returned on each next() call.
2906    */
2907   @Test
2908   public void testScanner_JoinedScannersWithLimits() throws IOException {
2909     final byte[] cf_first = Bytes.toBytes("first");
2910     final byte[] cf_second = Bytes.toBytes("second");
2911 
2912     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2913     try {
2914       final byte[] col_a = Bytes.toBytes("a");
2915       final byte[] col_b = Bytes.toBytes("b");
2916 
2917       Put put;
2918 
2919       for (int i = 0; i < 10; i++) {
2920         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2921         put.add(cf_first, col_a, Bytes.toBytes(i));
2922         if (i < 5) {
2923           put.add(cf_first, col_b, Bytes.toBytes(i));
2924           put.add(cf_second, col_a, Bytes.toBytes(i));
2925           put.add(cf_second, col_b, Bytes.toBytes(i));
2926         }
2927         region.put(put);
2928       }
2929 
2930       Scan scan = new Scan();
2931       scan.setLoadColumnFamiliesOnDemand(true);
2932       Filter bogusFilter = new FilterBase() {
2933         @Override
2934         public boolean isFamilyEssential(byte[] name) {
2935           return Bytes.equals(name, cf_first);
2936         }
2937       };
2938 
2939       scan.setFilter(bogusFilter);
2940       InternalScanner s = region.getScanner(scan);
2941 
2942       // Our data looks like this:
2943       // r0: first:a, first:b, second:a, second:b
2944       // r1: first:a, first:b, second:a, second:b
2945       // r2: first:a, first:b, second:a, second:b
2946       // r3: first:a, first:b, second:a, second:b
2947       // r4: first:a, first:b, second:a, second:b
2948       // r5: first:a
2949       // r6: first:a
2950       // r7: first:a
2951       // r8: first:a
2952       // r9: first:a
2953 
2954       // But due to next's limit set to 3, we should get this:
2955       // r0: first:a, first:b, second:a
2956       // r0: second:b
2957       // r1: first:a, first:b, second:a
2958       // r1: second:b
2959       // r2: first:a, first:b, second:a
2960       // r2: second:b
2961       // r3: first:a, first:b, second:a
2962       // r3: second:b
2963       // r4: first:a, first:b, second:a
2964       // r4: second:b
2965       // r5: first:a
2966       // r6: first:a
2967       // r7: first:a
2968       // r8: first:a
2969       // r9: first:a
2970 
2971       List<Cell> results = new ArrayList<Cell>();
2972       int index = 0;
2973       while (true) {
2974         boolean more = s.next(results, 3);
2975         if ((index >> 1) < 5) {
2976           if (index % 2 == 0)
2977             assertEquals(results.size(), 3);
2978           else
2979             assertEquals(results.size(), 1);
2980         } else
2981           assertEquals(results.size(), 1);
2982         results.clear();
2983         index++;
2984         if (!more)
2985           break;
2986       }
2987     } finally {
2988       HRegion.closeHRegion(this.region);
2989       this.region = null;
2990     }
2991   }
2992 
2993   // ////////////////////////////////////////////////////////////////////////////
2994   // Split test
2995   // ////////////////////////////////////////////////////////////////////////////
2996   /**
2997    * Splits twice and verifies getting from each of the split regions.
2998    * 
2999    * @throws Exception
3000    */
3001   @Test
3002   public void testBasicSplit() throws Exception {
3003     byte[][] families = { fam1, fam2, fam3 };
3004 
3005     Configuration hc = initSplit();
3006     // Setting up region
3007     String method = this.getName();
3008     this.region = initHRegion(tableName, method, hc, families);
3009 
3010     try {
3011       LOG.info("" + HBaseTestCase.addContent(region, fam3));
3012       region.flushcache();
3013       region.compactStores();
3014       byte[] splitRow = region.checkSplit();
3015       assertNotNull(splitRow);
3016       LOG.info("SplitRow: " + Bytes.toString(splitRow));
3017       HRegion[] regions = splitRegion(region, splitRow);
3018       try {
3019         // Need to open the regions.
3020         // TODO: Add an 'open' to HRegion... don't do open by constructing
3021         // instance.
3022         for (int i = 0; i < regions.length; i++) {
3023           regions[i] = HRegion.openHRegion(regions[i], null);
3024         }
3025         // Assert can get rows out of new regions. Should be able to get first
3026         // row from first region and the midkey from second region.
3027         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3028         assertGet(regions[1], fam3, splitRow);
3029         // Test I can get scanner and that it starts at right place.
3030         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3031         assertScan(regions[1], fam3, splitRow);
3032         // Now prove can't split regions that have references.
3033         for (int i = 0; i < regions.length; i++) {
3034           // Add so much data to this region, we create a store file that is >
3035           // than one of our unsplitable references. it will.
3036           for (int j = 0; j < 2; j++) {
3037             HBaseTestCase.addContent(regions[i], fam3);
3038           }
3039           HBaseTestCase.addContent(regions[i], fam2);
3040           HBaseTestCase.addContent(regions[i], fam1);
3041           regions[i].flushcache();
3042         }
3043 
3044         byte[][] midkeys = new byte[regions.length][];
3045         // To make regions splitable force compaction.
3046         for (int i = 0; i < regions.length; i++) {
3047           regions[i].compactStores();
3048           midkeys[i] = regions[i].checkSplit();
3049         }
3050 
3051         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3052         // Split these two daughter regions so then I'll have 4 regions. Will
3053         // split because added data above.
3054         for (int i = 0; i < regions.length; i++) {
3055           HRegion[] rs = null;
3056           if (midkeys[i] != null) {
3057             rs = splitRegion(regions[i], midkeys[i]);
3058             for (int j = 0; j < rs.length; j++) {
3059               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3060             }
3061           }
3062         }
3063         LOG.info("Made 4 regions");
3064         // The splits should have been even. Test I can get some arbitrary row
3065         // out of each.
3066         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3067         byte[] b = Bytes.toBytes(START_KEY);
3068         for (HRegion r : sortedMap.values()) {
3069           assertGet(r, fam3, b);
3070           b[0] += interval;
3071         }
3072       } finally {
3073         for (int i = 0; i < regions.length; i++) {
3074           try {
3075             regions[i].close();
3076           } catch (IOException e) {
3077             // Ignore.
3078           }
3079         }
3080       }
3081     } finally {
3082       HRegion.closeHRegion(this.region);
3083       this.region = null;
3084     }
3085   }
3086 
3087   @Test
3088   public void testSplitRegion() throws IOException {
3089     byte[] qualifier = Bytes.toBytes("qualifier");
3090     Configuration hc = initSplit();
3091     int numRows = 10;
3092     byte[][] families = { fam1, fam3 };
3093 
3094     // Setting up region
3095     String method = this.getName();
3096     this.region = initHRegion(tableName, method, hc, families);
3097 
3098     // Put data in region
3099     int startRow = 100;
3100     putData(startRow, numRows, qualifier, families);
3101     int splitRow = startRow + numRows;
3102     putData(splitRow, numRows, qualifier, families);
3103     region.flushcache();
3104 
3105     HRegion[] regions = null;
3106     try {
3107       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3108       // Opening the regions returned.
3109       for (int i = 0; i < regions.length; i++) {
3110         regions[i] = HRegion.openHRegion(regions[i], null);
3111       }
3112       // Verifying that the region has been split
3113       assertEquals(2, regions.length);
3114 
3115       // Verifying that all data is still there and that data is in the right
3116       // place
3117       verifyData(regions[0], startRow, numRows, qualifier, families);
3118       verifyData(regions[1], splitRow, numRows, qualifier, families);
3119 
3120     } finally {
3121       HRegion.closeHRegion(this.region);
3122       this.region = null;
3123     }
3124   }
3125 
3126   @Test
3127   public void testClearForceSplit() throws IOException {
3128     byte[] qualifier = Bytes.toBytes("qualifier");
3129     Configuration hc = initSplit();
3130     int numRows = 10;
3131     byte[][] families = { fam1, fam3 };
3132 
3133     // Setting up region
3134     String method = this.getName();
3135     this.region = initHRegion(tableName, method, hc, families);
3136 
3137     // Put data in region
3138     int startRow = 100;
3139     putData(startRow, numRows, qualifier, families);
3140     int splitRow = startRow + numRows;
3141     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3142     putData(splitRow, numRows, qualifier, families);
3143     region.flushcache();
3144 
3145     HRegion[] regions = null;
3146     try {
3147       // Set force split
3148       region.forceSplit(splitRowBytes);
3149       assertTrue(region.shouldForceSplit());
3150       // Split point should be the force split row
3151       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3152 
3153       // Add a store that has references.
3154       HStore storeMock = Mockito.mock(HStore.class);
3155       Mockito.when(storeMock.hasReferences()).thenReturn(true);
3156       Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3157       Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3158       Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3159       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3160       assertTrue(region.hasReferences());
3161 
3162       // Will not split since the store has references.
3163       regions = splitRegion(region, splitRowBytes);
3164       assertNull(regions);
3165 
3166       // Region force split should be cleared after the split try.
3167       assertFalse(region.shouldForceSplit());
3168 
3169       // Remove the store that has references.
3170       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3171       assertFalse(region.hasReferences());
3172 
3173       // Now we can split.
3174       regions = splitRegion(region, splitRowBytes);
3175 
3176       // Opening the regions returned.
3177       for (int i = 0; i < regions.length; i++) {
3178         regions[i] = HRegion.openHRegion(regions[i], null);
3179       }
3180       // Verifying that the region has been split
3181       assertEquals(2, regions.length);
3182 
3183       // Verifying that all data is still there and that data is in the right
3184       // place
3185       verifyData(regions[0], startRow, numRows, qualifier, families);
3186       verifyData(regions[1], splitRow, numRows, qualifier, families);
3187 
3188     } finally {
3189       HRegion.closeHRegion(this.region);
3190       this.region = null;
3191     }
3192   }
3193 
3194   /**
3195    * Flushes the cache in a thread while scanning. The tests verify that the
3196    * scan is coherent - e.g. the returned results are always of the same or
3197    * later update as the previous results.
3198    * 
3199    * @throws IOException
3200    *           scan / compact
3201    * @throws InterruptedException
3202    *           thread join
3203    */
3204   @Test
3205   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3206     byte[] family = Bytes.toBytes("family");
3207     int numRows = 1000;
3208     int flushAndScanInterval = 10;
3209     int compactInterval = 10 * flushAndScanInterval;
3210 
3211     String method = "testFlushCacheWhileScanning";
3212     this.region = initHRegion(tableName, method, CONF, family);
3213     try {
3214       FlushThread flushThread = new FlushThread();
3215       flushThread.start();
3216 
3217       Scan scan = new Scan();
3218       scan.addFamily(family);
3219       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3220           new BinaryComparator(Bytes.toBytes(5L))));
3221 
3222       int expectedCount = 0;
3223       List<Cell> res = new ArrayList<Cell>();
3224 
3225       boolean toggle = true;
3226       for (long i = 0; i < numRows; i++) {
3227         Put put = new Put(Bytes.toBytes(i));
3228         put.setDurability(Durability.SKIP_WAL);
3229         put.add(family, qual1, Bytes.toBytes(i % 10));
3230         region.put(put);
3231 
3232         if (i != 0 && i % compactInterval == 0) {
3233           // System.out.println("iteration = " + i);
3234           region.compactStores(true);
3235         }
3236 
3237         if (i % 10 == 5L) {
3238           expectedCount++;
3239         }
3240 
3241         if (i != 0 && i % flushAndScanInterval == 0) {
3242           res.clear();
3243           InternalScanner scanner = region.getScanner(scan);
3244           if (toggle) {
3245             flushThread.flush();
3246           }
3247           while (scanner.next(res))
3248             ;
3249           if (!toggle) {
3250             flushThread.flush();
3251           }
3252           assertEquals("i=" + i, expectedCount, res.size());
3253           toggle = !toggle;
3254         }
3255       }
3256 
3257       flushThread.done();
3258       flushThread.join();
3259       flushThread.checkNoError();
3260     } finally {
3261       HRegion.closeHRegion(this.region);
3262       this.region = null;
3263     }
3264   }
3265 
3266   protected class FlushThread extends Thread {
3267     private volatile boolean done;
3268     private Throwable error = null;
3269 
3270     public void done() {
3271       done = true;
3272       synchronized (this) {
3273         interrupt();
3274       }
3275     }
3276 
3277     public void checkNoError() {
3278       if (error != null) {
3279         assertNull(error);
3280       }
3281     }
3282 
3283     @Override
3284     public void run() {
3285       done = false;
3286       while (!done) {
3287         synchronized (this) {
3288           try {
3289             wait();
3290           } catch (InterruptedException ignored) {
3291             if (done) {
3292               break;
3293             }
3294           }
3295         }
3296         try {
3297           region.flushcache();
3298         } catch (IOException e) {
3299           if (!done) {
3300             LOG.error("Error while flusing cache", e);
3301             error = e;
3302           }
3303           break;
3304         }
3305       }
3306 
3307     }
3308 
3309     public void flush() {
3310       synchronized (this) {
3311         notify();
3312       }
3313 
3314     }
3315   }
3316 
3317   /**
3318    * Writes very wide records and scans for the latest every time.. Flushes and
3319    * compacts the region every now and then to keep things realistic.
3320    * 
3321    * @throws IOException
3322    *           by flush / scan / compaction
3323    * @throws InterruptedException
3324    *           when joining threads
3325    */
3326   @Test
3327   public void testWritesWhileScanning() throws IOException, InterruptedException {
3328     int testCount = 100;
3329     int numRows = 1;
3330     int numFamilies = 10;
3331     int numQualifiers = 100;
3332     int flushInterval = 7;
3333     int compactInterval = 5 * flushInterval;
3334     byte[][] families = new byte[numFamilies][];
3335     for (int i = 0; i < numFamilies; i++) {
3336       families[i] = Bytes.toBytes("family" + i);
3337     }
3338     byte[][] qualifiers = new byte[numQualifiers][];
3339     for (int i = 0; i < numQualifiers; i++) {
3340       qualifiers[i] = Bytes.toBytes("qual" + i);
3341     }
3342 
3343     String method = "testWritesWhileScanning";
3344     this.region = initHRegion(tableName, method, CONF, families);
3345     try {
3346       PutThread putThread = new PutThread(numRows, families, qualifiers);
3347       putThread.start();
3348       putThread.waitForFirstPut();
3349 
3350       FlushThread flushThread = new FlushThread();
3351       flushThread.start();
3352 
3353       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3354 
3355       int expectedCount = numFamilies * numQualifiers;
3356       List<Cell> res = new ArrayList<Cell>();
3357 
3358       long prevTimestamp = 0L;
3359       for (int i = 0; i < testCount; i++) {
3360 
3361         if (i != 0 && i % compactInterval == 0) {
3362           region.compactStores(true);
3363         }
3364 
3365         if (i != 0 && i % flushInterval == 0) {
3366           flushThread.flush();
3367         }
3368 
3369         boolean previousEmpty = res.isEmpty();
3370         res.clear();
3371         InternalScanner scanner = region.getScanner(scan);
3372         while (scanner.next(res))
3373           ;
3374         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3375           assertEquals("i=" + i, expectedCount, res.size());
3376           long timestamp = res.get(0).getTimestamp();
3377           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3378               timestamp >= prevTimestamp);
3379           prevTimestamp = timestamp;
3380         }
3381       }
3382 
3383       putThread.done();
3384 
3385       region.flushcache();
3386 
3387       putThread.join();
3388       putThread.checkNoError();
3389 
3390       flushThread.done();
3391       flushThread.join();
3392       flushThread.checkNoError();
3393     } finally {
3394       try {
3395         HRegion.closeHRegion(this.region);
3396       } catch (DroppedSnapshotException dse) {
3397         // We could get this on way out because we interrupt the background flusher and it could
3398         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3399         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3400         // flush because the accounting is 'off' since original DSE.
3401       }
3402       this.region = null;
3403     }
3404   }
3405 
3406   protected class PutThread extends Thread {
3407     private volatile boolean done;
3408     private volatile int numPutsFinished = 0;
3409 
3410     private Throwable error = null;
3411     private int numRows;
3412     private byte[][] families;
3413     private byte[][] qualifiers;
3414 
3415     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3416       this.numRows = numRows;
3417       this.families = families;
3418       this.qualifiers = qualifiers;
3419     }
3420 
3421     /**
3422      * Block until this thread has put at least one row.
3423      */
3424     public void waitForFirstPut() throws InterruptedException {
3425       // wait until put thread actually puts some data
3426       while (numPutsFinished == 0) {
3427         checkNoError();
3428         Thread.sleep(50);
3429       }
3430     }
3431 
3432     public void done() {
3433       done = true;
3434       synchronized (this) {
3435         interrupt();
3436       }
3437     }
3438 
3439     public void checkNoError() {
3440       if (error != null) {
3441         assertNull(error);
3442       }
3443     }
3444 
3445     @Override
3446     public void run() {
3447       done = false;
3448       while (!done) {
3449         try {
3450           for (int r = 0; r < numRows; r++) {
3451             byte[] row = Bytes.toBytes("row" + r);
3452             Put put = new Put(row);
3453             put.setDurability(Durability.SKIP_WAL);
3454             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3455             for (byte[] family : families) {
3456               for (byte[] qualifier : qualifiers) {
3457                 put.add(family, qualifier, (long) numPutsFinished, value);
3458               }
3459             }
3460             region.put(put);
3461             numPutsFinished++;
3462             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3463               System.out.println("put iteration = " + numPutsFinished);
3464               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3465               region.delete(delete);
3466             }
3467             numPutsFinished++;
3468           }
3469         } catch (InterruptedIOException e) {
3470           // This is fine. It means we are done, or didn't get the lock on time
3471         } catch (IOException e) {
3472           LOG.error("error while putting records", e);
3473           error = e;
3474           break;
3475         }
3476       }
3477 
3478     }
3479 
3480   }
3481 
3482   /**
3483    * Writes very wide records and gets the latest row every time.. Flushes and
3484    * compacts the region aggressivly to catch issues.
3485    * 
3486    * @throws IOException
3487    *           by flush / scan / compaction
3488    * @throws InterruptedException
3489    *           when joining threads
3490    */
3491   @Test
3492   public void testWritesWhileGetting() throws Exception {
3493     int testCount = 100;
3494     int numRows = 1;
3495     int numFamilies = 10;
3496     int numQualifiers = 100;
3497     int compactInterval = 100;
3498     byte[][] families = new byte[numFamilies][];
3499     for (int i = 0; i < numFamilies; i++) {
3500       families[i] = Bytes.toBytes("family" + i);
3501     }
3502     byte[][] qualifiers = new byte[numQualifiers][];
3503     for (int i = 0; i < numQualifiers; i++) {
3504       qualifiers[i] = Bytes.toBytes("qual" + i);
3505     }
3506 
3507 
3508     String method = "testWritesWhileGetting";
3509     // This test flushes constantly and can cause many files to be created,
3510     // possibly
3511     // extending over the ulimit. Make sure compactions are aggressive in
3512     // reducing
3513     // the number of HFiles created.
3514     Configuration conf = HBaseConfiguration.create(CONF);
3515     conf.setInt("hbase.hstore.compaction.min", 1);
3516     conf.setInt("hbase.hstore.compaction.max", 1000);
3517     this.region = initHRegion(tableName, method, conf, families);
3518     PutThread putThread = null;
3519     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3520     try {
3521       putThread = new PutThread(numRows, families, qualifiers);
3522       putThread.start();
3523       putThread.waitForFirstPut();
3524 
3525       // Add a thread that flushes as fast as possible
3526       ctx.addThread(new RepeatingTestThread(ctx) {
3527         private int flushesSinceCompact = 0;
3528         private final int maxFlushesSinceCompact = 20;
3529 
3530         @Override
3531         public void doAnAction() throws Exception {
3532           if (region.flushcache().isCompactionNeeded()) {
3533             ++flushesSinceCompact;
3534           }
3535           // Compact regularly to avoid creating too many files and exceeding
3536           // the ulimit.
3537           if (flushesSinceCompact == maxFlushesSinceCompact) {
3538             region.compactStores(false);
3539             flushesSinceCompact = 0;
3540           }
3541         }
3542       });
3543       ctx.startThreads();
3544 
3545       Get get = new Get(Bytes.toBytes("row0"));
3546       Result result = null;
3547 
3548       int expectedCount = numFamilies * numQualifiers;
3549 
3550       long prevTimestamp = 0L;
3551       for (int i = 0; i < testCount; i++) {
3552 
3553         boolean previousEmpty = result == null || result.isEmpty();
3554         result = region.get(get);
3555         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3556           assertEquals("i=" + i, expectedCount, result.size());
3557           // TODO this was removed, now what dangit?!
3558           // search looking for the qualifier in question?
3559           long timestamp = 0;
3560           for (Cell kv : result.rawCells()) {
3561             if (CellUtil.matchingFamily(kv, families[0])
3562                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3563               timestamp = kv.getTimestamp();
3564             }
3565           }
3566           assertTrue(timestamp >= prevTimestamp);
3567           prevTimestamp = timestamp;
3568           Cell previousKV = null;
3569 
3570           for (Cell kv : result.rawCells()) {
3571             byte[] thisValue = CellUtil.cloneValue(kv);
3572             if (previousKV != null) {
3573               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3574                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3575                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3576                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3577                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3578               }
3579             }
3580             previousKV = kv;
3581           }
3582         }
3583       }
3584     } finally {
3585       if (putThread != null)
3586         putThread.done();
3587 
3588       region.flushcache();
3589 
3590       if (putThread != null) {
3591         putThread.join();
3592         putThread.checkNoError();
3593       }
3594 
3595       ctx.stop();
3596       HRegion.closeHRegion(this.region);
3597       this.region = null;
3598     }
3599   }
3600 
3601   @Test
3602   public void testHolesInMeta() throws Exception {
3603     byte[] family = Bytes.toBytes("family");
3604     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3605         false, family);
3606     try {
3607       byte[] rowNotServed = Bytes.toBytes("a");
3608       Get g = new Get(rowNotServed);
3609       try {
3610         region.get(g);
3611         fail();
3612       } catch (WrongRegionException x) {
3613         // OK
3614       }
3615       byte[] row = Bytes.toBytes("y");
3616       g = new Get(row);
3617       region.get(g);
3618     } finally {
3619       HRegion.closeHRegion(this.region);
3620       this.region = null;
3621     }
3622   }
3623 
3624   @Test
3625   public void testIndexesScanWithOneDeletedRow() throws IOException {
3626     byte[] family = Bytes.toBytes("family");
3627 
3628     // Setting up region
3629     String method = "testIndexesScanWithOneDeletedRow";
3630     this.region = initHRegion(tableName, method, CONF, family);
3631     try {
3632       Put put = new Put(Bytes.toBytes(1L));
3633       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3634       region.put(put);
3635 
3636       region.flushcache();
3637 
3638       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3639       region.delete(delete);
3640 
3641       put = new Put(Bytes.toBytes(2L));
3642       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3643       region.put(put);
3644 
3645       Scan idxScan = new Scan();
3646       idxScan.addFamily(family);
3647       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3648           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3649               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3650               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3651       InternalScanner scanner = region.getScanner(idxScan);
3652       List<Cell> res = new ArrayList<Cell>();
3653 
3654       while (scanner.next(res))
3655         ;
3656       assertEquals(1L, res.size());
3657     } finally {
3658       HRegion.closeHRegion(this.region);
3659       this.region = null;
3660     }
3661   }
3662 
3663   // ////////////////////////////////////////////////////////////////////////////
3664   // Bloom filter test
3665   // ////////////////////////////////////////////////////////////////////////////
3666   @Test
3667   public void testBloomFilterSize() throws IOException {
3668     byte[] fam1 = Bytes.toBytes("fam1");
3669     byte[] qf1 = Bytes.toBytes("col");
3670     byte[] val1 = Bytes.toBytes("value1");
3671     // Create Table
3672     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3673         .setBloomFilterType(BloomType.ROWCOL);
3674 
3675     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3676     htd.addFamily(hcd);
3677     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3678     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3679     try {
3680       int num_unique_rows = 10;
3681       int duplicate_multiplier = 2;
3682       int num_storefiles = 4;
3683 
3684       int version = 0;
3685       for (int f = 0; f < num_storefiles; f++) {
3686         for (int i = 0; i < duplicate_multiplier; i++) {
3687           for (int j = 0; j < num_unique_rows; j++) {
3688             Put put = new Put(Bytes.toBytes("row" + j));
3689             put.setDurability(Durability.SKIP_WAL);
3690             put.add(fam1, qf1, version++, val1);
3691             region.put(put);
3692           }
3693         }
3694         region.flushcache();
3695       }
3696       // before compaction
3697       HStore store = (HStore) region.getStore(fam1);
3698       Collection<StoreFile> storeFiles = store.getStorefiles();
3699       for (StoreFile storefile : storeFiles) {
3700         StoreFile.Reader reader = storefile.getReader();
3701         reader.loadFileInfo();
3702         reader.loadBloomfilter();
3703         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3704         assertEquals(num_unique_rows, reader.getFilterEntries());
3705       }
3706 
3707       region.compactStores(true);
3708 
3709       // after compaction
3710       storeFiles = store.getStorefiles();
3711       for (StoreFile storefile : storeFiles) {
3712         StoreFile.Reader reader = storefile.getReader();
3713         reader.loadFileInfo();
3714         reader.loadBloomfilter();
3715         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3716         assertEquals(num_unique_rows, reader.getFilterEntries());
3717       }
3718     } finally {
3719       HRegion.closeHRegion(this.region);
3720       this.region = null;
3721     }
3722   }
3723 
3724   @Test
3725   public void testAllColumnsWithBloomFilter() throws IOException {
3726     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3727     byte[] FAMILY = Bytes.toBytes("family");
3728 
3729     // Create table
3730     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3731         .setBloomFilterType(BloomType.ROWCOL);
3732     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3733     htd.addFamily(hcd);
3734     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3735     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3736     try {
3737       // For row:0, col:0: insert versions 1 through 5.
3738       byte row[] = Bytes.toBytes("row:" + 0);
3739       byte column[] = Bytes.toBytes("column:" + 0);
3740       Put put = new Put(row);
3741       put.setDurability(Durability.SKIP_WAL);
3742       for (long idx = 1; idx <= 4; idx++) {
3743         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3744       }
3745       region.put(put);
3746 
3747       // Flush
3748       region.flushcache();
3749 
3750       // Get rows
3751       Get get = new Get(row);
3752       get.setMaxVersions();
3753       Cell[] kvs = region.get(get).rawCells();
3754 
3755       // Check if rows are correct
3756       assertEquals(4, kvs.length);
3757       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3758       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3759       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3760       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3761     } finally {
3762       HRegion.closeHRegion(this.region);
3763       this.region = null;
3764     }
3765   }
3766 
3767   /**
3768    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3769    * issuing delete row on columns with bloom filter set to row+col
3770    * (BloomType.ROWCOL)
3771    */
3772   @Test
3773   public void testDeleteRowWithBloomFilter() throws IOException {
3774     byte[] familyName = Bytes.toBytes("familyName");
3775 
3776     // Create Table
3777     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3778         .setBloomFilterType(BloomType.ROWCOL);
3779 
3780     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3781     htd.addFamily(hcd);
3782     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3783     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3784     try {
3785       // Insert some data
3786       byte row[] = Bytes.toBytes("row1");
3787       byte col[] = Bytes.toBytes("col1");
3788 
3789       Put put = new Put(row);
3790       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3791       region.put(put);
3792       region.flushcache();
3793 
3794       Delete del = new Delete(row);
3795       region.delete(del);
3796       region.flushcache();
3797 
3798       // Get remaining rows (should have none)
3799       Get get = new Get(row);
3800       get.addColumn(familyName, col);
3801 
3802       Cell[] keyValues = region.get(get).rawCells();
3803       assertTrue(keyValues.length == 0);
3804     } finally {
3805       HRegion.closeHRegion(this.region);
3806       this.region = null;
3807     }
3808   }
3809 
3810   @Test
3811   public void testgetHDFSBlocksDistribution() throws Exception {
3812     HBaseTestingUtility htu = new HBaseTestingUtility();
3813     final int DEFAULT_BLOCK_SIZE = 1024;
3814     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3815     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3816     htu.getConfiguration().setInt("dfs.replication", 2);
3817 
3818     // set up a cluster with 3 nodes
3819     MiniHBaseCluster cluster = null;
3820     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3821     int regionServersCount = 3;
3822 
3823     try {
3824       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3825       byte[][] families = { fam1, fam2 };
3826       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3827 
3828       // Setting up region
3829       byte row[] = Bytes.toBytes("row1");
3830       byte col[] = Bytes.toBytes("col1");
3831 
3832       Put put = new Put(row);
3833       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3834       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3835       ht.put(put);
3836 
3837       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3838           .get(0);
3839       firstRegion.flushcache();
3840       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3841 
3842       // given the default replication factor is 2 and we have 2 HFiles,
3843       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3844       // must be at least one host that have replica for 2 HFiles. That host's
3845       // weight will be equal to the unique block weight.
3846       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3847 
3848       String topHost = blocksDistribution1.getTopHosts().get(0);
3849       long topHostWeight = blocksDistribution1.getWeight(topHost);
3850       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3851 
3852       // use the static method to compute the value, it should be the same.
3853       // static method is used by load balancer or other components
3854       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3855           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3856       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3857 
3858       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3859 
3860       ht.close();
3861     } finally {
3862       if (cluster != null) {
3863         htu.shutdownMiniCluster();
3864       }
3865     }
3866   }
3867 
3868   /**
3869    * Testcase to check state of region initialization task set to ABORTED or not
3870    * if any exceptions during initialization
3871    * 
3872    * @throws Exception
3873    */
3874   @Test
3875   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3876     TableName tableName = TableName.valueOf(name.getMethodName());
3877     HRegionInfo info = null;
3878     try {
3879       FileSystem fs = Mockito.mock(FileSystem.class);
3880       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3881       HTableDescriptor htd = new HTableDescriptor(tableName);
3882       htd.addFamily(new HColumnDescriptor("cf"));
3883       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3884           HConstants.EMPTY_BYTE_ARRAY, false);
3885       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3886       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3887       // region initialization throws IOException and set task state to ABORTED.
3888       region.initialize();
3889       fail("Region initialization should fail due to IOException");
3890     } catch (IOException io) {
3891       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3892       for (MonitoredTask monitoredTask : tasks) {
3893         if (!(monitoredTask instanceof MonitoredRPCHandler)
3894             && monitoredTask.getDescription().contains(region.toString())) {
3895           assertTrue("Region state should be ABORTED.",
3896               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3897           break;
3898         }
3899       }
3900     } finally {
3901       HRegion.closeHRegion(region);
3902     }
3903   }
3904 
3905   /**
3906    * Verifies that the .regioninfo file is written on region creation and that
3907    * is recreated if missing during region opening.
3908    */
3909   @Test
3910   public void testRegionInfoFileCreation() throws IOException {
3911     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3912 
3913     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3914     htd.addFamily(new HColumnDescriptor("cf"));
3915 
3916     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3917 
3918     // Create a region and skip the initialization (like CreateTableHandler)
3919     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3920 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3921     Path regionDir = region.getRegionFileSystem().getRegionDir();
3922     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3923     HRegion.closeHRegion(region);
3924 
3925     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3926 
3927     // Verify that the .regioninfo file is present
3928     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3929         fs.exists(regionInfoFile));
3930 
3931     // Try to open the region
3932     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3933     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3934     HRegion.closeHRegion(region);
3935 
3936     // Verify that the .regioninfo file is still there
3937     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3938         fs.exists(regionInfoFile));
3939 
3940     // Remove the .regioninfo file and verify is recreated on region open
3941     fs.delete(regionInfoFile);
3942     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3943         fs.exists(regionInfoFile));
3944 
3945     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3946 //    region = TEST_UTIL.openHRegion(hri, htd);
3947     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3948     HRegion.closeHRegion(region);
3949 
3950     // Verify that the .regioninfo file is still there
3951     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3952         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3953   }
3954 
3955   /**
3956    * TestCase for increment
3957    */
3958   private static class Incrementer implements Runnable {
3959     private HRegion region;
3960     private final static byte[] incRow = Bytes.toBytes("incRow");
3961     private final static byte[] family = Bytes.toBytes("family");
3962     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3963     private final static long ONE = 1l;
3964     private int incCounter;
3965 
3966     public Incrementer(HRegion region, int incCounter) {
3967       this.region = region;
3968       this.incCounter = incCounter;
3969     }
3970 
3971     @Override
3972     public void run() {
3973       int count = 0;
3974       while (count < incCounter) {
3975         Increment inc = new Increment(incRow);
3976         inc.addColumn(family, qualifier, ONE);
3977         count++;
3978         try {
3979           region.increment(inc);
3980         } catch (IOException e) {
3981           e.printStackTrace();
3982           break;
3983         }
3984       }
3985     }
3986   }
3987 
3988   /**
3989    * Test case to check increment function with memstore flushing
3990    * @throws Exception
3991    */
3992   @Test
3993   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3994     byte[] family = Incrementer.family;
3995     this.region = initHRegion(tableName, method, CONF, family);
3996     final HRegion region = this.region;
3997     final AtomicBoolean incrementDone = new AtomicBoolean(false);
3998     Runnable flusher = new Runnable() {
3999       @Override
4000       public void run() {
4001         while (!incrementDone.get()) {
4002           try {
4003             region.flushcache();
4004           } catch (Exception e) {
4005             e.printStackTrace();
4006           }
4007         }
4008       }
4009     };
4010 
4011     // after all increment finished, the row will increment to 20*100 = 2000
4012     int threadNum = 20;
4013     int incCounter = 100;
4014     long expected = threadNum * incCounter;
4015     Thread[] incrementers = new Thread[threadNum];
4016     Thread flushThread = new Thread(flusher);
4017     for (int i = 0; i < threadNum; i++) {
4018       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4019       incrementers[i].start();
4020     }
4021     flushThread.start();
4022     for (int i = 0; i < threadNum; i++) {
4023       incrementers[i].join();
4024     }
4025 
4026     incrementDone.set(true);
4027     flushThread.join();
4028 
4029     Get get = new Get(Incrementer.incRow);
4030     get.addColumn(Incrementer.family, Incrementer.qualifier);
4031     get.setMaxVersions(1);
4032     Result res = this.region.get(get);
4033     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4034 
4035     // we just got the latest version
4036     assertEquals(kvs.size(), 1);
4037     Cell kv = kvs.get(0);
4038     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4039     this.region = null;
4040   }
4041 
4042   /**
4043    * TestCase for append
4044    */
4045   private static class Appender implements Runnable {
4046     private HRegion region;
4047     private final static byte[] appendRow = Bytes.toBytes("appendRow");
4048     private final static byte[] family = Bytes.toBytes("family");
4049     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4050     private final static byte[] CHAR = Bytes.toBytes("a");
4051     private int appendCounter;
4052 
4053     public Appender(HRegion region, int appendCounter) {
4054       this.region = region;
4055       this.appendCounter = appendCounter;
4056     }
4057 
4058     @Override
4059     public void run() {
4060       int count = 0;
4061       while (count < appendCounter) {
4062         Append app = new Append(appendRow);
4063         app.add(family, qualifier, CHAR);
4064         count++;
4065         try {
4066           region.append(app);
4067         } catch (IOException e) {
4068           e.printStackTrace();
4069           break;
4070         }
4071       }
4072     }
4073   }
4074 
4075   /**
4076    * Test case to check append function with memstore flushing
4077    * @throws Exception
4078    */
4079   @Test
4080   public void testParallelAppendWithMemStoreFlush() throws Exception {
4081     byte[] family = Appender.family;
4082     this.region = initHRegion(tableName, method, CONF, family);
4083     final HRegion region = this.region;
4084     final AtomicBoolean appendDone = new AtomicBoolean(false);
4085     Runnable flusher = new Runnable() {
4086       @Override
4087       public void run() {
4088         while (!appendDone.get()) {
4089           try {
4090             region.flushcache();
4091           } catch (Exception e) {
4092             e.printStackTrace();
4093           }
4094         }
4095       }
4096     };
4097 
4098     // after all append finished, the value will append to threadNum *
4099     // appendCounter Appender.CHAR
4100     int threadNum = 20;
4101     int appendCounter = 100;
4102     byte[] expected = new byte[threadNum * appendCounter];
4103     for (int i = 0; i < threadNum * appendCounter; i++) {
4104       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4105     }
4106     Thread[] appenders = new Thread[threadNum];
4107     Thread flushThread = new Thread(flusher);
4108     for (int i = 0; i < threadNum; i++) {
4109       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4110       appenders[i].start();
4111     }
4112     flushThread.start();
4113     for (int i = 0; i < threadNum; i++) {
4114       appenders[i].join();
4115     }
4116 
4117     appendDone.set(true);
4118     flushThread.join();
4119 
4120     Get get = new Get(Appender.appendRow);
4121     get.addColumn(Appender.family, Appender.qualifier);
4122     get.setMaxVersions(1);
4123     Result res = this.region.get(get);
4124     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4125 
4126     // we just got the latest version
4127     assertEquals(kvs.size(), 1);
4128     Cell kv = kvs.get(0);
4129     byte[] appendResult = new byte[kv.getValueLength()];
4130     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4131     assertArrayEquals(expected, appendResult);
4132     this.region = null;
4133   }
4134 
4135   /**
4136    * Test case to check put function with memstore flushing for same row, same ts
4137    * @throws Exception
4138    */
4139   @Test
4140   public void testPutWithMemStoreFlush() throws Exception {
4141     byte[] family = Bytes.toBytes("family");
4142     ;
4143     byte[] qualifier = Bytes.toBytes("qualifier");
4144     byte[] row = Bytes.toBytes("putRow");
4145     byte[] value = null;
4146     this.region = initHRegion(tableName, method, CONF, family);
4147     Put put = null;
4148     Get get = null;
4149     List<Cell> kvs = null;
4150     Result res = null;
4151 
4152     put = new Put(row);
4153     value = Bytes.toBytes("value0");
4154     put.add(family, qualifier, 1234567l, value);
4155     region.put(put);
4156     get = new Get(row);
4157     get.addColumn(family, qualifier);
4158     get.setMaxVersions();
4159     res = this.region.get(get);
4160     kvs = res.getColumnCells(family, qualifier);
4161     assertEquals(1, kvs.size());
4162     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4163 
4164     region.flushcache();
4165     get = new Get(row);
4166     get.addColumn(family, qualifier);
4167     get.setMaxVersions();
4168     res = this.region.get(get);
4169     kvs = res.getColumnCells(family, qualifier);
4170     assertEquals(1, kvs.size());
4171     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4172 
4173     put = new Put(row);
4174     value = Bytes.toBytes("value1");
4175     put.add(family, qualifier, 1234567l, value);
4176     region.put(put);
4177     get = new Get(row);
4178     get.addColumn(family, qualifier);
4179     get.setMaxVersions();
4180     res = this.region.get(get);
4181     kvs = res.getColumnCells(family, qualifier);
4182     assertEquals(1, kvs.size());
4183     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4184 
4185     region.flushcache();
4186     get = new Get(row);
4187     get.addColumn(family, qualifier);
4188     get.setMaxVersions();
4189     res = this.region.get(get);
4190     kvs = res.getColumnCells(family, qualifier);
4191     assertEquals(1, kvs.size());
4192     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4193   }
4194 
4195   @Test
4196   public void testDurability() throws Exception {
4197     String method = "testDurability";
4198     // there are 5 x 5 cases:
4199     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4200     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4201 
4202     // expected cases for append and sync wal
4203     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4204     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4205     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4206 
4207     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4208     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4209     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4210 
4211     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4212     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4213 
4214     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4215     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4216 
4217     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4218     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4219     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4220 
4221     // expected cases for async wal
4222     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4223     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4224     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4225     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4226     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4227     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4228 
4229     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4230     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4231     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4232     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4233     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4234     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4235 
4236     // expect skip wal cases
4237     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4238     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4239     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4240     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4241     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4242     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4243 
4244   }
4245 
4246   private void durabilityTest(String method, Durability tableDurability,
4247       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4248       final boolean expectSyncFromLogSyncer) throws Exception {
4249     Configuration conf = HBaseConfiguration.create(CONF);
4250     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4251     TableName tableName = TableName.valueOf(method);
4252     byte[] family = Bytes.toBytes("family");
4253     Path logDir = new Path(new Path(dir + method), "log");
4254     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4255     final HLog log = spy(hlog);
4256     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4257         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4258         new byte[][] { family });
4259 
4260     Put put = new Put(Bytes.toBytes("r1"));
4261     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4262     put.setDurability(mutationDurability);
4263     region.put(put);
4264 
4265     //verify append called or not
4266     verify(log, expectAppend ? times(1) : never())
4267       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4268         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4269 
4270     // verify sync called or not
4271     if (expectSync || expectSyncFromLogSyncer) {
4272       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4273         @Override
4274         public boolean evaluate() throws Exception {
4275           try {
4276             if (expectSync) {
4277               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4278             } else if (expectSyncFromLogSyncer) {
4279               verify(log, times(1)).sync(); // log syncer calls this one
4280             }
4281           } catch (Throwable ignore) {
4282           }
4283           return true;
4284         }
4285       });
4286     } else {
4287       verify(log, never()).sync(anyLong());
4288       verify(log, never()).sync();
4289     }
4290 
4291     HRegion.closeHRegion(this.region);
4292     this.region = null;
4293   }
4294 
4295   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4296     for (int i = startRow; i < startRow + numRows; i++) {
4297       Put put = new Put(Bytes.toBytes("" + i));
4298       put.setDurability(Durability.SKIP_WAL);
4299       for (byte[] family : families) {
4300         put.add(family, qf, null);
4301       }
4302       region.put(put);
4303     }
4304   }
4305 
4306   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4307       throws IOException {
4308     for (int i = startRow; i < startRow + numRows; i++) {
4309       byte[] row = Bytes.toBytes("" + i);
4310       Get get = new Get(row);
4311       for (byte[] family : families) {
4312         get.addColumn(family, qf);
4313       }
4314       Result result = newReg.get(get);
4315       Cell[] raw = result.rawCells();
4316       assertEquals(families.length, result.size());
4317       for (int j = 0; j < families.length; j++) {
4318         assertTrue(CellUtil.matchingRow(raw[j], row));
4319         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4320         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4321       }
4322     }
4323   }
4324 
4325   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4326     // Now I have k, get values out and assert they are as expected.
4327     Get get = new Get(k).addFamily(family).setMaxVersions();
4328     Cell[] results = r.get(get).rawCells();
4329     for (int j = 0; j < results.length; j++) {
4330       byte[] tmp = CellUtil.cloneValue(results[j]);
4331       // Row should be equal to value every time.
4332       assertTrue(Bytes.equals(k, tmp));
4333     }
4334   }
4335 
4336   /*
4337    * Assert first value in the passed region is <code>firstValue</code>.
4338    * 
4339    * @param r
4340    * 
4341    * @param fs
4342    * 
4343    * @param firstValue
4344    * 
4345    * @throws IOException
4346    */
4347   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4348       throws IOException {
4349     byte[][] families = { fs };
4350     Scan scan = new Scan();
4351     for (int i = 0; i < families.length; i++)
4352       scan.addFamily(families[i]);
4353     InternalScanner s = r.getScanner(scan);
4354     try {
4355       List<Cell> curVals = new ArrayList<Cell>();
4356       boolean first = true;
4357       OUTER_LOOP: while (s.next(curVals)) {
4358         for (Cell kv : curVals) {
4359           byte[] val = CellUtil.cloneValue(kv);
4360           byte[] curval = val;
4361           if (first) {
4362             first = false;
4363             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4364           } else {
4365             // Not asserting anything. Might as well break.
4366             break OUTER_LOOP;
4367           }
4368         }
4369       }
4370     } finally {
4371       s.close();
4372     }
4373   }
4374 
4375   /**
4376    * Test that we get the expected flush results back
4377    * @throws IOException
4378    */
4379   @Test
4380   public void testFlushResult() throws IOException {
4381     String method = name.getMethodName();
4382     byte[] tableName = Bytes.toBytes(method);
4383     byte[] family = Bytes.toBytes("family");
4384 
4385     this.region = initHRegion(tableName, method, family);
4386 
4387     // empty memstore, flush doesn't run
4388     HRegion.FlushResult fr = region.flushcache();
4389     assertFalse(fr.isFlushSucceeded());
4390     assertFalse(fr.isCompactionNeeded());
4391 
4392     // Flush enough files to get up to the threshold, doesn't need compactions
4393     for (int i = 0; i < 2; i++) {
4394       Put put = new Put(tableName).add(family, family, tableName);
4395       region.put(put);
4396       fr = region.flushcache();
4397       assertTrue(fr.isFlushSucceeded());
4398       assertFalse(fr.isCompactionNeeded());
4399     }
4400 
4401     // Two flushes after the threshold, compactions are needed
4402     for (int i = 0; i < 2; i++) {
4403       Put put = new Put(tableName).add(family, family, tableName);
4404       region.put(put);
4405       fr = region.flushcache();
4406       assertTrue(fr.isFlushSucceeded());
4407       assertTrue(fr.isCompactionNeeded());
4408     }
4409   }
4410 
4411   private Configuration initSplit() {
4412     // Always compact if there is more than one store file.
4413     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4414 
4415     // Make lease timeout longer, lease checks less frequent
4416     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4417 
4418     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4419 
4420     // Increase the amount of time between client retries
4421     CONF.setLong("hbase.client.pause", 15 * 1000);
4422 
4423     // This size should make it so we always split using the addContent
4424     // below. After adding all data, the first region is 1.3M
4425     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4426     return CONF;
4427   }
4428 
4429   /**
4430    * @param tableName
4431    * @param callingMethod
4432    * @param conf
4433    * @param families
4434    * @throws IOException
4435    * @return A region on which you must call
4436    *         {@link HRegion#closeHRegion(HRegion)} when done.
4437    */
4438   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4439       byte[]... families) throws IOException {
4440     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4441   }
4442 
4443   /**
4444    * @param tableName
4445    * @param callingMethod
4446    * @param conf
4447    * @param families
4448    * @throws IOException
4449    * @return A region on which you must call
4450    *         {@link HRegion#closeHRegion(HRegion)} when done.
4451    */
4452   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4453       byte[]... families) throws IOException {
4454     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4455   }
4456 
4457   /**
4458    * @param tableName
4459    * @param callingMethod
4460    * @param conf
4461    * @param isReadOnly
4462    * @param families
4463    * @throws IOException
4464    * @return A region on which you must call
4465    *         {@link HRegion#closeHRegion(HRegion)} when done.
4466    */
4467   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4468       boolean isReadOnly, byte[]... families) throws IOException {
4469     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4470   }
4471 
4472   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4473       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4474       throws IOException {
4475     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4476         Durability.SYNC_WAL, null, families);
4477   }
4478 
4479   /**
4480    * @param tableName
4481    * @param startKey
4482    * @param stopKey
4483    * @param callingMethod
4484    * @param conf
4485    * @param isReadOnly
4486    * @param families
4487    * @throws IOException
4488    * @return A region on which you must call
4489    *         {@link HRegion#closeHRegion(HRegion)} when done.
4490    */
4491   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4492       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4493       HLog hlog, byte[]... families) throws IOException {
4494     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4495   }
4496 
4497   /**
4498    * Assert that the passed in Cell has expected contents for the specified row,
4499    * column & timestamp.
4500    */
4501   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4502     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4503     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4504         Bytes.toString(CellUtil.cloneRow(kv)));
4505     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4506         Bytes.toString(CellUtil.cloneFamily(kv)));
4507     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4508         Bytes.toString(CellUtil.cloneQualifier(kv)));
4509     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4510     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4511         Bytes.toString(CellUtil.cloneValue(kv)));
4512   }
4513 
4514   @Test (timeout=60000)
4515   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4516       throws IOException {
4517     byte[] rowC = Bytes.toBytes("rowC");
4518     byte[] rowA = Bytes.toBytes("rowA");
4519     byte[] rowB = Bytes.toBytes("rowB");
4520     byte[] cf = Bytes.toBytes("CF");
4521     byte[][] families = { cf };
4522     byte[] col = Bytes.toBytes("C");
4523     long ts = 1;
4524     String method = this.getName();
4525     this.region = initHRegion(tableName, method, families);
4526     try {
4527       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4528       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4529           null);
4530       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4531       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4532       Put put = null;
4533       put = new Put(rowC);
4534       put.add(kv1);
4535       put.add(kv11);
4536       region.put(put);
4537       put = new Put(rowA);
4538       put.add(kv2);
4539       region.put(put);
4540       put = new Put(rowB);
4541       put.add(kv3);
4542       region.put(put);
4543 
4544       Scan scan = new Scan(rowC);
4545       scan.setMaxVersions(5);
4546       scan.setReversed(true);
4547       InternalScanner scanner = region.getScanner(scan);
4548       List<Cell> currRow = new ArrayList<Cell>();
4549       boolean hasNext = scanner.next(currRow);
4550       assertEquals(2, currRow.size());
4551       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4552       assertTrue(hasNext);
4553       currRow.clear();
4554       hasNext = scanner.next(currRow);
4555       assertEquals(1, currRow.size());
4556       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4557       assertTrue(hasNext);
4558       currRow.clear();
4559       hasNext = scanner.next(currRow);
4560       assertEquals(1, currRow.size());
4561       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4562       assertFalse(hasNext);
4563       scanner.close();
4564     } finally {
4565       HRegion.closeHRegion(this.region);
4566       this.region = null;
4567     }
4568   }
4569 
4570   @Test (timeout=60000)
4571   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4572       throws IOException {
4573     byte[] rowC = Bytes.toBytes("rowC");
4574     byte[] rowA = Bytes.toBytes("rowA");
4575     byte[] rowB = Bytes.toBytes("rowB");
4576     byte[] rowD = Bytes.toBytes("rowD");
4577     byte[] cf = Bytes.toBytes("CF");
4578     byte[][] families = { cf };
4579     byte[] col = Bytes.toBytes("C");
4580     long ts = 1;
4581     String method = this.getName();
4582     this.region = initHRegion(tableName, method, families);
4583     try {
4584       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4585       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4586           null);
4587       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4588       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4589       Put put = null;
4590       put = new Put(rowC);
4591       put.add(kv1);
4592       put.add(kv11);
4593       region.put(put);
4594       put = new Put(rowA);
4595       put.add(kv2);
4596       region.put(put);
4597       put = new Put(rowB);
4598       put.add(kv3);
4599       region.put(put);
4600 
4601       Scan scan = new Scan(rowD);
4602       List<Cell> currRow = new ArrayList<Cell>();
4603       scan.setReversed(true);
4604       scan.setMaxVersions(5);
4605       InternalScanner scanner = region.getScanner(scan);
4606       boolean hasNext = scanner.next(currRow);
4607       assertEquals(2, currRow.size());
4608       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4609       assertTrue(hasNext);
4610       currRow.clear();
4611       hasNext = scanner.next(currRow);
4612       assertEquals(1, currRow.size());
4613       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4614       assertTrue(hasNext);
4615       currRow.clear();
4616       hasNext = scanner.next(currRow);
4617       assertEquals(1, currRow.size());
4618       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4619       assertFalse(hasNext);
4620       scanner.close();
4621     } finally {
4622       HRegion.closeHRegion(this.region);
4623       this.region = null;
4624     }
4625   }
4626 
4627   @Test (timeout=60000)
4628   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4629       throws IOException {
4630     byte[] rowC = Bytes.toBytes("rowC");
4631     byte[] rowA = Bytes.toBytes("rowA");
4632     byte[] rowB = Bytes.toBytes("rowB");
4633     byte[] cf = Bytes.toBytes("CF");
4634     byte[][] families = { cf };
4635     byte[] col = Bytes.toBytes("C");
4636     long ts = 1;
4637     String method = this.getName();
4638     this.region = initHRegion(tableName, method, families);
4639     try {
4640       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4641       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4642           null);
4643       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4644       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4645       Put put = null;
4646       put = new Put(rowC);
4647       put.add(kv1);
4648       put.add(kv11);
4649       region.put(put);
4650       put = new Put(rowA);
4651       put.add(kv2);
4652       region.put(put);
4653       put = new Put(rowB);
4654       put.add(kv3);
4655       region.put(put);
4656       Scan scan = new Scan();
4657       List<Cell> currRow = new ArrayList<Cell>();
4658       scan.setReversed(true);
4659       InternalScanner scanner = region.getScanner(scan);
4660       boolean hasNext = scanner.next(currRow);
4661       assertEquals(1, currRow.size());
4662       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4663       assertTrue(hasNext);
4664       currRow.clear();
4665       hasNext = scanner.next(currRow);
4666       assertEquals(1, currRow.size());
4667       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4668       assertTrue(hasNext);
4669       currRow.clear();
4670       hasNext = scanner.next(currRow);
4671       assertEquals(1, currRow.size());
4672       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4673       assertFalse(hasNext);
4674       scanner.close();
4675     } finally {
4676       HRegion.closeHRegion(this.region);
4677       this.region = null;
4678     }
4679   }
4680 
4681   @Test (timeout=60000)
4682   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4683     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4684     byte[] rowA = Bytes.toBytes("rowA");
4685     byte[] rowB = Bytes.toBytes("rowB");
4686     byte[] rowC = Bytes.toBytes("rowC");
4687     byte[] rowD = Bytes.toBytes("rowD");
4688     byte[] rowE = Bytes.toBytes("rowE");
4689     byte[] cf = Bytes.toBytes("CF");
4690     byte[][] families = { cf };
4691     byte[] col1 = Bytes.toBytes("col1");
4692     byte[] col2 = Bytes.toBytes("col2");
4693     long ts = 1;
4694     String method = this.getName();
4695     this.region = initHRegion(tableName, method, families);
4696     try {
4697       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4698       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4699       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4700       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4701       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4702       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4703       Put put = null;
4704       put = new Put(rowA);
4705       put.add(kv1);
4706       region.put(put);
4707       put = new Put(rowB);
4708       put.add(kv2);
4709       region.put(put);
4710       put = new Put(rowC);
4711       put.add(kv3);
4712       region.put(put);
4713       put = new Put(rowD);
4714       put.add(kv4_1);
4715       region.put(put);
4716       put = new Put(rowD);
4717       put.add(kv4_2);
4718       region.put(put);
4719       put = new Put(rowE);
4720       put.add(kv5);
4721       region.put(put);
4722       region.flushcache();
4723       Scan scan = new Scan(rowD, rowA);
4724       scan.addColumn(families[0], col1);
4725       scan.setReversed(true);
4726       List<Cell> currRow = new ArrayList<Cell>();
4727       InternalScanner scanner = region.getScanner(scan);
4728       boolean hasNext = scanner.next(currRow);
4729       assertEquals(1, currRow.size());
4730       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4731       assertTrue(hasNext);
4732       currRow.clear();
4733       hasNext = scanner.next(currRow);
4734       assertEquals(1, currRow.size());
4735       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4736       assertTrue(hasNext);
4737       currRow.clear();
4738       hasNext = scanner.next(currRow);
4739       assertEquals(1, currRow.size());
4740       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4741       assertFalse(hasNext);
4742       scanner.close();
4743 
4744       scan = new Scan(rowD, rowA);
4745       scan.addColumn(families[0], col2);
4746       scan.setReversed(true);
4747       currRow.clear();
4748       scanner = region.getScanner(scan);
4749       hasNext = scanner.next(currRow);
4750       assertEquals(1, currRow.size());
4751       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4752       scanner.close();
4753     } finally {
4754       HRegion.closeHRegion(this.region);
4755       this.region = null;
4756     }
4757   }
4758 
4759   @Test (timeout=60000)
4760   public void testReverseScanner_smaller_blocksize() throws IOException {
4761     // case to ensure no conflict with HFile index optimization
4762     byte[] rowA = Bytes.toBytes("rowA");
4763     byte[] rowB = Bytes.toBytes("rowB");
4764     byte[] rowC = Bytes.toBytes("rowC");
4765     byte[] rowD = Bytes.toBytes("rowD");
4766     byte[] rowE = Bytes.toBytes("rowE");
4767     byte[] cf = Bytes.toBytes("CF");
4768     byte[][] families = { cf };
4769     byte[] col1 = Bytes.toBytes("col1");
4770     byte[] col2 = Bytes.toBytes("col2");
4771     long ts = 1;
4772     String method = this.getName();
4773     HBaseConfiguration config = new HBaseConfiguration();
4774     config.setInt("test.block.size", 1);
4775     this.region = initHRegion(tableName, method, config, families);
4776     try {
4777       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4778       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4779       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4780       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4781       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4782       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4783       Put put = null;
4784       put = new Put(rowA);
4785       put.add(kv1);
4786       region.put(put);
4787       put = new Put(rowB);
4788       put.add(kv2);
4789       region.put(put);
4790       put = new Put(rowC);
4791       put.add(kv3);
4792       region.put(put);
4793       put = new Put(rowD);
4794       put.add(kv4_1);
4795       region.put(put);
4796       put = new Put(rowD);
4797       put.add(kv4_2);
4798       region.put(put);
4799       put = new Put(rowE);
4800       put.add(kv5);
4801       region.put(put);
4802       region.flushcache();
4803       Scan scan = new Scan(rowD, rowA);
4804       scan.addColumn(families[0], col1);
4805       scan.setReversed(true);
4806       List<Cell> currRow = new ArrayList<Cell>();
4807       InternalScanner scanner = region.getScanner(scan);
4808       boolean hasNext = scanner.next(currRow);
4809       assertEquals(1, currRow.size());
4810       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4811       assertTrue(hasNext);
4812       currRow.clear();
4813       hasNext = scanner.next(currRow);
4814       assertEquals(1, currRow.size());
4815       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4816       assertTrue(hasNext);
4817       currRow.clear();
4818       hasNext = scanner.next(currRow);
4819       assertEquals(1, currRow.size());
4820       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4821       assertFalse(hasNext);
4822       scanner.close();
4823 
4824       scan = new Scan(rowD, rowA);
4825       scan.addColumn(families[0], col2);
4826       scan.setReversed(true);
4827       currRow.clear();
4828       scanner = region.getScanner(scan);
4829       hasNext = scanner.next(currRow);
4830       assertEquals(1, currRow.size());
4831       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4832       scanner.close();
4833     } finally {
4834       HRegion.closeHRegion(this.region);
4835       this.region = null;
4836     }
4837   }
4838 
4839   @Test (timeout=60000)
4840   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4841       throws IOException {
4842     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4843     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4844     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4845     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4846     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4847     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4848     byte[] cf1 = Bytes.toBytes("CF1");
4849     byte[] cf2 = Bytes.toBytes("CF2");
4850     byte[] cf3 = Bytes.toBytes("CF3");
4851     byte[][] families = { cf1, cf2, cf3 };
4852     byte[] col = Bytes.toBytes("C");
4853     long ts = 1;
4854     String method = this.getName();
4855     HBaseConfiguration conf = new HBaseConfiguration();
4856     // disable compactions in this test.
4857     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4858     this.region = initHRegion(tableName, method, conf, families);
4859     try {
4860       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4861       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4862           null);
4863       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4864           null);
4865       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4866           KeyValue.Type.Put, null);
4867       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4868           null);
4869       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4870           null);
4871       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4872           null);
4873       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4874           KeyValue.Type.Put, null);
4875       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4876           null);
4877       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4878           KeyValue.Type.Put, null);
4879       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4880           null);
4881       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4882           null);
4883       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4884           KeyValue.Type.Put, null);
4885       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4886           null);
4887       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4888           KeyValue.Type.Put, null);
4889       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4890           null);
4891       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4892           null);
4893       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4894       Put put = null;
4895       put = new Put(row1);
4896       put.add(kv1_2_1);
4897       region.put(put);
4898       put = new Put(row2);
4899       put.add(kv2_4_1);
4900       region.put(put);
4901       put = new Put(row4);
4902       put.add(kv4_5_4);
4903       put.add(kv4_5_5);
4904       region.put(put);
4905       region.flushcache();
4906       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4907       put = new Put(row4);
4908       put.add(kv4_5_1);
4909       put.add(kv4_5_3);
4910       region.put(put);
4911       put = new Put(row1);
4912       put.add(kv1_2_2);
4913       region.put(put);
4914       put = new Put(row2);
4915       put.add(kv2_4_4);
4916       region.put(put);
4917       region.flushcache();
4918       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4919       put = new Put(row4);
4920       put.add(kv4_5_2);
4921       region.put(put);
4922       put = new Put(row2);
4923       put.add(kv2_4_2);
4924       put.add(kv2_4_3);
4925       region.put(put);
4926       put = new Put(row3);
4927       put.add(kv3_2_2);
4928       region.put(put);
4929       region.flushcache();
4930       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
4931       // ( 2 kv)
4932       put = new Put(row0);
4933       put.add(kv0_1_1);
4934       region.put(put);
4935       put = new Put(row3);
4936       put.add(kv3_2_1);
4937       region.put(put);
4938       put = new Put(row5);
4939       put.add(kv5_2_1);
4940       put.add(kv5_2_2);
4941       region.put(put);
4942       // scan range = ["row4", min), skip the max "row5"
4943       Scan scan = new Scan(row4);
4944       scan.setMaxVersions(5);
4945       scan.setBatch(3);
4946       scan.setReversed(true);
4947       InternalScanner scanner = region.getScanner(scan);
4948       List<Cell> currRow = new ArrayList<Cell>();
4949       boolean hasNext = false;
4950       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
4951       // included in scan range
4952       // "row4" takes 2 next() calls since batch=3
4953       hasNext = scanner.next(currRow);
4954       assertEquals(3, currRow.size());
4955       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4956       assertTrue(hasNext);
4957       currRow.clear();
4958       hasNext = scanner.next(currRow);
4959       assertEquals(2, currRow.size());
4960       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4961       assertTrue(hasNext);
4962       // 2. scan out "row3" (2 kv)
4963       currRow.clear();
4964       hasNext = scanner.next(currRow);
4965       assertEquals(2, currRow.size());
4966       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4967       assertTrue(hasNext);
4968       // 3. scan out "row2" (4 kvs)
4969       // "row2" takes 2 next() calls since batch=3
4970       currRow.clear();
4971       hasNext = scanner.next(currRow);
4972       assertEquals(3, currRow.size());
4973       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4974       assertTrue(hasNext);
4975       currRow.clear();
4976       hasNext = scanner.next(currRow);
4977       assertEquals(1, currRow.size());
4978       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4979       assertTrue(hasNext);
4980       // 4. scan out "row1" (2 kv)
4981       currRow.clear();
4982       hasNext = scanner.next(currRow);
4983       assertEquals(2, currRow.size());
4984       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4985       assertTrue(hasNext);
4986       // 5. scan out "row0" (1 kv)
4987       currRow.clear();
4988       hasNext = scanner.next(currRow);
4989       assertEquals(1, currRow.size());
4990       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4991       assertFalse(hasNext);
4992 
4993       scanner.close();
4994     } finally {
4995       HRegion.closeHRegion(this.region);
4996       this.region = null;
4997     }
4998   }
4999 
5000   @Test (timeout=60000)
5001   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5002       throws IOException {
5003     byte[] row1 = Bytes.toBytes("row1");
5004     byte[] row2 = Bytes.toBytes("row2");
5005     byte[] row3 = Bytes.toBytes("row3");
5006     byte[] row4 = Bytes.toBytes("row4");
5007     byte[] cf1 = Bytes.toBytes("CF1");
5008     byte[] cf2 = Bytes.toBytes("CF2");
5009     byte[] cf3 = Bytes.toBytes("CF3");
5010     byte[] cf4 = Bytes.toBytes("CF4");
5011     byte[][] families = { cf1, cf2, cf3, cf4 };
5012     byte[] col = Bytes.toBytes("C");
5013     long ts = 1;
5014     String method = this.getName();
5015     HBaseConfiguration conf = new HBaseConfiguration();
5016     // disable compactions in this test.
5017     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5018     this.region = initHRegion(tableName, method, conf, families);
5019     try {
5020       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5021       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5022       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5023       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5024       // storefile1
5025       Put put = new Put(row1);
5026       put.add(kv1);
5027       region.put(put);
5028       region.flushcache();
5029       // storefile2
5030       put = new Put(row2);
5031       put.add(kv2);
5032       region.put(put);
5033       region.flushcache();
5034       // storefile3
5035       put = new Put(row3);
5036       put.add(kv3);
5037       region.put(put);
5038       region.flushcache();
5039       // memstore
5040       put = new Put(row4);
5041       put.add(kv4);
5042       region.put(put);
5043       // scan range = ["row4", min)
5044       Scan scan = new Scan(row4);
5045       scan.setReversed(true);
5046       scan.setBatch(10);
5047       InternalScanner scanner = region.getScanner(scan);
5048       List<Cell> currRow = new ArrayList<Cell>();
5049       boolean hasNext = scanner.next(currRow);
5050       assertEquals(1, currRow.size());
5051       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5052       assertTrue(hasNext);
5053       currRow.clear();
5054       hasNext = scanner.next(currRow);
5055       assertEquals(1, currRow.size());
5056       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5057       assertTrue(hasNext);
5058       currRow.clear();
5059       hasNext = scanner.next(currRow);
5060       assertEquals(1, currRow.size());
5061       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5062       assertTrue(hasNext);
5063       currRow.clear();
5064       hasNext = scanner.next(currRow);
5065       assertEquals(1, currRow.size());
5066       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5067       assertFalse(hasNext);
5068     } finally {
5069       HRegion.closeHRegion(this.region);
5070       this.region = null;
5071     }
5072   }
5073 
5074   @Test (timeout=60000)
5075   public void testSplitRegionWithReverseScan() throws IOException {
5076     byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5077     byte [] qualifier = Bytes.toBytes("qualifier");
5078     Configuration hc = initSplit();
5079     int numRows = 3;
5080     byte [][] families = {fam1};
5081 
5082     //Setting up region
5083     String method = this.getName();
5084     this.region = initHRegion(tableName, method, hc, families);
5085 
5086     //Put data in region
5087     int startRow = 100;
5088     putData(startRow, numRows, qualifier, families);
5089     int splitRow = startRow + numRows;
5090     putData(splitRow, numRows, qualifier, families);
5091     int endRow = splitRow + numRows;
5092     region.flushcache();
5093 
5094     HRegion [] regions = null;
5095     try {
5096       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5097       //Opening the regions returned.
5098       for (int i = 0; i < regions.length; i++) {
5099         regions[i] = HRegion.openHRegion(regions[i], null);
5100       }
5101       //Verifying that the region has been split
5102       assertEquals(2, regions.length);
5103 
5104       //Verifying that all data is still there and that data is in the right
5105       //place
5106       verifyData(regions[0], startRow, numRows, qualifier, families);
5107       verifyData(regions[1], splitRow, numRows, qualifier, families);
5108 
5109       //fire the reverse scan1:  top range, and larger than the last row
5110       Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5111       scan.setReversed(true);
5112       InternalScanner scanner = regions[1].getScanner(scan);
5113       List<Cell> currRow = new ArrayList<Cell>();
5114       boolean more = false;
5115       int verify = startRow + 2 * numRows - 1;
5116       do {
5117         more = scanner.next(currRow);
5118         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5119         verify--;
5120         currRow.clear();
5121       } while(more);
5122       assertEquals(verify, startRow + numRows - 1);
5123       scanner.close();
5124       //fire the reverse scan2:  top range, and equals to the last row
5125       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5126       scan.setReversed(true);
5127       scanner = regions[1].getScanner(scan);
5128       verify = startRow + 2 * numRows - 1;
5129       do {
5130         more = scanner.next(currRow);
5131         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5132         verify--;
5133         currRow.clear();
5134       } while(more);
5135       assertEquals(verify, startRow + numRows - 1);
5136       scanner.close();
5137       //fire the reverse scan3:  bottom range, and larger than the last row
5138       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5139       scan.setReversed(true);
5140       scanner = regions[0].getScanner(scan);
5141       verify = startRow + numRows - 1;
5142       do {
5143         more = scanner.next(currRow);
5144         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5145         verify--;
5146         currRow.clear();
5147       } while(more);
5148       assertEquals(verify, 99);
5149       scanner.close();
5150       //fire the reverse scan4:  bottom range, and equals to the last row
5151       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5152       scan.setReversed(true);
5153       scanner = regions[0].getScanner(scan);
5154       verify = startRow + numRows - 1;
5155       do {
5156         more = scanner.next(currRow);
5157         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5158         verify--;
5159         currRow.clear();
5160       } while(more);
5161       assertEquals(verify, startRow - 1);
5162       scanner.close();
5163     } finally {
5164       HRegion.closeHRegion(this.region);
5165       this.region = null;
5166     }
5167   }
5168 
5169   @Test
5170   public void testWriteRequestsCounter() throws IOException {
5171     byte[] fam = Bytes.toBytes("info");
5172     byte[][] families = { fam };
5173     this.region = initHRegion(tableName, method, CONF, families);
5174 
5175     Assert.assertEquals(0L, region.getWriteRequestsCount());
5176 
5177     Put put = new Put(row);
5178     put.add(fam, fam, fam);
5179 
5180     Assert.assertEquals(0L, region.getWriteRequestsCount());
5181     region.put(put);
5182     Assert.assertEquals(1L, region.getWriteRequestsCount());
5183     region.put(put);
5184     Assert.assertEquals(2L, region.getWriteRequestsCount());
5185     region.put(put);
5186     Assert.assertEquals(3L, region.getWriteRequestsCount());
5187 
5188     region.delete(new Delete(row));
5189     Assert.assertEquals(4L, region.getWriteRequestsCount());
5190 
5191     HRegion.closeHRegion(this.region);
5192     this.region = null;
5193   }
5194 
5195   /**
5196    * Test RegionTooBusyException thrown when region is busy
5197    */
5198   @Test (timeout=24000)
5199   public void testRegionTooBusy() throws IOException {
5200     String method = "testRegionTooBusy";
5201     byte[] tableName = Bytes.toBytes(method);
5202     byte[] family = Bytes.toBytes("family");
5203     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5204       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5205     CONF.setLong("hbase.busy.wait.duration", 1000);
5206     region = initHRegion(tableName, method, CONF, family);
5207     final AtomicBoolean stopped = new AtomicBoolean(true);
5208     Thread t = new Thread(new Runnable() {
5209       @Override
5210       public void run() {
5211         try {
5212           region.lock.writeLock().lock();
5213           stopped.set(false);
5214           while (!stopped.get()) {
5215             Thread.sleep(100);
5216           }
5217         } catch (InterruptedException ie) {
5218         } finally {
5219           region.lock.writeLock().unlock();
5220         }
5221       }
5222     });
5223     t.start();
5224     Get get = new Get(row);
5225     try {
5226       while (stopped.get()) {
5227         Thread.sleep(100);
5228       }
5229       region.get(get);
5230       fail("Should throw RegionTooBusyException");
5231     } catch (InterruptedException ie) {
5232       fail("test interrupted");
5233     } catch (RegionTooBusyException e) {
5234       // Good, expected
5235     } finally {
5236       stopped.set(true);
5237       try {
5238         t.join();
5239       } catch (Throwable e) {
5240       }
5241 
5242       HRegion.closeHRegion(region);
5243       region = null;
5244       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5245     }
5246   }
5247 
5248   @Test
5249   public void testCellTTLs() throws IOException {
5250     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5251     EnvironmentEdgeManager.injectEdge(edge);
5252 
5253     final byte[] row = Bytes.toBytes("testRow");
5254     final byte[] q1 = Bytes.toBytes("q1");
5255     final byte[] q2 = Bytes.toBytes("q2");
5256     final byte[] q3 = Bytes.toBytes("q3");
5257     final byte[] q4 = Bytes.toBytes("q4");
5258 
5259     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5260     HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5261     hcd.setTimeToLive(10); // 10 seconds
5262     htd.addFamily(hcd);
5263 
5264     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5265     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5266 
5267     HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5268         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5269       TEST_UTIL.getDataTestDir(), conf, htd);
5270     assertNotNull(region);
5271     try {
5272       long now = EnvironmentEdgeManager.currentTimeMillis();
5273       // Add a cell that will expire in 5 seconds via cell TTL
5274       region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5275         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5276           // TTL tags specify ts in milliseconds
5277           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5278       // Add a cell that will expire after 10 seconds via family setting
5279       region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5280       // Add a cell that will expire in 15 seconds via cell TTL
5281       region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5282         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5283           // TTL tags specify ts in milliseconds
5284           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5285       // Add a cell that will expire in 20 seconds via family setting
5286       region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5287 
5288       // Flush so we are sure store scanning gets this right
5289       region.flushcache();
5290 
5291       // A query at time T+0 should return all cells
5292       Result r = region.get(new Get(row));
5293       assertNotNull(r.getValue(fam1, q1));
5294       assertNotNull(r.getValue(fam1, q2));
5295       assertNotNull(r.getValue(fam1, q3));
5296       assertNotNull(r.getValue(fam1, q4));
5297 
5298       // Increment time to T+5 seconds
5299       edge.incrementTime(5000);
5300 
5301       r = region.get(new Get(row));
5302       assertNull(r.getValue(fam1, q1));
5303       assertNotNull(r.getValue(fam1, q2));
5304       assertNotNull(r.getValue(fam1, q3));
5305       assertNotNull(r.getValue(fam1, q4));
5306 
5307       // Increment time to T+10 seconds
5308       edge.incrementTime(5000);
5309 
5310       r = region.get(new Get(row));
5311       assertNull(r.getValue(fam1, q1));
5312       assertNull(r.getValue(fam1, q2));
5313       assertNotNull(r.getValue(fam1, q3));
5314       assertNotNull(r.getValue(fam1, q4));
5315 
5316       // Increment time to T+15 seconds
5317       edge.incrementTime(5000);
5318 
5319       r = region.get(new Get(row));
5320       assertNull(r.getValue(fam1, q1));
5321       assertNull(r.getValue(fam1, q2));
5322       assertNull(r.getValue(fam1, q3));
5323       assertNotNull(r.getValue(fam1, q4));
5324 
5325       // Increment time to T+20 seconds
5326       edge.incrementTime(10000);
5327 
5328       r = region.get(new Get(row));
5329       assertNull(r.getValue(fam1, q1));
5330       assertNull(r.getValue(fam1, q2));
5331       assertNull(r.getValue(fam1, q3));
5332       assertNull(r.getValue(fam1, q4));
5333 
5334       // Fun with disappearing increments
5335 
5336       // Start at 1
5337       region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5338       r = region.get(new Get(row));
5339       byte[] val = r.getValue(fam1, q1);
5340       assertNotNull(val);
5341       assertEquals(Bytes.toLong(val), 1L);
5342 
5343       // Increment with a TTL of 5 seconds
5344       Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5345       incr.setTTL(5000);
5346       region.increment(incr); // 2
5347 
5348       // New value should be 2
5349       r = region.get(new Get(row));
5350       val = r.getValue(fam1, q1);
5351       assertNotNull(val);
5352       assertEquals(Bytes.toLong(val), 2L);
5353 
5354       // Increment time to T+25 seconds
5355       edge.incrementTime(5000);
5356 
5357       // Value should be back to 1
5358       r = region.get(new Get(row));
5359       val = r.getValue(fam1, q1);
5360       assertNotNull(val);
5361       assertEquals(Bytes.toLong(val), 1L);
5362 
5363       // Increment time to T+30 seconds
5364       edge.incrementTime(5000);
5365 
5366       // Original value written at T+20 should be gone now via family TTL
5367       r = region.get(new Get(row));
5368       assertNull(r.getValue(fam1, q1));
5369 
5370     } finally {
5371       HRegion.closeHRegion(region);
5372     }
5373   }
5374 
5375   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5376       byte[]... families) throws IOException {
5377     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5378         families);
5379   }
5380 }