1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.RegionTooBusyException;
79 import org.apache.hadoop.hbase.Tag;
80 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
81 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
82 import org.apache.hadoop.hbase.HRegionInfo;
83 import org.apache.hadoop.hbase.HTableDescriptor;
84 import org.apache.hadoop.hbase.KeyValue;
85 import org.apache.hadoop.hbase.testclassification.MediumTests;
86 import org.apache.hadoop.hbase.MiniHBaseCluster;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil;
88 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
89 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
90 import org.apache.hadoop.hbase.NotServingRegionException;
91 import org.apache.hadoop.hbase.TableName;
92 import org.apache.hadoop.hbase.TagType;
93 import org.apache.hadoop.hbase.Waiter;
94 import org.apache.hadoop.hbase.client.Append;
95 import org.apache.hadoop.hbase.client.Delete;
96 import org.apache.hadoop.hbase.client.Durability;
97 import org.apache.hadoop.hbase.client.Get;
98 import org.apache.hadoop.hbase.client.HTable;
99 import org.apache.hadoop.hbase.client.Increment;
100 import org.apache.hadoop.hbase.client.Put;
101 import org.apache.hadoop.hbase.client.Result;
102 import org.apache.hadoop.hbase.client.Scan;
103 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
104 import org.apache.hadoop.hbase.filter.BinaryComparator;
105 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
106 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
107 import org.apache.hadoop.hbase.filter.Filter;
108 import org.apache.hadoop.hbase.filter.FilterBase;
109 import org.apache.hadoop.hbase.filter.FilterList;
110 import org.apache.hadoop.hbase.filter.NullComparator;
111 import org.apache.hadoop.hbase.filter.PrefixFilter;
112 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
113 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
114 import org.apache.hadoop.hbase.io.hfile.HFile;
115 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
116 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
117 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
118 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
119 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
120 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
121 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
122 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
123 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
124 import org.apache.hadoop.hbase.regionserver.wal.HLog;
125 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
126 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
128 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
129 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
130 import org.apache.hadoop.hbase.security.User;
131 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
132 import org.apache.hadoop.hbase.util.Bytes;
133 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
134 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
135 import org.apache.hadoop.hbase.util.FSUtils;
136 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
137 import org.apache.hadoop.hbase.util.PairOfSameType;
138 import org.apache.hadoop.hbase.util.Threads;
139 import org.junit.After;
140 import org.junit.Assert;
141 import org.junit.Before;
142 import org.junit.Rule;
143 import org.junit.Test;
144 import org.junit.experimental.categories.Category;
145 import org.junit.rules.TestName;
146 import org.mockito.Mockito;
147
148 import com.google.common.collect.ImmutableList;
149 import com.google.common.collect.Lists;
150 import com.google.protobuf.ByteString;
151
152
153
154
155
156
157
158 @Category(MediumTests.class)
159 @SuppressWarnings("deprecation")
160 public class TestHRegion {
161
162
163 static final Log LOG = LogFactory.getLog(TestHRegion.class);
164 @Rule public TestName name = new TestName();
165
166 private static final String COLUMN_FAMILY = "MyCF";
167 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
168
169 HRegion region = null;
170
171 private static HBaseTestingUtility TEST_UTIL;
172 public static Configuration CONF ;
173 private String dir;
174 private static FileSystem FILESYSTEM;
175 private final int MAX_VERSIONS = 2;
176
177
178 protected byte[] tableName;
179 protected String method;
180 protected final byte[] qual1 = Bytes.toBytes("qual1");
181 protected final byte[] qual2 = Bytes.toBytes("qual2");
182 protected final byte[] qual3 = Bytes.toBytes("qual3");
183 protected final byte[] value1 = Bytes.toBytes("value1");
184 protected final byte[] value2 = Bytes.toBytes("value2");
185 protected final byte[] row = Bytes.toBytes("rowA");
186 protected final byte[] row2 = Bytes.toBytes("rowB");
187
188 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
189 .getInstance(MetricsAssertHelper.class);
190
191 @Before
192 public void setup() throws IOException {
193 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
194 FILESYSTEM = TEST_UTIL.getTestFileSystem();
195 CONF = TEST_UTIL.getConfiguration();
196 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
197 method = name.getMethodName();
198 tableName = Bytes.toBytes(name.getMethodName());
199 }
200
201 @After
202 public void tearDown() throws Exception {
203 EnvironmentEdgeManagerTestHelper.reset();
204 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
205 TEST_UTIL.cleanupTestDir();
206 }
207
208 String getName() {
209 return name.getMethodName();
210 }
211
212
213
214
215
216
217
218
219
220
221
222 @Test (timeout=60000)
223 public void testCloseCarryingSnapshot() throws IOException {
224 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
225 Store store = region.getStore(COLUMN_FAMILY_BYTES);
226
227 byte [] value = Bytes.toBytes(name.getMethodName());
228
229 Put put = new Put(value);
230 put.add(COLUMN_FAMILY_BYTES, null, value);
231
232 region.put(put);
233 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
234 storeFlushCtx.prepare();
235
236 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
237 region.put(put);
238
239 region.close();
240 assertEquals(0, region.getMemstoreSize().get());
241 HRegion.closeHRegion(region);
242 }
243
244
245
246
247
248 @Test (timeout=60000)
249 public void testMemstoreSnapshotSize() throws IOException {
250 class MyFaultyHLog extends FaultyHLog {
251 StoreFlushContext storeFlushCtx;
252 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
253 throws IOException {
254 super(fs, rootDir, logName, conf);
255 }
256
257 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
258 this.storeFlushCtx = storeFlushCtx;
259 }
260
261 @Override
262 public void sync(long txid) throws IOException {
263 storeFlushCtx.prepare();
264 super.sync(txid);
265 }
266 }
267
268 FileSystem fs = FileSystem.get(CONF);
269 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
270 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
271 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
272 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
273
274 Store store = region.getStore(COLUMN_FAMILY_BYTES);
275
276 byte [] value = Bytes.toBytes(name.getMethodName());
277 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
278
279 Put put = new Put(value);
280 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
281 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
282
283 boolean threwIOE = false;
284 try {
285 region.put(put);
286 } catch (IOException ioe) {
287 threwIOE = true;
288 } finally {
289 assertTrue("The regionserver should have thrown an exception", threwIOE);
290 }
291 long sz = store.getFlushableSize();
292 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
293 HRegion.closeHRegion(region);
294 }
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311 @Test (timeout=60000)
312 public void testFlushSizeAccounting() throws Exception {
313 final Configuration conf = HBaseConfiguration.create(CONF);
314
315 conf.setInt("hbase.hstore.flush.retries.number", 1);
316 final User user =
317 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
318
319 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
320 user.runAs(new PrivilegedExceptionAction<Object>() {
321 @Override
322 public Object run() throws Exception {
323
324 FileSystem fs = FileSystem.get(conf);
325 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
326 FaultyFileSystem ffs = (FaultyFileSystem)fs;
327 HRegion region = null;
328 try {
329
330 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
331 long size = region.getMemstoreSize().get();
332 Assert.assertEquals(0, size);
333
334 Put p1 = new Put(row);
335 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
336 region.put(p1);
337 final long sizeOfOnePut = region.getMemstoreSize().get();
338
339 try {
340 LOG.info("Flushing");
341 region.flushcache();
342 Assert.fail("Didn't bubble up IOE!");
343 } catch (DroppedSnapshotException dse) {
344
345 }
346
347 ffs.fault.set(false);
348
349 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
350
351
352 Put p2 = new Put(row);
353 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
354 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
355 region.put(p2);
356 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
357
358
359
360 region.flushcache();
361
362 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
363 } finally {
364 HRegion.closeHRegion(region);
365 }
366 return null;
367 }
368 });
369 FileSystem.closeAllForUGI(user.getUGI());
370 }
371
372 @Test
373 public void testCompactionAffectedByScanners() throws Exception {
374 byte[] family = Bytes.toBytes("family");
375 this.region = initHRegion(tableName, method, CONF, family);
376
377 Put put = new Put(Bytes.toBytes("r1"));
378 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
379 region.put(put);
380 region.flushcache();
381
382 Scan scan = new Scan();
383 scan.setMaxVersions(3);
384
385 RegionScanner scanner1 = region.getScanner(scan);
386
387 Delete delete = new Delete(Bytes.toBytes("r1"));
388 region.delete(delete);
389 region.flushcache();
390
391
392 RegionScanner scanner2 = region.getScanner(scan);
393
394 List<Cell> results = new ArrayList<Cell>();
395
396 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
397
398
399 region.compactStores(true);
400
401
402 RegionScanner scanner3 = region.getScanner(scan);
403
404
405 scanner1.next(results);
406 System.out.println(results);
407 assertEquals(1, results.size());
408
409 results.clear();
410 scanner2.next(results);
411 System.out.println(results);
412 assertEquals(0, results.size());
413
414 results.clear();
415 scanner3.next(results);
416 System.out.println(results);
417 assertEquals(0, results.size());
418 }
419
420 @Test
421 public void testToShowNPEOnRegionScannerReseek() throws Exception {
422 byte[] family = Bytes.toBytes("family");
423 this.region = initHRegion(tableName, method, CONF, family);
424
425 Put put = new Put(Bytes.toBytes("r1"));
426 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
427 region.put(put);
428 put = new Put(Bytes.toBytes("r2"));
429 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
430 region.put(put);
431 region.flushcache();
432
433 Scan scan = new Scan();
434 scan.setMaxVersions(3);
435
436 RegionScanner scanner1 = region.getScanner(scan);
437
438 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
439
440 region.compactStores(true);
441
442 scanner1.reseek(Bytes.toBytes("r2"));
443 List<Cell> results = new ArrayList<Cell>();
444 scanner1.next(results);
445 Cell keyValue = results.get(0);
446 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
447 scanner1.close();
448 }
449
450 @Test
451 public void testSkipRecoveredEditsReplay() throws Exception {
452 String method = "testSkipRecoveredEditsReplay";
453 TableName tableName = TableName.valueOf(method);
454 byte[] family = Bytes.toBytes("family");
455 this.region = initHRegion(tableName, method, CONF, family);
456 try {
457 Path regiondir = region.getRegionFileSystem().getRegionDir();
458 FileSystem fs = region.getRegionFileSystem().getFileSystem();
459 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
460
461 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
462
463 long maxSeqId = 1050;
464 long minSeqId = 1000;
465
466 for (long i = minSeqId; i <= maxSeqId; i += 10) {
467 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
468 fs.create(recoveredEdits);
469 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
470
471 long time = System.nanoTime();
472 WALEdit edit = new WALEdit();
473 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
474 .toBytes(i)));
475 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
476 HConstants.DEFAULT_CLUSTER_ID), edit));
477
478 writer.close();
479 }
480 MonitoredTask status = TaskMonitor.get().createStatus(method);
481 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
482 for (Store store : region.getStores().values()) {
483 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
484 }
485 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
486 assertEquals(maxSeqId, seqId);
487 Get get = new Get(row);
488 Result result = region.get(get);
489 for (long i = minSeqId; i <= maxSeqId; i += 10) {
490 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
491 assertEquals(1, kvs.size());
492 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
493 }
494 } finally {
495 HRegion.closeHRegion(this.region);
496 this.region = null;
497 }
498 }
499
500 @Test
501 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
502 String method = "testSkipRecoveredEditsReplaySomeIgnored";
503 TableName tableName = TableName.valueOf(method);
504 byte[] family = Bytes.toBytes("family");
505 this.region = initHRegion(tableName, method, CONF, family);
506 try {
507 Path regiondir = region.getRegionFileSystem().getRegionDir();
508 FileSystem fs = region.getRegionFileSystem().getFileSystem();
509 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
510
511 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
512
513 long maxSeqId = 1050;
514 long minSeqId = 1000;
515
516 for (long i = minSeqId; i <= maxSeqId; i += 10) {
517 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
518 fs.create(recoveredEdits);
519 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
520
521 long time = System.nanoTime();
522 WALEdit edit = new WALEdit();
523 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
524 .toBytes(i)));
525 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
526 HConstants.DEFAULT_CLUSTER_ID), edit));
527
528 writer.close();
529 }
530 long recoverSeqId = 1030;
531 MonitoredTask status = TaskMonitor.get().createStatus(method);
532 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
533 for (Store store : region.getStores().values()) {
534 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
535 }
536 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
537 assertEquals(maxSeqId, seqId);
538 Get get = new Get(row);
539 Result result = region.get(get);
540 for (long i = minSeqId; i <= maxSeqId; i += 10) {
541 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
542 if (i < recoverSeqId) {
543 assertEquals(0, kvs.size());
544 } else {
545 assertEquals(1, kvs.size());
546 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
547 }
548 }
549 } finally {
550 HRegion.closeHRegion(this.region);
551 this.region = null;
552 }
553 }
554
555 @Test
556 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
557 byte[] family = Bytes.toBytes("family");
558 this.region = initHRegion(tableName, method, CONF, family);
559 try {
560 Path regiondir = region.getRegionFileSystem().getRegionDir();
561 FileSystem fs = region.getRegionFileSystem().getFileSystem();
562
563 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
564 for (int i = 1000; i < 1050; i += 10) {
565 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
566 FSDataOutputStream dos = fs.create(recoveredEdits);
567 dos.writeInt(i);
568 dos.close();
569 }
570 long minSeqId = 2000;
571 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
572 FSDataOutputStream dos = fs.create(recoveredEdits);
573 dos.close();
574
575 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
576 for (Store store : region.getStores().values()) {
577 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
578 }
579 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
580 assertEquals(minSeqId, seqId);
581 } finally {
582 HRegion.closeHRegion(this.region);
583 this.region = null;
584 }
585 }
586
587 @Test
588 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
589 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
590 TableName tableName = TableName.valueOf(method);
591 byte[] family = Bytes.toBytes("family");
592 this.region = initHRegion(tableName, method, CONF, family);
593 try {
594 Path regiondir = region.getRegionFileSystem().getRegionDir();
595 FileSystem fs = region.getRegionFileSystem().getFileSystem();
596 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
597
598 assertEquals(0, region.getStoreFileList(
599 region.getStores().keySet().toArray(new byte[0][])).size());
600
601 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
602
603 long maxSeqId = 1050;
604 long minSeqId = 1000;
605
606 for (long i = minSeqId; i <= maxSeqId; i += 10) {
607 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
608 fs.create(recoveredEdits);
609 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
610
611 long time = System.nanoTime();
612 WALEdit edit = null;
613 if (i == maxSeqId) {
614 edit = WALEdit.createCompaction(region.getRegionInfo(),
615 CompactionDescriptor.newBuilder()
616 .setTableName(ByteString.copyFrom(tableName.getName()))
617 .setFamilyName(ByteString.copyFrom(regionName))
618 .setEncodedRegionName(ByteString.copyFrom(regionName))
619 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
620 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
621 .build());
622 } else {
623 edit = new WALEdit();
624 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
625 .toBytes(i)));
626 }
627 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
628 HConstants.DEFAULT_CLUSTER_ID), edit));
629 writer.close();
630 }
631
632 long recoverSeqId = 1030;
633 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
634 MonitoredTask status = TaskMonitor.get().createStatus(method);
635 for (Store store : region.getStores().values()) {
636 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
637 }
638 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
639 assertEquals(maxSeqId, seqId);
640
641
642 assertEquals(1, region.getStoreFileList(
643 region.getStores().keySet().toArray(new byte[0][])).size());
644
645 } finally {
646 HRegion.closeHRegion(this.region);
647 this.region = null;
648 } }
649
650 @Test
651 public void testRecoveredEditsReplayCompaction() throws Exception {
652 String method = name.getMethodName();
653 TableName tableName = TableName.valueOf(method);
654 byte[] family = Bytes.toBytes("family");
655 this.region = initHRegion(tableName, method, CONF, family);
656 try {
657 Path regiondir = region.getRegionFileSystem().getRegionDir();
658 FileSystem fs = region.getRegionFileSystem().getFileSystem();
659 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
660
661 long maxSeqId = 3;
662 long minSeqId = 0;
663
664 for (long i = minSeqId; i < maxSeqId; i++) {
665 Put put = new Put(Bytes.toBytes(i));
666 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
667 region.put(put);
668 region.flushcache();
669 }
670
671
672 assertEquals(3, region.getStore(family).getStorefilesCount());
673 List<Path> storeFiles = new ArrayList<Path>(3);
674 for (StoreFile sf : region.getStore(family).getStorefiles()) {
675 storeFiles.add(sf.getPath());
676 }
677
678
679 CONF.setBoolean("hbase.hstore.compaction.complete", false);
680 region.compactStores();
681
682
683 assertEquals(3, region.getStore(family).getStorefilesCount());
684
685
686 Path tmpDir = region.getRegionFileSystem().getTempDir();
687 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
688 String errorMsg = "Expected to find 1 file in the region temp directory "
689 + "from the compaction, could not find any";
690 assertNotNull(errorMsg, files);
691 assertEquals(errorMsg, 1, files.length);
692
693 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
694 files[0].getPath());
695
696 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
697 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
698 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
699
700 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
701 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
702
703 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
704
705 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
706 fs.create(recoveredEdits);
707 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
708
709 long time = System.nanoTime();
710
711 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
712 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
713 compactionDescriptor)));
714 writer.close();
715
716
717 region.getTableDesc();
718 region.getRegionInfo();
719 region.close();
720 region = HRegion.openHRegion(region, null);
721
722
723 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
724 for (StoreFile sf : sfs) {
725 LOG.info(sf.getPath());
726 }
727 assertEquals(1, region.getStore(family).getStorefilesCount());
728 files = FSUtils.listStatus(fs, tmpDir);
729 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
730
731 for (long i = minSeqId; i < maxSeqId; i++) {
732 Get get = new Get(Bytes.toBytes(i));
733 Result result = region.get(get);
734 byte[] value = result.getValue(family, Bytes.toBytes(i));
735 assertArrayEquals(Bytes.toBytes(i), value);
736 }
737 } finally {
738 HRegion.closeHRegion(this.region);
739 this.region = null;
740 }
741 }
742
743 @Test
744 public void testGetWhileRegionClose() throws IOException {
745 TableName tableName = TableName.valueOf(name.getMethodName());
746 Configuration hc = initSplit();
747 int numRows = 100;
748 byte[][] families = { fam1, fam2, fam3 };
749
750
751 String method = name.getMethodName();
752 this.region = initHRegion(tableName, method, hc, families);
753 try {
754
755 final int startRow = 100;
756 putData(startRow, numRows, qual1, families);
757 putData(startRow, numRows, qual2, families);
758 putData(startRow, numRows, qual3, families);
759 final AtomicBoolean done = new AtomicBoolean(false);
760 final AtomicInteger gets = new AtomicInteger(0);
761 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
762 try {
763
764 for (int i = 0; i < threads.length / 2; i++) {
765 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
766 threads[i].setDaemon(true);
767 threads[i].start();
768 }
769
770
771 this.region.closing.set(true);
772 for (int i = threads.length / 2; i < threads.length; i++) {
773 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
774 threads[i].setDaemon(true);
775 threads[i].start();
776 }
777 } finally {
778 if (this.region != null) {
779 HRegion.closeHRegion(this.region);
780 }
781 }
782 done.set(true);
783 for (GetTillDoneOrException t : threads) {
784 try {
785 t.join();
786 } catch (InterruptedException e) {
787 e.printStackTrace();
788 }
789 if (t.e != null) {
790 LOG.info("Exception=" + t.e);
791 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
792 }
793 }
794 } finally {
795 HRegion.closeHRegion(this.region);
796 this.region = null;
797 }
798 }
799
800
801
802
803
804 class GetTillDoneOrException extends Thread {
805 private final Get g;
806 private final AtomicBoolean done;
807 private final AtomicInteger count;
808 private Exception e;
809
810 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
811 super("getter." + i);
812 this.g = new Get(r);
813 this.done = d;
814 this.count = c;
815 }
816
817 @Override
818 public void run() {
819 while (!this.done.get()) {
820 try {
821 assertTrue(region.get(g).size() > 0);
822 this.count.incrementAndGet();
823 } catch (Exception e) {
824 this.e = e;
825 break;
826 }
827 }
828 }
829 }
830
831
832
833
834 @Test
835 public void testWeirdCacheBehaviour() throws Exception {
836 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
837 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
838 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
839 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
840 try {
841 String value = "this is the value";
842 String value2 = "this is some other value";
843 String keyPrefix1 = "prefix1";
844 String keyPrefix2 = "prefix2";
845 String keyPrefix3 = "prefix3";
846 putRows(this.region, 3, value, keyPrefix1);
847 putRows(this.region, 3, value, keyPrefix2);
848 putRows(this.region, 3, value, keyPrefix3);
849 putRows(this.region, 3, value2, keyPrefix1);
850 putRows(this.region, 3, value2, keyPrefix2);
851 putRows(this.region, 3, value2, keyPrefix3);
852 System.out.println("Checking values for key: " + keyPrefix1);
853 assertEquals("Got back incorrect number of rows from scan", 3,
854 getNumberOfRows(keyPrefix1, value2, this.region));
855 System.out.println("Checking values for key: " + keyPrefix2);
856 assertEquals("Got back incorrect number of rows from scan", 3,
857 getNumberOfRows(keyPrefix2, value2, this.region));
858 System.out.println("Checking values for key: " + keyPrefix3);
859 assertEquals("Got back incorrect number of rows from scan", 3,
860 getNumberOfRows(keyPrefix3, value2, this.region));
861 deleteColumns(this.region, value2, keyPrefix1);
862 deleteColumns(this.region, value2, keyPrefix2);
863 deleteColumns(this.region, value2, keyPrefix3);
864 System.out.println("Starting important checks.....");
865 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
866 getNumberOfRows(keyPrefix1, value2, this.region));
867 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
868 getNumberOfRows(keyPrefix2, value2, this.region));
869 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
870 getNumberOfRows(keyPrefix3, value2, this.region));
871 } finally {
872 HRegion.closeHRegion(this.region);
873 this.region = null;
874 }
875 }
876
877 @Test
878 public void testAppendWithReadOnlyTable() throws Exception {
879 byte[] TABLE = Bytes.toBytes("readOnlyTable");
880 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
881 boolean exceptionCaught = false;
882 Append append = new Append(Bytes.toBytes("somerow"));
883 append.setDurability(Durability.SKIP_WAL);
884 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
885 Bytes.toBytes("somevalue"));
886 try {
887 region.append(append);
888 } catch (IOException e) {
889 exceptionCaught = true;
890 } finally {
891 HRegion.closeHRegion(this.region);
892 this.region = null;
893 }
894 assertTrue(exceptionCaught == true);
895 }
896
897 @Test
898 public void testIncrWithReadOnlyTable() throws Exception {
899 byte[] TABLE = Bytes.toBytes("readOnlyTable");
900 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
901 boolean exceptionCaught = false;
902 Increment inc = new Increment(Bytes.toBytes("somerow"));
903 inc.setDurability(Durability.SKIP_WAL);
904 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
905 try {
906 region.increment(inc);
907 } catch (IOException e) {
908 exceptionCaught = true;
909 } finally {
910 HRegion.closeHRegion(this.region);
911 this.region = null;
912 }
913 assertTrue(exceptionCaught == true);
914 }
915
916 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
917 InternalScanner scanner = buildScanner(keyPrefix, value, r);
918 int count = 0;
919 boolean more = false;
920 List<Cell> results = new ArrayList<Cell>();
921 do {
922 more = scanner.next(results);
923 if (results != null && !results.isEmpty())
924 count++;
925 else
926 break;
927 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
928 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
929 r.delete(delete);
930 results.clear();
931 } while (more);
932 assertEquals("Did not perform correct number of deletes", 3, count);
933 }
934
935 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
936 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
937 int numberOfResults = 0;
938 List<Cell> results = new ArrayList<Cell>();
939 boolean more = false;
940 do {
941 more = resultScanner.next(results);
942 if (results != null && !results.isEmpty())
943 numberOfResults++;
944 else
945 break;
946 for (Cell kv : results) {
947 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
948 }
949 results.clear();
950 } while (more);
951 return numberOfResults;
952 }
953
954 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
955 throws IOException {
956
957 FilterList allFilters = new FilterList();
958 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
959
960 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
961 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
962 filter.setFilterIfMissing(true);
963 allFilters.addFilter(filter);
964 Scan scan = new Scan();
965 scan.addFamily(Bytes.toBytes("trans-blob"));
966 scan.addFamily(Bytes.toBytes("trans-type"));
967 scan.addFamily(Bytes.toBytes("trans-date"));
968 scan.addFamily(Bytes.toBytes("trans-tags"));
969 scan.addFamily(Bytes.toBytes("trans-group"));
970 scan.setFilter(allFilters);
971 return r.getScanner(scan);
972 }
973
974 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
975 for (int i = 0; i < numRows; i++) {
976 String row = key + "_" + i
977 System.out.println(String.format("Saving row: %s, with value %s", row, value));
978 Put put = new Put(Bytes.toBytes(row));
979 put.setDurability(Durability.SKIP_WAL);
980 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
981 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
982 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
983 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
984 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
985 r.put(put);
986 }
987 }
988
989 @Test
990 public void testFamilyWithAndWithoutColon() throws Exception {
991 byte[] b = Bytes.toBytes(getName());
992 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
993 this.region = initHRegion(b, getName(), CONF, cf);
994 try {
995 Put p = new Put(b);
996 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
997 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
998 boolean exception = false;
999 try {
1000 this.region.put(p);
1001 } catch (NoSuchColumnFamilyException e) {
1002 exception = true;
1003 }
1004 assertTrue(exception);
1005 } finally {
1006 HRegion.closeHRegion(this.region);
1007 this.region = null;
1008 }
1009 }
1010
1011 @Test
1012 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1013 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1014 byte[] qual = Bytes.toBytes("qual");
1015 byte[] val = Bytes.toBytes("val");
1016 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1017 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1018 try {
1019 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1020 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1021
1022 LOG.info("First a batch put with all valid puts");
1023 final Put[] puts = new Put[10];
1024 for (int i = 0; i < 10; i++) {
1025 puts[i] = new Put(Bytes.toBytes("row_" + i));
1026 puts[i].add(cf, qual, val);
1027 }
1028
1029 OperationStatus[] codes = this.region.batchMutate(puts);
1030 assertEquals(10, codes.length);
1031 for (int i = 0; i < 10; i++) {
1032 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1033 }
1034 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1035
1036 LOG.info("Next a batch put with one invalid family");
1037 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1038 codes = this.region.batchMutate(puts);
1039 assertEquals(10, codes.length);
1040 for (int i = 0; i < 10; i++) {
1041 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1042 codes[i].getOperationStatusCode());
1043 }
1044
1045 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1046 } finally {
1047 HRegion.closeHRegion(this.region);
1048 this.region = null;
1049 }
1050 }
1051
1052 @Test
1053 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1054 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1055 byte[] qual = Bytes.toBytes("qual");
1056 byte[] val = Bytes.toBytes("val");
1057 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1058 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1059 try {
1060 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1061 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1062
1063 final Put[] puts = new Put[10];
1064 for (int i = 0; i < 10; i++) {
1065 puts[i] = new Put(Bytes.toBytes("row_" + i));
1066 puts[i].add(cf, qual, val);
1067 }
1068 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1069
1070 LOG.info("batchPut will have to break into four batches to avoid row locks");
1071 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1072 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1073 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1074
1075 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1076 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1077 TestThread putter = new TestThread(ctx) {
1078 @Override
1079 public void doWork() throws IOException {
1080 retFromThread.set(region.batchMutate(puts));
1081 }
1082 };
1083 LOG.info("...starting put thread while holding locks");
1084 ctx.addThread(putter);
1085 ctx.startThreads();
1086
1087 LOG.info("...waiting for put thread to sync 1st time");
1088 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1089
1090
1091
1092
1093
1094
1095 Thread regionCloseThread = new Thread() {
1096 @Override
1097 public void run() {
1098 try {
1099 HRegion.closeHRegion(region);
1100 } catch (IOException e) {
1101 throw new RuntimeException(e);
1102 }
1103 }
1104 };
1105 regionCloseThread.start();
1106
1107 LOG.info("...releasing row lock 1, which should let put thread continue");
1108 rowLock1.release();
1109
1110 LOG.info("...waiting for put thread to sync 2nd time");
1111 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1112
1113 LOG.info("...releasing row lock 2, which should let put thread continue");
1114 rowLock2.release();
1115
1116 LOG.info("...waiting for put thread to sync 3rd time");
1117 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1118
1119 LOG.info("...releasing row lock 3, which should let put thread continue");
1120 rowLock3.release();
1121
1122 LOG.info("...waiting for put thread to sync 4th time");
1123 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1124
1125 LOG.info("...joining on put thread");
1126 ctx.stop();
1127 regionCloseThread.join();
1128
1129 OperationStatus[] codes = retFromThread.get();
1130 for (int i = 0; i < codes.length; i++) {
1131 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1132 codes[i].getOperationStatusCode());
1133 }
1134 } finally {
1135 HRegion.closeHRegion(this.region);
1136 this.region = null;
1137 }
1138 }
1139
1140 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1141 throws InterruptedException {
1142 long startWait = System.currentTimeMillis();
1143 long currentCount;
1144 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1145 Thread.sleep(100);
1146 if (System.currentTimeMillis() - startWait > 10000) {
1147 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1148 expectedCount, currentCount));
1149 }
1150 }
1151 }
1152
1153
1154 @Test
1155 public void testBatchPutWithTsSlop() throws Exception {
1156 byte[] b = Bytes.toBytes(getName());
1157 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1158 byte[] qual = Bytes.toBytes("qual");
1159 byte[] val = Bytes.toBytes("val");
1160
1161
1162 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1163 this.region = initHRegion(b, getName(), CONF, cf);
1164
1165 try {
1166 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1167 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1168 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1169
1170 final Put[] puts = new Put[10];
1171 for (int i = 0; i < 10; i++) {
1172 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1173 puts[i].add(cf, qual, val);
1174 }
1175
1176 OperationStatus[] codes = this.region.batchMutate(puts);
1177 assertEquals(10, codes.length);
1178 for (int i = 0; i < 10; i++) {
1179 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1180 }
1181 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1182
1183 } finally {
1184 HRegion.closeHRegion(this.region);
1185 this.region = null;
1186 }
1187
1188 }
1189
1190
1191
1192
1193 @Test
1194 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1195 byte[] row1 = Bytes.toBytes("row1");
1196 byte[] fam1 = Bytes.toBytes("fam1");
1197 byte[] qf1 = Bytes.toBytes("qualifier");
1198 byte[] emptyVal = new byte[] {};
1199 byte[] val1 = Bytes.toBytes("value1");
1200 byte[] val2 = Bytes.toBytes("value2");
1201
1202
1203 String method = this.getName();
1204 this.region = initHRegion(tableName, method, CONF, fam1);
1205 try {
1206
1207 Put put = new Put(row1);
1208 put.add(fam1, qf1, emptyVal);
1209
1210
1211 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1212 emptyVal), put, true);
1213 assertTrue(res);
1214
1215
1216 put = new Put(row1);
1217 put.add(fam1, qf1, val1);
1218
1219
1220 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1221 put, true);
1222 assertTrue(res);
1223
1224
1225 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1226 put, true);
1227 assertFalse(res);
1228
1229 Delete delete = new Delete(row1);
1230 delete.deleteColumn(fam1, qf1);
1231 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1232 delete, true);
1233 assertFalse(res);
1234
1235 put = new Put(row1);
1236 put.add(fam1, qf1, val2);
1237
1238 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1239 put, true);
1240 assertTrue(res);
1241
1242
1243 delete = new Delete(row1);
1244 delete.deleteColumn(fam1, qf1);
1245 delete.deleteColumn(fam1, qf1);
1246 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1247 delete, true);
1248 assertTrue(res);
1249
1250 delete = new Delete(row1);
1251 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1252 delete, true);
1253 assertTrue(res);
1254
1255
1256 put = new Put(row1);
1257 put.add(fam1, qf1, val1);
1258
1259 res = region
1260 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1261 assertTrue(res);
1262 } finally {
1263 HRegion.closeHRegion(this.region);
1264 this.region = null;
1265 }
1266 }
1267
1268 @Test
1269 public void testCheckAndMutate_WithWrongValue() throws IOException {
1270 byte[] row1 = Bytes.toBytes("row1");
1271 byte[] fam1 = Bytes.toBytes("fam1");
1272 byte[] qf1 = Bytes.toBytes("qualifier");
1273 byte[] val1 = Bytes.toBytes("value1");
1274 byte[] val2 = Bytes.toBytes("value2");
1275
1276
1277 String method = this.getName();
1278 this.region = initHRegion(tableName, method, CONF, fam1);
1279 try {
1280
1281 Put put = new Put(row1);
1282 put.add(fam1, qf1, val1);
1283 region.put(put);
1284
1285
1286 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1287 val2), put, true);
1288 assertEquals(false, res);
1289
1290
1291 Delete delete = new Delete(row1);
1292 delete.deleteFamily(fam1);
1293 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1294 put, true);
1295 assertEquals(false, res);
1296 } finally {
1297 HRegion.closeHRegion(this.region);
1298 this.region = null;
1299 }
1300 }
1301
1302 @Test
1303 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1304 byte[] row1 = Bytes.toBytes("row1");
1305 byte[] fam1 = Bytes.toBytes("fam1");
1306 byte[] qf1 = Bytes.toBytes("qualifier");
1307 byte[] val1 = Bytes.toBytes("value1");
1308
1309
1310 String method = this.getName();
1311 this.region = initHRegion(tableName, method, CONF, fam1);
1312 try {
1313
1314 Put put = new Put(row1);
1315 put.add(fam1, qf1, val1);
1316 region.put(put);
1317
1318
1319 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1320 val1), put, true);
1321 assertEquals(true, res);
1322
1323
1324 Delete delete = new Delete(row1);
1325 delete.deleteColumn(fam1, qf1);
1326 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1327 delete, true);
1328 assertEquals(true, res);
1329 } finally {
1330 HRegion.closeHRegion(this.region);
1331 this.region = null;
1332 }
1333 }
1334
1335 @Test
1336 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1337 byte[] row1 = Bytes.toBytes("row1");
1338 byte[] fam1 = Bytes.toBytes("fam1");
1339 byte[] qf1 = Bytes.toBytes("qualifier");
1340 byte[] val1 = Bytes.toBytes("value1");
1341 byte[] val2 = Bytes.toBytes("value2");
1342 byte[] val3 = Bytes.toBytes("value3");
1343 byte[] val4 = Bytes.toBytes("value4");
1344
1345
1346 String method = this.getName();
1347 this.region = initHRegion(tableName, method, CONF, fam1);
1348 try {
1349
1350 Put put = new Put(row1);
1351 put.add(fam1, qf1, val3);
1352 region.put(put);
1353
1354
1355 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1356 new BinaryComparator(val3), put, true);
1357 assertEquals(false, res);
1358
1359
1360 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1361 new BinaryComparator(val4), put, true);
1362 assertEquals(false, res);
1363
1364
1365
1366 put = new Put(row1);
1367 put.add(fam1, qf1, val2);
1368 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1369 new BinaryComparator(val2), put, true);
1370 assertEquals(true, res);
1371
1372
1373 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1374 new BinaryComparator(val3), put, true);
1375 assertEquals(false, res);
1376
1377
1378
1379 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1380 new BinaryComparator(val2), put, true);
1381 assertEquals(true, res);
1382
1383
1384
1385 put = new Put(row1);
1386 put.add(fam1, qf1, val3);
1387 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1388 new BinaryComparator(val1), put, true);
1389 assertEquals(true, res);
1390
1391
1392 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1393 new BinaryComparator(val3), put, true);
1394 assertEquals(false, res);
1395
1396
1397 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1398 new BinaryComparator(val2), put, true);
1399 assertEquals(false, res);
1400
1401
1402
1403 put = new Put(row1);
1404 put.add(fam1, qf1, val2);
1405 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1406 new BinaryComparator(val4), put, true);
1407 assertEquals(true, res);
1408
1409
1410 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1411 new BinaryComparator(val1), put, true);
1412 assertEquals(false, res);
1413
1414
1415
1416 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1417 new BinaryComparator(val2), put, true);
1418 assertEquals(true, res);
1419
1420
1421 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1422 new BinaryComparator(val3), put, true);
1423 assertEquals(true, res);
1424 } finally {
1425 HRegion.closeHRegion(this.region);
1426 this.region = null;
1427 }
1428 }
1429
1430 @Test
1431 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1432 byte[] row1 = Bytes.toBytes("row1");
1433 byte[] fam1 = Bytes.toBytes("fam1");
1434 byte[] fam2 = Bytes.toBytes("fam2");
1435 byte[] qf1 = Bytes.toBytes("qualifier");
1436 byte[] val1 = Bytes.toBytes("value1");
1437 byte[] val2 = Bytes.toBytes("value2");
1438
1439 byte[][] families = { fam1, fam2 };
1440
1441
1442 String method = this.getName();
1443 this.region = initHRegion(tableName, method, CONF, families);
1444 try {
1445
1446 Put put = new Put(row1);
1447 put.add(fam1, qf1, val1);
1448 region.put(put);
1449
1450
1451 long ts = System.currentTimeMillis();
1452 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1453 put = new Put(row1);
1454 put.add(kv);
1455
1456
1457 HStore store = (HStore) region.getStore(fam1);
1458 store.memstore.kvset.size();
1459
1460 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1461 val1), put, true);
1462 assertEquals(true, res);
1463 store.memstore.kvset.size();
1464
1465 Get get = new Get(row1);
1466 get.addColumn(fam2, qf1);
1467 Cell[] actual = region.get(get).rawCells();
1468
1469 Cell[] expected = { kv };
1470
1471 assertEquals(expected.length, actual.length);
1472 for (int i = 0; i < actual.length; i++) {
1473 assertEquals(expected[i], actual[i]);
1474 }
1475 } finally {
1476 HRegion.closeHRegion(this.region);
1477 this.region = null;
1478 }
1479 }
1480
1481 @Test
1482 public void testCheckAndPut_wrongRowInPut() throws IOException {
1483 TableName tableName = TableName.valueOf(name.getMethodName());
1484 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1485 try {
1486 Put put = new Put(row2);
1487 put.add(fam1, qual1, value1);
1488 try {
1489 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1490 new BinaryComparator(value2), put, false);
1491 fail();
1492 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1493
1494 }
1495 } finally {
1496 HRegion.closeHRegion(this.region);
1497 this.region = null;
1498 }
1499 }
1500
1501 @Test
1502 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1503 byte[] row1 = Bytes.toBytes("row1");
1504 byte[] fam1 = Bytes.toBytes("fam1");
1505 byte[] fam2 = Bytes.toBytes("fam2");
1506 byte[] qf1 = Bytes.toBytes("qualifier1");
1507 byte[] qf2 = Bytes.toBytes("qualifier2");
1508 byte[] qf3 = Bytes.toBytes("qualifier3");
1509 byte[] val1 = Bytes.toBytes("value1");
1510 byte[] val2 = Bytes.toBytes("value2");
1511 byte[] val3 = Bytes.toBytes("value3");
1512 byte[] emptyVal = new byte[] {};
1513
1514 byte[][] families = { fam1, fam2 };
1515
1516
1517 String method = this.getName();
1518 this.region = initHRegion(tableName, method, CONF, families);
1519 try {
1520
1521 Put put = new Put(row1);
1522 put.add(fam1, qf1, val1);
1523 region.put(put);
1524 Threads.sleep(2);
1525
1526 put = new Put(row1);
1527 put.add(fam1, qf1, val2);
1528 put.add(fam2, qf1, val3);
1529 put.add(fam2, qf2, val2);
1530 put.add(fam2, qf3, val1);
1531 put.add(fam1, qf3, val1);
1532 region.put(put);
1533
1534
1535 Delete delete = new Delete(row1);
1536 delete.deleteColumn(fam1, qf1);
1537 delete.deleteColumn(fam2, qf1);
1538 delete.deleteColumn(fam1, qf3);
1539 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1540 val2), delete, true);
1541 assertEquals(true, res);
1542
1543 Get get = new Get(row1);
1544 get.addColumn(fam1, qf1);
1545 get.addColumn(fam1, qf3);
1546 get.addColumn(fam2, qf2);
1547 Result r = region.get(get);
1548 assertEquals(2, r.size());
1549 assertArrayEquals(val1, r.getValue(fam1, qf1));
1550 assertArrayEquals(val2, r.getValue(fam2, qf2));
1551
1552
1553 delete = new Delete(row1);
1554 delete.deleteFamily(fam2);
1555 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1556 delete, true);
1557 assertEquals(true, res);
1558
1559 get = new Get(row1);
1560 r = region.get(get);
1561 assertEquals(1, r.size());
1562 assertArrayEquals(val1, r.getValue(fam1, qf1));
1563
1564
1565 delete = new Delete(row1);
1566 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1567 delete, true);
1568 assertEquals(true, res);
1569 get = new Get(row1);
1570 r = region.get(get);
1571 assertEquals(0, r.size());
1572 } finally {
1573 HRegion.closeHRegion(this.region);
1574 this.region = null;
1575 }
1576 }
1577
1578
1579
1580
1581 @Test
1582 public void testDelete_multiDeleteColumn() throws IOException {
1583 byte[] row1 = Bytes.toBytes("row1");
1584 byte[] fam1 = Bytes.toBytes("fam1");
1585 byte[] qual = Bytes.toBytes("qualifier");
1586 byte[] value = Bytes.toBytes("value");
1587
1588 Put put = new Put(row1);
1589 put.add(fam1, qual, 1, value);
1590 put.add(fam1, qual, 2, value);
1591
1592 String method = this.getName();
1593 this.region = initHRegion(tableName, method, CONF, fam1);
1594 try {
1595 region.put(put);
1596
1597
1598 Delete delete = new Delete(row1);
1599 delete.deleteColumn(fam1, qual);
1600 delete.deleteColumn(fam1, qual);
1601 region.delete(delete);
1602
1603 Get get = new Get(row1);
1604 get.addFamily(fam1);
1605 Result r = region.get(get);
1606 assertEquals(0, r.size());
1607 } finally {
1608 HRegion.closeHRegion(this.region);
1609 this.region = null;
1610 }
1611 }
1612
1613 @Test
1614 public void testDelete_CheckFamily() throws IOException {
1615 byte[] row1 = Bytes.toBytes("row1");
1616 byte[] fam1 = Bytes.toBytes("fam1");
1617 byte[] fam2 = Bytes.toBytes("fam2");
1618 byte[] fam3 = Bytes.toBytes("fam3");
1619 byte[] fam4 = Bytes.toBytes("fam4");
1620
1621
1622 String method = this.getName();
1623 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1624 try {
1625 List<Cell> kvs = new ArrayList<Cell>();
1626 kvs.add(new KeyValue(row1, fam4, null, null));
1627
1628
1629 byte[] family = fam2;
1630 try {
1631 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1632 Bytes.BYTES_COMPARATOR);
1633 deleteMap.put(family, kvs);
1634 region.delete(deleteMap, Durability.SYNC_WAL);
1635 } catch (Exception e) {
1636 assertTrue("Family " + new String(family) + " does not exist", false);
1637 }
1638
1639
1640 boolean ok = false;
1641 family = fam4;
1642 try {
1643 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1644 Bytes.BYTES_COMPARATOR);
1645 deleteMap.put(family, kvs);
1646 region.delete(deleteMap, Durability.SYNC_WAL);
1647 } catch (Exception e) {
1648 ok = true;
1649 }
1650 assertEquals("Family " + new String(family) + " does exist", true, ok);
1651 } finally {
1652 HRegion.closeHRegion(this.region);
1653 this.region = null;
1654 }
1655 }
1656
1657 @Test
1658 public void testDelete_mixed() throws IOException, InterruptedException {
1659 byte[] fam = Bytes.toBytes("info");
1660 byte[][] families = { fam };
1661 String method = this.getName();
1662 this.region = initHRegion(tableName, method, CONF, families);
1663 try {
1664 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1665
1666 byte[] row = Bytes.toBytes("table_name");
1667
1668 byte[] serverinfo = Bytes.toBytes("serverinfo");
1669 byte[] splitA = Bytes.toBytes("splitA");
1670 byte[] splitB = Bytes.toBytes("splitB");
1671
1672
1673 Put put = new Put(row);
1674 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1675 region.put(put);
1676
1677 put = new Put(row);
1678 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1679 region.put(put);
1680
1681 put = new Put(row);
1682 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1683 region.put(put);
1684
1685
1686 Delete delete = new Delete(row);
1687 delete.deleteColumns(fam, splitA);
1688 region.delete(delete);
1689
1690
1691 Get get = new Get(row).addColumn(fam, serverinfo);
1692 Result result = region.get(get);
1693 assertEquals(1, result.size());
1694
1695 get = new Get(row).addColumn(fam, splitA);
1696 result = region.get(get);
1697 assertEquals(0, result.size());
1698
1699 get = new Get(row).addColumn(fam, splitB);
1700 result = region.get(get);
1701 assertEquals(1, result.size());
1702
1703
1704 put = new Put(row);
1705 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1706 region.put(put);
1707 get = new Get(row);
1708 result = region.get(get);
1709 assertEquals(3, result.size());
1710
1711
1712 delete = new Delete(row);
1713 region.delete(delete);
1714 assertEquals(0, region.get(get).size());
1715
1716 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1717 result = region.get(get);
1718 assertEquals(1, result.size());
1719 } finally {
1720 HRegion.closeHRegion(this.region);
1721 this.region = null;
1722 }
1723 }
1724
1725 @Test
1726 public void testDeleteRowWithFutureTs() throws IOException {
1727 byte[] fam = Bytes.toBytes("info");
1728 byte[][] families = { fam };
1729 String method = this.getName();
1730 this.region = initHRegion(tableName, method, CONF, families);
1731 try {
1732 byte[] row = Bytes.toBytes("table_name");
1733
1734 byte[] serverinfo = Bytes.toBytes("serverinfo");
1735
1736
1737 Put put = new Put(row);
1738 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1739 region.put(put);
1740
1741
1742 Delete delete = new Delete(row);
1743 region.delete(delete);
1744
1745
1746 Get get = new Get(row).addColumn(fam, serverinfo);
1747 Result result = region.get(get);
1748 assertEquals(1, result.size());
1749
1750
1751 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1752 region.delete(delete);
1753
1754
1755 get = new Get(row).addColumn(fam, serverinfo);
1756 result = region.get(get);
1757 assertEquals(0, result.size());
1758 } finally {
1759 HRegion.closeHRegion(this.region);
1760 this.region = null;
1761 }
1762 }
1763
1764
1765
1766
1767
1768 @Test
1769 public void testPutWithLatestTS() throws IOException {
1770 byte[] fam = Bytes.toBytes("info");
1771 byte[][] families = { fam };
1772 String method = this.getName();
1773 this.region = initHRegion(tableName, method, CONF, families);
1774 try {
1775 byte[] row = Bytes.toBytes("row1");
1776
1777 byte[] qual = Bytes.toBytes("qual");
1778
1779
1780 Put put = new Put(row);
1781 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1782 region.put(put);
1783
1784
1785 Get get = new Get(row).addColumn(fam, qual);
1786 Result result = region.get(get);
1787 assertEquals(1, result.size());
1788 Cell kv = result.rawCells()[0];
1789 LOG.info("Got: " + kv);
1790 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1791 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1792
1793
1794
1795 row = Bytes.toBytes("row2");
1796 put = new Put(row);
1797 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1798 region.put(put);
1799
1800
1801 get = new Get(row).addColumn(fam, qual);
1802 result = region.get(get);
1803 assertEquals(1, result.size());
1804 kv = result.rawCells()[0];
1805 LOG.info("Got: " + kv);
1806 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1807 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1808 } finally {
1809 HRegion.closeHRegion(this.region);
1810 this.region = null;
1811 }
1812
1813 }
1814
1815
1816
1817
1818
1819
1820 @Test
1821 public void testPutWithTsSlop() throws IOException {
1822 byte[] fam = Bytes.toBytes("info");
1823 byte[][] families = { fam };
1824 String method = this.getName();
1825
1826
1827 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1828 this.region = initHRegion(tableName, method, CONF, families);
1829 boolean caughtExcep = false;
1830 try {
1831 try {
1832
1833 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1834
1835 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1836 Bytes.toBytes("value")));
1837 fail("Expected IOE for TS out of configured timerange");
1838 } catch (FailedSanityCheckException ioe) {
1839 LOG.debug("Received expected exception", ioe);
1840 caughtExcep = true;
1841 }
1842 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1843 } finally {
1844 HRegion.closeHRegion(this.region);
1845 this.region = null;
1846 }
1847 }
1848
1849 @Test
1850 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1851 byte[] fam1 = Bytes.toBytes("columnA");
1852 byte[] fam2 = Bytes.toBytes("columnB");
1853 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1854 try {
1855 byte[] rowA = Bytes.toBytes("rowA");
1856 byte[] rowB = Bytes.toBytes("rowB");
1857
1858 byte[] value = Bytes.toBytes("value");
1859
1860 Delete delete = new Delete(rowA);
1861 delete.deleteFamily(fam1);
1862
1863 region.delete(delete);
1864
1865
1866 Put put = new Put(rowA);
1867 put.add(fam2, null, value);
1868 region.put(put);
1869
1870 put = new Put(rowB);
1871 put.add(fam1, null, value);
1872 put.add(fam2, null, value);
1873 region.put(put);
1874
1875 Scan scan = new Scan();
1876 scan.addFamily(fam1).addFamily(fam2);
1877 InternalScanner s = region.getScanner(scan);
1878 List<Cell> results = new ArrayList<Cell>();
1879 s.next(results);
1880 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1881
1882 results.clear();
1883 s.next(results);
1884 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1885 } finally {
1886 HRegion.closeHRegion(this.region);
1887 this.region = null;
1888 }
1889 }
1890
1891 @Test
1892 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1893 Delete delete = new Delete(row);
1894 delete.deleteColumns(fam1, qual1);
1895 doTestDelete_AndPostInsert(delete);
1896 }
1897
1898 @Test
1899 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1900 Delete delete = new Delete(row);
1901 delete.deleteFamily(fam1);
1902 doTestDelete_AndPostInsert(delete);
1903 }
1904
1905 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1906 TableName tableName = TableName.valueOf(name.getMethodName());
1907 this.region = initHRegion(tableName, getName(), CONF, fam1);
1908 try {
1909 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1910 Put put = new Put(row);
1911 put.add(fam1, qual1, value1);
1912 region.put(put);
1913
1914
1915 region.delete(delete);
1916
1917
1918 put = new Put(row);
1919 put.add(fam1, qual1, value2);
1920 region.put(put);
1921
1922
1923 Get get = new Get(row);
1924 get.addColumn(fam1, qual1);
1925
1926 Result r = region.get(get);
1927 assertEquals(1, r.size());
1928 assertArrayEquals(value2, r.getValue(fam1, qual1));
1929
1930
1931 Scan scan = new Scan(row);
1932 scan.addColumn(fam1, qual1);
1933 InternalScanner s = region.getScanner(scan);
1934
1935 List<Cell> results = new ArrayList<Cell>();
1936 assertEquals(false, s.next(results));
1937 assertEquals(1, results.size());
1938 Cell kv = results.get(0);
1939
1940 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1941 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1942 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1943 assertArrayEquals(row, CellUtil.cloneRow(kv));
1944 } finally {
1945 HRegion.closeHRegion(this.region);
1946 this.region = null;
1947 }
1948 }
1949
1950 @Test
1951 public void testDelete_CheckTimestampUpdated() throws IOException {
1952 TableName tableName = TableName.valueOf(name.getMethodName());
1953 byte[] row1 = Bytes.toBytes("row1");
1954 byte[] col1 = Bytes.toBytes("col1");
1955 byte[] col2 = Bytes.toBytes("col2");
1956 byte[] col3 = Bytes.toBytes("col3");
1957
1958
1959 String method = this.getName();
1960 this.region = initHRegion(tableName, method, CONF, fam1);
1961 try {
1962
1963 List<Cell> kvs = new ArrayList<Cell>();
1964 kvs.add(new KeyValue(row1, fam1, col1, null));
1965 kvs.add(new KeyValue(row1, fam1, col2, null));
1966 kvs.add(new KeyValue(row1, fam1, col3, null));
1967
1968 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1969 Bytes.BYTES_COMPARATOR);
1970 deleteMap.put(fam1, kvs);
1971 region.delete(deleteMap, Durability.SYNC_WAL);
1972
1973
1974
1975 long now = System.currentTimeMillis();
1976 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1977 assertTrue(firstKv.getTimestamp() <= now);
1978 now = firstKv.getTimestamp();
1979 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1980 assertTrue(kv.getTimestamp() <= now);
1981 now = kv.getTimestamp();
1982 }
1983 } finally {
1984 HRegion.closeHRegion(this.region);
1985 this.region = null;
1986 }
1987 }
1988
1989
1990
1991
1992 @Test
1993 public void testGet_FamilyChecker() throws IOException {
1994 byte[] row1 = Bytes.toBytes("row1");
1995 byte[] fam1 = Bytes.toBytes("fam1");
1996 byte[] fam2 = Bytes.toBytes("False");
1997 byte[] col1 = Bytes.toBytes("col1");
1998
1999
2000 String method = this.getName();
2001 this.region = initHRegion(tableName, method, CONF, fam1);
2002 try {
2003 Get get = new Get(row1);
2004 get.addColumn(fam2, col1);
2005
2006
2007 try {
2008 region.get(get);
2009 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2010 assertFalse(false);
2011 return;
2012 }
2013 assertFalse(true);
2014 } finally {
2015 HRegion.closeHRegion(this.region);
2016 this.region = null;
2017 }
2018 }
2019
2020 @Test
2021 public void testGet_Basic() throws IOException {
2022 byte[] row1 = Bytes.toBytes("row1");
2023 byte[] fam1 = Bytes.toBytes("fam1");
2024 byte[] col1 = Bytes.toBytes("col1");
2025 byte[] col2 = Bytes.toBytes("col2");
2026 byte[] col3 = Bytes.toBytes("col3");
2027 byte[] col4 = Bytes.toBytes("col4");
2028 byte[] col5 = Bytes.toBytes("col5");
2029
2030
2031 String method = this.getName();
2032 this.region = initHRegion(tableName, method, CONF, fam1);
2033 try {
2034
2035 Put put = new Put(row1);
2036 put.add(fam1, col1, null);
2037 put.add(fam1, col2, null);
2038 put.add(fam1, col3, null);
2039 put.add(fam1, col4, null);
2040 put.add(fam1, col5, null);
2041 region.put(put);
2042
2043 Get get = new Get(row1);
2044 get.addColumn(fam1, col2);
2045 get.addColumn(fam1, col4);
2046
2047 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2048 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2049 KeyValue[] expected = { kv1, kv2 };
2050
2051
2052 Result res = region.get(get);
2053 assertEquals(expected.length, res.size());
2054 for (int i = 0; i < res.size(); i++) {
2055 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2056 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2057 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2058 }
2059
2060
2061 Get g = new Get(row1);
2062 final int count = 2;
2063 g.setFilter(new ColumnCountGetFilter(count));
2064 res = region.get(g);
2065 assertEquals(count, res.size());
2066 } finally {
2067 HRegion.closeHRegion(this.region);
2068 this.region = null;
2069 }
2070 }
2071
2072 @Test
2073 public void testGet_Empty() throws IOException {
2074 byte[] row = Bytes.toBytes("row");
2075 byte[] fam = Bytes.toBytes("fam");
2076
2077 String method = this.getName();
2078 this.region = initHRegion(tableName, method, CONF, fam);
2079 try {
2080 Get get = new Get(row);
2081 get.addFamily(fam);
2082 Result r = region.get(get);
2083
2084 assertTrue(r.isEmpty());
2085 } finally {
2086 HRegion.closeHRegion(this.region);
2087 this.region = null;
2088 }
2089 }
2090
2091
2092
2093
2094 @Test
2095 public void testMerge() throws IOException {
2096 byte[][] families = { fam1, fam2, fam3 };
2097 Configuration hc = initSplit();
2098
2099 String method = this.getName();
2100 this.region = initHRegion(tableName, method, hc, families);
2101 try {
2102 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2103 region.flushcache();
2104 region.compactStores();
2105 byte[] splitRow = region.checkSplit();
2106 assertNotNull(splitRow);
2107 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2108 HRegion[] subregions = splitRegion(region, splitRow);
2109 try {
2110
2111 for (int i = 0; i < subregions.length; i++) {
2112 HRegion.openHRegion(subregions[i], null);
2113 subregions[i].compactStores();
2114 }
2115 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2116 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2117 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2118 long startTime = System.currentTimeMillis();
2119 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2120 LOG.info("Merge regions elapsed time: "
2121 + ((System.currentTimeMillis() - startTime) / 1000.0));
2122 FILESYSTEM.delete(oldRegion1, true);
2123 FILESYSTEM.delete(oldRegion2, true);
2124 FILESYSTEM.delete(oldRegionPath, true);
2125 LOG.info("splitAndMerge completed.");
2126 } finally {
2127 for (int i = 0; i < subregions.length; i++) {
2128 try {
2129 HRegion.closeHRegion(subregions[i]);
2130 } catch (IOException e) {
2131
2132 }
2133 }
2134 }
2135 } finally {
2136 HRegion.closeHRegion(this.region);
2137 this.region = null;
2138 }
2139 }
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2150 PairOfSameType<HRegion> result = null;
2151 SplitTransaction st = new SplitTransaction(parent, midkey);
2152
2153
2154 if (!st.prepare()) {
2155 parent.clearSplit();
2156 return null;
2157 }
2158 try {
2159 result = st.execute(null, null);
2160 } catch (IOException ioe) {
2161 try {
2162 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2163 + ioe.getMessage());
2164 st.rollback(null, null);
2165 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2166 return null;
2167 } catch (RuntimeException e) {
2168
2169 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2170 + " -- aborting server", e);
2171 }
2172 }
2173 finally {
2174 parent.clearSplit();
2175 }
2176 return new HRegion[] { result.getFirst(), result.getSecond() };
2177 }
2178
2179
2180
2181
2182 @Test
2183 public void testGetScanner_WithOkFamilies() throws IOException {
2184 byte[] fam1 = Bytes.toBytes("fam1");
2185 byte[] fam2 = Bytes.toBytes("fam2");
2186
2187 byte[][] families = { fam1, fam2 };
2188
2189
2190 String method = this.getName();
2191 this.region = initHRegion(tableName, method, CONF, families);
2192 try {
2193 Scan scan = new Scan();
2194 scan.addFamily(fam1);
2195 scan.addFamily(fam2);
2196 try {
2197 region.getScanner(scan);
2198 } catch (Exception e) {
2199 assertTrue("Families could not be found in Region", false);
2200 }
2201 } finally {
2202 HRegion.closeHRegion(this.region);
2203 this.region = null;
2204 }
2205 }
2206
2207 @Test
2208 public void testGetScanner_WithNotOkFamilies() throws IOException {
2209 byte[] fam1 = Bytes.toBytes("fam1");
2210 byte[] fam2 = Bytes.toBytes("fam2");
2211
2212 byte[][] families = { fam1 };
2213
2214
2215 String method = this.getName();
2216 this.region = initHRegion(tableName, method, CONF, families);
2217 try {
2218 Scan scan = new Scan();
2219 scan.addFamily(fam2);
2220 boolean ok = false;
2221 try {
2222 region.getScanner(scan);
2223 } catch (Exception e) {
2224 ok = true;
2225 }
2226 assertTrue("Families could not be found in Region", ok);
2227 } finally {
2228 HRegion.closeHRegion(this.region);
2229 this.region = null;
2230 }
2231 }
2232
2233 @Test
2234 public void testGetScanner_WithNoFamilies() throws IOException {
2235 byte[] row1 = Bytes.toBytes("row1");
2236 byte[] fam1 = Bytes.toBytes("fam1");
2237 byte[] fam2 = Bytes.toBytes("fam2");
2238 byte[] fam3 = Bytes.toBytes("fam3");
2239 byte[] fam4 = Bytes.toBytes("fam4");
2240
2241 byte[][] families = { fam1, fam2, fam3, fam4 };
2242
2243
2244 String method = this.getName();
2245 this.region = initHRegion(tableName, method, CONF, families);
2246 try {
2247
2248
2249 Put put = new Put(row1);
2250 put.add(fam1, null, null);
2251 put.add(fam2, null, null);
2252 put.add(fam3, null, null);
2253 put.add(fam4, null, null);
2254 region.put(put);
2255
2256 Scan scan = null;
2257 HRegion.RegionScannerImpl is = null;
2258
2259
2260
2261
2262 scan = new Scan();
2263 scan.addFamily(fam2);
2264 scan.addFamily(fam4);
2265 is = (RegionScannerImpl) region.getScanner(scan);
2266 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2267
2268 scan = new Scan();
2269 is = (RegionScannerImpl) region.getScanner(scan);
2270 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2271 } finally {
2272 HRegion.closeHRegion(this.region);
2273 this.region = null;
2274 }
2275 }
2276
2277
2278
2279
2280
2281
2282 @Test
2283 public void testGetScanner_WithRegionClosed() throws IOException {
2284 byte[] fam1 = Bytes.toBytes("fam1");
2285 byte[] fam2 = Bytes.toBytes("fam2");
2286
2287 byte[][] families = { fam1, fam2 };
2288
2289
2290 String method = this.getName();
2291 try {
2292 this.region = initHRegion(tableName, method, CONF, families);
2293 } catch (IOException e) {
2294 e.printStackTrace();
2295 fail("Got IOException during initHRegion, " + e.getMessage());
2296 }
2297 try {
2298 region.closed.set(true);
2299 try {
2300 region.getScanner(null);
2301 fail("Expected to get an exception during getScanner on a region that is closed");
2302 } catch (NotServingRegionException e) {
2303
2304 } catch (IOException e) {
2305 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2306 + e.getMessage());
2307 }
2308 } finally {
2309 HRegion.closeHRegion(this.region);
2310 this.region = null;
2311 }
2312 }
2313
2314 @Test
2315 public void testRegionScanner_Next() throws IOException {
2316 byte[] row1 = Bytes.toBytes("row1");
2317 byte[] row2 = Bytes.toBytes("row2");
2318 byte[] fam1 = Bytes.toBytes("fam1");
2319 byte[] fam2 = Bytes.toBytes("fam2");
2320 byte[] fam3 = Bytes.toBytes("fam3");
2321 byte[] fam4 = Bytes.toBytes("fam4");
2322
2323 byte[][] families = { fam1, fam2, fam3, fam4 };
2324 long ts = System.currentTimeMillis();
2325
2326
2327 String method = this.getName();
2328 this.region = initHRegion(tableName, method, CONF, families);
2329 try {
2330
2331 Put put = null;
2332 put = new Put(row1);
2333 put.add(fam1, (byte[]) null, ts, null);
2334 put.add(fam2, (byte[]) null, ts, null);
2335 put.add(fam3, (byte[]) null, ts, null);
2336 put.add(fam4, (byte[]) null, ts, null);
2337 region.put(put);
2338
2339 put = new Put(row2);
2340 put.add(fam1, (byte[]) null, ts, null);
2341 put.add(fam2, (byte[]) null, ts, null);
2342 put.add(fam3, (byte[]) null, ts, null);
2343 put.add(fam4, (byte[]) null, ts, null);
2344 region.put(put);
2345
2346 Scan scan = new Scan();
2347 scan.addFamily(fam2);
2348 scan.addFamily(fam4);
2349 InternalScanner is = region.getScanner(scan);
2350
2351 List<Cell> res = null;
2352
2353
2354 List<Cell> expected1 = new ArrayList<Cell>();
2355 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2356 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2357
2358 res = new ArrayList<Cell>();
2359 is.next(res);
2360 for (int i = 0; i < res.size(); i++) {
2361 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2362 }
2363
2364
2365 List<Cell> expected2 = new ArrayList<Cell>();
2366 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2367 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2368
2369 res = new ArrayList<Cell>();
2370 is.next(res);
2371 for (int i = 0; i < res.size(); i++) {
2372 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2373 }
2374 } finally {
2375 HRegion.closeHRegion(this.region);
2376 this.region = null;
2377 }
2378 }
2379
2380 @Test
2381 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2382 byte[] row1 = Bytes.toBytes("row1");
2383 byte[] qf1 = Bytes.toBytes("qualifier1");
2384 byte[] qf2 = Bytes.toBytes("qualifier2");
2385 byte[] fam1 = Bytes.toBytes("fam1");
2386 byte[][] families = { fam1 };
2387
2388 long ts1 = System.currentTimeMillis();
2389 long ts2 = ts1 + 1;
2390 long ts3 = ts1 + 2;
2391
2392
2393 String method = this.getName();
2394 this.region = initHRegion(tableName, method, CONF, families);
2395 try {
2396
2397 Put put = null;
2398 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2399 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2400 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2401
2402 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2403 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2404 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2405
2406 put = new Put(row1);
2407 put.add(kv13);
2408 put.add(kv12);
2409 put.add(kv11);
2410 put.add(kv23);
2411 put.add(kv22);
2412 put.add(kv21);
2413 region.put(put);
2414
2415
2416 List<Cell> expected = new ArrayList<Cell>();
2417 expected.add(kv13);
2418 expected.add(kv12);
2419
2420 Scan scan = new Scan(row1);
2421 scan.addColumn(fam1, qf1);
2422 scan.setMaxVersions(MAX_VERSIONS);
2423 List<Cell> actual = new ArrayList<Cell>();
2424 InternalScanner scanner = region.getScanner(scan);
2425
2426 boolean hasNext = scanner.next(actual);
2427 assertEquals(false, hasNext);
2428
2429
2430 for (int i = 0; i < expected.size(); i++) {
2431 assertEquals(expected.get(i), actual.get(i));
2432 }
2433 } finally {
2434 HRegion.closeHRegion(this.region);
2435 this.region = null;
2436 }
2437 }
2438
2439 @Test
2440 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2441 byte[] row1 = Bytes.toBytes("row1");
2442 byte[] qf1 = Bytes.toBytes("qualifier1");
2443 byte[] qf2 = Bytes.toBytes("qualifier2");
2444 byte[] fam1 = Bytes.toBytes("fam1");
2445 byte[][] families = { fam1 };
2446
2447 long ts1 = 1;
2448 long ts2 = ts1 + 1;
2449 long ts3 = ts1 + 2;
2450
2451
2452 String method = this.getName();
2453 this.region = initHRegion(tableName, method, CONF, families);
2454 try {
2455
2456 Put put = null;
2457 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2458 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2459 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2460
2461 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2462 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2463 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2464
2465 put = new Put(row1);
2466 put.add(kv13);
2467 put.add(kv12);
2468 put.add(kv11);
2469 put.add(kv23);
2470 put.add(kv22);
2471 put.add(kv21);
2472 region.put(put);
2473 region.flushcache();
2474
2475
2476 List<Cell> expected = new ArrayList<Cell>();
2477 expected.add(kv13);
2478 expected.add(kv12);
2479 expected.add(kv23);
2480 expected.add(kv22);
2481
2482 Scan scan = new Scan(row1);
2483 scan.addColumn(fam1, qf1);
2484 scan.addColumn(fam1, qf2);
2485 scan.setMaxVersions(MAX_VERSIONS);
2486 List<Cell> actual = new ArrayList<Cell>();
2487 InternalScanner scanner = region.getScanner(scan);
2488
2489 boolean hasNext = scanner.next(actual);
2490 assertEquals(false, hasNext);
2491
2492
2493 for (int i = 0; i < expected.size(); i++) {
2494 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2495 }
2496 } finally {
2497 HRegion.closeHRegion(this.region);
2498 this.region = null;
2499 }
2500 }
2501
2502 @Test
2503 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2504 byte[] row1 = Bytes.toBytes("row1");
2505 byte[] fam1 = Bytes.toBytes("fam1");
2506 byte[][] families = { fam1 };
2507 byte[] qf1 = Bytes.toBytes("qualifier1");
2508 byte[] qf2 = Bytes.toBytes("qualifier2");
2509
2510 long ts1 = 1;
2511 long ts2 = ts1 + 1;
2512 long ts3 = ts1 + 2;
2513 long ts4 = ts1 + 3;
2514
2515
2516 String method = this.getName();
2517 this.region = initHRegion(tableName, method, CONF, families);
2518 try {
2519
2520 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2521 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2522 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2523 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2524
2525 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2526 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2527 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2528 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2529
2530 Put put = null;
2531 put = new Put(row1);
2532 put.add(kv14);
2533 put.add(kv24);
2534 region.put(put);
2535 region.flushcache();
2536
2537 put = new Put(row1);
2538 put.add(kv23);
2539 put.add(kv13);
2540 region.put(put);
2541 region.flushcache();
2542
2543 put = new Put(row1);
2544 put.add(kv22);
2545 put.add(kv12);
2546 region.put(put);
2547 region.flushcache();
2548
2549 put = new Put(row1);
2550 put.add(kv21);
2551 put.add(kv11);
2552 region.put(put);
2553
2554
2555 List<Cell> expected = new ArrayList<Cell>();
2556 expected.add(kv14);
2557 expected.add(kv13);
2558 expected.add(kv12);
2559 expected.add(kv24);
2560 expected.add(kv23);
2561 expected.add(kv22);
2562
2563 Scan scan = new Scan(row1);
2564 scan.addColumn(fam1, qf1);
2565 scan.addColumn(fam1, qf2);
2566 int versions = 3;
2567 scan.setMaxVersions(versions);
2568 List<Cell> actual = new ArrayList<Cell>();
2569 InternalScanner scanner = region.getScanner(scan);
2570
2571 boolean hasNext = scanner.next(actual);
2572 assertEquals(false, hasNext);
2573
2574
2575 for (int i = 0; i < expected.size(); i++) {
2576 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2577 }
2578 } finally {
2579 HRegion.closeHRegion(this.region);
2580 this.region = null;
2581 }
2582 }
2583
2584 @Test
2585 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2586 byte[] row1 = Bytes.toBytes("row1");
2587 byte[] qf1 = Bytes.toBytes("qualifier1");
2588 byte[] qf2 = Bytes.toBytes("qualifier2");
2589 byte[] fam1 = Bytes.toBytes("fam1");
2590 byte[][] families = { fam1 };
2591
2592 long ts1 = System.currentTimeMillis();
2593 long ts2 = ts1 + 1;
2594 long ts3 = ts1 + 2;
2595
2596
2597 String method = this.getName();
2598 this.region = initHRegion(tableName, method, CONF, families);
2599 try {
2600
2601 Put put = null;
2602 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2603 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2604 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2605
2606 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2607 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2608 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2609
2610 put = new Put(row1);
2611 put.add(kv13);
2612 put.add(kv12);
2613 put.add(kv11);
2614 put.add(kv23);
2615 put.add(kv22);
2616 put.add(kv21);
2617 region.put(put);
2618
2619
2620 List<Cell> expected = new ArrayList<Cell>();
2621 expected.add(kv13);
2622 expected.add(kv12);
2623 expected.add(kv23);
2624 expected.add(kv22);
2625
2626 Scan scan = new Scan(row1);
2627 scan.addFamily(fam1);
2628 scan.setMaxVersions(MAX_VERSIONS);
2629 List<Cell> actual = new ArrayList<Cell>();
2630 InternalScanner scanner = region.getScanner(scan);
2631
2632 boolean hasNext = scanner.next(actual);
2633 assertEquals(false, hasNext);
2634
2635
2636 for (int i = 0; i < expected.size(); i++) {
2637 assertEquals(expected.get(i), actual.get(i));
2638 }
2639 } finally {
2640 HRegion.closeHRegion(this.region);
2641 this.region = null;
2642 }
2643 }
2644
2645 @Test
2646 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2647 byte[] row1 = Bytes.toBytes("row1");
2648 byte[] qf1 = Bytes.toBytes("qualifier1");
2649 byte[] qf2 = Bytes.toBytes("qualifier2");
2650 byte[] fam1 = Bytes.toBytes("fam1");
2651
2652 long ts1 = 1;
2653 long ts2 = ts1 + 1;
2654 long ts3 = ts1 + 2;
2655
2656
2657 String method = this.getName();
2658 this.region = initHRegion(tableName, method, CONF, fam1);
2659 try {
2660
2661 Put put = null;
2662 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2663 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2664 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2665
2666 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2667 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2668 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2669
2670 put = new Put(row1);
2671 put.add(kv13);
2672 put.add(kv12);
2673 put.add(kv11);
2674 put.add(kv23);
2675 put.add(kv22);
2676 put.add(kv21);
2677 region.put(put);
2678 region.flushcache();
2679
2680
2681 List<Cell> expected = new ArrayList<Cell>();
2682 expected.add(kv13);
2683 expected.add(kv12);
2684 expected.add(kv23);
2685 expected.add(kv22);
2686
2687 Scan scan = new Scan(row1);
2688 scan.addFamily(fam1);
2689 scan.setMaxVersions(MAX_VERSIONS);
2690 List<Cell> actual = new ArrayList<Cell>();
2691 InternalScanner scanner = region.getScanner(scan);
2692
2693 boolean hasNext = scanner.next(actual);
2694 assertEquals(false, hasNext);
2695
2696
2697 for (int i = 0; i < expected.size(); i++) {
2698 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2699 }
2700 } finally {
2701 HRegion.closeHRegion(this.region);
2702 this.region = null;
2703 }
2704 }
2705
2706 @Test
2707 public void testScanner_StopRow1542() throws IOException {
2708 byte[] family = Bytes.toBytes("testFamily");
2709 this.region = initHRegion(tableName, getName(), CONF, family);
2710 try {
2711 byte[] row1 = Bytes.toBytes("row111");
2712 byte[] row2 = Bytes.toBytes("row222");
2713 byte[] row3 = Bytes.toBytes("row333");
2714 byte[] row4 = Bytes.toBytes("row444");
2715 byte[] row5 = Bytes.toBytes("row555");
2716
2717 byte[] col1 = Bytes.toBytes("Pub111");
2718 byte[] col2 = Bytes.toBytes("Pub222");
2719
2720 Put put = new Put(row1);
2721 put.add(family, col1, Bytes.toBytes(10L));
2722 region.put(put);
2723
2724 put = new Put(row2);
2725 put.add(family, col1, Bytes.toBytes(15L));
2726 region.put(put);
2727
2728 put = new Put(row3);
2729 put.add(family, col2, Bytes.toBytes(20L));
2730 region.put(put);
2731
2732 put = new Put(row4);
2733 put.add(family, col2, Bytes.toBytes(30L));
2734 region.put(put);
2735
2736 put = new Put(row5);
2737 put.add(family, col1, Bytes.toBytes(40L));
2738 region.put(put);
2739
2740 Scan scan = new Scan(row3, row4);
2741 scan.setMaxVersions();
2742 scan.addColumn(family, col1);
2743 InternalScanner s = region.getScanner(scan);
2744
2745 List<Cell> results = new ArrayList<Cell>();
2746 assertEquals(false, s.next(results));
2747 assertEquals(0, results.size());
2748 } finally {
2749 HRegion.closeHRegion(this.region);
2750 this.region = null;
2751 }
2752 }
2753
2754 @Test
2755 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2756 byte[] row1 = Bytes.toBytes("row1");
2757 byte[] fam1 = Bytes.toBytes("fam1");
2758 byte[] qf1 = Bytes.toBytes("qualifier1");
2759 byte[] qf2 = Bytes.toBytes("quateslifier2");
2760
2761 long ts1 = 1;
2762 long ts2 = ts1 + 1;
2763 long ts3 = ts1 + 2;
2764 long ts4 = ts1 + 3;
2765
2766
2767 String method = this.getName();
2768 this.region = initHRegion(tableName, method, CONF, fam1);
2769 try {
2770
2771 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2772 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2773 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2774 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2775
2776 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2777 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2778 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2779 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2780
2781 Put put = null;
2782 put = new Put(row1);
2783 put.add(kv14);
2784 put.add(kv24);
2785 region.put(put);
2786 region.flushcache();
2787
2788 put = new Put(row1);
2789 put.add(kv23);
2790 put.add(kv13);
2791 region.put(put);
2792 region.flushcache();
2793
2794 put = new Put(row1);
2795 put.add(kv22);
2796 put.add(kv12);
2797 region.put(put);
2798 region.flushcache();
2799
2800 put = new Put(row1);
2801 put.add(kv21);
2802 put.add(kv11);
2803 region.put(put);
2804
2805
2806 List<KeyValue> expected = new ArrayList<KeyValue>();
2807 expected.add(kv14);
2808 expected.add(kv13);
2809 expected.add(kv12);
2810 expected.add(kv24);
2811 expected.add(kv23);
2812 expected.add(kv22);
2813
2814 Scan scan = new Scan(row1);
2815 int versions = 3;
2816 scan.setMaxVersions(versions);
2817 List<Cell> actual = new ArrayList<Cell>();
2818 InternalScanner scanner = region.getScanner(scan);
2819
2820 boolean hasNext = scanner.next(actual);
2821 assertEquals(false, hasNext);
2822
2823
2824 for (int i = 0; i < expected.size(); i++) {
2825 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2826 }
2827 } finally {
2828 HRegion.closeHRegion(this.region);
2829 this.region = null;
2830 }
2831 }
2832
2833
2834
2835
2836
2837
2838
2839 @Test
2840 public void testScanner_JoinedScanners() throws IOException {
2841 byte[] cf_essential = Bytes.toBytes("essential");
2842 byte[] cf_joined = Bytes.toBytes("joined");
2843 byte[] cf_alpha = Bytes.toBytes("alpha");
2844 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2845 try {
2846 byte[] row1 = Bytes.toBytes("row1");
2847 byte[] row2 = Bytes.toBytes("row2");
2848 byte[] row3 = Bytes.toBytes("row3");
2849
2850 byte[] col_normal = Bytes.toBytes("d");
2851 byte[] col_alpha = Bytes.toBytes("a");
2852
2853 byte[] filtered_val = Bytes.toBytes(3);
2854
2855 Put put = new Put(row1);
2856 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2857 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2858 region.put(put);
2859
2860 put = new Put(row2);
2861 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2862 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2863 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2864 region.put(put);
2865
2866 put = new Put(row3);
2867 put.add(cf_essential, col_normal, filtered_val);
2868 put.add(cf_joined, col_normal, filtered_val);
2869 region.put(put);
2870
2871
2872
2873
2874
2875 Scan scan = new Scan();
2876 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2877 CompareOp.NOT_EQUAL, filtered_val);
2878 scan.setFilter(filter);
2879 scan.setLoadColumnFamiliesOnDemand(true);
2880 InternalScanner s = region.getScanner(scan);
2881
2882 List<Cell> results = new ArrayList<Cell>();
2883 assertTrue(s.next(results));
2884 assertEquals(results.size(), 1);
2885 results.clear();
2886
2887 assertTrue(s.next(results));
2888 assertEquals(results.size(), 3);
2889 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2890 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2891 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2892 results.clear();
2893
2894 assertFalse(s.next(results));
2895 assertEquals(results.size(), 0);
2896 } finally {
2897 HRegion.closeHRegion(this.region);
2898 this.region = null;
2899 }
2900 }
2901
2902
2903
2904
2905
2906
2907 @Test
2908 public void testScanner_JoinedScannersWithLimits() throws IOException {
2909 final byte[] cf_first = Bytes.toBytes("first");
2910 final byte[] cf_second = Bytes.toBytes("second");
2911
2912 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2913 try {
2914 final byte[] col_a = Bytes.toBytes("a");
2915 final byte[] col_b = Bytes.toBytes("b");
2916
2917 Put put;
2918
2919 for (int i = 0; i < 10; i++) {
2920 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2921 put.add(cf_first, col_a, Bytes.toBytes(i));
2922 if (i < 5) {
2923 put.add(cf_first, col_b, Bytes.toBytes(i));
2924 put.add(cf_second, col_a, Bytes.toBytes(i));
2925 put.add(cf_second, col_b, Bytes.toBytes(i));
2926 }
2927 region.put(put);
2928 }
2929
2930 Scan scan = new Scan();
2931 scan.setLoadColumnFamiliesOnDemand(true);
2932 Filter bogusFilter = new FilterBase() {
2933 @Override
2934 public boolean isFamilyEssential(byte[] name) {
2935 return Bytes.equals(name, cf_first);
2936 }
2937 };
2938
2939 scan.setFilter(bogusFilter);
2940 InternalScanner s = region.getScanner(scan);
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971 List<Cell> results = new ArrayList<Cell>();
2972 int index = 0;
2973 while (true) {
2974 boolean more = s.next(results, 3);
2975 if ((index >> 1) < 5) {
2976 if (index % 2 == 0)
2977 assertEquals(results.size(), 3);
2978 else
2979 assertEquals(results.size(), 1);
2980 } else
2981 assertEquals(results.size(), 1);
2982 results.clear();
2983 index++;
2984 if (!more)
2985 break;
2986 }
2987 } finally {
2988 HRegion.closeHRegion(this.region);
2989 this.region = null;
2990 }
2991 }
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001 @Test
3002 public void testBasicSplit() throws Exception {
3003 byte[][] families = { fam1, fam2, fam3 };
3004
3005 Configuration hc = initSplit();
3006
3007 String method = this.getName();
3008 this.region = initHRegion(tableName, method, hc, families);
3009
3010 try {
3011 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3012 region.flushcache();
3013 region.compactStores();
3014 byte[] splitRow = region.checkSplit();
3015 assertNotNull(splitRow);
3016 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3017 HRegion[] regions = splitRegion(region, splitRow);
3018 try {
3019
3020
3021
3022 for (int i = 0; i < regions.length; i++) {
3023 regions[i] = HRegion.openHRegion(regions[i], null);
3024 }
3025
3026
3027 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3028 assertGet(regions[1], fam3, splitRow);
3029
3030 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3031 assertScan(regions[1], fam3, splitRow);
3032
3033 for (int i = 0; i < regions.length; i++) {
3034
3035
3036 for (int j = 0; j < 2; j++) {
3037 HBaseTestCase.addContent(regions[i], fam3);
3038 }
3039 HBaseTestCase.addContent(regions[i], fam2);
3040 HBaseTestCase.addContent(regions[i], fam1);
3041 regions[i].flushcache();
3042 }
3043
3044 byte[][] midkeys = new byte[regions.length][];
3045
3046 for (int i = 0; i < regions.length; i++) {
3047 regions[i].compactStores();
3048 midkeys[i] = regions[i].checkSplit();
3049 }
3050
3051 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3052
3053
3054 for (int i = 0; i < regions.length; i++) {
3055 HRegion[] rs = null;
3056 if (midkeys[i] != null) {
3057 rs = splitRegion(regions[i], midkeys[i]);
3058 for (int j = 0; j < rs.length; j++) {
3059 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3060 }
3061 }
3062 }
3063 LOG.info("Made 4 regions");
3064
3065
3066 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3067 byte[] b = Bytes.toBytes(START_KEY);
3068 for (HRegion r : sortedMap.values()) {
3069 assertGet(r, fam3, b);
3070 b[0] += interval;
3071 }
3072 } finally {
3073 for (int i = 0; i < regions.length; i++) {
3074 try {
3075 regions[i].close();
3076 } catch (IOException e) {
3077
3078 }
3079 }
3080 }
3081 } finally {
3082 HRegion.closeHRegion(this.region);
3083 this.region = null;
3084 }
3085 }
3086
3087 @Test
3088 public void testSplitRegion() throws IOException {
3089 byte[] qualifier = Bytes.toBytes("qualifier");
3090 Configuration hc = initSplit();
3091 int numRows = 10;
3092 byte[][] families = { fam1, fam3 };
3093
3094
3095 String method = this.getName();
3096 this.region = initHRegion(tableName, method, hc, families);
3097
3098
3099 int startRow = 100;
3100 putData(startRow, numRows, qualifier, families);
3101 int splitRow = startRow + numRows;
3102 putData(splitRow, numRows, qualifier, families);
3103 region.flushcache();
3104
3105 HRegion[] regions = null;
3106 try {
3107 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3108
3109 for (int i = 0; i < regions.length; i++) {
3110 regions[i] = HRegion.openHRegion(regions[i], null);
3111 }
3112
3113 assertEquals(2, regions.length);
3114
3115
3116
3117 verifyData(regions[0], startRow, numRows, qualifier, families);
3118 verifyData(regions[1], splitRow, numRows, qualifier, families);
3119
3120 } finally {
3121 HRegion.closeHRegion(this.region);
3122 this.region = null;
3123 }
3124 }
3125
3126 @Test
3127 public void testClearForceSplit() throws IOException {
3128 byte[] qualifier = Bytes.toBytes("qualifier");
3129 Configuration hc = initSplit();
3130 int numRows = 10;
3131 byte[][] families = { fam1, fam3 };
3132
3133
3134 String method = this.getName();
3135 this.region = initHRegion(tableName, method, hc, families);
3136
3137
3138 int startRow = 100;
3139 putData(startRow, numRows, qualifier, families);
3140 int splitRow = startRow + numRows;
3141 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3142 putData(splitRow, numRows, qualifier, families);
3143 region.flushcache();
3144
3145 HRegion[] regions = null;
3146 try {
3147
3148 region.forceSplit(splitRowBytes);
3149 assertTrue(region.shouldForceSplit());
3150
3151 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3152
3153
3154 HStore storeMock = Mockito.mock(HStore.class);
3155 Mockito.when(storeMock.hasReferences()).thenReturn(true);
3156 Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3157 Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3158 Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3159 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3160 assertTrue(region.hasReferences());
3161
3162
3163 regions = splitRegion(region, splitRowBytes);
3164 assertNull(regions);
3165
3166
3167 assertFalse(region.shouldForceSplit());
3168
3169
3170 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3171 assertFalse(region.hasReferences());
3172
3173
3174 regions = splitRegion(region, splitRowBytes);
3175
3176
3177 for (int i = 0; i < regions.length; i++) {
3178 regions[i] = HRegion.openHRegion(regions[i], null);
3179 }
3180
3181 assertEquals(2, regions.length);
3182
3183
3184
3185 verifyData(regions[0], startRow, numRows, qualifier, families);
3186 verifyData(regions[1], splitRow, numRows, qualifier, families);
3187
3188 } finally {
3189 HRegion.closeHRegion(this.region);
3190 this.region = null;
3191 }
3192 }
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204 @Test
3205 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3206 byte[] family = Bytes.toBytes("family");
3207 int numRows = 1000;
3208 int flushAndScanInterval = 10;
3209 int compactInterval = 10 * flushAndScanInterval;
3210
3211 String method = "testFlushCacheWhileScanning";
3212 this.region = initHRegion(tableName, method, CONF, family);
3213 try {
3214 FlushThread flushThread = new FlushThread();
3215 flushThread.start();
3216
3217 Scan scan = new Scan();
3218 scan.addFamily(family);
3219 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3220 new BinaryComparator(Bytes.toBytes(5L))));
3221
3222 int expectedCount = 0;
3223 List<Cell> res = new ArrayList<Cell>();
3224
3225 boolean toggle = true;
3226 for (long i = 0; i < numRows; i++) {
3227 Put put = new Put(Bytes.toBytes(i));
3228 put.setDurability(Durability.SKIP_WAL);
3229 put.add(family, qual1, Bytes.toBytes(i % 10));
3230 region.put(put);
3231
3232 if (i != 0 && i % compactInterval == 0) {
3233
3234 region.compactStores(true);
3235 }
3236
3237 if (i % 10 == 5L) {
3238 expectedCount++;
3239 }
3240
3241 if (i != 0 && i % flushAndScanInterval == 0) {
3242 res.clear();
3243 InternalScanner scanner = region.getScanner(scan);
3244 if (toggle) {
3245 flushThread.flush();
3246 }
3247 while (scanner.next(res))
3248 ;
3249 if (!toggle) {
3250 flushThread.flush();
3251 }
3252 assertEquals("i=" + i, expectedCount, res.size());
3253 toggle = !toggle;
3254 }
3255 }
3256
3257 flushThread.done();
3258 flushThread.join();
3259 flushThread.checkNoError();
3260 } finally {
3261 HRegion.closeHRegion(this.region);
3262 this.region = null;
3263 }
3264 }
3265
3266 protected class FlushThread extends Thread {
3267 private volatile boolean done;
3268 private Throwable error = null;
3269
3270 public void done() {
3271 done = true;
3272 synchronized (this) {
3273 interrupt();
3274 }
3275 }
3276
3277 public void checkNoError() {
3278 if (error != null) {
3279 assertNull(error);
3280 }
3281 }
3282
3283 @Override
3284 public void run() {
3285 done = false;
3286 while (!done) {
3287 synchronized (this) {
3288 try {
3289 wait();
3290 } catch (InterruptedException ignored) {
3291 if (done) {
3292 break;
3293 }
3294 }
3295 }
3296 try {
3297 region.flushcache();
3298 } catch (IOException e) {
3299 if (!done) {
3300 LOG.error("Error while flusing cache", e);
3301 error = e;
3302 }
3303 break;
3304 }
3305 }
3306
3307 }
3308
3309 public void flush() {
3310 synchronized (this) {
3311 notify();
3312 }
3313
3314 }
3315 }
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326 @Test
3327 public void testWritesWhileScanning() throws IOException, InterruptedException {
3328 int testCount = 100;
3329 int numRows = 1;
3330 int numFamilies = 10;
3331 int numQualifiers = 100;
3332 int flushInterval = 7;
3333 int compactInterval = 5 * flushInterval;
3334 byte[][] families = new byte[numFamilies][];
3335 for (int i = 0; i < numFamilies; i++) {
3336 families[i] = Bytes.toBytes("family" + i);
3337 }
3338 byte[][] qualifiers = new byte[numQualifiers][];
3339 for (int i = 0; i < numQualifiers; i++) {
3340 qualifiers[i] = Bytes.toBytes("qual" + i);
3341 }
3342
3343 String method = "testWritesWhileScanning";
3344 this.region = initHRegion(tableName, method, CONF, families);
3345 try {
3346 PutThread putThread = new PutThread(numRows, families, qualifiers);
3347 putThread.start();
3348 putThread.waitForFirstPut();
3349
3350 FlushThread flushThread = new FlushThread();
3351 flushThread.start();
3352
3353 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3354
3355 int expectedCount = numFamilies * numQualifiers;
3356 List<Cell> res = new ArrayList<Cell>();
3357
3358 long prevTimestamp = 0L;
3359 for (int i = 0; i < testCount; i++) {
3360
3361 if (i != 0 && i % compactInterval == 0) {
3362 region.compactStores(true);
3363 }
3364
3365 if (i != 0 && i % flushInterval == 0) {
3366 flushThread.flush();
3367 }
3368
3369 boolean previousEmpty = res.isEmpty();
3370 res.clear();
3371 InternalScanner scanner = region.getScanner(scan);
3372 while (scanner.next(res))
3373 ;
3374 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3375 assertEquals("i=" + i, expectedCount, res.size());
3376 long timestamp = res.get(0).getTimestamp();
3377 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3378 timestamp >= prevTimestamp);
3379 prevTimestamp = timestamp;
3380 }
3381 }
3382
3383 putThread.done();
3384
3385 region.flushcache();
3386
3387 putThread.join();
3388 putThread.checkNoError();
3389
3390 flushThread.done();
3391 flushThread.join();
3392 flushThread.checkNoError();
3393 } finally {
3394 try {
3395 HRegion.closeHRegion(this.region);
3396 } catch (DroppedSnapshotException dse) {
3397
3398
3399
3400
3401 }
3402 this.region = null;
3403 }
3404 }
3405
3406 protected class PutThread extends Thread {
3407 private volatile boolean done;
3408 private volatile int numPutsFinished = 0;
3409
3410 private Throwable error = null;
3411 private int numRows;
3412 private byte[][] families;
3413 private byte[][] qualifiers;
3414
3415 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3416 this.numRows = numRows;
3417 this.families = families;
3418 this.qualifiers = qualifiers;
3419 }
3420
3421
3422
3423
3424 public void waitForFirstPut() throws InterruptedException {
3425
3426 while (numPutsFinished == 0) {
3427 checkNoError();
3428 Thread.sleep(50);
3429 }
3430 }
3431
3432 public void done() {
3433 done = true;
3434 synchronized (this) {
3435 interrupt();
3436 }
3437 }
3438
3439 public void checkNoError() {
3440 if (error != null) {
3441 assertNull(error);
3442 }
3443 }
3444
3445 @Override
3446 public void run() {
3447 done = false;
3448 while (!done) {
3449 try {
3450 for (int r = 0; r < numRows; r++) {
3451 byte[] row = Bytes.toBytes("row" + r);
3452 Put put = new Put(row);
3453 put.setDurability(Durability.SKIP_WAL);
3454 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3455 for (byte[] family : families) {
3456 for (byte[] qualifier : qualifiers) {
3457 put.add(family, qualifier, (long) numPutsFinished, value);
3458 }
3459 }
3460 region.put(put);
3461 numPutsFinished++;
3462 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3463 System.out.println("put iteration = " + numPutsFinished);
3464 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3465 region.delete(delete);
3466 }
3467 numPutsFinished++;
3468 }
3469 } catch (InterruptedIOException e) {
3470
3471 } catch (IOException e) {
3472 LOG.error("error while putting records", e);
3473 error = e;
3474 break;
3475 }
3476 }
3477
3478 }
3479
3480 }
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491 @Test
3492 public void testWritesWhileGetting() throws Exception {
3493 int testCount = 100;
3494 int numRows = 1;
3495 int numFamilies = 10;
3496 int numQualifiers = 100;
3497 int compactInterval = 100;
3498 byte[][] families = new byte[numFamilies][];
3499 for (int i = 0; i < numFamilies; i++) {
3500 families[i] = Bytes.toBytes("family" + i);
3501 }
3502 byte[][] qualifiers = new byte[numQualifiers][];
3503 for (int i = 0; i < numQualifiers; i++) {
3504 qualifiers[i] = Bytes.toBytes("qual" + i);
3505 }
3506
3507
3508 String method = "testWritesWhileGetting";
3509
3510
3511
3512
3513
3514 Configuration conf = HBaseConfiguration.create(CONF);
3515 conf.setInt("hbase.hstore.compaction.min", 1);
3516 conf.setInt("hbase.hstore.compaction.max", 1000);
3517 this.region = initHRegion(tableName, method, conf, families);
3518 PutThread putThread = null;
3519 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3520 try {
3521 putThread = new PutThread(numRows, families, qualifiers);
3522 putThread.start();
3523 putThread.waitForFirstPut();
3524
3525
3526 ctx.addThread(new RepeatingTestThread(ctx) {
3527 private int flushesSinceCompact = 0;
3528 private final int maxFlushesSinceCompact = 20;
3529
3530 @Override
3531 public void doAnAction() throws Exception {
3532 if (region.flushcache().isCompactionNeeded()) {
3533 ++flushesSinceCompact;
3534 }
3535
3536
3537 if (flushesSinceCompact == maxFlushesSinceCompact) {
3538 region.compactStores(false);
3539 flushesSinceCompact = 0;
3540 }
3541 }
3542 });
3543 ctx.startThreads();
3544
3545 Get get = new Get(Bytes.toBytes("row0"));
3546 Result result = null;
3547
3548 int expectedCount = numFamilies * numQualifiers;
3549
3550 long prevTimestamp = 0L;
3551 for (int i = 0; i < testCount; i++) {
3552
3553 boolean previousEmpty = result == null || result.isEmpty();
3554 result = region.get(get);
3555 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3556 assertEquals("i=" + i, expectedCount, result.size());
3557
3558
3559 long timestamp = 0;
3560 for (Cell kv : result.rawCells()) {
3561 if (CellUtil.matchingFamily(kv, families[0])
3562 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3563 timestamp = kv.getTimestamp();
3564 }
3565 }
3566 assertTrue(timestamp >= prevTimestamp);
3567 prevTimestamp = timestamp;
3568 Cell previousKV = null;
3569
3570 for (Cell kv : result.rawCells()) {
3571 byte[] thisValue = CellUtil.cloneValue(kv);
3572 if (previousKV != null) {
3573 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3574 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3575 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3576 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3577 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3578 }
3579 }
3580 previousKV = kv;
3581 }
3582 }
3583 }
3584 } finally {
3585 if (putThread != null)
3586 putThread.done();
3587
3588 region.flushcache();
3589
3590 if (putThread != null) {
3591 putThread.join();
3592 putThread.checkNoError();
3593 }
3594
3595 ctx.stop();
3596 HRegion.closeHRegion(this.region);
3597 this.region = null;
3598 }
3599 }
3600
3601 @Test
3602 public void testHolesInMeta() throws Exception {
3603 byte[] family = Bytes.toBytes("family");
3604 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3605 false, family);
3606 try {
3607 byte[] rowNotServed = Bytes.toBytes("a");
3608 Get g = new Get(rowNotServed);
3609 try {
3610 region.get(g);
3611 fail();
3612 } catch (WrongRegionException x) {
3613
3614 }
3615 byte[] row = Bytes.toBytes("y");
3616 g = new Get(row);
3617 region.get(g);
3618 } finally {
3619 HRegion.closeHRegion(this.region);
3620 this.region = null;
3621 }
3622 }
3623
3624 @Test
3625 public void testIndexesScanWithOneDeletedRow() throws IOException {
3626 byte[] family = Bytes.toBytes("family");
3627
3628
3629 String method = "testIndexesScanWithOneDeletedRow";
3630 this.region = initHRegion(tableName, method, CONF, family);
3631 try {
3632 Put put = new Put(Bytes.toBytes(1L));
3633 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3634 region.put(put);
3635
3636 region.flushcache();
3637
3638 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3639 region.delete(delete);
3640
3641 put = new Put(Bytes.toBytes(2L));
3642 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3643 region.put(put);
3644
3645 Scan idxScan = new Scan();
3646 idxScan.addFamily(family);
3647 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3648 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3649 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3650 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3651 InternalScanner scanner = region.getScanner(idxScan);
3652 List<Cell> res = new ArrayList<Cell>();
3653
3654 while (scanner.next(res))
3655 ;
3656 assertEquals(1L, res.size());
3657 } finally {
3658 HRegion.closeHRegion(this.region);
3659 this.region = null;
3660 }
3661 }
3662
3663
3664
3665
3666 @Test
3667 public void testBloomFilterSize() throws IOException {
3668 byte[] fam1 = Bytes.toBytes("fam1");
3669 byte[] qf1 = Bytes.toBytes("col");
3670 byte[] val1 = Bytes.toBytes("value1");
3671
3672 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3673 .setBloomFilterType(BloomType.ROWCOL);
3674
3675 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3676 htd.addFamily(hcd);
3677 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3678 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3679 try {
3680 int num_unique_rows = 10;
3681 int duplicate_multiplier = 2;
3682 int num_storefiles = 4;
3683
3684 int version = 0;
3685 for (int f = 0; f < num_storefiles; f++) {
3686 for (int i = 0; i < duplicate_multiplier; i++) {
3687 for (int j = 0; j < num_unique_rows; j++) {
3688 Put put = new Put(Bytes.toBytes("row" + j));
3689 put.setDurability(Durability.SKIP_WAL);
3690 put.add(fam1, qf1, version++, val1);
3691 region.put(put);
3692 }
3693 }
3694 region.flushcache();
3695 }
3696
3697 HStore store = (HStore) region.getStore(fam1);
3698 Collection<StoreFile> storeFiles = store.getStorefiles();
3699 for (StoreFile storefile : storeFiles) {
3700 StoreFile.Reader reader = storefile.getReader();
3701 reader.loadFileInfo();
3702 reader.loadBloomfilter();
3703 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3704 assertEquals(num_unique_rows, reader.getFilterEntries());
3705 }
3706
3707 region.compactStores(true);
3708
3709
3710 storeFiles = store.getStorefiles();
3711 for (StoreFile storefile : storeFiles) {
3712 StoreFile.Reader reader = storefile.getReader();
3713 reader.loadFileInfo();
3714 reader.loadBloomfilter();
3715 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3716 assertEquals(num_unique_rows, reader.getFilterEntries());
3717 }
3718 } finally {
3719 HRegion.closeHRegion(this.region);
3720 this.region = null;
3721 }
3722 }
3723
3724 @Test
3725 public void testAllColumnsWithBloomFilter() throws IOException {
3726 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3727 byte[] FAMILY = Bytes.toBytes("family");
3728
3729
3730 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3731 .setBloomFilterType(BloomType.ROWCOL);
3732 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3733 htd.addFamily(hcd);
3734 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3735 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3736 try {
3737
3738 byte row[] = Bytes.toBytes("row:" + 0);
3739 byte column[] = Bytes.toBytes("column:" + 0);
3740 Put put = new Put(row);
3741 put.setDurability(Durability.SKIP_WAL);
3742 for (long idx = 1; idx <= 4; idx++) {
3743 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3744 }
3745 region.put(put);
3746
3747
3748 region.flushcache();
3749
3750
3751 Get get = new Get(row);
3752 get.setMaxVersions();
3753 Cell[] kvs = region.get(get).rawCells();
3754
3755
3756 assertEquals(4, kvs.length);
3757 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3758 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3759 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3760 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3761 } finally {
3762 HRegion.closeHRegion(this.region);
3763 this.region = null;
3764 }
3765 }
3766
3767
3768
3769
3770
3771
3772 @Test
3773 public void testDeleteRowWithBloomFilter() throws IOException {
3774 byte[] familyName = Bytes.toBytes("familyName");
3775
3776
3777 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3778 .setBloomFilterType(BloomType.ROWCOL);
3779
3780 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3781 htd.addFamily(hcd);
3782 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3783 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3784 try {
3785
3786 byte row[] = Bytes.toBytes("row1");
3787 byte col[] = Bytes.toBytes("col1");
3788
3789 Put put = new Put(row);
3790 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3791 region.put(put);
3792 region.flushcache();
3793
3794 Delete del = new Delete(row);
3795 region.delete(del);
3796 region.flushcache();
3797
3798
3799 Get get = new Get(row);
3800 get.addColumn(familyName, col);
3801
3802 Cell[] keyValues = region.get(get).rawCells();
3803 assertTrue(keyValues.length == 0);
3804 } finally {
3805 HRegion.closeHRegion(this.region);
3806 this.region = null;
3807 }
3808 }
3809
3810 @Test
3811 public void testgetHDFSBlocksDistribution() throws Exception {
3812 HBaseTestingUtility htu = new HBaseTestingUtility();
3813 final int DEFAULT_BLOCK_SIZE = 1024;
3814 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3815 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3816 htu.getConfiguration().setInt("dfs.replication", 2);
3817
3818
3819 MiniHBaseCluster cluster = null;
3820 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3821 int regionServersCount = 3;
3822
3823 try {
3824 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3825 byte[][] families = { fam1, fam2 };
3826 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3827
3828
3829 byte row[] = Bytes.toBytes("row1");
3830 byte col[] = Bytes.toBytes("col1");
3831
3832 Put put = new Put(row);
3833 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3834 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3835 ht.put(put);
3836
3837 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3838 .get(0);
3839 firstRegion.flushcache();
3840 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3841
3842
3843
3844
3845
3846 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3847
3848 String topHost = blocksDistribution1.getTopHosts().get(0);
3849 long topHostWeight = blocksDistribution1.getWeight(topHost);
3850 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3851
3852
3853
3854 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3855 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3856 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3857
3858 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3859
3860 ht.close();
3861 } finally {
3862 if (cluster != null) {
3863 htu.shutdownMiniCluster();
3864 }
3865 }
3866 }
3867
3868
3869
3870
3871
3872
3873
3874 @Test
3875 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3876 TableName tableName = TableName.valueOf(name.getMethodName());
3877 HRegionInfo info = null;
3878 try {
3879 FileSystem fs = Mockito.mock(FileSystem.class);
3880 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3881 HTableDescriptor htd = new HTableDescriptor(tableName);
3882 htd.addFamily(new HColumnDescriptor("cf"));
3883 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3884 HConstants.EMPTY_BYTE_ARRAY, false);
3885 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3886 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3887
3888 region.initialize();
3889 fail("Region initialization should fail due to IOException");
3890 } catch (IOException io) {
3891 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3892 for (MonitoredTask monitoredTask : tasks) {
3893 if (!(monitoredTask instanceof MonitoredRPCHandler)
3894 && monitoredTask.getDescription().contains(region.toString())) {
3895 assertTrue("Region state should be ABORTED.",
3896 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3897 break;
3898 }
3899 }
3900 } finally {
3901 HRegion.closeHRegion(region);
3902 }
3903 }
3904
3905
3906
3907
3908
3909 @Test
3910 public void testRegionInfoFileCreation() throws IOException {
3911 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3912
3913 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3914 htd.addFamily(new HColumnDescriptor("cf"));
3915
3916 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3917
3918
3919 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3920
3921 Path regionDir = region.getRegionFileSystem().getRegionDir();
3922 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3923 HRegion.closeHRegion(region);
3924
3925 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3926
3927
3928 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3929 fs.exists(regionInfoFile));
3930
3931
3932 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3933 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3934 HRegion.closeHRegion(region);
3935
3936
3937 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3938 fs.exists(regionInfoFile));
3939
3940
3941 fs.delete(regionInfoFile);
3942 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3943 fs.exists(regionInfoFile));
3944
3945 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3946
3947 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3948 HRegion.closeHRegion(region);
3949
3950
3951 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3952 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3953 }
3954
3955
3956
3957
3958 private static class Incrementer implements Runnable {
3959 private HRegion region;
3960 private final static byte[] incRow = Bytes.toBytes("incRow");
3961 private final static byte[] family = Bytes.toBytes("family");
3962 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3963 private final static long ONE = 1l;
3964 private int incCounter;
3965
3966 public Incrementer(HRegion region, int incCounter) {
3967 this.region = region;
3968 this.incCounter = incCounter;
3969 }
3970
3971 @Override
3972 public void run() {
3973 int count = 0;
3974 while (count < incCounter) {
3975 Increment inc = new Increment(incRow);
3976 inc.addColumn(family, qualifier, ONE);
3977 count++;
3978 try {
3979 region.increment(inc);
3980 } catch (IOException e) {
3981 e.printStackTrace();
3982 break;
3983 }
3984 }
3985 }
3986 }
3987
3988
3989
3990
3991
3992 @Test
3993 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3994 byte[] family = Incrementer.family;
3995 this.region = initHRegion(tableName, method, CONF, family);
3996 final HRegion region = this.region;
3997 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3998 Runnable flusher = new Runnable() {
3999 @Override
4000 public void run() {
4001 while (!incrementDone.get()) {
4002 try {
4003 region.flushcache();
4004 } catch (Exception e) {
4005 e.printStackTrace();
4006 }
4007 }
4008 }
4009 };
4010
4011
4012 int threadNum = 20;
4013 int incCounter = 100;
4014 long expected = threadNum * incCounter;
4015 Thread[] incrementers = new Thread[threadNum];
4016 Thread flushThread = new Thread(flusher);
4017 for (int i = 0; i < threadNum; i++) {
4018 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4019 incrementers[i].start();
4020 }
4021 flushThread.start();
4022 for (int i = 0; i < threadNum; i++) {
4023 incrementers[i].join();
4024 }
4025
4026 incrementDone.set(true);
4027 flushThread.join();
4028
4029 Get get = new Get(Incrementer.incRow);
4030 get.addColumn(Incrementer.family, Incrementer.qualifier);
4031 get.setMaxVersions(1);
4032 Result res = this.region.get(get);
4033 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4034
4035
4036 assertEquals(kvs.size(), 1);
4037 Cell kv = kvs.get(0);
4038 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4039 this.region = null;
4040 }
4041
4042
4043
4044
4045 private static class Appender implements Runnable {
4046 private HRegion region;
4047 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4048 private final static byte[] family = Bytes.toBytes("family");
4049 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4050 private final static byte[] CHAR = Bytes.toBytes("a");
4051 private int appendCounter;
4052
4053 public Appender(HRegion region, int appendCounter) {
4054 this.region = region;
4055 this.appendCounter = appendCounter;
4056 }
4057
4058 @Override
4059 public void run() {
4060 int count = 0;
4061 while (count < appendCounter) {
4062 Append app = new Append(appendRow);
4063 app.add(family, qualifier, CHAR);
4064 count++;
4065 try {
4066 region.append(app);
4067 } catch (IOException e) {
4068 e.printStackTrace();
4069 break;
4070 }
4071 }
4072 }
4073 }
4074
4075
4076
4077
4078
4079 @Test
4080 public void testParallelAppendWithMemStoreFlush() throws Exception {
4081 byte[] family = Appender.family;
4082 this.region = initHRegion(tableName, method, CONF, family);
4083 final HRegion region = this.region;
4084 final AtomicBoolean appendDone = new AtomicBoolean(false);
4085 Runnable flusher = new Runnable() {
4086 @Override
4087 public void run() {
4088 while (!appendDone.get()) {
4089 try {
4090 region.flushcache();
4091 } catch (Exception e) {
4092 e.printStackTrace();
4093 }
4094 }
4095 }
4096 };
4097
4098
4099
4100 int threadNum = 20;
4101 int appendCounter = 100;
4102 byte[] expected = new byte[threadNum * appendCounter];
4103 for (int i = 0; i < threadNum * appendCounter; i++) {
4104 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4105 }
4106 Thread[] appenders = new Thread[threadNum];
4107 Thread flushThread = new Thread(flusher);
4108 for (int i = 0; i < threadNum; i++) {
4109 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4110 appenders[i].start();
4111 }
4112 flushThread.start();
4113 for (int i = 0; i < threadNum; i++) {
4114 appenders[i].join();
4115 }
4116
4117 appendDone.set(true);
4118 flushThread.join();
4119
4120 Get get = new Get(Appender.appendRow);
4121 get.addColumn(Appender.family, Appender.qualifier);
4122 get.setMaxVersions(1);
4123 Result res = this.region.get(get);
4124 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4125
4126
4127 assertEquals(kvs.size(), 1);
4128 Cell kv = kvs.get(0);
4129 byte[] appendResult = new byte[kv.getValueLength()];
4130 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4131 assertArrayEquals(expected, appendResult);
4132 this.region = null;
4133 }
4134
4135
4136
4137
4138
4139 @Test
4140 public void testPutWithMemStoreFlush() throws Exception {
4141 byte[] family = Bytes.toBytes("family");
4142 ;
4143 byte[] qualifier = Bytes.toBytes("qualifier");
4144 byte[] row = Bytes.toBytes("putRow");
4145 byte[] value = null;
4146 this.region = initHRegion(tableName, method, CONF, family);
4147 Put put = null;
4148 Get get = null;
4149 List<Cell> kvs = null;
4150 Result res = null;
4151
4152 put = new Put(row);
4153 value = Bytes.toBytes("value0");
4154 put.add(family, qualifier, 1234567l, value);
4155 region.put(put);
4156 get = new Get(row);
4157 get.addColumn(family, qualifier);
4158 get.setMaxVersions();
4159 res = this.region.get(get);
4160 kvs = res.getColumnCells(family, qualifier);
4161 assertEquals(1, kvs.size());
4162 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4163
4164 region.flushcache();
4165 get = new Get(row);
4166 get.addColumn(family, qualifier);
4167 get.setMaxVersions();
4168 res = this.region.get(get);
4169 kvs = res.getColumnCells(family, qualifier);
4170 assertEquals(1, kvs.size());
4171 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4172
4173 put = new Put(row);
4174 value = Bytes.toBytes("value1");
4175 put.add(family, qualifier, 1234567l, value);
4176 region.put(put);
4177 get = new Get(row);
4178 get.addColumn(family, qualifier);
4179 get.setMaxVersions();
4180 res = this.region.get(get);
4181 kvs = res.getColumnCells(family, qualifier);
4182 assertEquals(1, kvs.size());
4183 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4184
4185 region.flushcache();
4186 get = new Get(row);
4187 get.addColumn(family, qualifier);
4188 get.setMaxVersions();
4189 res = this.region.get(get);
4190 kvs = res.getColumnCells(family, qualifier);
4191 assertEquals(1, kvs.size());
4192 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4193 }
4194
4195 @Test
4196 public void testDurability() throws Exception {
4197 String method = "testDurability";
4198
4199
4200
4201
4202
4203 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4204 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4205 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4206
4207 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4208 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4209 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4210
4211 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4212 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4213
4214 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4215 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4216
4217 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4218 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4219 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4220
4221
4222 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4223 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4224 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4225 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4226 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4227 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4228
4229 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4230 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4231 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4232 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4233 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4234 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4235
4236
4237 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4238 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4239 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4240 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4241 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4242 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4243
4244 }
4245
4246 private void durabilityTest(String method, Durability tableDurability,
4247 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4248 final boolean expectSyncFromLogSyncer) throws Exception {
4249 Configuration conf = HBaseConfiguration.create(CONF);
4250 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4251 TableName tableName = TableName.valueOf(method);
4252 byte[] family = Bytes.toBytes("family");
4253 Path logDir = new Path(new Path(dir + method), "log");
4254 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4255 final HLog log = spy(hlog);
4256 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4257 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4258 new byte[][] { family });
4259
4260 Put put = new Put(Bytes.toBytes("r1"));
4261 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4262 put.setDurability(mutationDurability);
4263 region.put(put);
4264
4265
4266 verify(log, expectAppend ? times(1) : never())
4267 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4268 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4269
4270
4271 if (expectSync || expectSyncFromLogSyncer) {
4272 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4273 @Override
4274 public boolean evaluate() throws Exception {
4275 try {
4276 if (expectSync) {
4277 verify(log, times(1)).sync(anyLong());
4278 } else if (expectSyncFromLogSyncer) {
4279 verify(log, times(1)).sync();
4280 }
4281 } catch (Throwable ignore) {
4282 }
4283 return true;
4284 }
4285 });
4286 } else {
4287 verify(log, never()).sync(anyLong());
4288 verify(log, never()).sync();
4289 }
4290
4291 HRegion.closeHRegion(this.region);
4292 this.region = null;
4293 }
4294
4295 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4296 for (int i = startRow; i < startRow + numRows; i++) {
4297 Put put = new Put(Bytes.toBytes("" + i));
4298 put.setDurability(Durability.SKIP_WAL);
4299 for (byte[] family : families) {
4300 put.add(family, qf, null);
4301 }
4302 region.put(put);
4303 }
4304 }
4305
4306 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4307 throws IOException {
4308 for (int i = startRow; i < startRow + numRows; i++) {
4309 byte[] row = Bytes.toBytes("" + i);
4310 Get get = new Get(row);
4311 for (byte[] family : families) {
4312 get.addColumn(family, qf);
4313 }
4314 Result result = newReg.get(get);
4315 Cell[] raw = result.rawCells();
4316 assertEquals(families.length, result.size());
4317 for (int j = 0; j < families.length; j++) {
4318 assertTrue(CellUtil.matchingRow(raw[j], row));
4319 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4320 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4321 }
4322 }
4323 }
4324
4325 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4326
4327 Get get = new Get(k).addFamily(family).setMaxVersions();
4328 Cell[] results = r.get(get).rawCells();
4329 for (int j = 0; j < results.length; j++) {
4330 byte[] tmp = CellUtil.cloneValue(results[j]);
4331
4332 assertTrue(Bytes.equals(k, tmp));
4333 }
4334 }
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4348 throws IOException {
4349 byte[][] families = { fs };
4350 Scan scan = new Scan();
4351 for (int i = 0; i < families.length; i++)
4352 scan.addFamily(families[i]);
4353 InternalScanner s = r.getScanner(scan);
4354 try {
4355 List<Cell> curVals = new ArrayList<Cell>();
4356 boolean first = true;
4357 OUTER_LOOP: while (s.next(curVals)) {
4358 for (Cell kv : curVals) {
4359 byte[] val = CellUtil.cloneValue(kv);
4360 byte[] curval = val;
4361 if (first) {
4362 first = false;
4363 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4364 } else {
4365
4366 break OUTER_LOOP;
4367 }
4368 }
4369 }
4370 } finally {
4371 s.close();
4372 }
4373 }
4374
4375
4376
4377
4378
4379 @Test
4380 public void testFlushResult() throws IOException {
4381 String method = name.getMethodName();
4382 byte[] tableName = Bytes.toBytes(method);
4383 byte[] family = Bytes.toBytes("family");
4384
4385 this.region = initHRegion(tableName, method, family);
4386
4387
4388 HRegion.FlushResult fr = region.flushcache();
4389 assertFalse(fr.isFlushSucceeded());
4390 assertFalse(fr.isCompactionNeeded());
4391
4392
4393 for (int i = 0; i < 2; i++) {
4394 Put put = new Put(tableName).add(family, family, tableName);
4395 region.put(put);
4396 fr = region.flushcache();
4397 assertTrue(fr.isFlushSucceeded());
4398 assertFalse(fr.isCompactionNeeded());
4399 }
4400
4401
4402 for (int i = 0; i < 2; i++) {
4403 Put put = new Put(tableName).add(family, family, tableName);
4404 region.put(put);
4405 fr = region.flushcache();
4406 assertTrue(fr.isFlushSucceeded());
4407 assertTrue(fr.isCompactionNeeded());
4408 }
4409 }
4410
4411 private Configuration initSplit() {
4412
4413 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4414
4415
4416 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4417
4418 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4419
4420
4421 CONF.setLong("hbase.client.pause", 15 * 1000);
4422
4423
4424
4425 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4426 return CONF;
4427 }
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4439 byte[]... families) throws IOException {
4440 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4441 }
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4453 byte[]... families) throws IOException {
4454 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4455 }
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4468 boolean isReadOnly, byte[]... families) throws IOException {
4469 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4470 }
4471
4472 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4473 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4474 throws IOException {
4475 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4476 Durability.SYNC_WAL, null, families);
4477 }
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4492 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4493 HLog hlog, byte[]... families) throws IOException {
4494 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4495 }
4496
4497
4498
4499
4500
4501 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4502 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4503 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4504 Bytes.toString(CellUtil.cloneRow(kv)));
4505 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4506 Bytes.toString(CellUtil.cloneFamily(kv)));
4507 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4508 Bytes.toString(CellUtil.cloneQualifier(kv)));
4509 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4510 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4511 Bytes.toString(CellUtil.cloneValue(kv)));
4512 }
4513
4514 @Test (timeout=60000)
4515 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4516 throws IOException {
4517 byte[] rowC = Bytes.toBytes("rowC");
4518 byte[] rowA = Bytes.toBytes("rowA");
4519 byte[] rowB = Bytes.toBytes("rowB");
4520 byte[] cf = Bytes.toBytes("CF");
4521 byte[][] families = { cf };
4522 byte[] col = Bytes.toBytes("C");
4523 long ts = 1;
4524 String method = this.getName();
4525 this.region = initHRegion(tableName, method, families);
4526 try {
4527 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4528 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4529 null);
4530 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4531 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4532 Put put = null;
4533 put = new Put(rowC);
4534 put.add(kv1);
4535 put.add(kv11);
4536 region.put(put);
4537 put = new Put(rowA);
4538 put.add(kv2);
4539 region.put(put);
4540 put = new Put(rowB);
4541 put.add(kv3);
4542 region.put(put);
4543
4544 Scan scan = new Scan(rowC);
4545 scan.setMaxVersions(5);
4546 scan.setReversed(true);
4547 InternalScanner scanner = region.getScanner(scan);
4548 List<Cell> currRow = new ArrayList<Cell>();
4549 boolean hasNext = scanner.next(currRow);
4550 assertEquals(2, currRow.size());
4551 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4552 assertTrue(hasNext);
4553 currRow.clear();
4554 hasNext = scanner.next(currRow);
4555 assertEquals(1, currRow.size());
4556 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4557 assertTrue(hasNext);
4558 currRow.clear();
4559 hasNext = scanner.next(currRow);
4560 assertEquals(1, currRow.size());
4561 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4562 assertFalse(hasNext);
4563 scanner.close();
4564 } finally {
4565 HRegion.closeHRegion(this.region);
4566 this.region = null;
4567 }
4568 }
4569
4570 @Test (timeout=60000)
4571 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4572 throws IOException {
4573 byte[] rowC = Bytes.toBytes("rowC");
4574 byte[] rowA = Bytes.toBytes("rowA");
4575 byte[] rowB = Bytes.toBytes("rowB");
4576 byte[] rowD = Bytes.toBytes("rowD");
4577 byte[] cf = Bytes.toBytes("CF");
4578 byte[][] families = { cf };
4579 byte[] col = Bytes.toBytes("C");
4580 long ts = 1;
4581 String method = this.getName();
4582 this.region = initHRegion(tableName, method, families);
4583 try {
4584 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4585 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4586 null);
4587 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4588 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4589 Put put = null;
4590 put = new Put(rowC);
4591 put.add(kv1);
4592 put.add(kv11);
4593 region.put(put);
4594 put = new Put(rowA);
4595 put.add(kv2);
4596 region.put(put);
4597 put = new Put(rowB);
4598 put.add(kv3);
4599 region.put(put);
4600
4601 Scan scan = new Scan(rowD);
4602 List<Cell> currRow = new ArrayList<Cell>();
4603 scan.setReversed(true);
4604 scan.setMaxVersions(5);
4605 InternalScanner scanner = region.getScanner(scan);
4606 boolean hasNext = scanner.next(currRow);
4607 assertEquals(2, currRow.size());
4608 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4609 assertTrue(hasNext);
4610 currRow.clear();
4611 hasNext = scanner.next(currRow);
4612 assertEquals(1, currRow.size());
4613 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4614 assertTrue(hasNext);
4615 currRow.clear();
4616 hasNext = scanner.next(currRow);
4617 assertEquals(1, currRow.size());
4618 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4619 assertFalse(hasNext);
4620 scanner.close();
4621 } finally {
4622 HRegion.closeHRegion(this.region);
4623 this.region = null;
4624 }
4625 }
4626
4627 @Test (timeout=60000)
4628 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4629 throws IOException {
4630 byte[] rowC = Bytes.toBytes("rowC");
4631 byte[] rowA = Bytes.toBytes("rowA");
4632 byte[] rowB = Bytes.toBytes("rowB");
4633 byte[] cf = Bytes.toBytes("CF");
4634 byte[][] families = { cf };
4635 byte[] col = Bytes.toBytes("C");
4636 long ts = 1;
4637 String method = this.getName();
4638 this.region = initHRegion(tableName, method, families);
4639 try {
4640 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4641 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4642 null);
4643 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4644 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4645 Put put = null;
4646 put = new Put(rowC);
4647 put.add(kv1);
4648 put.add(kv11);
4649 region.put(put);
4650 put = new Put(rowA);
4651 put.add(kv2);
4652 region.put(put);
4653 put = new Put(rowB);
4654 put.add(kv3);
4655 region.put(put);
4656 Scan scan = new Scan();
4657 List<Cell> currRow = new ArrayList<Cell>();
4658 scan.setReversed(true);
4659 InternalScanner scanner = region.getScanner(scan);
4660 boolean hasNext = scanner.next(currRow);
4661 assertEquals(1, currRow.size());
4662 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4663 assertTrue(hasNext);
4664 currRow.clear();
4665 hasNext = scanner.next(currRow);
4666 assertEquals(1, currRow.size());
4667 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4668 assertTrue(hasNext);
4669 currRow.clear();
4670 hasNext = scanner.next(currRow);
4671 assertEquals(1, currRow.size());
4672 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4673 assertFalse(hasNext);
4674 scanner.close();
4675 } finally {
4676 HRegion.closeHRegion(this.region);
4677 this.region = null;
4678 }
4679 }
4680
4681 @Test (timeout=60000)
4682 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4683
4684 byte[] rowA = Bytes.toBytes("rowA");
4685 byte[] rowB = Bytes.toBytes("rowB");
4686 byte[] rowC = Bytes.toBytes("rowC");
4687 byte[] rowD = Bytes.toBytes("rowD");
4688 byte[] rowE = Bytes.toBytes("rowE");
4689 byte[] cf = Bytes.toBytes("CF");
4690 byte[][] families = { cf };
4691 byte[] col1 = Bytes.toBytes("col1");
4692 byte[] col2 = Bytes.toBytes("col2");
4693 long ts = 1;
4694 String method = this.getName();
4695 this.region = initHRegion(tableName, method, families);
4696 try {
4697 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4698 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4699 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4700 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4701 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4702 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4703 Put put = null;
4704 put = new Put(rowA);
4705 put.add(kv1);
4706 region.put(put);
4707 put = new Put(rowB);
4708 put.add(kv2);
4709 region.put(put);
4710 put = new Put(rowC);
4711 put.add(kv3);
4712 region.put(put);
4713 put = new Put(rowD);
4714 put.add(kv4_1);
4715 region.put(put);
4716 put = new Put(rowD);
4717 put.add(kv4_2);
4718 region.put(put);
4719 put = new Put(rowE);
4720 put.add(kv5);
4721 region.put(put);
4722 region.flushcache();
4723 Scan scan = new Scan(rowD, rowA);
4724 scan.addColumn(families[0], col1);
4725 scan.setReversed(true);
4726 List<Cell> currRow = new ArrayList<Cell>();
4727 InternalScanner scanner = region.getScanner(scan);
4728 boolean hasNext = scanner.next(currRow);
4729 assertEquals(1, currRow.size());
4730 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4731 assertTrue(hasNext);
4732 currRow.clear();
4733 hasNext = scanner.next(currRow);
4734 assertEquals(1, currRow.size());
4735 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4736 assertTrue(hasNext);
4737 currRow.clear();
4738 hasNext = scanner.next(currRow);
4739 assertEquals(1, currRow.size());
4740 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4741 assertFalse(hasNext);
4742 scanner.close();
4743
4744 scan = new Scan(rowD, rowA);
4745 scan.addColumn(families[0], col2);
4746 scan.setReversed(true);
4747 currRow.clear();
4748 scanner = region.getScanner(scan);
4749 hasNext = scanner.next(currRow);
4750 assertEquals(1, currRow.size());
4751 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4752 scanner.close();
4753 } finally {
4754 HRegion.closeHRegion(this.region);
4755 this.region = null;
4756 }
4757 }
4758
4759 @Test (timeout=60000)
4760 public void testReverseScanner_smaller_blocksize() throws IOException {
4761
4762 byte[] rowA = Bytes.toBytes("rowA");
4763 byte[] rowB = Bytes.toBytes("rowB");
4764 byte[] rowC = Bytes.toBytes("rowC");
4765 byte[] rowD = Bytes.toBytes("rowD");
4766 byte[] rowE = Bytes.toBytes("rowE");
4767 byte[] cf = Bytes.toBytes("CF");
4768 byte[][] families = { cf };
4769 byte[] col1 = Bytes.toBytes("col1");
4770 byte[] col2 = Bytes.toBytes("col2");
4771 long ts = 1;
4772 String method = this.getName();
4773 HBaseConfiguration config = new HBaseConfiguration();
4774 config.setInt("test.block.size", 1);
4775 this.region = initHRegion(tableName, method, config, families);
4776 try {
4777 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4778 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4779 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4780 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4781 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4782 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4783 Put put = null;
4784 put = new Put(rowA);
4785 put.add(kv1);
4786 region.put(put);
4787 put = new Put(rowB);
4788 put.add(kv2);
4789 region.put(put);
4790 put = new Put(rowC);
4791 put.add(kv3);
4792 region.put(put);
4793 put = new Put(rowD);
4794 put.add(kv4_1);
4795 region.put(put);
4796 put = new Put(rowD);
4797 put.add(kv4_2);
4798 region.put(put);
4799 put = new Put(rowE);
4800 put.add(kv5);
4801 region.put(put);
4802 region.flushcache();
4803 Scan scan = new Scan(rowD, rowA);
4804 scan.addColumn(families[0], col1);
4805 scan.setReversed(true);
4806 List<Cell> currRow = new ArrayList<Cell>();
4807 InternalScanner scanner = region.getScanner(scan);
4808 boolean hasNext = scanner.next(currRow);
4809 assertEquals(1, currRow.size());
4810 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4811 assertTrue(hasNext);
4812 currRow.clear();
4813 hasNext = scanner.next(currRow);
4814 assertEquals(1, currRow.size());
4815 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4816 assertTrue(hasNext);
4817 currRow.clear();
4818 hasNext = scanner.next(currRow);
4819 assertEquals(1, currRow.size());
4820 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4821 assertFalse(hasNext);
4822 scanner.close();
4823
4824 scan = new Scan(rowD, rowA);
4825 scan.addColumn(families[0], col2);
4826 scan.setReversed(true);
4827 currRow.clear();
4828 scanner = region.getScanner(scan);
4829 hasNext = scanner.next(currRow);
4830 assertEquals(1, currRow.size());
4831 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4832 scanner.close();
4833 } finally {
4834 HRegion.closeHRegion(this.region);
4835 this.region = null;
4836 }
4837 }
4838
4839 @Test (timeout=60000)
4840 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4841 throws IOException {
4842 byte[] row0 = Bytes.toBytes("row0");
4843 byte[] row1 = Bytes.toBytes("row1");
4844 byte[] row2 = Bytes.toBytes("row2");
4845 byte[] row3 = Bytes.toBytes("row3");
4846 byte[] row4 = Bytes.toBytes("row4");
4847 byte[] row5 = Bytes.toBytes("row5");
4848 byte[] cf1 = Bytes.toBytes("CF1");
4849 byte[] cf2 = Bytes.toBytes("CF2");
4850 byte[] cf3 = Bytes.toBytes("CF3");
4851 byte[][] families = { cf1, cf2, cf3 };
4852 byte[] col = Bytes.toBytes("C");
4853 long ts = 1;
4854 String method = this.getName();
4855 HBaseConfiguration conf = new HBaseConfiguration();
4856
4857 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4858 this.region = initHRegion(tableName, method, conf, families);
4859 try {
4860
4861 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4862 null);
4863 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4864 null);
4865 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4866 KeyValue.Type.Put, null);
4867 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4868 null);
4869 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4870 null);
4871 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4872 null);
4873 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4874 KeyValue.Type.Put, null);
4875 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4876 null);
4877 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4878 KeyValue.Type.Put, null);
4879 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4880 null);
4881 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4882 null);
4883 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4884 KeyValue.Type.Put, null);
4885 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4886 null);
4887 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4888 KeyValue.Type.Put, null);
4889 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4890 null);
4891 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4892 null);
4893
4894 Put put = null;
4895 put = new Put(row1);
4896 put.add(kv1_2_1);
4897 region.put(put);
4898 put = new Put(row2);
4899 put.add(kv2_4_1);
4900 region.put(put);
4901 put = new Put(row4);
4902 put.add(kv4_5_4);
4903 put.add(kv4_5_5);
4904 region.put(put);
4905 region.flushcache();
4906
4907 put = new Put(row4);
4908 put.add(kv4_5_1);
4909 put.add(kv4_5_3);
4910 region.put(put);
4911 put = new Put(row1);
4912 put.add(kv1_2_2);
4913 region.put(put);
4914 put = new Put(row2);
4915 put.add(kv2_4_4);
4916 region.put(put);
4917 region.flushcache();
4918
4919 put = new Put(row4);
4920 put.add(kv4_5_2);
4921 region.put(put);
4922 put = new Put(row2);
4923 put.add(kv2_4_2);
4924 put.add(kv2_4_3);
4925 region.put(put);
4926 put = new Put(row3);
4927 put.add(kv3_2_2);
4928 region.put(put);
4929 region.flushcache();
4930
4931
4932 put = new Put(row0);
4933 put.add(kv0_1_1);
4934 region.put(put);
4935 put = new Put(row3);
4936 put.add(kv3_2_1);
4937 region.put(put);
4938 put = new Put(row5);
4939 put.add(kv5_2_1);
4940 put.add(kv5_2_2);
4941 region.put(put);
4942
4943 Scan scan = new Scan(row4);
4944 scan.setMaxVersions(5);
4945 scan.setBatch(3);
4946 scan.setReversed(true);
4947 InternalScanner scanner = region.getScanner(scan);
4948 List<Cell> currRow = new ArrayList<Cell>();
4949 boolean hasNext = false;
4950
4951
4952
4953 hasNext = scanner.next(currRow);
4954 assertEquals(3, currRow.size());
4955 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4956 assertTrue(hasNext);
4957 currRow.clear();
4958 hasNext = scanner.next(currRow);
4959 assertEquals(2, currRow.size());
4960 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4961 assertTrue(hasNext);
4962
4963 currRow.clear();
4964 hasNext = scanner.next(currRow);
4965 assertEquals(2, currRow.size());
4966 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4967 assertTrue(hasNext);
4968
4969
4970 currRow.clear();
4971 hasNext = scanner.next(currRow);
4972 assertEquals(3, currRow.size());
4973 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4974 assertTrue(hasNext);
4975 currRow.clear();
4976 hasNext = scanner.next(currRow);
4977 assertEquals(1, currRow.size());
4978 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4979 assertTrue(hasNext);
4980
4981 currRow.clear();
4982 hasNext = scanner.next(currRow);
4983 assertEquals(2, currRow.size());
4984 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4985 assertTrue(hasNext);
4986
4987 currRow.clear();
4988 hasNext = scanner.next(currRow);
4989 assertEquals(1, currRow.size());
4990 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4991 assertFalse(hasNext);
4992
4993 scanner.close();
4994 } finally {
4995 HRegion.closeHRegion(this.region);
4996 this.region = null;
4997 }
4998 }
4999
5000 @Test (timeout=60000)
5001 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5002 throws IOException {
5003 byte[] row1 = Bytes.toBytes("row1");
5004 byte[] row2 = Bytes.toBytes("row2");
5005 byte[] row3 = Bytes.toBytes("row3");
5006 byte[] row4 = Bytes.toBytes("row4");
5007 byte[] cf1 = Bytes.toBytes("CF1");
5008 byte[] cf2 = Bytes.toBytes("CF2");
5009 byte[] cf3 = Bytes.toBytes("CF3");
5010 byte[] cf4 = Bytes.toBytes("CF4");
5011 byte[][] families = { cf1, cf2, cf3, cf4 };
5012 byte[] col = Bytes.toBytes("C");
5013 long ts = 1;
5014 String method = this.getName();
5015 HBaseConfiguration conf = new HBaseConfiguration();
5016
5017 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5018 this.region = initHRegion(tableName, method, conf, families);
5019 try {
5020 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5021 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5022 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5023 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5024
5025 Put put = new Put(row1);
5026 put.add(kv1);
5027 region.put(put);
5028 region.flushcache();
5029
5030 put = new Put(row2);
5031 put.add(kv2);
5032 region.put(put);
5033 region.flushcache();
5034
5035 put = new Put(row3);
5036 put.add(kv3);
5037 region.put(put);
5038 region.flushcache();
5039
5040 put = new Put(row4);
5041 put.add(kv4);
5042 region.put(put);
5043
5044 Scan scan = new Scan(row4);
5045 scan.setReversed(true);
5046 scan.setBatch(10);
5047 InternalScanner scanner = region.getScanner(scan);
5048 List<Cell> currRow = new ArrayList<Cell>();
5049 boolean hasNext = scanner.next(currRow);
5050 assertEquals(1, currRow.size());
5051 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5052 assertTrue(hasNext);
5053 currRow.clear();
5054 hasNext = scanner.next(currRow);
5055 assertEquals(1, currRow.size());
5056 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5057 assertTrue(hasNext);
5058 currRow.clear();
5059 hasNext = scanner.next(currRow);
5060 assertEquals(1, currRow.size());
5061 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5062 assertTrue(hasNext);
5063 currRow.clear();
5064 hasNext = scanner.next(currRow);
5065 assertEquals(1, currRow.size());
5066 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5067 assertFalse(hasNext);
5068 } finally {
5069 HRegion.closeHRegion(this.region);
5070 this.region = null;
5071 }
5072 }
5073
5074 @Test (timeout=60000)
5075 public void testSplitRegionWithReverseScan() throws IOException {
5076 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5077 byte [] qualifier = Bytes.toBytes("qualifier");
5078 Configuration hc = initSplit();
5079 int numRows = 3;
5080 byte [][] families = {fam1};
5081
5082
5083 String method = this.getName();
5084 this.region = initHRegion(tableName, method, hc, families);
5085
5086
5087 int startRow = 100;
5088 putData(startRow, numRows, qualifier, families);
5089 int splitRow = startRow + numRows;
5090 putData(splitRow, numRows, qualifier, families);
5091 int endRow = splitRow + numRows;
5092 region.flushcache();
5093
5094 HRegion [] regions = null;
5095 try {
5096 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5097
5098 for (int i = 0; i < regions.length; i++) {
5099 regions[i] = HRegion.openHRegion(regions[i], null);
5100 }
5101
5102 assertEquals(2, regions.length);
5103
5104
5105
5106 verifyData(regions[0], startRow, numRows, qualifier, families);
5107 verifyData(regions[1], splitRow, numRows, qualifier, families);
5108
5109
5110 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5111 scan.setReversed(true);
5112 InternalScanner scanner = regions[1].getScanner(scan);
5113 List<Cell> currRow = new ArrayList<Cell>();
5114 boolean more = false;
5115 int verify = startRow + 2 * numRows - 1;
5116 do {
5117 more = scanner.next(currRow);
5118 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5119 verify--;
5120 currRow.clear();
5121 } while(more);
5122 assertEquals(verify, startRow + numRows - 1);
5123 scanner.close();
5124
5125 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5126 scan.setReversed(true);
5127 scanner = regions[1].getScanner(scan);
5128 verify = startRow + 2 * numRows - 1;
5129 do {
5130 more = scanner.next(currRow);
5131 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5132 verify--;
5133 currRow.clear();
5134 } while(more);
5135 assertEquals(verify, startRow + numRows - 1);
5136 scanner.close();
5137
5138 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5139 scan.setReversed(true);
5140 scanner = regions[0].getScanner(scan);
5141 verify = startRow + numRows - 1;
5142 do {
5143 more = scanner.next(currRow);
5144 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5145 verify--;
5146 currRow.clear();
5147 } while(more);
5148 assertEquals(verify, 99);
5149 scanner.close();
5150
5151 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5152 scan.setReversed(true);
5153 scanner = regions[0].getScanner(scan);
5154 verify = startRow + numRows - 1;
5155 do {
5156 more = scanner.next(currRow);
5157 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5158 verify--;
5159 currRow.clear();
5160 } while(more);
5161 assertEquals(verify, startRow - 1);
5162 scanner.close();
5163 } finally {
5164 HRegion.closeHRegion(this.region);
5165 this.region = null;
5166 }
5167 }
5168
5169 @Test
5170 public void testWriteRequestsCounter() throws IOException {
5171 byte[] fam = Bytes.toBytes("info");
5172 byte[][] families = { fam };
5173 this.region = initHRegion(tableName, method, CONF, families);
5174
5175 Assert.assertEquals(0L, region.getWriteRequestsCount());
5176
5177 Put put = new Put(row);
5178 put.add(fam, fam, fam);
5179
5180 Assert.assertEquals(0L, region.getWriteRequestsCount());
5181 region.put(put);
5182 Assert.assertEquals(1L, region.getWriteRequestsCount());
5183 region.put(put);
5184 Assert.assertEquals(2L, region.getWriteRequestsCount());
5185 region.put(put);
5186 Assert.assertEquals(3L, region.getWriteRequestsCount());
5187
5188 region.delete(new Delete(row));
5189 Assert.assertEquals(4L, region.getWriteRequestsCount());
5190
5191 HRegion.closeHRegion(this.region);
5192 this.region = null;
5193 }
5194
5195
5196
5197
5198 @Test (timeout=24000)
5199 public void testRegionTooBusy() throws IOException {
5200 String method = "testRegionTooBusy";
5201 byte[] tableName = Bytes.toBytes(method);
5202 byte[] family = Bytes.toBytes("family");
5203 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5204 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5205 CONF.setLong("hbase.busy.wait.duration", 1000);
5206 region = initHRegion(tableName, method, CONF, family);
5207 final AtomicBoolean stopped = new AtomicBoolean(true);
5208 Thread t = new Thread(new Runnable() {
5209 @Override
5210 public void run() {
5211 try {
5212 region.lock.writeLock().lock();
5213 stopped.set(false);
5214 while (!stopped.get()) {
5215 Thread.sleep(100);
5216 }
5217 } catch (InterruptedException ie) {
5218 } finally {
5219 region.lock.writeLock().unlock();
5220 }
5221 }
5222 });
5223 t.start();
5224 Get get = new Get(row);
5225 try {
5226 while (stopped.get()) {
5227 Thread.sleep(100);
5228 }
5229 region.get(get);
5230 fail("Should throw RegionTooBusyException");
5231 } catch (InterruptedException ie) {
5232 fail("test interrupted");
5233 } catch (RegionTooBusyException e) {
5234
5235 } finally {
5236 stopped.set(true);
5237 try {
5238 t.join();
5239 } catch (Throwable e) {
5240 }
5241
5242 HRegion.closeHRegion(region);
5243 region = null;
5244 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5245 }
5246 }
5247
5248 @Test
5249 public void testCellTTLs() throws IOException {
5250 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5251 EnvironmentEdgeManager.injectEdge(edge);
5252
5253 final byte[] row = Bytes.toBytes("testRow");
5254 final byte[] q1 = Bytes.toBytes("q1");
5255 final byte[] q2 = Bytes.toBytes("q2");
5256 final byte[] q3 = Bytes.toBytes("q3");
5257 final byte[] q4 = Bytes.toBytes("q4");
5258
5259 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5260 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5261 hcd.setTimeToLive(10);
5262 htd.addFamily(hcd);
5263
5264 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5265 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5266
5267 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5268 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5269 TEST_UTIL.getDataTestDir(), conf, htd);
5270 assertNotNull(region);
5271 try {
5272 long now = EnvironmentEdgeManager.currentTimeMillis();
5273
5274 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5275 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5276
5277 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5278
5279 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5280
5281 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5282 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5283
5284 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5285
5286 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5287
5288
5289 region.flushcache();
5290
5291
5292 Result r = region.get(new Get(row));
5293 assertNotNull(r.getValue(fam1, q1));
5294 assertNotNull(r.getValue(fam1, q2));
5295 assertNotNull(r.getValue(fam1, q3));
5296 assertNotNull(r.getValue(fam1, q4));
5297
5298
5299 edge.incrementTime(5000);
5300
5301 r = region.get(new Get(row));
5302 assertNull(r.getValue(fam1, q1));
5303 assertNotNull(r.getValue(fam1, q2));
5304 assertNotNull(r.getValue(fam1, q3));
5305 assertNotNull(r.getValue(fam1, q4));
5306
5307
5308 edge.incrementTime(5000);
5309
5310 r = region.get(new Get(row));
5311 assertNull(r.getValue(fam1, q1));
5312 assertNull(r.getValue(fam1, q2));
5313 assertNotNull(r.getValue(fam1, q3));
5314 assertNotNull(r.getValue(fam1, q4));
5315
5316
5317 edge.incrementTime(5000);
5318
5319 r = region.get(new Get(row));
5320 assertNull(r.getValue(fam1, q1));
5321 assertNull(r.getValue(fam1, q2));
5322 assertNull(r.getValue(fam1, q3));
5323 assertNotNull(r.getValue(fam1, q4));
5324
5325
5326 edge.incrementTime(10000);
5327
5328 r = region.get(new Get(row));
5329 assertNull(r.getValue(fam1, q1));
5330 assertNull(r.getValue(fam1, q2));
5331 assertNull(r.getValue(fam1, q3));
5332 assertNull(r.getValue(fam1, q4));
5333
5334
5335
5336
5337 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5338 r = region.get(new Get(row));
5339 byte[] val = r.getValue(fam1, q1);
5340 assertNotNull(val);
5341 assertEquals(Bytes.toLong(val), 1L);
5342
5343
5344 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5345 incr.setTTL(5000);
5346 region.increment(incr);
5347
5348
5349 r = region.get(new Get(row));
5350 val = r.getValue(fam1, q1);
5351 assertNotNull(val);
5352 assertEquals(Bytes.toLong(val), 2L);
5353
5354
5355 edge.incrementTime(5000);
5356
5357
5358 r = region.get(new Get(row));
5359 val = r.getValue(fam1, q1);
5360 assertNotNull(val);
5361 assertEquals(Bytes.toLong(val), 1L);
5362
5363
5364 edge.incrementTime(5000);
5365
5366
5367 r = region.get(new Get(row));
5368 assertNull(r.getValue(fam1, q1));
5369
5370 } finally {
5371 HRegion.closeHRegion(region);
5372 }
5373 }
5374
5375 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5376 byte[]... families) throws IOException {
5377 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5378 families);
5379 }
5380 }