1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32 import java.util.concurrent.atomic.AtomicBoolean;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FilterFileSystem;
41 import org.apache.hadoop.fs.LocalFileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.permission.FsPermission;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.KeyValue.KVComparator;
53 import org.apache.hadoop.hbase.KeyValueUtil;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.client.Get;
56 import org.apache.hadoop.hbase.io.compress.Compression;
57 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
58 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
59 import org.apache.hadoop.hbase.io.hfile.HFile;
60 import org.apache.hadoop.hbase.io.hfile.HFileContext;
61 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
62 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
64 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
65 import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
66 import org.apache.hadoop.hbase.regionserver.wal.HLog;
67 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
68 import org.apache.hadoop.hbase.security.User;
69 import org.apache.hadoop.hbase.testclassification.MediumTests;
70 import org.apache.hadoop.hbase.util.Bytes;
71 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
73 import org.apache.hadoop.hbase.util.FSUtils;
74 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
75 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
76 import org.apache.hadoop.util.Progressable;
77 import org.junit.After;
78 import org.junit.Assert;
79 import org.junit.Before;
80 import org.junit.Rule;
81 import org.junit.Test;
82 import org.junit.experimental.categories.Category;
83 import org.junit.rules.TestName;
84 import org.mockito.Mockito;
85
86
87
88
89 @Category(MediumTests.class)
90 public class TestStore {
91 public static final Log LOG = LogFactory.getLog(TestStore.class);
92 @Rule public TestName name = new TestName();
93
94 HStore store;
95 byte [] table = Bytes.toBytes("table");
96 byte [] family = Bytes.toBytes("family");
97
98 byte [] row = Bytes.toBytes("row");
99 byte [] row2 = Bytes.toBytes("row2");
100 byte [] qf1 = Bytes.toBytes("qf1");
101 byte [] qf2 = Bytes.toBytes("qf2");
102 byte [] qf3 = Bytes.toBytes("qf3");
103 byte [] qf4 = Bytes.toBytes("qf4");
104 byte [] qf5 = Bytes.toBytes("qf5");
105 byte [] qf6 = Bytes.toBytes("qf6");
106
107 NavigableSet<byte[]> qualifiers =
108 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
109
110 List<Cell> expected = new ArrayList<Cell>();
111 List<Cell> result = new ArrayList<Cell>();
112
113 long id = System.currentTimeMillis();
114 Get get = new Get(row);
115
116 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
117 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
118
119
120
121
122
123
124 @Before
125 public void setUp() throws IOException {
126 qualifiers.add(qf1);
127 qualifiers.add(qf3);
128 qualifiers.add(qf5);
129
130 Iterator<byte[]> iter = qualifiers.iterator();
131 while(iter.hasNext()){
132 byte [] next = iter.next();
133 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
134 get.addColumn(family, next);
135 }
136 }
137
138 private void init(String methodName) throws IOException {
139 init(methodName, HBaseConfiguration.create());
140 }
141
142 private void init(String methodName, Configuration conf)
143 throws IOException {
144 HColumnDescriptor hcd = new HColumnDescriptor(family);
145
146
147 hcd.setMaxVersions(4);
148 init(methodName, conf, hcd);
149 }
150
151 private void init(String methodName, Configuration conf,
152 HColumnDescriptor hcd) throws IOException {
153 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
154 init(methodName, conf, htd, hcd);
155 }
156
157 @SuppressWarnings("deprecation")
158 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
159 HColumnDescriptor hcd) throws IOException {
160
161 Path basedir = new Path(DIR+methodName);
162 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
163 String logName = "logs";
164 Path logdir = new Path(basedir, logName);
165
166 FileSystem fs = FileSystem.get(conf);
167
168 fs.delete(logdir, true);
169
170 htd.addFamily(hcd);
171 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
172 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
173 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
174
175 store = new HStore(region, hcd, conf);
176 return store;
177 }
178
179
180
181
182
183
184 @Test
185 public void testFlushSizeAccounting() throws Exception {
186 LOG.info("Setting up a faulty file system that cannot write in " +
187 this.name.getMethodName());
188 final Configuration conf = HBaseConfiguration.create();
189
190 conf.setInt("hbase.hstore.flush.retries.number", 1);
191 User user = User.createUserForTesting(conf, this.name.getMethodName(),
192 new String[]{"foo"});
193
194 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
195 user.runAs(new PrivilegedExceptionAction<Object>() {
196 public Object run() throws Exception {
197
198 FileSystem fs = FileSystem.get(conf);
199 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
200 FaultyFileSystem ffs = (FaultyFileSystem)fs;
201
202
203 init(name.getMethodName(), conf);
204
205 long size = store.memstore.getFlushableSize();
206 Assert.assertEquals(0, size);
207 LOG.info("Adding some data");
208 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
209 size = store.memstore.getFlushableSize();
210 Assert.assertEquals(kvSize, size);
211
212 try {
213 LOG.info("Flushing");
214 flushStore(store, id++);
215 Assert.fail("Didn't bubble up IOE!");
216 } catch (IOException ioe) {
217 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
218 }
219 size = store.memstore.getFlushableSize();
220 Assert.assertEquals(kvSize, size);
221 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
222
223
224 Assert.assertEquals(kvSize, size);
225 ffs.fault.set(false);
226 flushStore(store, id++);
227 size = store.memstore.getFlushableSize();
228
229 Assert.assertEquals(kvSize, size);
230 flushStore(store, id++);
231 size = store.memstore.getFlushableSize();
232 Assert.assertEquals(0, size);
233 return null;
234 }
235 });
236 }
237
238
239
240
241
242 @Test
243 public void testCreateWriter() throws Exception {
244 Configuration conf = HBaseConfiguration.create();
245 FileSystem fs = FileSystem.get(conf);
246
247 HColumnDescriptor hcd = new HColumnDescriptor(family);
248 hcd.setCompressionType(Compression.Algorithm.GZ);
249 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
250 init(name.getMethodName(), conf, hcd);
251
252
253 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
254 Path path = writer.getPath();
255 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
256 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
257 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
258 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
259 writer.close();
260
261
262 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
263 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
264 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
265 reader.close();
266 }
267
268 @Test
269 public void testDeleteExpiredStoreFiles() throws Exception {
270 testDeleteExpiredStoreFiles(0);
271 testDeleteExpiredStoreFiles(1);
272 }
273
274
275
276
277 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
278 int storeFileNum = 4;
279 int ttl = 4;
280 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
281 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
282
283 Configuration conf = HBaseConfiguration.create();
284
285 conf.setBoolean("hbase.store.delete.expired.storefile", true);
286
287 conf.setInt(CompactionConfiguration.MIN_KEY, 5);
288
289 HColumnDescriptor hcd = new HColumnDescriptor(family);
290 hcd.setMinVersions(minVersions);
291 hcd.setTimeToLive(ttl);
292 init(name.getMethodName(), conf, hcd);
293
294 long storeTtl = this.store.getScanInfo().getTtl();
295 long sleepTime = storeTtl / storeFileNum;
296 long timeStamp;
297
298
299 for (int i = 1; i <= storeFileNum; i++) {
300 LOG.info("Adding some data for the store file #" + i);
301 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
302 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
303 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
304 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
305 flush(i);
306 edge.incrementTime(sleepTime);
307 }
308
309
310 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
311
312
313
314 for (int i = 1; i <= storeFileNum - 1; i++) {
315
316 Assert.assertNull(this.store.requestCompaction());
317 Collection<StoreFile> sfs = this.store.getStorefiles();
318
319 if (minVersions == 0) {
320 Assert.assertEquals(storeFileNum - i, sfs.size());
321
322 for (StoreFile sf : sfs) {
323 Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
324 }
325 } else {
326 Assert.assertEquals(storeFileNum, sfs.size());
327 }
328
329 edge.incrementTime(sleepTime);
330 }
331 Assert.assertNull(this.store.requestCompaction());
332 Collection<StoreFile> sfs = this.store.getStorefiles();
333
334 if (minVersions == 0) {
335 Assert.assertEquals(1, sfs.size());
336 }
337 long ts = sfs.iterator().next().getReader().getMaxTimestamp();
338 Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
339 }
340
341 @Test
342 public void testLowestModificationTime() throws Exception {
343 Configuration conf = HBaseConfiguration.create();
344 FileSystem fs = FileSystem.get(conf);
345
346 init(name.getMethodName(), conf);
347
348 int storeFileNum = 4;
349 for (int i = 1; i <= storeFileNum; i++) {
350 LOG.info("Adding some data for the store file #"+i);
351 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
352 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
353 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
354 flush(i);
355 }
356
357 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
358 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
359 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
360
361
362 store.compact(store.requestCompaction(), NoLimitCompactionThroughputController.INSTANCE);
363 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
364 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
365 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
366 }
367
368 private static long getLowestTimeStampFromFS(FileSystem fs,
369 final Collection<StoreFile> candidates) throws IOException {
370 long minTs = Long.MAX_VALUE;
371 if (candidates.isEmpty()) {
372 return minTs;
373 }
374 Path[] p = new Path[candidates.size()];
375 int i = 0;
376 for (StoreFile sf : candidates) {
377 p[i] = sf.getPath();
378 ++i;
379 }
380
381 FileStatus[] stats = fs.listStatus(p);
382 if (stats == null || stats.length == 0) {
383 return minTs;
384 }
385 for (FileStatus s : stats) {
386 minTs = Math.min(minTs, s.getModificationTime());
387 }
388 return minTs;
389 }
390
391
392
393
394
395 private static final int BLOCKSIZE_SMALL = 8192;
396
397
398
399
400 @Test
401 public void testEmptyStoreFile() throws IOException {
402 init(this.name.getMethodName());
403
404 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
405 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
406 flush(1);
407
408
409 StoreFile f = this.store.getStorefiles().iterator().next();
410 Path storedir = f.getPath().getParent();
411 long seqid = f.getMaxSequenceId();
412 Configuration c = HBaseConfiguration.create();
413 FileSystem fs = FileSystem.get(c);
414 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
415 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
416 fs)
417 .withOutputDir(storedir)
418 .withFileContext(meta)
419 .build();
420 w.appendMetadata(seqid + 1, false);
421 w.close();
422 this.store.close();
423
424 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
425 Assert.assertEquals(2, this.store.getStorefilesCount());
426
427 result = HBaseTestingUtility.getFromStoreFile(store,
428 get.getRow(),
429 qualifiers);
430 Assert.assertEquals(1, result.size());
431 }
432
433
434
435
436
437 @Test
438 public void testGet_FromMemStoreOnly() throws IOException {
439 init(this.name.getMethodName());
440
441
442 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
443 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
444 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
445 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
446 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
447 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
448
449
450 result = HBaseTestingUtility.getFromStoreFile(store,
451 get.getRow(), qualifiers);
452
453
454 assertCheck();
455 }
456
457
458
459
460
461 @Test
462 public void testGet_FromFilesOnly() throws IOException {
463 init(this.name.getMethodName());
464
465
466 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
467 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
468
469 flush(1);
470
471
472 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
473 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
474
475 flush(2);
476
477
478 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
479 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
480
481 flush(3);
482
483
484 result = HBaseTestingUtility.getFromStoreFile(store,
485 get.getRow(),
486 qualifiers);
487
488
489
490 Collections.sort(result, KeyValue.COMPARATOR);
491
492
493 assertCheck();
494 }
495
496
497
498
499
500 @Test
501 public void testGet_FromMemStoreAndFiles() throws IOException {
502 init(this.name.getMethodName());
503
504
505 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
506 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
507
508 flush(1);
509
510
511 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
512 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
513
514 flush(2);
515
516
517 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
518 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
519
520
521 result = HBaseTestingUtility.getFromStoreFile(store,
522 get.getRow(), qualifiers);
523
524
525 Collections.sort(result, KeyValue.COMPARATOR);
526
527
528 assertCheck();
529 }
530
531 private void flush(int storeFilessize) throws IOException{
532 this.store.snapshot();
533 flushStore(store, id++);
534 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
535 Assert.assertEquals(0, this.store.memstore.kvset.size());
536 }
537
538 private void assertCheck() {
539 Assert.assertEquals(expected.size(), result.size());
540 for(int i=0; i<expected.size(); i++) {
541 Assert.assertEquals(expected.get(i), result.get(i));
542 }
543 }
544
545
546
547
548
549
550
551 @Test
552 public void testIncrementColumnValue_ICVDuringFlush()
553 throws IOException, InterruptedException {
554 init(this.name.getMethodName());
555
556 long oldValue = 1L;
557 long newValue = 3L;
558 this.store.add(new KeyValue(row, family, qf1,
559 System.currentTimeMillis(),
560 Bytes.toBytes(oldValue)));
561
562
563 this.store.snapshot();
564
565
566 this.store.add(new KeyValue(row, family, qf2,
567 System.currentTimeMillis(),
568 Bytes.toBytes(oldValue)));
569
570
571 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
572
573
574 Assert.assertTrue(ret > 0);
575
576
577 flushStore(store, id++);
578 Assert.assertEquals(1, this.store.getStorefiles().size());
579
580 Assert.assertEquals(2, this.store.memstore.kvset.size());
581
582
583 Get get = new Get(row);
584 get.addColumn(family, qf1);
585 get.setMaxVersions();
586 List<Cell> results = new ArrayList<Cell>();
587
588 results = HBaseTestingUtility.getFromStoreFile(store, get);
589 Assert.assertEquals(2, results.size());
590
591 long ts1 = results.get(0).getTimestamp();
592 long ts2 = results.get(1).getTimestamp();
593
594 Assert.assertTrue(ts1 > ts2);
595
596 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
597 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
598 }
599
600 @After
601 public void tearDown() throws Exception {
602 EnvironmentEdgeManagerTestHelper.reset();
603 }
604
605 @Test
606 public void testICV_negMemstoreSize() throws IOException {
607 init(this.name.getMethodName());
608
609 long time = 100;
610 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
611 ee.setValue(time);
612 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
613 long newValue = 3L;
614 long size = 0;
615
616
617 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
618 System.currentTimeMillis(),
619 Bytes.toBytes(newValue)));
620 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
621 System.currentTimeMillis(),
622 Bytes.toBytes(newValue)));
623 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
624 System.currentTimeMillis(),
625 Bytes.toBytes(newValue)));
626 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
627 System.currentTimeMillis(),
628 Bytes.toBytes(newValue)));
629 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
630 System.currentTimeMillis(),
631 Bytes.toBytes(newValue)));
632
633
634 for ( int i = 0 ; i < 10000 ; ++i) {
635 newValue++;
636
637 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
638 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
639
640 if (ret != 0) System.out.println("ret: " + ret);
641 if (ret2 != 0) System.out.println("ret2: " + ret2);
642
643 Assert.assertTrue("ret: " + ret, ret >= 0);
644 size += ret;
645 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
646 size += ret2;
647
648
649 if (i % 1000 == 0)
650 ee.setValue(++time);
651 }
652
653 long computedSize=0;
654 for (KeyValue kv : this.store.memstore.kvset) {
655 long kvsize = MemStore.heapSizeChange(kv, true);
656
657 computedSize += kvsize;
658 }
659 Assert.assertEquals(computedSize, size);
660 }
661
662 @Test
663 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
664 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
665 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
666 init(this.name.getMethodName());
667
668 long oldValue = 1L;
669 long newValue = 3L;
670 this.store.add(new KeyValue(row, family, qf1,
671 EnvironmentEdgeManager.currentTimeMillis(),
672 Bytes.toBytes(oldValue)));
673
674
675 this.store.snapshot();
676
677
678 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
679
680
681 Assert.assertTrue(ret > 0);
682
683
684 flushStore(store, id++);
685 Assert.assertEquals(1, this.store.getStorefiles().size());
686 Assert.assertEquals(1, this.store.memstore.kvset.size());
687
688
689 newValue += 1;
690 this.store.updateColumnValue(row, family, qf1, newValue);
691
692
693 newValue += 1;
694 this.store.updateColumnValue(row, family, qf1, newValue);
695
696
697
698
699
700 Get get = new Get(row);
701 get.addColumn(family, qf1);
702 get.setMaxVersions();
703 List<Cell> results = new ArrayList<Cell>();
704
705 results = HBaseTestingUtility.getFromStoreFile(store, get);
706 Assert.assertEquals(2, results.size());
707
708 long ts1 = results.get(0).getTimestamp();
709 long ts2 = results.get(1).getTimestamp();
710
711 Assert.assertTrue(ts1 > ts2);
712 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
713 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
714
715 mee.setValue(2);
716 newValue += 1;
717 this.store.updateColumnValue(row, family, qf1, newValue);
718
719 results = HBaseTestingUtility.getFromStoreFile(store, get);
720 Assert.assertEquals(2, results.size());
721
722 ts1 = results.get(0).getTimestamp();
723 ts2 = results.get(1).getTimestamp();
724
725 Assert.assertTrue(ts1 > ts2);
726 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
727 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
728 }
729
730 @Test
731 public void testHandleErrorsInFlush() throws Exception {
732 LOG.info("Setting up a faulty file system that cannot write");
733
734 final Configuration conf = HBaseConfiguration.create();
735 User user = User.createUserForTesting(conf,
736 "testhandleerrorsinflush", new String[]{"foo"});
737
738 conf.setClass("fs.file.impl", FaultyFileSystem.class,
739 FileSystem.class);
740 user.runAs(new PrivilegedExceptionAction<Object>() {
741 public Object run() throws Exception {
742
743 FileSystem fs = FileSystem.get(conf);
744 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
745
746
747 init(name.getMethodName(), conf);
748
749 LOG.info("Adding some data");
750 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
751 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
752 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
753
754 LOG.info("Before flush, we should have no files");
755
756 Collection<StoreFileInfo> files =
757 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
758 Assert.assertEquals(0, files != null ? files.size() : 0);
759
760
761 try {
762 LOG.info("Flushing");
763 flush(1);
764 Assert.fail("Didn't bubble up IOE!");
765 } catch (IOException ioe) {
766 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
767 }
768
769 LOG.info("After failed flush, we should still have no files!");
770 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
771 Assert.assertEquals(0, files != null ? files.size() : 0);
772 store.getHRegion().getLog().closeAndDelete();
773 return null;
774 }
775 });
776 FileSystem.closeAllForUGI(user.getUGI());
777 }
778
779
780
781
782
783 static class FaultyFileSystem extends FilterFileSystem {
784 List<SoftReference<FaultyOutputStream>> outStreams =
785 new ArrayList<SoftReference<FaultyOutputStream>>();
786 private long faultPos = 200;
787 AtomicBoolean fault = new AtomicBoolean(true);
788
789 public FaultyFileSystem() {
790 super(new LocalFileSystem());
791 System.err.println("Creating faulty!");
792 }
793
794 @Override
795 public FSDataOutputStream create(Path p) throws IOException {
796 return new FaultyOutputStream(super.create(p), faultPos, fault);
797 }
798
799 @Override
800 public FSDataOutputStream create(Path f, FsPermission permission,
801 boolean overwrite, int bufferSize, short replication, long blockSize,
802 Progressable progress) throws IOException {
803 return new FaultyOutputStream(super.create(f, permission,
804 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
805 }
806
807 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
808 int bufferSize, short replication, long blockSize, Progressable progress)
809 throws IOException {
810
811
812 return create(f, overwrite, bufferSize, replication, blockSize, progress);
813 }
814 }
815
816 static class FaultyOutputStream extends FSDataOutputStream {
817 volatile long faultPos = Long.MAX_VALUE;
818 private final AtomicBoolean fault;
819
820 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
821 throws IOException {
822 super(out, null);
823 this.faultPos = faultPos;
824 this.fault = fault;
825 }
826
827 @Override
828 public void write(byte[] buf, int offset, int length) throws IOException {
829 System.err.println("faulty stream write at pos " + getPos());
830 injectFault();
831 super.write(buf, offset, length);
832 }
833
834 private void injectFault() throws IOException {
835 if (this.fault.get() && getPos() >= faultPos) {
836 throw new IOException("Fault injected");
837 }
838 }
839 }
840
841 private static void flushStore(HStore store, long id) throws IOException {
842 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
843 storeFlushCtx.prepare();
844 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
845 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
846 }
847
848
849
850
851
852
853
854
855
856 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
857 byte[] qualifier, byte[] family) {
858 List<Cell> kvList = new ArrayList<Cell>();
859 for (int i=1;i<=numRows;i++) {
860 byte[] b = Bytes.toBytes(i);
861 for (long timestamp: timestamps) {
862 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
863 }
864 }
865 return kvList;
866 }
867
868
869
870
871
872 @Test
873 public void testMultipleTimestamps() throws IOException {
874 int numRows = 1;
875 long[] timestamps1 = new long[] {1,5,10,20};
876 long[] timestamps2 = new long[] {30,80};
877
878 init(this.name.getMethodName());
879
880 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
881 for (Cell kv : kvList1) {
882 this.store.add(KeyValueUtil.ensureKeyValue(kv));
883 }
884
885 this.store.snapshot();
886 flushStore(store, id++);
887
888 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
889 for(Cell kv : kvList2) {
890 this.store.add(KeyValueUtil.ensureKeyValue(kv));
891 }
892
893 List<Cell> result;
894 Get get = new Get(Bytes.toBytes(1));
895 get.addColumn(family,qf1);
896
897 get.setTimeRange(0,15);
898 result = HBaseTestingUtility.getFromStoreFile(store, get);
899 Assert.assertTrue(result.size()>0);
900
901 get.setTimeRange(40,90);
902 result = HBaseTestingUtility.getFromStoreFile(store, get);
903 Assert.assertTrue(result.size()>0);
904
905 get.setTimeRange(10,45);
906 result = HBaseTestingUtility.getFromStoreFile(store, get);
907 Assert.assertTrue(result.size()>0);
908
909 get.setTimeRange(80,145);
910 result = HBaseTestingUtility.getFromStoreFile(store, get);
911 Assert.assertTrue(result.size()>0);
912
913 get.setTimeRange(1,2);
914 result = HBaseTestingUtility.getFromStoreFile(store, get);
915 Assert.assertTrue(result.size()>0);
916
917 get.setTimeRange(90,200);
918 result = HBaseTestingUtility.getFromStoreFile(store, get);
919 Assert.assertTrue(result.size()==0);
920 }
921
922
923
924
925
926
927 @Test
928 public void testSplitWithEmptyColFam() throws IOException {
929 init(this.name.getMethodName());
930 Assert.assertNull(store.getSplitPoint());
931 store.getHRegion().forceSplit(null);
932 Assert.assertNull(store.getSplitPoint());
933 store.getHRegion().clearSplit();
934 }
935
936 @Test
937 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
938 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
939 long anyValue = 10;
940
941
942
943
944 Configuration conf = HBaseConfiguration.create();
945 conf.setLong(CONFIG_KEY, anyValue);
946 init(name.getMethodName() + "-xml", conf);
947 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
948 Assert.assertFalse(store.throttleCompaction(anyValue));
949
950
951 --anyValue;
952 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
953 HColumnDescriptor hcd = new HColumnDescriptor(family);
954 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
955 init(name.getMethodName() + "-htd", conf, htd, hcd);
956 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
957 Assert.assertFalse(store.throttleCompaction(anyValue));
958
959
960 --anyValue;
961 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
962 init(name.getMethodName() + "-hcd", conf, htd, hcd);
963 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
964 Assert.assertFalse(store.throttleCompaction(anyValue));
965 }
966
967 public static class DummyStoreEngine extends DefaultStoreEngine {
968 public static DefaultCompactor lastCreatedCompactor = null;
969 @Override
970 protected void createComponents(
971 Configuration conf, Store store, KVComparator comparator) throws IOException {
972 super.createComponents(conf, store, comparator);
973 lastCreatedCompactor = this.compactor;
974 }
975 }
976
977 @Test
978 public void testStoreUsesSearchEngineOverride() throws Exception {
979 Configuration conf = HBaseConfiguration.create();
980 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
981 init(this.name.getMethodName(), conf);
982 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
983 this.store.storeEngine.getCompactor());
984 }
985 }