1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile.bucket;
20
21 import static org.junit.Assert.assertEquals;
22
23 import java.io.FileNotFoundException;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.List;
28 import java.util.Random;
29
30 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
31 import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
32 import org.apache.hadoop.hbase.io.hfile.Cacheable;
33 import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.BucketSizeInfo;
34 import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.IndexStatistics;
35 import org.apache.hadoop.hbase.testclassification.SmallTests;
36 import org.junit.After;
37 import org.junit.Before;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40 import org.junit.runner.RunWith;
41 import org.junit.runners.Parameterized;
42
43
44
45
46
47
48
49 @RunWith(Parameterized.class)
50 @Category(SmallTests.class)
51 public class TestBucketCache {
52
53 private static final Random RAND = new Random();
54
55 @Parameterized.Parameters(name="{index}: blockSize={0}, bucketSizes={1}")
56 public static Iterable<Object[]> data() {
57 return Arrays.asList(new Object[][] {
58 { 8192, null },
59 { 16 * 1024, new int[] {
60 2 * 1024 + 1024, 4 * 1024 + 1024, 8 * 1024 + 1024, 16 * 1024 + 1024,
61 28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024,
62 128 * 1024 + 1024 } }
63 });
64 }
65
66 @Parameterized.Parameter(0)
67 public int constructedBlockSize;
68
69 @Parameterized.Parameter(1)
70 public int[] constructedBlockSizes;
71
72 BucketCache cache;
73 final int CACHE_SIZE = 1000000;
74 final int NUM_BLOCKS = 100;
75 final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS;
76 final int NUM_THREADS = 1000;
77 final int NUM_QUERIES = 10000;
78
79 final long capacitySize = 32 * 1024 * 1024;
80 final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS;
81 final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS;
82 String ioEngineName = "heap";
83 String persistencePath = null;
84
85 private class MockedBucketCache extends BucketCache {
86
87 public MockedBucketCache(String ioEngineName, long capacity, int blockSize, int[] bucketSizes,
88 int writerThreads, int writerQLen, String persistencePath)
89 throws FileNotFoundException, IOException {
90 super(ioEngineName, capacity, blockSize, bucketSizes, writerThreads, writerQLen,
91 persistencePath);
92 super.wait_when_cache = true;
93 }
94
95 @Override
96 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf,
97 boolean inMemory) {
98 if (super.getBlock(cacheKey, true, false, true) != null) {
99 throw new RuntimeException("Cached an already cached block");
100 }
101 super.cacheBlock(cacheKey, buf, inMemory);
102 }
103
104 @Override
105 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
106 if (super.getBlock(cacheKey, true, false, true) != null) {
107 throw new RuntimeException("Cached an already cached block");
108 }
109 super.cacheBlock(cacheKey, buf);
110 }
111 }
112
113 @Before
114 public void setup() throws FileNotFoundException, IOException {
115 cache = new MockedBucketCache(ioEngineName, capacitySize, constructedBlockSize,
116 constructedBlockSizes, writeThreads, writerQLen, persistencePath);
117 }
118
119 @After
120 public void tearDown() {
121 cache.shutdown();
122 }
123
124
125
126
127 private static <T> T randFrom(List<T> a) {
128 return a.get(RAND.nextInt(a.size()));
129 }
130
131 @Test
132 public void testBucketAllocator() throws BucketAllocatorException {
133 BucketAllocator mAllocator = cache.getAllocator();
134
135
136
137 final List<Integer> BLOCKSIZES = Arrays.asList(4 * 1024, 8 * 1024, 64 * 1024, 96 * 1024);
138
139 boolean full = false;
140 ArrayList<Long> allocations = new ArrayList<Long>();
141
142
143 List<Integer> tmp = new ArrayList<Integer>(BLOCKSIZES);
144 for (int i = 0; !full; i++) {
145 Integer blockSize = null;
146 try {
147 blockSize = randFrom(tmp);
148 allocations.add(mAllocator.allocateBlock(blockSize));
149 } catch (CacheFullException cfe) {
150 tmp.remove(blockSize);
151 if (tmp.isEmpty()) full = true;
152 }
153 }
154
155 for (Integer blockSize : BLOCKSIZES) {
156 BucketSizeInfo bucketSizeInfo = mAllocator.roundUpToBucketSizeInfo(blockSize);
157 IndexStatistics indexStatistics = bucketSizeInfo.statistics();
158 assertEquals(
159 "unexpected freeCount for " + bucketSizeInfo,
160 0, indexStatistics.freeCount());
161 }
162
163 for (long offset : allocations) {
164 assertEquals(mAllocator.sizeOfAllocation(offset), mAllocator.freeBlock(offset));
165 }
166 assertEquals(0, mAllocator.getUsedSize());
167 }
168
169 @Test
170 public void testCacheSimple() throws Exception {
171 CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES);
172 }
173
174 @Test
175 public void testCacheMultiThreadedSingleKey() throws Exception {
176 CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES);
177 }
178
179 @Test
180 public void testHeapSizeChanges() throws Exception {
181 cache.stopWriterThreads();
182 CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
183 }
184
185 }