View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile.bucket;
20  
21  import static org.junit.Assert.assertEquals;
22  
23  import java.io.FileNotFoundException;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.List;
28  import java.util.Random;
29  
30  import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
31  import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
32  import org.apache.hadoop.hbase.io.hfile.Cacheable;
33  import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.BucketSizeInfo;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.IndexStatistics;
35  import org.apache.hadoop.hbase.testclassification.SmallTests;
36  import org.junit.After;
37  import org.junit.Before;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  import org.junit.runner.RunWith;
41  import org.junit.runners.Parameterized;
42  
43  /**
44   * Basic test of BucketCache.Puts and gets.
45   * <p>
46   * Tests will ensure that blocks' data correctness under several threads
47   * concurrency
48   */
49  @RunWith(Parameterized.class)
50  @Category(SmallTests.class)
51  public class TestBucketCache {
52  
53    private static final Random RAND = new Random();
54  
55    @Parameterized.Parameters(name="{index}: blockSize={0}, bucketSizes={1}")
56    public static Iterable<Object[]> data() {
57      return Arrays.asList(new Object[][] {
58        { 8192, null }, // TODO: why is 8k the default blocksize for these tests?
59        { 16 * 1024, new int[] {
60          2 * 1024 + 1024, 4 * 1024 + 1024, 8 * 1024 + 1024, 16 * 1024 + 1024,
61          28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024,
62          128 * 1024 + 1024 } }
63      });
64    }
65  
66    @Parameterized.Parameter(0)
67    public int constructedBlockSize;
68  
69    @Parameterized.Parameter(1)
70    public int[] constructedBlockSizes;
71  
72    BucketCache cache;
73    final int CACHE_SIZE = 1000000;
74    final int NUM_BLOCKS = 100;
75    final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS;
76    final int NUM_THREADS = 1000;
77    final int NUM_QUERIES = 10000;
78    
79    final long capacitySize = 32 * 1024 * 1024;
80    final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS;
81    final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS;
82    String ioEngineName = "heap";
83    String persistencePath = null;
84  
85    private class MockedBucketCache extends BucketCache {
86  
87      public MockedBucketCache(String ioEngineName, long capacity, int blockSize, int[] bucketSizes,
88        int writerThreads, int writerQLen, String persistencePath)
89          throws FileNotFoundException, IOException {
90        super(ioEngineName, capacity, blockSize, bucketSizes, writerThreads, writerQLen,
91          persistencePath);
92        super.wait_when_cache = true;
93      }
94  
95      @Override
96      public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf,
97          boolean inMemory) {
98        if (super.getBlock(cacheKey, true, false, true) != null) {
99          throw new RuntimeException("Cached an already cached block");
100       }
101       super.cacheBlock(cacheKey, buf, inMemory);
102     }
103 
104     @Override
105     public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
106       if (super.getBlock(cacheKey, true, false, true) != null) {
107         throw new RuntimeException("Cached an already cached block");
108       }
109       super.cacheBlock(cacheKey, buf);
110     }
111   }
112 
113   @Before
114   public void setup() throws FileNotFoundException, IOException {
115     cache = new MockedBucketCache(ioEngineName, capacitySize, constructedBlockSize,
116       constructedBlockSizes, writeThreads, writerQLen, persistencePath);
117   }
118 
119   @After
120   public void tearDown() {
121     cache.shutdown();
122   }
123 
124   /**
125    * Return a random element from {@code a}.
126    */
127   private static <T> T randFrom(List<T> a) {
128     return a.get(RAND.nextInt(a.size()));
129   }
130 
131   @Test
132   public void testBucketAllocator() throws BucketAllocatorException {
133     BucketAllocator mAllocator = cache.getAllocator();
134     /*
135      * Test the allocator first
136      */
137     final List<Integer> BLOCKSIZES = Arrays.asList(4 * 1024, 8 * 1024, 64 * 1024, 96 * 1024);
138 
139     boolean full = false;
140     ArrayList<Long> allocations = new ArrayList<Long>();
141     // Fill the allocated extents by choosing a random blocksize. Continues selecting blocks until
142     // the cache is completely filled.
143     List<Integer> tmp = new ArrayList<Integer>(BLOCKSIZES);
144     for (int i = 0; !full; i++) {
145       Integer blockSize = null;
146       try {
147         blockSize = randFrom(tmp);
148         allocations.add(mAllocator.allocateBlock(blockSize));
149       } catch (CacheFullException cfe) {
150         tmp.remove(blockSize);
151         if (tmp.isEmpty()) full = true;
152       }
153     }
154 
155     for (Integer blockSize : BLOCKSIZES) {
156       BucketSizeInfo bucketSizeInfo = mAllocator.roundUpToBucketSizeInfo(blockSize);
157       IndexStatistics indexStatistics = bucketSizeInfo.statistics();
158       assertEquals(
159         "unexpected freeCount for " + bucketSizeInfo,
160         0, indexStatistics.freeCount());
161     }
162 
163     for (long offset : allocations) {
164       assertEquals(mAllocator.sizeOfAllocation(offset), mAllocator.freeBlock(offset));
165     }
166     assertEquals(0, mAllocator.getUsedSize());
167   }
168 
169   @Test
170   public void testCacheSimple() throws Exception {
171     CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES);
172   }
173 
174   @Test
175   public void testCacheMultiThreadedSingleKey() throws Exception {
176     CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES);
177   }
178 
179   @Test
180   public void testHeapSizeChanges() throws Exception {
181     cache.stopWriterThreads();
182     CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
183   }
184 
185 }