View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.io;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.RuntimeMXBean;
25  import java.nio.ByteBuffer;
26  import java.util.ArrayList;
27  import java.util.Map;
28  import java.util.TreeMap;
29  import java.util.concurrent.ConcurrentHashMap;
30  import java.util.concurrent.ConcurrentSkipListMap;
31  import java.util.concurrent.CopyOnWriteArrayList;
32  import java.util.concurrent.CopyOnWriteArraySet;
33  import java.util.concurrent.atomic.AtomicBoolean;
34  import java.util.concurrent.atomic.AtomicInteger;
35  import java.util.concurrent.atomic.AtomicLong;
36  import java.util.concurrent.locks.ReentrantReadWriteLock;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.client.Delete;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
44  import org.apache.hadoop.hbase.io.hfile.LruCachedBlock;
45  import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
46  import org.apache.hadoop.hbase.regionserver.HRegion;
47  import org.apache.hadoop.hbase.regionserver.HStore;
48  import org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet;
49  import org.apache.hadoop.hbase.regionserver.MemStore;
50  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
51  import org.apache.hadoop.hbase.testclassification.SmallTests;
52  import org.apache.hadoop.hbase.util.ClassSize;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  import static org.junit.Assert.assertEquals;
58  
59  /**
60   * Testing the sizing that HeapSize offers and compares to the size given by
61   * ClassSize.
62   */
63  @Category(SmallTests.class)
64  public class TestHeapSize  {
65    static final Log LOG = LogFactory.getLog(TestHeapSize.class);
66    // List of classes implementing HeapSize
67    // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
68    // KeyValue, LruBlockCache, LruHashMap<K,V>, Put, HLogKey
69    
70    @BeforeClass
71    public static void beforeClass() throws Exception {
72      // Print detail on jvm so we know what is different should below test fail.
73      RuntimeMXBean b = ManagementFactory.getRuntimeMXBean();
74      LOG.info("name=" + b.getName()); 
75      LOG.info("specname=" + b.getSpecName()); 
76      LOG.info("specvendor=" + b.getSpecVendor()); 
77      LOG.info("vmname=" + b.getVmName()); 
78      LOG.info("vmversion=" + b.getVmVersion()); 
79      LOG.info("vmvendor=" + b.getVmVendor()); 
80      Map<String, String> p = b.getSystemProperties();
81      LOG.info("properties=" + p);
82    }
83  
84    /**
85     * Test our hard-coded sizing of native java objects
86     */
87    @Test
88    public void testNativeSizes() throws IOException {
89      Class<?> cl;
90      long expected;
91      long actual;
92  
93      // ArrayList
94      cl = ArrayList.class;
95      expected = ClassSize.estimateBase(cl, false);
96      actual = ClassSize.ARRAYLIST;
97      if(expected != actual) {
98        ClassSize.estimateBase(cl, true);
99        assertEquals(expected, actual);
100     }
101 
102     // ByteBuffer
103     cl = ByteBuffer.class;
104     expected = ClassSize.estimateBase(cl, false);
105     actual = ClassSize.BYTE_BUFFER;
106     if(expected != actual) {
107       ClassSize.estimateBase(cl, true);
108       assertEquals(expected, actual);
109     }
110 
111     // Integer
112     cl = Integer.class;
113     expected = ClassSize.estimateBase(cl, false);
114     actual = ClassSize.INTEGER;
115     if(expected != actual) {
116       ClassSize.estimateBase(cl, true);
117       assertEquals(expected, actual);
118     }
119 
120     // Map.Entry
121     // Interface is public, all others are not.  Hard to size via ClassSize
122 //    cl = Map.Entry.class;
123 //    expected = ClassSize.estimateBase(cl, false);
124 //    actual = ClassSize.MAP_ENTRY;
125 //    if(expected != actual) {
126 //      ClassSize.estimateBase(cl, true);
127 //      assertEquals(expected, actual);
128 //    }
129 
130     // Object
131     cl = Object.class;
132     expected = ClassSize.estimateBase(cl, false);
133     actual = ClassSize.OBJECT;
134     if(expected != actual) {
135       ClassSize.estimateBase(cl, true);
136       assertEquals(expected, actual);
137     }
138 
139     // TreeMap
140     cl = TreeMap.class;
141     expected = ClassSize.estimateBase(cl, false);
142     actual = ClassSize.TREEMAP;
143     if(expected != actual) {
144       ClassSize.estimateBase(cl, true);
145       assertEquals(expected, actual);
146     }
147 
148     // String
149     cl = String.class;
150     expected = ClassSize.estimateBase(cl, false);
151     actual = ClassSize.STRING;
152     if(expected != actual) {
153       ClassSize.estimateBase(cl, true);
154       assertEquals(expected, actual);
155     }
156 
157     // ConcurrentHashMap
158     cl = ConcurrentHashMap.class;
159     expected = ClassSize.estimateBase(cl, false);
160     actual = ClassSize.CONCURRENT_HASHMAP;
161     if(expected != actual) {
162       ClassSize.estimateBase(cl, true);
163       assertEquals(expected, actual);
164     }
165 
166     // ConcurrentSkipListMap
167     cl = ConcurrentSkipListMap.class;
168     expected = ClassSize.estimateBase(cl, false);
169     actual = ClassSize.CONCURRENT_SKIPLISTMAP;
170     if(expected != actual) {
171       ClassSize.estimateBase(cl, true);
172       assertEquals(expected, actual);
173     }
174 
175     // ReentrantReadWriteLock
176     cl = ReentrantReadWriteLock.class;
177     expected = ClassSize.estimateBase(cl, false);
178     actual = ClassSize.REENTRANT_LOCK;
179     if(expected != actual) {
180       ClassSize.estimateBase(cl, true);
181       assertEquals(expected, actual);
182     }
183 
184     // AtomicLong
185     cl = AtomicLong.class;
186     expected = ClassSize.estimateBase(cl, false);
187     actual = ClassSize.ATOMIC_LONG;
188     if(expected != actual) {
189       ClassSize.estimateBase(cl, true);
190       assertEquals(expected, actual);
191     }
192 
193     // AtomicInteger
194     cl = AtomicInteger.class;
195     expected = ClassSize.estimateBase(cl, false);
196     actual = ClassSize.ATOMIC_INTEGER;
197     if(expected != actual) {
198       ClassSize.estimateBase(cl, true);
199       assertEquals(expected, actual);
200     }
201 
202     // AtomicBoolean
203     cl = AtomicBoolean.class;
204     expected = ClassSize.estimateBase(cl, false);
205     actual = ClassSize.ATOMIC_BOOLEAN;
206     if(expected != actual) {
207       ClassSize.estimateBase(cl, true);
208       assertEquals(expected, actual);
209     }
210 
211     // CopyOnWriteArraySet
212     cl = CopyOnWriteArraySet.class;
213     expected = ClassSize.estimateBase(cl, false);
214     actual = ClassSize.COPYONWRITE_ARRAYSET;
215     if(expected != actual) {
216       ClassSize.estimateBase(cl, true);
217       assertEquals(expected, actual);
218     }
219 
220     // CopyOnWriteArrayList
221     cl = CopyOnWriteArrayList.class;
222     expected = ClassSize.estimateBase(cl, false);
223     actual = ClassSize.COPYONWRITE_ARRAYLIST;
224     if(expected != actual) {
225       ClassSize.estimateBase(cl, true);
226       assertEquals(expected, actual);
227     }
228 
229     // TimeRangeTracker
230     cl = TimeRangeTracker.class;
231     expected = ClassSize.estimateBase(cl, false);
232     actual = ClassSize.TIMERANGE_TRACKER;
233     if (expected != actual) {
234       ClassSize.estimateBase(cl, true);
235       assertEquals(expected, actual);
236     }
237 
238     // KeyValueSkipListSet
239     cl = KeyValueSkipListSet.class;
240     expected = ClassSize.estimateBase(cl, false);
241     actual = ClassSize.KEYVALUE_SKIPLIST_SET;
242     if (expected != actual) {
243       ClassSize.estimateBase(cl, true);
244       assertEquals(expected, actual);
245     }
246   }
247 
248   /**
249    * Testing the classes that implements HeapSize and are a part of 0.20.
250    * Some are not tested here for example BlockIndex which is tested in
251    * TestHFile since it is a non public class
252    * @throws IOException
253    */
254   @Test
255   public void testSizes() throws IOException {
256     Class<?> cl;
257     long expected;
258     long actual;
259 
260     //KeyValue
261     cl = KeyValue.class;
262     expected = ClassSize.estimateBase(cl, false);
263     KeyValue kv = new KeyValue();
264     actual = kv.heapSize();
265     if(expected != actual) {
266       ClassSize.estimateBase(cl, true);
267       assertEquals(expected, actual);
268     }
269 
270     //LruBlockCache Overhead
271     cl = LruBlockCache.class;
272     actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
273     expected = ClassSize.estimateBase(cl, false);
274     if(expected != actual) {
275       ClassSize.estimateBase(cl, true);
276       assertEquals(expected, actual);
277     }
278 
279     // CachedBlock Fixed Overhead
280     // We really need "deep" sizing but ClassSize does not do this.
281     // Perhaps we should do all these more in this style....
282     cl = LruCachedBlock.class;
283     actual = LruCachedBlock.PER_BLOCK_OVERHEAD;
284     expected = ClassSize.estimateBase(cl, false);
285     expected += ClassSize.estimateBase(String.class, false);
286     expected += ClassSize.estimateBase(ByteBuffer.class, false);
287     if(expected != actual) {
288       ClassSize.estimateBase(cl, true);
289       ClassSize.estimateBase(String.class, true);
290       ClassSize.estimateBase(ByteBuffer.class, true);
291       assertEquals(expected, actual);
292     }
293 
294     // MemStore Overhead
295     cl = MemStore.class;
296     actual = MemStore.FIXED_OVERHEAD;
297     expected = ClassSize.estimateBase(cl, false);
298     if(expected != actual) {
299       ClassSize.estimateBase(cl, true);
300       assertEquals(expected, actual);
301     }
302 
303     // MemStore Deep Overhead
304     actual = MemStore.DEEP_OVERHEAD;
305     expected = ClassSize.estimateBase(cl, false);
306     expected += ClassSize.estimateBase(AtomicLong.class, false);
307     expected += (2 * ClassSize.estimateBase(KeyValueSkipListSet.class, false));
308     expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false));
309     expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false));
310     if(expected != actual) {
311       ClassSize.estimateBase(cl, true);
312       ClassSize.estimateBase(AtomicLong.class, true);
313       ClassSize.estimateBase(KeyValueSkipListSet.class, true);
314       ClassSize.estimateBase(KeyValueSkipListSet.class, true);
315       ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
316       ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
317       ClassSize.estimateBase(TimeRangeTracker.class, true);
318       ClassSize.estimateBase(TimeRangeTracker.class, true);
319       assertEquals(expected, actual);
320     }
321 
322     // Store Overhead
323     cl = HStore.class;
324     actual = HStore.FIXED_OVERHEAD;
325     expected = ClassSize.estimateBase(cl, false);
326     if(expected != actual) {
327       ClassSize.estimateBase(cl, true);
328       assertEquals(expected, actual);
329     }
330 
331     // Region Overhead
332     cl = HRegion.class;
333     actual = HRegion.FIXED_OVERHEAD;
334     expected = ClassSize.estimateBase(cl, false);
335     if (expected != actual) {
336       ClassSize.estimateBase(cl, true);
337       assertEquals(expected, actual);
338     }
339 
340     // Block cache key overhead
341     cl = BlockCacheKey.class;
342     // Passing zero length file name, because estimateBase does not handle
343     // deep overhead.
344     actual = new BlockCacheKey("", 0).heapSize();
345     expected  = ClassSize.estimateBase(cl, false);
346     if (expected != actual) {
347       ClassSize.estimateBase(cl, true);
348       assertEquals(expected, actual);
349     }
350 
351     // Currently NOT testing Deep Overheads of many of these classes.
352     // Deep overheads cover a vast majority of stuff, but will not be 100%
353     // accurate because it's unclear when we're referencing stuff that's already
354     // accounted for.  But we have satisfied our two core requirements.
355     // Sizing is quite accurate now, and our tests will throw errors if
356     // any of these classes are modified without updating overhead sizes.
357   }
358 
359   @Test
360   public void testMutations(){
361     Class<?> cl;
362     long expected;
363     long actual;
364 
365     cl = TimeRange.class;
366     actual = ClassSize.TIMERANGE;
367     expected  = ClassSize.estimateBase(cl, false);
368     if (expected != actual) {
369       ClassSize.estimateBase(cl, true);
370       assertEquals(expected, actual);
371     }
372 
373     byte[] row = new byte[] { 0 };
374     cl = Put.class;
375     actual = new Put(row).MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
376     expected = ClassSize.estimateBase(cl, false);
377     //The actual TreeMap is not included in the above calculation
378     expected += ClassSize.align(ClassSize.TREEMAP);
379     if (expected != actual) {
380       ClassSize.estimateBase(cl, true);
381       assertEquals(expected, actual);
382     }
383 
384     cl = Delete.class;
385     actual = new Delete(row).MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY);
386     expected  = ClassSize.estimateBase(cl, false);
387     //The actual TreeMap is not included in the above calculation
388     expected += ClassSize.align(ClassSize.TREEMAP);
389     if (expected != actual) {
390       ClassSize.estimateBase(cl, true);
391       assertEquals(expected, actual);
392     }
393   }
394 
395 }
396