1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import com.google.common.annotations.VisibleForTesting;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67 "hbase.block.data.cachecompressed";
68
69
70
71
72
73 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74 "hbase.rs.evictblocksonclose";
75
76
77
78
79 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
82 "hbase.bucketcache.persistent.path";
83 public static final String BUCKET_CACHE_COMBINED_KEY =
84 "hbase.bucketcache.combinedcache.enabled";
85 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
86 "hbase.bucketcache.percentage.in.combinedcache";
87 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
89 "hbase.bucketcache.writer.queuelength";
90
91
92
93
94 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95
96
97
98
99 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103
104
105
106
107
108 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109 "hbase.rs.prefetchblocksonopen";
110
111
112
113
114
115
116
117 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118
119
120
121 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
122 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
123 public static final boolean DEFAULT_IN_MEMORY = false;
124 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
125 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
126 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
127 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
128 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
129
130
131 private final BlockCache blockCache;
132
133
134
135
136
137 private boolean cacheDataOnRead;
138
139
140 private final boolean inMemory;
141
142
143 private boolean cacheDataOnWrite;
144
145
146 private final boolean cacheIndexesOnWrite;
147
148
149 private final boolean cacheBloomsOnWrite;
150
151
152 private boolean evictOnClose;
153
154
155 private final boolean cacheDataCompressed;
156
157
158 private final boolean prefetchOnOpen;
159
160
161
162
163
164
165
166 public CacheConfig(Configuration conf, HColumnDescriptor family) {
167 this(CacheConfig.instantiateBlockCache(conf),
168 family.isBlockCacheEnabled(),
169 family.isInMemory(),
170
171
172 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
173 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
174 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
175 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
176 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
177 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
178 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
179 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
180 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
181 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
182 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
183 );
184 }
185
186
187
188
189
190
191 public CacheConfig(Configuration conf) {
192 this(CacheConfig.instantiateBlockCache(conf),
193 DEFAULT_CACHE_DATA_ON_READ,
194 DEFAULT_IN_MEMORY,
195
196 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
197 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
198 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
199 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
200 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
201 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
202 );
203 }
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218 CacheConfig(final BlockCache blockCache,
219 final boolean cacheDataOnRead, final boolean inMemory,
220 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
221 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
222 final boolean cacheDataCompressed, final boolean prefetchOnOpen) {
223 this.blockCache = blockCache;
224 this.cacheDataOnRead = cacheDataOnRead;
225 this.inMemory = inMemory;
226 this.cacheDataOnWrite = cacheDataOnWrite;
227 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
228 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
229 this.evictOnClose = evictOnClose;
230 this.cacheDataCompressed = cacheDataCompressed;
231 this.prefetchOnOpen = prefetchOnOpen;
232 }
233
234
235
236
237
238 public CacheConfig(CacheConfig cacheConf) {
239 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
240 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
241 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
242 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen);
243 }
244
245
246
247
248 public boolean isBlockCacheEnabled() {
249 return this.blockCache != null;
250 }
251
252
253
254
255
256 public BlockCache getBlockCache() {
257 return this.blockCache;
258 }
259
260
261
262
263
264 public boolean shouldCacheDataOnRead() {
265 return isBlockCacheEnabled() && cacheDataOnRead;
266 }
267
268
269
270
271
272
273 public boolean shouldCacheBlockOnRead(BlockCategory category) {
274 return isBlockCacheEnabled()
275 && (cacheDataOnRead ||
276 category == BlockCategory.INDEX ||
277 category == BlockCategory.BLOOM ||
278 (prefetchOnOpen &&
279 (category != BlockCategory.META &&
280 category != BlockCategory.UNKNOWN)));
281 }
282
283
284
285
286 public boolean isInMemory() {
287 return isBlockCacheEnabled() && this.inMemory;
288 }
289
290
291
292
293
294 public boolean shouldCacheDataOnWrite() {
295 return isBlockCacheEnabled() && this.cacheDataOnWrite;
296 }
297
298
299
300
301
302
303 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
304 this.cacheDataOnWrite = cacheDataOnWrite;
305 }
306
307
308
309
310
311 public boolean shouldCacheIndexesOnWrite() {
312 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
313 }
314
315
316
317
318
319 public boolean shouldCacheBloomsOnWrite() {
320 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
321 }
322
323
324
325
326
327 public boolean shouldEvictOnClose() {
328 return isBlockCacheEnabled() && this.evictOnClose;
329 }
330
331
332
333
334
335
336 public void setEvictOnClose(boolean evictOnClose) {
337 this.evictOnClose = evictOnClose;
338 }
339
340
341
342
343 public boolean shouldCacheDataCompressed() {
344 return isBlockCacheEnabled() && this.cacheDataCompressed;
345 }
346
347
348
349
350 public boolean shouldCacheCompressed(BlockCategory category) {
351 if (!isBlockCacheEnabled()) return false;
352 switch (category) {
353 case DATA:
354 return this.cacheDataCompressed;
355 default:
356 return false;
357 }
358 }
359
360
361
362
363 public boolean shouldPrefetchOnOpen() {
364 return isBlockCacheEnabled() && this.prefetchOnOpen;
365 }
366
367 @Override
368 public String toString() {
369 if (!isBlockCacheEnabled()) {
370 return "CacheConfig:disabled";
371 }
372 return "CacheConfig:enabled " +
373 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
374 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
375 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
376 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
377 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
378 "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
379 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
380 }
381
382
383
384
385
386
387
388 @VisibleForTesting
389 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
390
391
392 private static boolean blockCacheDisabled = false;
393
394
395
396
397
398
399
400 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
401 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
402 if (blockCacheDisabled) return null;
403
404 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
405 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
406 if (cachePercentage == 0L) {
407 blockCacheDisabled = true;
408 return null;
409 }
410 if (cachePercentage > 1.0) {
411 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
412 " must be between 0.0 and 1.0, and not > 1.0");
413 }
414
415
416 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
417 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
418 int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
419 long offHeapCacheSize =
420 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
421 DirectMemoryUtils.getDirectMemorySize());
422 if (offHeapCacheSize <= 0) {
423 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
424 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
425
426 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
427 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
428
429 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
430 DEFAULT_BUCKET_CACHE_COMBINED);
431 BucketCache bucketCache = null;
432 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
433 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
434 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
435 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
436 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
437 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
438 float combinedPercentage = conf.getFloat(
439 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
440 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
441 String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
442 int[] bucketSizes = null;
443 if (configuredBucketSizes != null) {
444 bucketSizes = new int[configuredBucketSizes.length];
445 for (int i = 0; i < configuredBucketSizes.length; i++) {
446 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
447 }
448 }
449 if (combinedWithLru) {
450 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
451 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
452 }
453 try {
454 int ioErrorsTolerationDuration = conf.getInt(
455 "hbase.bucketcache.ioengine.errors.tolerated.duration",
456 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
457 bucketCache = new BucketCache(bucketCacheIOEngineName,
458 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
459 ioErrorsTolerationDuration);
460 } catch (IOException ioex) {
461 LOG.error("Can't instantiate bucket cache", ioex);
462 throw new RuntimeException(ioex);
463 }
464 }
465 LOG.info("Allocating LruBlockCache with maximum size " +
466 StringUtils.humanReadableInt(lruCacheSize));
467 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
468 lruCache.setVictimCache(bucketCache);
469 if (bucketCache != null && combinedWithLru) {
470 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
471 } else {
472 GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
473 }
474 } else {
475 LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
476 GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
477 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
478 }
479 return GLOBAL_BLOCK_CACHE_INSTANCE;
480 }
481 }