1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.util.NavigableMap;
22 import java.util.NavigableSet;
23 import java.util.concurrent.ConcurrentSkipListMap;
24 import java.util.concurrent.ConcurrentSkipListSet;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.codehaus.jackson.JsonGenerationException;
29 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
30 import org.codehaus.jackson.map.JsonMappingException;
31 import org.codehaus.jackson.map.ObjectMapper;
32 import org.codehaus.jackson.map.SerializationConfig;
33
34 import com.yammer.metrics.core.Histogram;
35 import com.yammer.metrics.core.MetricsRegistry;
36 import com.yammer.metrics.stats.Snapshot;
37
38
39
40
41
42 @InterfaceAudience.Private
43 public class BlockCacheUtil {
44
45
46
47 private static final MetricsRegistry METRICS = new MetricsRegistry();
48
49
50
51
52 private static final ObjectMapper MAPPER = new ObjectMapper();
53 static {
54 MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
55 MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
56 MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
57 }
58
59
60
61
62
63 public static String toString(final CachedBlock cb, final long now) {
64 return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
65 }
66
67
68
69
70
71 static class CachedBlockCountsPerFile {
72 private int count = 0;
73 private long size = 0;
74 private int countData = 0;
75 private long sizeData = 0;
76 private final String filename;
77
78 CachedBlockCountsPerFile(final String filename) {
79 this.filename = filename;
80 }
81
82 public int getCount() {
83 return count;
84 }
85
86 public long getSize() {
87 return size;
88 }
89
90 public int getCountData() {
91 return countData;
92 }
93
94 public long getSizeData() {
95 return sizeData;
96 }
97
98 public String getFilename() {
99 return filename;
100 }
101 }
102
103
104
105
106
107
108
109
110
111 public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
112 throws JsonGenerationException, JsonMappingException, IOException {
113 CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
114 for (CachedBlock cb: blocks) {
115 counts.count++;
116 counts.size += cb.getSize();
117 BlockType bt = cb.getBlockType();
118 if (bt != null && bt.isData()) {
119 counts.countData++;
120 counts.sizeData += cb.getSize();
121 }
122 }
123 return MAPPER.writeValueAsString(counts);
124 }
125
126
127
128
129
130
131
132
133 public static String toJSON(final CachedBlocksByFile cbsbf)
134 throws JsonGenerationException, JsonMappingException, IOException {
135 return MAPPER.writeValueAsString(cbsbf);
136 }
137
138
139
140
141
142
143
144
145 public static String toJSON(final BlockCache bc)
146 throws JsonGenerationException, JsonMappingException, IOException {
147 return MAPPER.writeValueAsString(bc);
148 }
149
150
151
152
153
154 public static String toStringMinusFileName(final CachedBlock cb, final long now) {
155 return "offset=" + cb.getOffset() +
156 ", size=" + cb.getSize() +
157 ", age=" + (now - cb.getCachedTime()) +
158 ", type=" + cb.getBlockType() +
159 ", priority=" + cb.getBlockPriority();
160 }
161
162
163
164
165
166 @JsonIgnoreProperties({"ageHistogram", "snapshot"})
167 public static class AgeSnapshot {
168 private final Histogram ageHistogram;
169 private final Snapshot snapshot;
170
171 AgeSnapshot(final Histogram ageHistogram) {
172 this.ageHistogram = ageHistogram;
173 this.snapshot = ageHistogram.getSnapshot();
174 }
175
176 public double get75thPercentile() {
177 return snapshot.get75thPercentile();
178 }
179
180 public double get95thPercentile() {
181 return snapshot.get95thPercentile();
182 }
183
184 public double get98thPercentile() {
185 return snapshot.get98thPercentile();
186 }
187
188 public double get999thPercentile() {
189 return snapshot.get999thPercentile();
190 }
191
192 public double get99thPercentile() {
193 return snapshot.get99thPercentile();
194 }
195
196 public double getMean() {
197 return this.ageHistogram.mean();
198 }
199
200 public double getMax() {
201 return ageHistogram.max();
202 }
203
204 public double getMin() {
205 return ageHistogram.min();
206 }
207
208 public double getStdDev() {
209 return ageHistogram.stdDev();
210 }
211 }
212
213
214
215
216
217
218
219
220 public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
221 final BlockCache bc) {
222 CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
223 for (CachedBlock cb: bc) {
224 if (cbsbf.update(cb)) break;
225 }
226 return cbsbf;
227 }
228
229
230
231
232
233
234 @JsonIgnoreProperties({"cachedBlockStatsByFile"})
235 public static class CachedBlocksByFile {
236 private int count;
237 private int dataBlockCount;
238 private long size;
239 private long dataSize;
240 private final long now = System.nanoTime();
241 private final int max;
242 public static final int DEFAULT_MAX = 100000;
243
244 CachedBlocksByFile() {
245 this(null);
246 }
247
248 CachedBlocksByFile(final Configuration c) {
249 this.max = c == null? DEFAULT_MAX:
250 c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
251 }
252
253
254
255
256 private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
257 new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
258 Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
259
260
261
262
263
264 public boolean update(final CachedBlock cb) {
265 if (isFull()) return true;
266 NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
267 if (set == null) {
268 set = new ConcurrentSkipListSet<CachedBlock>();
269 this.cachedBlockByFile.put(cb.getFilename(), set);
270 }
271 set.add(cb);
272 this.size += cb.getSize();
273 this.count++;
274 BlockType bt = cb.getBlockType();
275 if (bt != null && bt.isData()) {
276 this.dataBlockCount++;
277 this.dataSize += cb.getSize();
278 }
279 long age = this.now - cb.getCachedTime();
280 this.age.update(age);
281 return false;
282 }
283
284
285
286
287
288
289 public boolean isFull() {
290 return this.count >= this.max;
291 }
292
293 public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
294 return this.cachedBlockByFile;
295 }
296
297
298
299
300 public int getCount() {
301 return count;
302 }
303
304 public int getDataCount() {
305 return dataBlockCount;
306 }
307
308
309
310
311 public long getSize() {
312 return size;
313 }
314
315
316
317
318 public long getDataSize() {
319 return dataSize;
320 }
321
322 public AgeSnapshot getAgeSnapshot() {
323 return new AgeSnapshot(this.age);
324 }
325
326 @Override
327 public String toString() {
328 Snapshot snapshot = this.age.getSnapshot();
329 return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
330 ", dataSize=" + getDataSize() +
331 ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
332 ", min age=" + this.age.min() + ", max age=" + this.age.max() +
333 ", 95th percentile age=" + snapshot.get95thPercentile() +
334 ", 99th percentile age=" + snapshot.get99thPercentile();
335 }
336 }
337 }