1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.util.NavigableMap;
22 import java.util.NavigableSet;
23 import java.util.concurrent.ConcurrentSkipListMap;
24 import java.util.concurrent.ConcurrentSkipListSet;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.codehaus.jackson.JsonGenerationException;
29 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
30 import org.codehaus.jackson.map.JsonMappingException;
31 import org.codehaus.jackson.map.ObjectMapper;
32 import org.codehaus.jackson.map.SerializationConfig;
33
34 import com.yammer.metrics.core.Histogram;
35 import com.yammer.metrics.core.MetricsRegistry;
36 import com.yammer.metrics.stats.Snapshot;
37
38
39
40
41
42 @InterfaceAudience.Private
43 public class BlockCacheUtil {
44
45 public static final long NANOS_PER_SECOND = 1000000000;
46
47
48
49 private static final MetricsRegistry METRICS = new MetricsRegistry();
50
51
52
53
54 private static final ObjectMapper MAPPER = new ObjectMapper();
55 static {
56 MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
57 MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
58 MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
59 }
60
61
62
63
64
65 public static String toString(final CachedBlock cb, final long now) {
66 return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
67 }
68
69
70
71
72
73 static class CachedBlockCountsPerFile {
74 private int count = 0;
75 private long size = 0;
76 private int countData = 0;
77 private long sizeData = 0;
78 private final String filename;
79
80 CachedBlockCountsPerFile(final String filename) {
81 this.filename = filename;
82 }
83
84 public int getCount() {
85 return count;
86 }
87
88 public long getSize() {
89 return size;
90 }
91
92 public int getCountData() {
93 return countData;
94 }
95
96 public long getSizeData() {
97 return sizeData;
98 }
99
100 public String getFilename() {
101 return filename;
102 }
103 }
104
105
106
107
108
109
110
111
112
113 public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
114 throws JsonGenerationException, JsonMappingException, IOException {
115 CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
116 for (CachedBlock cb: blocks) {
117 counts.count++;
118 counts.size += cb.getSize();
119 BlockType bt = cb.getBlockType();
120 if (bt != null && bt.isData()) {
121 counts.countData++;
122 counts.sizeData += cb.getSize();
123 }
124 }
125 return MAPPER.writeValueAsString(counts);
126 }
127
128
129
130
131
132
133
134
135 public static String toJSON(final CachedBlocksByFile cbsbf)
136 throws JsonGenerationException, JsonMappingException, IOException {
137 return MAPPER.writeValueAsString(cbsbf);
138 }
139
140
141
142
143
144
145
146
147 public static String toJSON(final BlockCache bc)
148 throws JsonGenerationException, JsonMappingException, IOException {
149 return MAPPER.writeValueAsString(bc);
150 }
151
152
153
154
155
156 public static String toStringMinusFileName(final CachedBlock cb, final long now) {
157 return "offset=" + cb.getOffset() +
158 ", size=" + cb.getSize() +
159 ", age=" + (now - cb.getCachedTime()) +
160 ", type=" + cb.getBlockType() +
161 ", priority=" + cb.getBlockPriority();
162 }
163
164
165
166
167
168
169
170
171 public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
172 final BlockCache bc) {
173 CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
174 for (CachedBlock cb: bc) {
175 if (cbsbf.update(cb)) break;
176 }
177 return cbsbf;
178 }
179
180
181
182
183
184
185 @JsonIgnoreProperties({"cachedBlockStatsByFile"})
186 public static class CachedBlocksByFile {
187 private int count;
188 private int dataBlockCount;
189 private long size;
190 private long dataSize;
191 private final long now = System.nanoTime();
192
193
194
195
196
197
198
199 private final int max;
200 public static final int DEFAULT_MAX = 1000000;
201
202 CachedBlocksByFile() {
203 this(null);
204 }
205
206 CachedBlocksByFile(final Configuration c) {
207 this.max = c == null? DEFAULT_MAX: c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
208 }
209
210
211
212
213 private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
214 new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
215 Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
216
217
218
219
220
221 public boolean update(final CachedBlock cb) {
222 if (isFull()) return true;
223 NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
224 if (set == null) {
225 set = new ConcurrentSkipListSet<CachedBlock>();
226 this.cachedBlockByFile.put(cb.getFilename(), set);
227 }
228 set.add(cb);
229 this.size += cb.getSize();
230 this.count++;
231 BlockType bt = cb.getBlockType();
232 if (bt != null && bt.isData()) {
233 this.dataBlockCount++;
234 this.dataSize += cb.getSize();
235 }
236 long age = (this.now - cb.getCachedTime())/NANOS_PER_SECOND;
237 this.age.update(age);
238 return false;
239 }
240
241
242
243
244
245
246 public boolean isFull() {
247 return this.count >= this.max;
248 }
249
250 public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
251 return this.cachedBlockByFile;
252 }
253
254
255
256
257 public int getCount() {
258 return count;
259 }
260
261 public int getDataCount() {
262 return dataBlockCount;
263 }
264
265
266
267
268 public long getSize() {
269 return size;
270 }
271
272
273
274
275 public long getDataSize() {
276 return dataSize;
277 }
278
279 public AgeSnapshot getAgeInCacheSnapshot() {
280 return new AgeSnapshot(this.age);
281 }
282
283 @Override
284 public String toString() {
285 Snapshot snapshot = this.age.getSnapshot();
286 return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
287 ", dataSize=" + getDataSize() +
288 ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
289 ", min age=" + this.age.min() + ", max age=" + this.age.max() +
290 ", 95th percentile age=" + snapshot.get95thPercentile() +
291 ", 99th percentile age=" + snapshot.get99thPercentile();
292 }
293 }
294 }