1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.NavigableMap;
26 import java.util.TreeMap;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.classification.InterfaceAudience;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValueUtil;
37 import org.apache.hadoop.hbase.codec.Codec;
38 import org.apache.hadoop.hbase.io.HeapSize;
39 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
40 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
41 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
42 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.ClassSize;
45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46 import org.apache.hadoop.io.Writable;
47
48 import com.google.common.annotations.VisibleForTesting;
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.REPLICATION,
86 HBaseInterfaceAudience.COPROC })
87 public class WALEdit implements Writable, HeapSize {
88 public static final Log LOG = LogFactory.getLog(WALEdit.class);
89
90
91 public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
92 static final byte [] METAROW = Bytes.toBytes("METAROW");
93 static final byte[] COMPACTION = Bytes.toBytes("HBASE::COMPACTION");
94 static final byte [] FLUSH = Bytes.toBytes("HBASE::FLUSH");
95 static final byte [] REGION_EVENT = Bytes.toBytes("HBASE::REGION_EVENT");
96 @VisibleForTesting
97 public static final byte [] BULK_LOAD = Bytes.toBytes("HBASE::BULK_LOAD");
98
99 private final int VERSION_2 = -1;
100 private final boolean isReplay;
101
102 private final ArrayList<Cell> cells = new ArrayList<Cell>(1);
103
104 public static final WALEdit EMPTY_WALEDIT = new WALEdit();
105
106
107 @Deprecated
108 private NavigableMap<byte[], Integer> scopes;
109
110 private CompressionContext compressionContext;
111
112 public WALEdit() {
113 this(false);
114 }
115
116 public WALEdit(boolean isReplay) {
117 this.isReplay = isReplay;
118 }
119
120
121
122
123
124 public static boolean isMetaEditFamily(final byte [] f) {
125 return Bytes.equals(METAFAMILY, f);
126 }
127
128 public static boolean isMetaEditFamily(Cell cell) {
129 return CellUtil.matchingFamily(cell, METAFAMILY);
130 }
131
132 public boolean isMetaEdit() {
133 for (Cell cell: cells) {
134 if (!isMetaEditFamily(cell)) {
135 return false;
136 }
137 }
138 return true;
139 }
140
141
142
143
144
145 public boolean isReplay() {
146 return this.isReplay;
147 }
148
149 public void setCompressionContext(final CompressionContext compressionContext) {
150 this.compressionContext = compressionContext;
151 }
152
153 public WALEdit add(Cell cell) {
154 this.cells.add(cell);
155 return this;
156 }
157
158 public boolean isEmpty() {
159 return cells.isEmpty();
160 }
161
162 public int size() {
163 return cells.size();
164 }
165
166 public ArrayList<Cell> getCells() {
167 return cells;
168 }
169
170 public NavigableMap<byte[], Integer> getAndRemoveScopes() {
171 NavigableMap<byte[], Integer> result = scopes;
172 scopes = null;
173 return result;
174 }
175
176 @Override
177 public void readFields(DataInput in) throws IOException {
178 cells.clear();
179 if (scopes != null) {
180 scopes.clear();
181 }
182 int versionOrLength = in.readInt();
183
184 if (versionOrLength == VERSION_2) {
185
186 int numEdits = in.readInt();
187 for (int idx = 0; idx < numEdits; idx++) {
188 if (compressionContext != null) {
189 this.add(KeyValueCompression.readKV(in, compressionContext));
190 } else {
191 this.add(KeyValue.create(in));
192 }
193 }
194 int numFamilies = in.readInt();
195 if (numFamilies > 0) {
196 if (scopes == null) {
197 scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
198 }
199 for (int i = 0; i < numFamilies; i++) {
200 byte[] fam = Bytes.readByteArray(in);
201 int scope = in.readInt();
202 scopes.put(fam, scope);
203 }
204 }
205 } else {
206
207
208 this.add(KeyValue.create(versionOrLength, in));
209 }
210 }
211
212 @Override
213 public void write(DataOutput out) throws IOException {
214 LOG.warn("WALEdit is being serialized to writable - only expected in test code");
215 out.writeInt(VERSION_2);
216 out.writeInt(cells.size());
217
218 for (Cell cell : cells) {
219
220 KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
221 if (compressionContext != null) {
222 KeyValueCompression.writeKV(out, kv, compressionContext);
223 } else{
224 KeyValue.write(kv, out);
225 }
226 }
227 if (scopes == null) {
228 out.writeInt(0);
229 } else {
230 out.writeInt(scopes.size());
231 for (byte[] key : scopes.keySet()) {
232 Bytes.writeByteArray(out, key);
233 out.writeInt(scopes.get(key));
234 }
235 }
236 }
237
238
239
240
241
242
243
244 public int readFromCells(Codec.Decoder cellDecoder, int expectedCount) throws IOException {
245 cells.clear();
246 cells.ensureCapacity(expectedCount);
247 while (cells.size() < expectedCount && cellDecoder.advance()) {
248 cells.add(cellDecoder.current());
249 }
250 return cells.size();
251 }
252
253 @Override
254 public long heapSize() {
255 long ret = ClassSize.ARRAYLIST;
256 for (Cell cell : cells) {
257 ret += CellUtil.estimatedHeapSizeOf(cell);
258 }
259 if (scopes != null) {
260 ret += ClassSize.TREEMAP;
261 ret += ClassSize.align(scopes.size() * ClassSize.MAP_ENTRY);
262
263 }
264 return ret;
265 }
266
267 @Override
268 public String toString() {
269 StringBuilder sb = new StringBuilder();
270
271 sb.append("[#edits: " + cells.size() + " = <");
272 for (Cell cell : cells) {
273 sb.append(cell);
274 sb.append("; ");
275 }
276 if (scopes != null) {
277 sb.append(" scopes: " + scopes.toString());
278 }
279 sb.append(">]");
280 return sb.toString();
281 }
282
283 public static WALEdit createFlushWALEdit(HRegionInfo hri, FlushDescriptor f) {
284 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, FLUSH,
285 EnvironmentEdgeManager.currentTime(), f.toByteArray());
286 return new WALEdit().add(kv);
287 }
288
289 public static FlushDescriptor getFlushDescriptor(Cell cell) throws IOException {
290 if (CellUtil.matchingColumn(cell, METAFAMILY, FLUSH)) {
291 return FlushDescriptor.parseFrom(cell.getValue());
292 }
293 return null;
294 }
295
296 public static WALEdit createRegionEventWALEdit(HRegionInfo hri,
297 RegionEventDescriptor regionEventDesc) {
298 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, REGION_EVENT,
299 EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray());
300 return new WALEdit().add(kv);
301 }
302
303 public static RegionEventDescriptor getRegionEventDescriptor(Cell cell) throws IOException {
304 if (CellUtil.matchingColumn(cell, METAFAMILY, REGION_EVENT)) {
305 return RegionEventDescriptor.parseFrom(cell.getValue());
306 }
307 return null;
308 }
309
310
311
312
313
314
315 public static WALEdit createCompaction(final HRegionInfo hri, final CompactionDescriptor c) {
316 byte [] pbbytes = c.toByteArray();
317 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION,
318 EnvironmentEdgeManager.currentTime(), pbbytes);
319 return new WALEdit().add(kv);
320 }
321
322 private static byte[] getRowForRegion(HRegionInfo hri) {
323 byte[] startKey = hri.getStartKey();
324 if (startKey.length == 0) {
325
326
327 return new byte[] {0};
328 }
329 return startKey;
330 }
331
332
333
334
335
336
337 public static CompactionDescriptor getCompaction(Cell kv) throws IOException {
338 if (CellUtil.matchingColumn(kv, METAFAMILY, COMPACTION)) {
339 return CompactionDescriptor.parseFrom(kv.getValue());
340 }
341 return null;
342 }
343
344
345
346
347
348
349
350
351 public static WALEdit createBulkLoadEvent(HRegionInfo hri,
352 WALProtos.BulkLoadDescriptor bulkLoadDescriptor) {
353 KeyValue kv = new KeyValue(getRowForRegion(hri),
354 METAFAMILY,
355 BULK_LOAD,
356 EnvironmentEdgeManager.currentTime(),
357 bulkLoadDescriptor.toByteArray());
358 return new WALEdit().add(kv);
359 }
360
361
362
363
364
365
366 public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) throws IOException {
367 if (CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD)) {
368 return WALProtos.BulkLoadDescriptor.parseFrom(cell.getValue());
369 }
370 return null;
371 }
372 }