001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.wal; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.List; 023import java.util.Map; 024import java.util.Set; 025import java.util.TreeSet; 026import org.apache.hadoop.hbase.Cell; 027import org.apache.hadoop.hbase.CellUtil; 028import org.apache.hadoop.hbase.HBaseInterfaceAudience; 029import org.apache.hadoop.hbase.KeyValue; 030import org.apache.hadoop.hbase.PrivateCellUtil; 031import org.apache.hadoop.hbase.client.RegionInfo; 032import org.apache.hadoop.hbase.codec.Codec; 033import org.apache.hadoop.hbase.io.HeapSize; 034import org.apache.hadoop.hbase.util.Bytes; 035import org.apache.hadoop.hbase.util.ClassSize; 036import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 037import org.apache.yetus.audience.InterfaceAudience; 038 039import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; 040import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 041import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 042import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 043 044/** 045 * Used in HBase's transaction log (WAL) to represent a collection of edits (Cell/KeyValue objects) 046 * that came in as a single transaction. All the edits for a given transaction are written out as a 047 * single record, in PB format, followed (optionally) by Cells written via the WALCellEncoder. 048 * <p> 049 * A particular WALEdit 'type' is the 'meta' type used to mark key operational events in the WAL 050 * such as compaction, flush, or region open. These meta types do not traverse hbase memstores. They 051 * are edits made by the hbase system rather than edit data submitted by clients. They only show in 052 * the WAL. These 'Meta' types have not been formally specified (or made into an explicit class 053 * type). They evolved organically. HBASE-8457 suggests codifying a WALEdit 'type' by adding a type 054 * field to WALEdit that gets serialized into the WAL. TODO. Would have to work on the 055 * consumption-side. Reading WALs on replay we seem to consume a Cell-at-a-time rather than by 056 * WALEdit. We are already in the below going out of our way to figure particular types -- e.g. if a 057 * compaction, replay, or close meta Marker -- during normal processing so would make sense to do 058 * this. Current system is an awkward marking of Cell columnfamily as {@link #METAFAMILY} and then 059 * setting qualifier based off meta edit type. For replay-time where we read Cell-at-a-time, there 060 * are utility methods below for figuring meta type. See also 061 * {@link #createBulkLoadEvent(RegionInfo, WALProtos.BulkLoadDescriptor)}, etc., for where we create 062 * meta WALEdit instances. 063 * </p> 064 * <p> 065 * WALEdit will accumulate a Set of all column family names referenced by the Cells 066 * {@link #add(Cell)}'d. This is an optimization. Usually when loading a WALEdit, we have the column 067 * family name to-hand.. just shove it into the WALEdit if available. Doing this, we can save on a 068 * parse of each Cell to figure column family down the line when we go to add the WALEdit to the WAL 069 * file. See the hand-off in FSWALEntry Constructor. 070 * @see WALKey 071 */ 072@InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.REPLICATION, 073 HBaseInterfaceAudience.COPROC }) 074public class WALEdit implements HeapSize { 075 // Below defines are for writing WALEdit 'meta' Cells.. 076 // TODO: Get rid of this system of special 'meta' Cells. See HBASE-8457. It suggests 077 // adding a type to WALEdit itself for use denoting meta Edits and their types. 078 public static final byte[] METAFAMILY = Bytes.toBytes("METAFAMILY"); 079 080 /** 081 * @deprecated Since 2.3.0. Not used. 082 */ 083 @Deprecated 084 public static final byte[] METAROW = Bytes.toBytes("METAROW"); 085 086 /** 087 * @deprecated Since 2.3.0. Make it protected, internal-use only. Use 088 * {@link #isCompactionMarker(Cell)} 089 */ 090 @Deprecated 091 @InterfaceAudience.Private 092 public static final byte[] COMPACTION = Bytes.toBytes("HBASE::COMPACTION"); 093 094 /** 095 * @deprecated Since 2.3.0. Make it protected, internal-use only. 096 */ 097 @Deprecated 098 @InterfaceAudience.Private 099 public static final byte[] FLUSH = Bytes.toBytes("HBASE::FLUSH"); 100 101 /** 102 * Qualifier for region event meta 'Marker' WALEdits start with the {@link #REGION_EVENT_PREFIX} 103 * prefix ('HBASE::REGION_EVENT::'). After the prefix, we note the type of the event which we get 104 * from the RegionEventDescriptor protobuf instance type (A RegionEventDescriptor protobuf 105 * instance is written as the meta Marker Cell value). Adding a type suffix means we do not have 106 * to deserialize the protobuf to figure out what type of event this is.. .just read the qualifier 107 * suffix. For example, a close region event descriptor will have a qualifier of 108 * HBASE::REGION_EVENT::REGION_CLOSE. See WAL.proto and the EventType in RegionEventDescriptor 109 * protos for all possible event types. 110 */ 111 private static final String REGION_EVENT_STR = "HBASE::REGION_EVENT"; 112 private static final String REGION_EVENT_PREFIX_STR = REGION_EVENT_STR + "::"; 113 private static final byte[] REGION_EVENT_PREFIX = Bytes.toBytes(REGION_EVENT_PREFIX_STR); 114 115 /** 116 * @deprecated Since 2.3.0. Remove. Not for external use. Not used. 117 */ 118 @Deprecated 119 public static final byte[] REGION_EVENT = Bytes.toBytes(REGION_EVENT_STR); 120 121 /** 122 * We use this define figuring if we are carrying a close event. 123 */ 124 private static final byte[] REGION_EVENT_CLOSE = 125 createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType.REGION_CLOSE); 126 127 @InterfaceAudience.Private 128 public static final byte[] BULK_LOAD = Bytes.toBytes("HBASE::BULK_LOAD"); 129 130 /** 131 * Periodically {@link org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore} 132 * will create marker edits with family as {@link WALEdit#METAFAMILY} and 133 * {@link WALEdit#REPLICATION_MARKER} as qualifier and an empty value. 134 * org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader will populate the 135 * Replication Marker edit with region_server_name, wal_name and wal_offset encoded in 136 * {@link org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.ReplicationMarkerDescriptor} 137 * object. {@link org.apache.hadoop.hbase.replication.regionserver.Replication} will change the 138 * REPLICATION_SCOPE for this edit to GLOBAL so that it can replicate. On the sink cluster, 139 * {@link org.apache.hadoop.hbase.replication.regionserver.ReplicationSink} will convert the 140 * ReplicationMarkerDescriptor into a Put mutation to REPLICATION_SINK_TRACKER_TABLE_NAME_STR 141 * table. 142 */ 143 @InterfaceAudience.Private 144 public static final byte[] REPLICATION_MARKER = Bytes.toBytes("HBASE::REPLICATION_MARKER"); 145 146 private final transient boolean replay; 147 148 private ArrayList<Cell> cells; 149 150 /** 151 * All the Cell families in <code>cells</code>. Updated by {@link #add(Cell)} and 152 * {@link #add(Map)}. This Set is passed to the FSWALEntry so it does not have to recalculate the 153 * Set of families in a transaction; makes for a bunch of CPU savings. 154 */ 155 private Set<byte[]> families = null; 156 157 public WALEdit() { 158 this(1, false); 159 } 160 161 /** 162 * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)} 163 * instead. 164 * @see #WALEdit(int, boolean) 165 * @see <a href="https://issues.apache.org/jira/browse/HBASE-20781">HBASE-20781</a> 166 */ 167 @Deprecated 168 public WALEdit(boolean replay) { 169 this(1, replay); 170 } 171 172 /** 173 * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)} 174 * instead. 175 * @see #WALEdit(int, boolean) 176 * @see <a href="https://issues.apache.org/jira/browse/HBASE-20781">HBASE-20781</a> 177 */ 178 @Deprecated 179 public WALEdit(int cellCount) { 180 this(cellCount, false); 181 } 182 183 /** 184 * @param cellCount Pass so can pre-size the WALEdit. Optimization. 185 */ 186 public WALEdit(int cellCount, boolean isReplay) { 187 this.replay = isReplay; 188 cells = new ArrayList<>(cellCount); 189 } 190 191 /** 192 * Create a new WALEdit from a existing {@link WALEdit}. 193 */ 194 public WALEdit(WALEdit walEdit) { 195 this.replay = walEdit.replay; 196 cells = new ArrayList<>(walEdit.cells); 197 if (walEdit.families != null) { 198 this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR); 199 this.families.addAll(walEdit.families); 200 } 201 202 } 203 204 private Set<byte[]> getOrCreateFamilies() { 205 if (this.families == null) { 206 this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR); 207 } 208 return this.families; 209 } 210 211 /** 212 * For use by FSWALEntry ONLY. An optimization. 213 * @return All families in {@link #getCells()}; may be null. 214 */ 215 public Set<byte[]> getFamilies() { 216 return this.families; 217 } 218 219 /** 220 * @return True is <code>f</code> is {@link #METAFAMILY} 221 * @deprecated Since 2.3.0. Do not expose. Make protected. 222 */ 223 @Deprecated 224 public static boolean isMetaEditFamily(final byte[] f) { 225 return Bytes.equals(METAFAMILY, f); 226 } 227 228 /** 229 * Replaying WALs can read Cell-at-a-time so need this method in those cases. 230 */ 231 public static boolean isMetaEditFamily(Cell cell) { 232 return CellUtil.matchingFamily(cell, METAFAMILY); 233 } 234 235 /** 236 * @return True if this is a meta edit; has one edit only and its columnfamily is 237 * {@link #METAFAMILY}. 238 */ 239 public boolean isMetaEdit() { 240 return this.families != null && this.families.size() == 1 && this.families.contains(METAFAMILY); 241 } 242 243 /** 244 * @return True when current WALEdit is created by log replay. Replication skips WALEdits from 245 * replay. 246 */ 247 public boolean isReplay() { 248 return this.replay; 249 } 250 251 public WALEdit add(Cell cell, byte[] family) { 252 getOrCreateFamilies().add(family); 253 return addCell(cell); 254 } 255 256 public WALEdit add(Cell cell) { 257 // We clone Family each time we add a Cell. Expensive but safe. For CPU savings, use 258 // add(Map) or add(Cell, family). 259 return add(cell, CellUtil.cloneFamily(cell)); 260 } 261 262 public WALEdit add(List<? extends Cell> cells) { 263 if (cells == null || cells.isEmpty()) { 264 return this; 265 } 266 for (Cell cell : cells) { 267 add(cell); 268 } 269 return this; 270 } 271 272 public boolean isEmpty() { 273 return cells.isEmpty(); 274 } 275 276 public int size() { 277 return cells.size(); 278 } 279 280 public ArrayList<Cell> getCells() { 281 return cells; 282 } 283 284 /** 285 * This is not thread safe. This will change the WALEdit and shouldn't be used unless you are sure 286 * that nothing else depends on the contents being immutable. 287 * @param cells the list of cells that this WALEdit now contains. 288 */ 289 // Used by replay. 290 public void setCells(ArrayList<Cell> cells) { 291 this.cells = cells; 292 this.families = null; 293 } 294 295 /** 296 * Reads WALEdit from cells. 297 * @param cellDecoder Cell decoder. 298 * @param expectedCount Expected cell count. 299 * @return Number of KVs read. 300 */ 301 public int readFromCells(Codec.Decoder cellDecoder, int expectedCount) throws IOException { 302 cells.clear(); 303 cells.ensureCapacity(expectedCount); 304 while (cells.size() < expectedCount && cellDecoder.advance()) { 305 add(cellDecoder.current()); 306 } 307 return cells.size(); 308 } 309 310 @Override 311 public long heapSize() { 312 long ret = ClassSize.ARRAYLIST; 313 for (Cell cell : cells) { 314 ret += cell.heapSize(); 315 } 316 return ret; 317 } 318 319 public long estimatedSerializedSizeOf() { 320 long ret = 0; 321 for (Cell cell : cells) { 322 ret += PrivateCellUtil.estimatedSerializedSizeOf(cell); 323 } 324 return ret; 325 } 326 327 @Override 328 public String toString() { 329 StringBuilder sb = new StringBuilder(); 330 331 sb.append("[#edits: ").append(cells.size()).append(" = <"); 332 for (Cell cell : cells) { 333 sb.append(cell); 334 sb.append("; "); 335 } 336 sb.append(">]"); 337 return sb.toString(); 338 } 339 340 public static WALEdit createFlushWALEdit(RegionInfo hri, FlushDescriptor f) { 341 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, FLUSH, 342 EnvironmentEdgeManager.currentTime(), f.toByteArray()); 343 return new WALEdit().add(kv, METAFAMILY); 344 } 345 346 public static FlushDescriptor getFlushDescriptor(Cell cell) throws IOException { 347 return CellUtil.matchingColumn(cell, METAFAMILY, FLUSH) 348 ? FlushDescriptor.parseFrom(CellUtil.cloneValue(cell)) 349 : null; 350 } 351 352 /** 353 * @return A meta Marker WALEdit that has a single Cell whose value is the passed in 354 * <code>regionEventDesc</code> serialized and whose row is this region, columnfamily is 355 * {@link #METAFAMILY} and qualifier is {@link #REGION_EVENT_PREFIX} + 356 * {@link RegionEventDescriptor#getEventType()}; for example 357 * HBASE::REGION_EVENT::REGION_CLOSE. 358 */ 359 public static WALEdit createRegionEventWALEdit(RegionInfo hri, 360 RegionEventDescriptor regionEventDesc) { 361 return createRegionEventWALEdit(getRowForRegion(hri), regionEventDesc); 362 } 363 364 @InterfaceAudience.Private 365 public static WALEdit createRegionEventWALEdit(byte[] rowForRegion, 366 RegionEventDescriptor regionEventDesc) { 367 KeyValue kv = new KeyValue(rowForRegion, METAFAMILY, 368 createRegionEventDescriptorQualifier(regionEventDesc.getEventType()), 369 EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray()); 370 return new WALEdit().add(kv, METAFAMILY); 371 } 372 373 /** 374 * @return Cell qualifier for the passed in RegionEventDescriptor Type; e.g. we'll return 375 * something like a byte array with HBASE::REGION_EVENT::REGION_OPEN in it. 376 */ 377 @InterfaceAudience.Private 378 public static byte[] createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType t) { 379 return Bytes.toBytes(REGION_EVENT_PREFIX_STR + t.toString()); 380 } 381 382 /** 383 * Public so can be accessed from regionserver.wal package. 384 * @return True if this is a Marker Edit and it is a RegionClose type. 385 */ 386 public boolean isRegionCloseMarker() { 387 return isMetaEdit() && PrivateCellUtil.matchingQualifier(this.cells.get(0), REGION_EVENT_CLOSE, 388 0, REGION_EVENT_CLOSE.length); 389 } 390 391 /** 392 * @return Returns a RegionEventDescriptor made by deserializing the content of the passed in 393 * <code>cell</code>, IFF the <code>cell</code> is a RegionEventDescriptor type WALEdit. 394 */ 395 public static RegionEventDescriptor getRegionEventDescriptor(Cell cell) throws IOException { 396 return CellUtil.matchingColumnFamilyAndQualifierPrefix(cell, METAFAMILY, REGION_EVENT_PREFIX) 397 ? RegionEventDescriptor.parseFrom(CellUtil.cloneValue(cell)) 398 : null; 399 } 400 401 /** Returns A Marker WALEdit that has <code>c</code> serialized as its value */ 402 public static WALEdit createCompaction(final RegionInfo hri, final CompactionDescriptor c) { 403 byte[] pbbytes = c.toByteArray(); 404 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION, 405 EnvironmentEdgeManager.currentTime(), pbbytes); 406 return new WALEdit().add(kv, METAFAMILY); // replication scope null so this won't be replicated 407 } 408 409 public static byte[] getRowForRegion(RegionInfo hri) { 410 byte[] startKey = hri.getStartKey(); 411 if (startKey.length == 0) { 412 // empty row key is not allowed in mutations because it is both the start key and the end key 413 // we return the smallest byte[] that is bigger (in lex comparison) than byte[0]. 414 return new byte[] { 0 }; 415 } 416 return startKey; 417 } 418 419 /** 420 * Deserialized and returns a CompactionDescriptor is the KeyValue contains one. 421 * @param kv the key value 422 * @return deserialized CompactionDescriptor or null. 423 */ 424 public static CompactionDescriptor getCompaction(Cell kv) throws IOException { 425 return isCompactionMarker(kv) ? CompactionDescriptor.parseFrom(CellUtil.cloneValue(kv)) : null; 426 } 427 428 /** 429 * Returns true if the given cell is a serialized {@link CompactionDescriptor} 430 * @see #getCompaction(Cell) 431 */ 432 public static boolean isCompactionMarker(Cell cell) { 433 return CellUtil.matchingColumn(cell, METAFAMILY, COMPACTION); 434 } 435 436 /** 437 * Create a bulk loader WALEdit 438 * @param hri The RegionInfo for the region in which we are bulk loading 439 * @param bulkLoadDescriptor The descriptor for the Bulk Loader 440 * @return The WALEdit for the BulkLoad 441 */ 442 public static WALEdit createBulkLoadEvent(RegionInfo hri, 443 WALProtos.BulkLoadDescriptor bulkLoadDescriptor) { 444 KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, BULK_LOAD, 445 EnvironmentEdgeManager.currentTime(), bulkLoadDescriptor.toByteArray()); 446 return new WALEdit().add(kv, METAFAMILY); 447 } 448 449 /** 450 * Deserialized and returns a BulkLoadDescriptor from the passed in Cell 451 * @param cell the key value 452 * @return deserialized BulkLoadDescriptor or null. 453 */ 454 public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) throws IOException { 455 return CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD) 456 ? WALProtos.BulkLoadDescriptor.parseFrom(CellUtil.cloneValue(cell)) 457 : null; 458 } 459 460 /** 461 * Append the given map of family->edits to a WALEdit data structure. This does not write to the 462 * WAL itself. Note that as an optimization, we will stamp the Set of column families into the 463 * WALEdit to save on our having to calculate column families subsequently down in the actual WAL 464 * writing. 465 * @param familyMap map of family->edits 466 */ 467 public void add(Map<byte[], List<Cell>> familyMap) { 468 for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) { 469 // 'foreach' loop NOT used. See HBASE-12023 "...creates too many iterator objects." 470 int listSize = e.getValue().size(); 471 // Add all Cells first and then at end, add the family rather than call {@link #add(Cell)} 472 // and have it clone family each time. Optimization! 473 for (int i = 0; i < listSize; i++) { 474 addCell(e.getValue().get(i)); 475 } 476 addFamily(e.getKey()); 477 } 478 } 479 480 private void addFamily(byte[] family) { 481 getOrCreateFamilies().add(family); 482 } 483 484 private WALEdit addCell(Cell cell) { 485 this.cells.add(cell); 486 return this; 487 } 488 489 /** 490 * Creates a replication tracker edit with {@link #METAFAMILY} family and 491 * {@link #REPLICATION_MARKER} qualifier and has null value. 492 * @param rowKey rowkey 493 * @param timestamp timestamp 494 */ 495 public static WALEdit createReplicationMarkerEdit(byte[] rowKey, long timestamp) { 496 KeyValue kv = 497 new KeyValue(rowKey, METAFAMILY, REPLICATION_MARKER, timestamp, KeyValue.Type.Put); 498 return new WALEdit().add(kv); 499 } 500 501 /** 502 * Checks whether this edit is a replication marker edit. 503 * @param edit edit 504 * @return true if the cell within an edit has column = METAFAMILY and qualifier = 505 * REPLICATION_MARKER, false otherwise 506 */ 507 public static boolean isReplicationMarkerEdit(WALEdit edit) { 508 // Check just the first cell from the edit. ReplicationMarker edit will have only 1 cell. 509 return edit.getCells().size() == 1 510 && CellUtil.matchingColumn(edit.getCells().get(0), METAFAMILY, REPLICATION_MARKER); 511 } 512}