001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import java.util.Collections; 021import java.util.Iterator; 022import java.util.List; 023import java.util.Objects; 024import java.util.SortedSet; 025import java.util.concurrent.atomic.AtomicReference; 026import java.util.concurrent.locks.ReentrantReadWriteLock; 027import org.apache.hadoop.hbase.Cell; 028import org.apache.hadoop.hbase.CellComparator; 029import org.apache.hadoop.hbase.KeyValue; 030import org.apache.hadoop.hbase.io.TimeRange; 031import org.apache.hadoop.hbase.util.Bytes; 032import org.apache.hadoop.hbase.util.ClassSize; 033import org.apache.yetus.audience.InterfaceAudience; 034import org.slf4j.Logger; 035 036/** 037 * This is an abstraction of a segment maintained in a memstore, e.g., the active cell set or its 038 * snapshot. This abstraction facilitates the management of the compaction pipeline and the shifts 039 * of these segments from active set to snapshot set in the default implementation. 040 */ 041@InterfaceAudience.Private 042public abstract class Segment implements MemStoreSizing { 043 044 public final static long FIXED_OVERHEAD = 045 ClassSize.align(ClassSize.OBJECT + 6 * ClassSize.REFERENCE // cellSet, comparator, updatesLock, 046 // memStoreLAB, memStoreSizing, 047 // and timeRangeTracker 048 + Bytes.SIZEOF_LONG // minSequenceId 049 + Bytes.SIZEOF_BOOLEAN); // tagsPresent 050 public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.ATOMIC_REFERENCE 051 + ClassSize.CELL_SET + 2 * ClassSize.ATOMIC_LONG + ClassSize.REENTRANT_LOCK; 052 053 private AtomicReference<CellSet> cellSet = new AtomicReference<>(); 054 private final CellComparator comparator; 055 private ReentrantReadWriteLock updatesLock; 056 protected long minSequenceId; 057 private MemStoreLAB memStoreLAB; 058 // Sum of sizes of all Cells added to this Segment. Cell's HeapSize is considered. This is not 059 // including the heap overhead of this class. 060 protected final MemStoreSizing memStoreSizing; 061 protected final TimeRangeTracker timeRangeTracker; 062 protected volatile boolean tagsPresent; 063 064 // Empty constructor to be used when Segment is used as interface, 065 // and there is no need in true Segments state 066 protected Segment(CellComparator comparator, TimeRangeTracker trt) { 067 this.comparator = comparator; 068 // Do we need to be thread safe always? What if ImmutableSegment? 069 // DITTO for the TimeRangeTracker below. 070 this.memStoreSizing = new ThreadSafeMemStoreSizing(); 071 this.timeRangeTracker = trt; 072 } 073 074 protected Segment(CellComparator comparator, List<ImmutableSegment> segments, 075 TimeRangeTracker trt) { 076 long dataSize = 0; 077 long heapSize = 0; 078 long OffHeapSize = 0; 079 int cellsCount = 0; 080 for (Segment segment : segments) { 081 MemStoreSize memStoreSize = segment.getMemStoreSize(); 082 dataSize += memStoreSize.getDataSize(); 083 heapSize += memStoreSize.getHeapSize(); 084 OffHeapSize += memStoreSize.getOffHeapSize(); 085 cellsCount += memStoreSize.getCellsCount(); 086 } 087 this.comparator = comparator; 088 this.updatesLock = new ReentrantReadWriteLock(); 089 // Do we need to be thread safe always? What if ImmutableSegment? 090 // DITTO for the TimeRangeTracker below. 091 this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount); 092 this.timeRangeTracker = trt; 093 } 094 095 // This constructor is used to create empty Segments. 096 protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, 097 TimeRangeTracker trt) { 098 this.cellSet.set(cellSet); 099 this.comparator = comparator; 100 this.updatesLock = new ReentrantReadWriteLock(); 101 this.minSequenceId = Long.MAX_VALUE; 102 this.memStoreLAB = memStoreLAB; 103 // Do we need to be thread safe always? What if ImmutableSegment? 104 // DITTO for the TimeRangeTracker below. 105 this.memStoreSizing = new ThreadSafeMemStoreSizing(); 106 this.tagsPresent = false; 107 this.timeRangeTracker = trt; 108 } 109 110 protected Segment(Segment segment) { 111 this.cellSet.set(segment.getCellSet()); 112 this.comparator = segment.getComparator(); 113 this.updatesLock = segment.getUpdatesLock(); 114 this.minSequenceId = segment.getMinSequenceId(); 115 this.memStoreLAB = segment.getMemStoreLAB(); 116 this.memStoreSizing = segment.memStoreSizing; 117 this.tagsPresent = segment.isTagsPresent(); 118 this.timeRangeTracker = segment.getTimeRangeTracker(); 119 } 120 121 /** 122 * Creates the scanner for the given read point 123 * @return a scanner for the given read point 124 */ 125 protected KeyValueScanner getScanner(long readPoint) { 126 return new SegmentScanner(this, readPoint); 127 } 128 129 public List<KeyValueScanner> getScanners(long readPoint) { 130 return Collections.singletonList(new SegmentScanner(this, readPoint)); 131 } 132 133 /** 134 * @return whether the segment has any cells 135 */ 136 public boolean isEmpty() { 137 return getCellSet().isEmpty(); 138 } 139 140 /** 141 * Closing a segment before it is being discarded 142 */ 143 public void close() { 144 if (this.memStoreLAB != null) { 145 this.memStoreLAB.close(); 146 } 147 // do not set MSLab to null as scanners may still be reading the data here and need to decrease 148 // the counter when they finish 149 } 150 151 /** 152 * If the segment has a memory allocator the cell is being cloned to this space, and returned; 153 * otherwise the given cell is returned When a cell's size is too big (bigger than maxAlloc), it 154 * is not allocated on MSLAB. Since the process of flattening to CellChunkMap assumes that all 155 * cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is 156 * set to 'true' and the cell is copied into MSLAB. 157 * @return either the given cell or its clone 158 */ 159 public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { 160 if (this.memStoreLAB == null) { 161 return cell; 162 } 163 164 Cell cellFromMslab; 165 if (forceCloneOfBigCell) { 166 cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell); 167 } else { 168 cellFromMslab = this.memStoreLAB.copyCellInto(cell); 169 } 170 return (cellFromMslab != null) ? cellFromMslab : cell; 171 } 172 173 /** 174 * Get cell length after serialized in {@link KeyValue} 175 */ 176 static int getCellLength(Cell cell) { 177 return cell.getSerializedSize(); 178 } 179 180 public boolean shouldSeek(TimeRange tr, long oldestUnexpiredTS) { 181 return !isEmpty() && (tr.isAllTime() || timeRangeTracker.includesTimeRange(tr)) 182 && timeRangeTracker.getMax() >= oldestUnexpiredTS; 183 } 184 185 public boolean isTagsPresent() { 186 return tagsPresent; 187 } 188 189 public void incScannerCount() { 190 if (this.memStoreLAB != null) { 191 this.memStoreLAB.incScannerCount(); 192 } 193 } 194 195 public void decScannerCount() { 196 if (this.memStoreLAB != null) { 197 this.memStoreLAB.decScannerCount(); 198 } 199 } 200 201 /** 202 * Setting the CellSet of the segment - used only for flat immutable segment for setting immutable 203 * CellSet after its creation in immutable segment constructor 204 * @return this object 205 */ 206 207 protected Segment setCellSet(CellSet cellSetOld, CellSet cellSetNew) { 208 this.cellSet.compareAndSet(cellSetOld, cellSetNew); 209 return this; 210 } 211 212 @Override 213 public MemStoreSize getMemStoreSize() { 214 return this.memStoreSizing.getMemStoreSize(); 215 } 216 217 @Override 218 public long getDataSize() { 219 return this.memStoreSizing.getDataSize(); 220 } 221 222 @Override 223 public long getHeapSize() { 224 return this.memStoreSizing.getHeapSize(); 225 } 226 227 @Override 228 public long getOffHeapSize() { 229 return this.memStoreSizing.getOffHeapSize(); 230 } 231 232 @Override 233 public int getCellsCount() { 234 return memStoreSizing.getCellsCount(); 235 } 236 237 @Override 238 public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) { 239 return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount); 240 } 241 242 public boolean sharedLock() { 243 return updatesLock.readLock().tryLock(); 244 } 245 246 public void sharedUnlock() { 247 updatesLock.readLock().unlock(); 248 } 249 250 public void waitForUpdates() { 251 if (!updatesLock.isWriteLocked()) { 252 updatesLock.writeLock().lock(); 253 } 254 } 255 256 @Override 257 public boolean compareAndSetDataSize(long expected, long updated) { 258 return memStoreSizing.compareAndSetDataSize(expected, updated); 259 } 260 261 public long getMinSequenceId() { 262 return minSequenceId; 263 } 264 265 public TimeRangeTracker getTimeRangeTracker() { 266 return this.timeRangeTracker; 267 } 268 269 // *** Methods for SegmentsScanner 270 public Cell last() { 271 return getCellSet().last(); 272 } 273 274 public Iterator<Cell> iterator() { 275 return getCellSet().iterator(); 276 } 277 278 public SortedSet<Cell> headSet(Cell firstKeyOnRow) { 279 return getCellSet().headSet(firstKeyOnRow); 280 } 281 282 public int compare(Cell left, Cell right) { 283 return getComparator().compare(left, right); 284 } 285 286 public int compareRows(Cell left, Cell right) { 287 return getComparator().compareRows(left, right); 288 } 289 290 /** 291 * @return a set of all cells in the segment 292 */ 293 protected CellSet getCellSet() { 294 return cellSet.get(); 295 } 296 297 /** 298 * Returns the Cell comparator used by this segment 299 * @return the Cell comparator used by this segment 300 */ 301 protected CellComparator getComparator() { 302 return comparator; 303 } 304 305 protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing, 306 boolean sizeAddedPreOperation) { 307 boolean succ = getCellSet().add(cell); 308 updateMetaInfo(cell, succ, mslabUsed, memstoreSizing, sizeAddedPreOperation); 309 } 310 311 protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, 312 MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { 313 long delta = 0; 314 long cellSize = getCellLength(cellToAdd); 315 int cellsCount = succ ? 1 : 0; 316 // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the 317 // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger 318 // than the counted number) 319 if (succ || mslabUsed) { 320 delta = cellSize; 321 } 322 if (sizeAddedPreOperation) { 323 delta -= cellSize; 324 } 325 long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed); 326 long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed); 327 incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); 328 if (memstoreSizing != null) { 329 memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); 330 } 331 getTimeRangeTracker().includeTimestamp(cellToAdd); 332 minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); 333 // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. 334 // When we use ACL CP or Visibility CP which deals with Tags during 335 // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not 336 // parse the byte[] to identify the tags length. 337 if (cellToAdd.getTagsLength() > 0) { 338 tagsPresent = true; 339 } 340 } 341 342 protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) { 343 updateMetaInfo(cellToAdd, succ, (getMemStoreLAB() != null), memstoreSizing, false); 344 } 345 346 /** 347 * @return The increase in heap size because of this cell addition. This includes this cell POJO's 348 * heap size itself and additional overhead because of addition on to CSLM. 349 */ 350 protected long heapSizeChange(Cell cell, boolean allocated) { 351 long res = 0; 352 if (allocated) { 353 boolean onHeap = true; 354 MemStoreLAB memStoreLAB = getMemStoreLAB(); 355 if (memStoreLAB != null) { 356 onHeap = memStoreLAB.isOnHeap(); 357 } 358 res += indexEntryOnHeapSize(onHeap); 359 if (onHeap) { 360 res += cell.heapSize(); 361 } 362 res = ClassSize.align(res); 363 } 364 return res; 365 } 366 367 protected long offHeapSizeChange(Cell cell, boolean allocated) { 368 long res = 0; 369 if (allocated) { 370 boolean offHeap = false; 371 MemStoreLAB memStoreLAB = getMemStoreLAB(); 372 if (memStoreLAB != null) { 373 offHeap = memStoreLAB.isOffHeap(); 374 } 375 res += indexEntryOffHeapSize(offHeap); 376 if (offHeap) { 377 res += cell.heapSize(); 378 } 379 res = ClassSize.align(res); 380 } 381 return res; 382 } 383 384 protected long indexEntryOnHeapSize(boolean onHeap) { 385 // in most cases index is allocated on-heap 386 // override this method when it is not always the case, e.g., in CCM 387 return indexEntrySize(); 388 } 389 390 protected long indexEntryOffHeapSize(boolean offHeap) { 391 // in most cases index is allocated on-heap 392 // override this method when it is not always the case, e.g., in CCM 393 return 0; 394 } 395 396 protected abstract long indexEntrySize(); 397 398 /** 399 * Returns a subset of the segment cell set, which starts with the given cell 400 * @param firstCell a cell in the segment 401 * @return a subset of the segment cell set, which starts with the given cell 402 */ 403 protected SortedSet<Cell> tailSet(Cell firstCell) { 404 return getCellSet().tailSet(firstCell); 405 } 406 407 MemStoreLAB getMemStoreLAB() { 408 return memStoreLAB; 409 } 410 411 // Debug methods 412 /** 413 * Dumps all cells of the segment into the given log 414 */ 415 void dump(Logger log) { 416 for (Cell cell : getCellSet()) { 417 log.debug(Objects.toString(cell)); 418 } 419 } 420 421 @Override 422 public String toString() { 423 String res = "type=" + this.getClass().getSimpleName() + ", "; 424 res += "empty=" + (isEmpty() ? "yes" : "no") + ", "; 425 res += "cellCount=" + getCellsCount() + ", "; 426 res += "cellSize=" + getDataSize() + ", "; 427 res += "totalHeapSize=" + getHeapSize() + ", "; 428 res += "min timestamp=" + timeRangeTracker.getMin() + ", "; 429 res += "max timestamp=" + timeRangeTracker.getMax(); 430 return res; 431 } 432 433 private ReentrantReadWriteLock getUpdatesLock() { 434 return updatesLock; 435 } 436}