001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import java.util.Collections; 021import java.util.Iterator; 022import java.util.List; 023import java.util.Objects; 024import java.util.SortedSet; 025import java.util.concurrent.atomic.AtomicReference; 026import java.util.concurrent.locks.ReentrantReadWriteLock; 027import org.apache.hadoop.hbase.Cell; 028import org.apache.hadoop.hbase.CellComparator; 029import org.apache.hadoop.hbase.KeyValue; 030import org.apache.hadoop.hbase.io.TimeRange; 031import org.apache.hadoop.hbase.util.Bytes; 032import org.apache.hadoop.hbase.util.ClassSize; 033import org.apache.yetus.audience.InterfaceAudience; 034import org.slf4j.Logger; 035 036/** 037 * This is an abstraction of a segment maintained in a memstore, e.g., the active cell set or its 038 * snapshot. This abstraction facilitates the management of the compaction pipeline and the shifts 039 * of these segments from active set to snapshot set in the default implementation. 040 */ 041@InterfaceAudience.Private 042public abstract class Segment implements MemStoreSizing { 043 044 public final static long FIXED_OVERHEAD = 045 ClassSize.align(ClassSize.OBJECT + 6 * ClassSize.REFERENCE // cellSet, comparator, updatesLock, 046 // memStoreLAB, memStoreSizing, 047 // and timeRangeTracker 048 + Bytes.SIZEOF_LONG // minSequenceId 049 + Bytes.SIZEOF_BOOLEAN); // tagsPresent 050 public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.ATOMIC_REFERENCE 051 + ClassSize.CELL_SET + 2 * ClassSize.ATOMIC_LONG + ClassSize.REENTRANT_LOCK; 052 053 private AtomicReference<CellSet> cellSet = new AtomicReference<>(); 054 private final CellComparator comparator; 055 private ReentrantReadWriteLock updatesLock; 056 protected long minSequenceId; 057 private MemStoreLAB memStoreLAB; 058 // Sum of sizes of all Cells added to this Segment. Cell's HeapSize is considered. This is not 059 // including the heap overhead of this class. 060 protected final MemStoreSizing memStoreSizing; 061 protected final TimeRangeTracker timeRangeTracker; 062 protected volatile boolean tagsPresent; 063 064 // Empty constructor to be used when Segment is used as interface, 065 // and there is no need in true Segments state 066 protected Segment(CellComparator comparator, TimeRangeTracker trt) { 067 this.comparator = comparator; 068 // Do we need to be thread safe always? What if ImmutableSegment? 069 // DITTO for the TimeRangeTracker below. 070 this.memStoreSizing = new ThreadSafeMemStoreSizing(); 071 this.timeRangeTracker = trt; 072 } 073 074 protected Segment(CellComparator comparator, List<ImmutableSegment> segments, 075 TimeRangeTracker trt) { 076 long dataSize = 0; 077 long heapSize = 0; 078 long OffHeapSize = 0; 079 int cellsCount = 0; 080 for (Segment segment : segments) { 081 MemStoreSize memStoreSize = segment.getMemStoreSize(); 082 dataSize += memStoreSize.getDataSize(); 083 heapSize += memStoreSize.getHeapSize(); 084 OffHeapSize += memStoreSize.getOffHeapSize(); 085 cellsCount += memStoreSize.getCellsCount(); 086 } 087 this.comparator = comparator; 088 this.updatesLock = new ReentrantReadWriteLock(); 089 // Do we need to be thread safe always? What if ImmutableSegment? 090 // DITTO for the TimeRangeTracker below. 091 this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount); 092 this.timeRangeTracker = trt; 093 } 094 095 // This constructor is used to create empty Segments. 096 protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, 097 TimeRangeTracker trt) { 098 this.cellSet.set(cellSet); 099 this.comparator = comparator; 100 this.updatesLock = new ReentrantReadWriteLock(); 101 this.minSequenceId = Long.MAX_VALUE; 102 this.memStoreLAB = memStoreLAB; 103 // Do we need to be thread safe always? What if ImmutableSegment? 104 // DITTO for the TimeRangeTracker below. 105 this.memStoreSizing = new ThreadSafeMemStoreSizing(); 106 this.tagsPresent = false; 107 this.timeRangeTracker = trt; 108 } 109 110 protected Segment(Segment segment) { 111 this.cellSet.set(segment.getCellSet()); 112 this.comparator = segment.getComparator(); 113 this.updatesLock = segment.getUpdatesLock(); 114 this.minSequenceId = segment.getMinSequenceId(); 115 this.memStoreLAB = segment.getMemStoreLAB(); 116 this.memStoreSizing = segment.memStoreSizing; 117 this.tagsPresent = segment.isTagsPresent(); 118 this.timeRangeTracker = segment.getTimeRangeTracker(); 119 } 120 121 /** 122 * Creates the scanner for the given read point 123 * @return a scanner for the given read point 124 */ 125 protected KeyValueScanner getScanner(long readPoint) { 126 return new SegmentScanner(this, readPoint); 127 } 128 129 public List<KeyValueScanner> getScanners(long readPoint) { 130 return Collections.singletonList(new SegmentScanner(this, readPoint)); 131 } 132 133 /** Returns whether the segment has any cells */ 134 public boolean isEmpty() { 135 return getCellSet().isEmpty(); 136 } 137 138 /** 139 * Closing a segment before it is being discarded 140 */ 141 public void close() { 142 if (this.memStoreLAB != null) { 143 this.memStoreLAB.close(); 144 } 145 // do not set MSLab to null as scanners may still be reading the data here and need to decrease 146 // the counter when they finish 147 } 148 149 /** 150 * If the segment has a memory allocator the cell is being cloned to this space, and returned; 151 * otherwise the given cell is returned When a cell's size is too big (bigger than maxAlloc), it 152 * is not allocated on MSLAB. Since the process of flattening to CellChunkMap assumes that all 153 * cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is 154 * set to 'true' and the cell is copied into MSLAB. 155 * @return either the given cell or its clone 156 */ 157 public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { 158 if (this.memStoreLAB == null) { 159 return cell; 160 } 161 162 Cell cellFromMslab; 163 if (forceCloneOfBigCell) { 164 cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell); 165 } else { 166 cellFromMslab = this.memStoreLAB.copyCellInto(cell); 167 } 168 return (cellFromMslab != null) ? cellFromMslab : cell; 169 } 170 171 /** 172 * Get cell length after serialized in {@link KeyValue} 173 */ 174 static int getCellLength(Cell cell) { 175 return cell.getSerializedSize(); 176 } 177 178 public boolean shouldSeek(TimeRange tr, long oldestUnexpiredTS) { 179 return !isEmpty() && (tr.isAllTime() || timeRangeTracker.includesTimeRange(tr)) 180 && timeRangeTracker.getMax() >= oldestUnexpiredTS; 181 } 182 183 public boolean isTagsPresent() { 184 return tagsPresent; 185 } 186 187 public void incScannerCount() { 188 if (this.memStoreLAB != null) { 189 this.memStoreLAB.incScannerCount(); 190 } 191 } 192 193 public void decScannerCount() { 194 if (this.memStoreLAB != null) { 195 this.memStoreLAB.decScannerCount(); 196 } 197 } 198 199 /** 200 * Setting the CellSet of the segment - used only for flat immutable segment for setting immutable 201 * CellSet after its creation in immutable segment constructor 202 * @return this object 203 */ 204 205 protected Segment setCellSet(CellSet cellSetOld, CellSet cellSetNew) { 206 this.cellSet.compareAndSet(cellSetOld, cellSetNew); 207 return this; 208 } 209 210 @Override 211 public MemStoreSize getMemStoreSize() { 212 return this.memStoreSizing.getMemStoreSize(); 213 } 214 215 @Override 216 public long getDataSize() { 217 return this.memStoreSizing.getDataSize(); 218 } 219 220 @Override 221 public long getHeapSize() { 222 return this.memStoreSizing.getHeapSize(); 223 } 224 225 @Override 226 public long getOffHeapSize() { 227 return this.memStoreSizing.getOffHeapSize(); 228 } 229 230 @Override 231 public int getCellsCount() { 232 return memStoreSizing.getCellsCount(); 233 } 234 235 @Override 236 public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) { 237 return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount); 238 } 239 240 public boolean sharedLock() { 241 return updatesLock.readLock().tryLock(); 242 } 243 244 public void sharedUnlock() { 245 updatesLock.readLock().unlock(); 246 } 247 248 public void waitForUpdates() { 249 if (!updatesLock.isWriteLocked()) { 250 updatesLock.writeLock().lock(); 251 } 252 } 253 254 @Override 255 public boolean compareAndSetDataSize(long expected, long updated) { 256 return memStoreSizing.compareAndSetDataSize(expected, updated); 257 } 258 259 public long getMinSequenceId() { 260 return minSequenceId; 261 } 262 263 public TimeRangeTracker getTimeRangeTracker() { 264 return this.timeRangeTracker; 265 } 266 267 // *** Methods for SegmentsScanner 268 public Cell last() { 269 return getCellSet().last(); 270 } 271 272 public Iterator<Cell> iterator() { 273 return getCellSet().iterator(); 274 } 275 276 public SortedSet<Cell> headSet(Cell firstKeyOnRow) { 277 return getCellSet().headSet(firstKeyOnRow); 278 } 279 280 public int compare(Cell left, Cell right) { 281 return getComparator().compare(left, right); 282 } 283 284 public int compareRows(Cell left, Cell right) { 285 return getComparator().compareRows(left, right); 286 } 287 288 /** Returns a set of all cells in the segment */ 289 protected CellSet getCellSet() { 290 return cellSet.get(); 291 } 292 293 /** 294 * Returns the Cell comparator used by this segment 295 * @return the Cell comparator used by this segment 296 */ 297 protected CellComparator getComparator() { 298 return comparator; 299 } 300 301 protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing, 302 boolean sizeAddedPreOperation) { 303 boolean succ = getCellSet().add(cell); 304 updateMetaInfo(cell, succ, mslabUsed, memstoreSizing, sizeAddedPreOperation); 305 } 306 307 protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, 308 MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { 309 long delta = 0; 310 long cellSize = getCellLength(cellToAdd); 311 int cellsCount = succ ? 1 : 0; 312 // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the 313 // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger 314 // than the counted number) 315 if (succ || mslabUsed) { 316 delta = cellSize; 317 } 318 if (sizeAddedPreOperation) { 319 delta -= cellSize; 320 } 321 long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed); 322 long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed); 323 incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); 324 if (memstoreSizing != null) { 325 memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); 326 } 327 getTimeRangeTracker().includeTimestamp(cellToAdd); 328 minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); 329 // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. 330 // When we use ACL CP or Visibility CP which deals with Tags during 331 // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not 332 // parse the byte[] to identify the tags length. 333 if (cellToAdd.getTagsLength() > 0) { 334 tagsPresent = true; 335 } 336 } 337 338 protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) { 339 updateMetaInfo(cellToAdd, succ, (getMemStoreLAB() != null), memstoreSizing, false); 340 } 341 342 /** 343 * @return The increase in heap size because of this cell addition. This includes this cell POJO's 344 * heap size itself and additional overhead because of addition on to CSLM. 345 */ 346 protected long heapSizeChange(Cell cell, boolean allocated) { 347 long res = 0; 348 if (allocated) { 349 boolean onHeap = true; 350 MemStoreLAB memStoreLAB = getMemStoreLAB(); 351 if (memStoreLAB != null) { 352 onHeap = memStoreLAB.isOnHeap(); 353 } 354 res += indexEntryOnHeapSize(onHeap); 355 if (onHeap) { 356 res += cell.heapSize(); 357 } 358 res = ClassSize.align(res); 359 } 360 return res; 361 } 362 363 protected long offHeapSizeChange(Cell cell, boolean allocated) { 364 long res = 0; 365 if (allocated) { 366 boolean offHeap = false; 367 MemStoreLAB memStoreLAB = getMemStoreLAB(); 368 if (memStoreLAB != null) { 369 offHeap = memStoreLAB.isOffHeap(); 370 } 371 res += indexEntryOffHeapSize(offHeap); 372 if (offHeap) { 373 res += cell.heapSize(); 374 } 375 res = ClassSize.align(res); 376 } 377 return res; 378 } 379 380 protected long indexEntryOnHeapSize(boolean onHeap) { 381 // in most cases index is allocated on-heap 382 // override this method when it is not always the case, e.g., in CCM 383 return indexEntrySize(); 384 } 385 386 protected long indexEntryOffHeapSize(boolean offHeap) { 387 // in most cases index is allocated on-heap 388 // override this method when it is not always the case, e.g., in CCM 389 return 0; 390 } 391 392 protected abstract long indexEntrySize(); 393 394 /** 395 * Returns a subset of the segment cell set, which starts with the given cell 396 * @param firstCell a cell in the segment 397 * @return a subset of the segment cell set, which starts with the given cell 398 */ 399 protected SortedSet<Cell> tailSet(Cell firstCell) { 400 return getCellSet().tailSet(firstCell); 401 } 402 403 MemStoreLAB getMemStoreLAB() { 404 return memStoreLAB; 405 } 406 407 // Debug methods 408 /** 409 * Dumps all cells of the segment into the given log 410 */ 411 void dump(Logger log) { 412 for (Cell cell : getCellSet()) { 413 log.debug(Objects.toString(cell)); 414 } 415 } 416 417 @Override 418 public String toString() { 419 String res = "type=" + this.getClass().getSimpleName() + ", "; 420 res += "empty=" + (isEmpty() ? "yes" : "no") + ", "; 421 res += "cellCount=" + getCellsCount() + ", "; 422 res += "cellSize=" + getDataSize() + ", "; 423 res += "totalHeapSize=" + getHeapSize() + ", "; 424 res += "min timestamp=" + timeRangeTracker.getMin() + ", "; 425 res += "max timestamp=" + timeRangeTracker.getMax(); 426 return res; 427 } 428 429 private ReentrantReadWriteLock getUpdatesLock() { 430 return updatesLock; 431 } 432}