001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.util.Collections;
021import java.util.Iterator;
022import java.util.List;
023import java.util.Objects;
024import java.util.SortedSet;
025import java.util.concurrent.atomic.AtomicReference;
026import java.util.concurrent.locks.ReentrantReadWriteLock;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.CellComparator;
029import org.apache.hadoop.hbase.KeyValue;
030import org.apache.hadoop.hbase.io.TimeRange;
031import org.apache.hadoop.hbase.util.Bytes;
032import org.apache.hadoop.hbase.util.ClassSize;
033import org.apache.yetus.audience.InterfaceAudience;
034import org.slf4j.Logger;
035
036/**
037 * This is an abstraction of a segment maintained in a memstore, e.g., the active cell set or its
038 * snapshot. This abstraction facilitates the management of the compaction pipeline and the shifts
039 * of these segments from active set to snapshot set in the default implementation.
040 */
041@InterfaceAudience.Private
042public abstract class Segment implements MemStoreSizing {
043
044  public final static long FIXED_OVERHEAD =
045    ClassSize.align(ClassSize.OBJECT + 6 * ClassSize.REFERENCE // cellSet, comparator, updatesLock,
046                                                               // memStoreLAB, memStoreSizing,
047                                                               // and timeRangeTracker
048      + Bytes.SIZEOF_LONG // minSequenceId
049      + Bytes.SIZEOF_BOOLEAN); // tagsPresent
050  public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.ATOMIC_REFERENCE
051    + ClassSize.CELL_SET + 2 * ClassSize.ATOMIC_LONG + ClassSize.REENTRANT_LOCK;
052
053  private AtomicReference<CellSet> cellSet = new AtomicReference<>();
054  private final CellComparator comparator;
055  private ReentrantReadWriteLock updatesLock;
056  protected long minSequenceId;
057  private MemStoreLAB memStoreLAB;
058  // Sum of sizes of all Cells added to this Segment. Cell's HeapSize is considered. This is not
059  // including the heap overhead of this class.
060  protected final MemStoreSizing memStoreSizing;
061  protected final TimeRangeTracker timeRangeTracker;
062  protected volatile boolean tagsPresent;
063
064  // Empty constructor to be used when Segment is used as interface,
065  // and there is no need in true Segments state
066  protected Segment(CellComparator comparator, TimeRangeTracker trt) {
067    this.comparator = comparator;
068    // Do we need to be thread safe always? What if ImmutableSegment?
069    // DITTO for the TimeRangeTracker below.
070    this.memStoreSizing = new ThreadSafeMemStoreSizing();
071    this.timeRangeTracker = trt;
072  }
073
074  protected Segment(CellComparator comparator, List<ImmutableSegment> segments,
075    TimeRangeTracker trt) {
076    long dataSize = 0;
077    long heapSize = 0;
078    long OffHeapSize = 0;
079    int cellsCount = 0;
080    for (Segment segment : segments) {
081      MemStoreSize memStoreSize = segment.getMemStoreSize();
082      dataSize += memStoreSize.getDataSize();
083      heapSize += memStoreSize.getHeapSize();
084      OffHeapSize += memStoreSize.getOffHeapSize();
085      cellsCount += memStoreSize.getCellsCount();
086    }
087    this.comparator = comparator;
088    this.updatesLock = new ReentrantReadWriteLock();
089    // Do we need to be thread safe always? What if ImmutableSegment?
090    // DITTO for the TimeRangeTracker below.
091    this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount);
092    this.timeRangeTracker = trt;
093  }
094
095  // This constructor is used to create empty Segments.
096  protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB,
097    TimeRangeTracker trt) {
098    this.cellSet.set(cellSet);
099    this.comparator = comparator;
100    this.updatesLock = new ReentrantReadWriteLock();
101    this.minSequenceId = Long.MAX_VALUE;
102    this.memStoreLAB = memStoreLAB;
103    // Do we need to be thread safe always? What if ImmutableSegment?
104    // DITTO for the TimeRangeTracker below.
105    this.memStoreSizing = new ThreadSafeMemStoreSizing();
106    this.tagsPresent = false;
107    this.timeRangeTracker = trt;
108  }
109
110  protected Segment(Segment segment) {
111    this.cellSet.set(segment.getCellSet());
112    this.comparator = segment.getComparator();
113    this.updatesLock = segment.getUpdatesLock();
114    this.minSequenceId = segment.getMinSequenceId();
115    this.memStoreLAB = segment.getMemStoreLAB();
116    this.memStoreSizing = segment.memStoreSizing;
117    this.tagsPresent = segment.isTagsPresent();
118    this.timeRangeTracker = segment.getTimeRangeTracker();
119  }
120
121  /**
122   * Creates the scanner for the given read point
123   * @return a scanner for the given read point
124   */
125  protected KeyValueScanner getScanner(long readPoint) {
126    return new SegmentScanner(this, readPoint);
127  }
128
129  public List<KeyValueScanner> getScanners(long readPoint) {
130    return Collections.singletonList(new SegmentScanner(this, readPoint));
131  }
132
133  /** Returns whether the segment has any cells */
134  public boolean isEmpty() {
135    return getCellSet().isEmpty();
136  }
137
138  /**
139   * Closing a segment before it is being discarded
140   */
141  public void close() {
142    if (this.memStoreLAB != null) {
143      this.memStoreLAB.close();
144    }
145    // do not set MSLab to null as scanners may still be reading the data here and need to decrease
146    // the counter when they finish
147  }
148
149  /**
150   * If the segment has a memory allocator the cell is being cloned to this space, and returned;
151   * otherwise the given cell is returned When a cell's size is too big (bigger than maxAlloc), it
152   * is not allocated on MSLAB. Since the process of flattening to CellChunkMap assumes that all
153   * cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is
154   * set to 'true' and the cell is copied into MSLAB.
155   * @return either the given cell or its clone
156   */
157  public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) {
158    if (this.memStoreLAB == null) {
159      return cell;
160    }
161
162    Cell cellFromMslab;
163    if (forceCloneOfBigCell) {
164      cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell);
165    } else {
166      cellFromMslab = this.memStoreLAB.copyCellInto(cell);
167    }
168    return (cellFromMslab != null) ? cellFromMslab : cell;
169  }
170
171  /**
172   * Get cell length after serialized in {@link KeyValue}
173   */
174  static int getCellLength(Cell cell) {
175    return cell.getSerializedSize();
176  }
177
178  public boolean shouldSeek(TimeRange tr, long oldestUnexpiredTS) {
179    return !isEmpty() && (tr.isAllTime() || timeRangeTracker.includesTimeRange(tr))
180      && timeRangeTracker.getMax() >= oldestUnexpiredTS;
181  }
182
183  public boolean isTagsPresent() {
184    return tagsPresent;
185  }
186
187  public void incScannerCount() {
188    if (this.memStoreLAB != null) {
189      this.memStoreLAB.incScannerCount();
190    }
191  }
192
193  public void decScannerCount() {
194    if (this.memStoreLAB != null) {
195      this.memStoreLAB.decScannerCount();
196    }
197  }
198
199  /**
200   * Setting the CellSet of the segment - used only for flat immutable segment for setting immutable
201   * CellSet after its creation in immutable segment constructor
202   * @return this object
203   */
204
205  protected Segment setCellSet(CellSet cellSetOld, CellSet cellSetNew) {
206    this.cellSet.compareAndSet(cellSetOld, cellSetNew);
207    return this;
208  }
209
210  @Override
211  public MemStoreSize getMemStoreSize() {
212    return this.memStoreSizing.getMemStoreSize();
213  }
214
215  @Override
216  public long getDataSize() {
217    return this.memStoreSizing.getDataSize();
218  }
219
220  @Override
221  public long getHeapSize() {
222    return this.memStoreSizing.getHeapSize();
223  }
224
225  @Override
226  public long getOffHeapSize() {
227    return this.memStoreSizing.getOffHeapSize();
228  }
229
230  @Override
231  public int getCellsCount() {
232    return memStoreSizing.getCellsCount();
233  }
234
235  @Override
236  public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) {
237    return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount);
238  }
239
240  public boolean sharedLock() {
241    return updatesLock.readLock().tryLock();
242  }
243
244  public void sharedUnlock() {
245    updatesLock.readLock().unlock();
246  }
247
248  public void waitForUpdates() {
249    if (!updatesLock.isWriteLocked()) {
250      updatesLock.writeLock().lock();
251    }
252  }
253
254  @Override
255  public boolean compareAndSetDataSize(long expected, long updated) {
256    return memStoreSizing.compareAndSetDataSize(expected, updated);
257  }
258
259  public long getMinSequenceId() {
260    return minSequenceId;
261  }
262
263  public TimeRangeTracker getTimeRangeTracker() {
264    return this.timeRangeTracker;
265  }
266
267  // *** Methods for SegmentsScanner
268  public Cell last() {
269    return getCellSet().last();
270  }
271
272  public Iterator<Cell> iterator() {
273    return getCellSet().iterator();
274  }
275
276  public SortedSet<Cell> headSet(Cell firstKeyOnRow) {
277    return getCellSet().headSet(firstKeyOnRow);
278  }
279
280  public int compare(Cell left, Cell right) {
281    return getComparator().compare(left, right);
282  }
283
284  public int compareRows(Cell left, Cell right) {
285    return getComparator().compareRows(left, right);
286  }
287
288  /** Returns a set of all cells in the segment */
289  protected CellSet getCellSet() {
290    return cellSet.get();
291  }
292
293  /**
294   * Returns the Cell comparator used by this segment
295   * @return the Cell comparator used by this segment
296   */
297  protected CellComparator getComparator() {
298    return comparator;
299  }
300
301  protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing,
302    boolean sizeAddedPreOperation) {
303    boolean succ = getCellSet().add(cell);
304    updateMetaInfo(cell, succ, mslabUsed, memstoreSizing, sizeAddedPreOperation);
305  }
306
307  protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
308    MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) {
309    long delta = 0;
310    long cellSize = getCellLength(cellToAdd);
311    int cellsCount = succ ? 1 : 0;
312    // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
313    // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
314    // than the counted number)
315    if (succ || mslabUsed) {
316      delta = cellSize;
317    }
318    if (sizeAddedPreOperation) {
319      delta -= cellSize;
320    }
321    long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed);
322    long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed);
323    incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
324    if (memstoreSizing != null) {
325      memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
326    }
327    getTimeRangeTracker().includeTimestamp(cellToAdd);
328    minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
329    // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call.
330    // When we use ACL CP or Visibility CP which deals with Tags during
331    // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not
332    // parse the byte[] to identify the tags length.
333    if (cellToAdd.getTagsLength() > 0) {
334      tagsPresent = true;
335    }
336  }
337
338  protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) {
339    updateMetaInfo(cellToAdd, succ, (getMemStoreLAB() != null), memstoreSizing, false);
340  }
341
342  /**
343   * @return The increase in heap size because of this cell addition. This includes this cell POJO's
344   *         heap size itself and additional overhead because of addition on to CSLM.
345   */
346  protected long heapSizeChange(Cell cell, boolean allocated) {
347    long res = 0;
348    if (allocated) {
349      boolean onHeap = true;
350      MemStoreLAB memStoreLAB = getMemStoreLAB();
351      if (memStoreLAB != null) {
352        onHeap = memStoreLAB.isOnHeap();
353      }
354      res += indexEntryOnHeapSize(onHeap);
355      if (onHeap) {
356        res += cell.heapSize();
357      }
358      res = ClassSize.align(res);
359    }
360    return res;
361  }
362
363  protected long offHeapSizeChange(Cell cell, boolean allocated) {
364    long res = 0;
365    if (allocated) {
366      boolean offHeap = false;
367      MemStoreLAB memStoreLAB = getMemStoreLAB();
368      if (memStoreLAB != null) {
369        offHeap = memStoreLAB.isOffHeap();
370      }
371      res += indexEntryOffHeapSize(offHeap);
372      if (offHeap) {
373        res += cell.heapSize();
374      }
375      res = ClassSize.align(res);
376    }
377    return res;
378  }
379
380  protected long indexEntryOnHeapSize(boolean onHeap) {
381    // in most cases index is allocated on-heap
382    // override this method when it is not always the case, e.g., in CCM
383    return indexEntrySize();
384  }
385
386  protected long indexEntryOffHeapSize(boolean offHeap) {
387    // in most cases index is allocated on-heap
388    // override this method when it is not always the case, e.g., in CCM
389    return 0;
390  }
391
392  protected abstract long indexEntrySize();
393
394  /**
395   * Returns a subset of the segment cell set, which starts with the given cell
396   * @param firstCell a cell in the segment
397   * @return a subset of the segment cell set, which starts with the given cell
398   */
399  protected SortedSet<Cell> tailSet(Cell firstCell) {
400    return getCellSet().tailSet(firstCell);
401  }
402
403  MemStoreLAB getMemStoreLAB() {
404    return memStoreLAB;
405  }
406
407  // Debug methods
408  /**
409   * Dumps all cells of the segment into the given log
410   */
411  void dump(Logger log) {
412    for (Cell cell : getCellSet()) {
413      log.debug(Objects.toString(cell));
414    }
415  }
416
417  @Override
418  public String toString() {
419    String res = "type=" + this.getClass().getSimpleName() + ", ";
420    res += "empty=" + (isEmpty() ? "yes" : "no") + ", ";
421    res += "cellCount=" + getCellsCount() + ", ";
422    res += "cellSize=" + getDataSize() + ", ";
423    res += "totalHeapSize=" + getHeapSize() + ", ";
424    res += "min timestamp=" + timeRangeTracker.getMin() + ", ";
425    res += "max timestamp=" + timeRangeTracker.getMax();
426    return res;
427  }
428
429  private ReentrantReadWriteLock getUpdatesLock() {
430    return updatesLock;
431  }
432}