001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.util.Collections;
021import java.util.Iterator;
022import java.util.List;
023import java.util.Objects;
024import java.util.SortedSet;
025import java.util.concurrent.atomic.AtomicReference;
026import java.util.concurrent.locks.ReentrantReadWriteLock;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.CellComparator;
029import org.apache.hadoop.hbase.KeyValue;
030import org.apache.hadoop.hbase.io.TimeRange;
031import org.apache.hadoop.hbase.util.Bytes;
032import org.apache.hadoop.hbase.util.ClassSize;
033import org.apache.yetus.audience.InterfaceAudience;
034import org.slf4j.Logger;
035
036/**
037 * This is an abstraction of a segment maintained in a memstore, e.g., the active cell set or its
038 * snapshot. This abstraction facilitates the management of the compaction pipeline and the shifts
039 * of these segments from active set to snapshot set in the default implementation.
040 */
041@InterfaceAudience.Private
042public abstract class Segment implements MemStoreSizing {
043
044  public final static long FIXED_OVERHEAD =
045    ClassSize.align(ClassSize.OBJECT + 6 * ClassSize.REFERENCE // cellSet, comparator, updatesLock,
046                                                               // memStoreLAB, memStoreSizing,
047                                                               // and timeRangeTracker
048      + Bytes.SIZEOF_LONG // minSequenceId
049      + Bytes.SIZEOF_BOOLEAN); // tagsPresent
050  public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.ATOMIC_REFERENCE
051    + ClassSize.CELL_SET + 2 * ClassSize.ATOMIC_LONG + ClassSize.REENTRANT_LOCK;
052
053  private AtomicReference<CellSet> cellSet = new AtomicReference<>();
054  private final CellComparator comparator;
055  private ReentrantReadWriteLock updatesLock;
056  protected long minSequenceId;
057  private MemStoreLAB memStoreLAB;
058  // Sum of sizes of all Cells added to this Segment. Cell's HeapSize is considered. This is not
059  // including the heap overhead of this class.
060  protected final MemStoreSizing memStoreSizing;
061  protected final TimeRangeTracker timeRangeTracker;
062  protected volatile boolean tagsPresent;
063
064  // Empty constructor to be used when Segment is used as interface,
065  // and there is no need in true Segments state
066  protected Segment(CellComparator comparator, TimeRangeTracker trt) {
067    this.comparator = comparator;
068    // Do we need to be thread safe always? What if ImmutableSegment?
069    // DITTO for the TimeRangeTracker below.
070    this.memStoreSizing = new ThreadSafeMemStoreSizing();
071    this.timeRangeTracker = trt;
072  }
073
074  protected Segment(CellComparator comparator, List<ImmutableSegment> segments,
075    TimeRangeTracker trt) {
076    long dataSize = 0;
077    long heapSize = 0;
078    long OffHeapSize = 0;
079    int cellsCount = 0;
080    for (Segment segment : segments) {
081      MemStoreSize memStoreSize = segment.getMemStoreSize();
082      dataSize += memStoreSize.getDataSize();
083      heapSize += memStoreSize.getHeapSize();
084      OffHeapSize += memStoreSize.getOffHeapSize();
085      cellsCount += memStoreSize.getCellsCount();
086    }
087    this.comparator = comparator;
088    this.updatesLock = new ReentrantReadWriteLock();
089    // Do we need to be thread safe always? What if ImmutableSegment?
090    // DITTO for the TimeRangeTracker below.
091    this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount);
092    this.timeRangeTracker = trt;
093  }
094
095  // This constructor is used to create empty Segments.
096  protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB,
097    TimeRangeTracker trt) {
098    this.cellSet.set(cellSet);
099    this.comparator = comparator;
100    this.updatesLock = new ReentrantReadWriteLock();
101    this.minSequenceId = Long.MAX_VALUE;
102    this.memStoreLAB = memStoreLAB;
103    // Do we need to be thread safe always? What if ImmutableSegment?
104    // DITTO for the TimeRangeTracker below.
105    this.memStoreSizing = new ThreadSafeMemStoreSizing();
106    this.tagsPresent = false;
107    this.timeRangeTracker = trt;
108  }
109
110  protected Segment(Segment segment) {
111    this.cellSet.set(segment.getCellSet());
112    this.comparator = segment.getComparator();
113    this.updatesLock = segment.getUpdatesLock();
114    this.minSequenceId = segment.getMinSequenceId();
115    this.memStoreLAB = segment.getMemStoreLAB();
116    this.memStoreSizing = segment.memStoreSizing;
117    this.tagsPresent = segment.isTagsPresent();
118    this.timeRangeTracker = segment.getTimeRangeTracker();
119  }
120
121  /**
122   * Creates the scanner for the given read point
123   * @return a scanner for the given read point
124   */
125  protected KeyValueScanner getScanner(long readPoint) {
126    return new SegmentScanner(this, readPoint);
127  }
128
129  public List<KeyValueScanner> getScanners(long readPoint) {
130    return Collections.singletonList(new SegmentScanner(this, readPoint));
131  }
132
133  /**
134   * @return whether the segment has any cells
135   */
136  public boolean isEmpty() {
137    return getCellSet().isEmpty();
138  }
139
140  /**
141   * Closing a segment before it is being discarded
142   */
143  public void close() {
144    if (this.memStoreLAB != null) {
145      this.memStoreLAB.close();
146    }
147    // do not set MSLab to null as scanners may still be reading the data here and need to decrease
148    // the counter when they finish
149  }
150
151  /**
152   * If the segment has a memory allocator the cell is being cloned to this space, and returned;
153   * otherwise the given cell is returned When a cell's size is too big (bigger than maxAlloc), it
154   * is not allocated on MSLAB. Since the process of flattening to CellChunkMap assumes that all
155   * cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is
156   * set to 'true' and the cell is copied into MSLAB.
157   * @return either the given cell or its clone
158   */
159  public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) {
160    if (this.memStoreLAB == null) {
161      return cell;
162    }
163
164    Cell cellFromMslab;
165    if (forceCloneOfBigCell) {
166      cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell);
167    } else {
168      cellFromMslab = this.memStoreLAB.copyCellInto(cell);
169    }
170    return (cellFromMslab != null) ? cellFromMslab : cell;
171  }
172
173  /**
174   * Get cell length after serialized in {@link KeyValue}
175   */
176  static int getCellLength(Cell cell) {
177    return cell.getSerializedSize();
178  }
179
180  public boolean shouldSeek(TimeRange tr, long oldestUnexpiredTS) {
181    return !isEmpty() && (tr.isAllTime() || timeRangeTracker.includesTimeRange(tr))
182      && timeRangeTracker.getMax() >= oldestUnexpiredTS;
183  }
184
185  public boolean isTagsPresent() {
186    return tagsPresent;
187  }
188
189  public void incScannerCount() {
190    if (this.memStoreLAB != null) {
191      this.memStoreLAB.incScannerCount();
192    }
193  }
194
195  public void decScannerCount() {
196    if (this.memStoreLAB != null) {
197      this.memStoreLAB.decScannerCount();
198    }
199  }
200
201  /**
202   * Setting the CellSet of the segment - used only for flat immutable segment for setting immutable
203   * CellSet after its creation in immutable segment constructor
204   * @return this object
205   */
206
207  protected Segment setCellSet(CellSet cellSetOld, CellSet cellSetNew) {
208    this.cellSet.compareAndSet(cellSetOld, cellSetNew);
209    return this;
210  }
211
212  @Override
213  public MemStoreSize getMemStoreSize() {
214    return this.memStoreSizing.getMemStoreSize();
215  }
216
217  @Override
218  public long getDataSize() {
219    return this.memStoreSizing.getDataSize();
220  }
221
222  @Override
223  public long getHeapSize() {
224    return this.memStoreSizing.getHeapSize();
225  }
226
227  @Override
228  public long getOffHeapSize() {
229    return this.memStoreSizing.getOffHeapSize();
230  }
231
232  @Override
233  public int getCellsCount() {
234    return memStoreSizing.getCellsCount();
235  }
236
237  @Override
238  public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) {
239    return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount);
240  }
241
242  public boolean sharedLock() {
243    return updatesLock.readLock().tryLock();
244  }
245
246  public void sharedUnlock() {
247    updatesLock.readLock().unlock();
248  }
249
250  public void waitForUpdates() {
251    if (!updatesLock.isWriteLocked()) {
252      updatesLock.writeLock().lock();
253    }
254  }
255
256  @Override
257  public boolean compareAndSetDataSize(long expected, long updated) {
258    return memStoreSizing.compareAndSetDataSize(expected, updated);
259  }
260
261  public long getMinSequenceId() {
262    return minSequenceId;
263  }
264
265  public TimeRangeTracker getTimeRangeTracker() {
266    return this.timeRangeTracker;
267  }
268
269  // *** Methods for SegmentsScanner
270  public Cell last() {
271    return getCellSet().last();
272  }
273
274  public Iterator<Cell> iterator() {
275    return getCellSet().iterator();
276  }
277
278  public SortedSet<Cell> headSet(Cell firstKeyOnRow) {
279    return getCellSet().headSet(firstKeyOnRow);
280  }
281
282  public int compare(Cell left, Cell right) {
283    return getComparator().compare(left, right);
284  }
285
286  public int compareRows(Cell left, Cell right) {
287    return getComparator().compareRows(left, right);
288  }
289
290  /**
291   * @return a set of all cells in the segment
292   */
293  protected CellSet getCellSet() {
294    return cellSet.get();
295  }
296
297  /**
298   * Returns the Cell comparator used by this segment
299   * @return the Cell comparator used by this segment
300   */
301  protected CellComparator getComparator() {
302    return comparator;
303  }
304
305  protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing,
306    boolean sizeAddedPreOperation) {
307    boolean succ = getCellSet().add(cell);
308    updateMetaInfo(cell, succ, mslabUsed, memstoreSizing, sizeAddedPreOperation);
309  }
310
311  protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
312    MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) {
313    long delta = 0;
314    long cellSize = getCellLength(cellToAdd);
315    int cellsCount = succ ? 1 : 0;
316    // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
317    // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
318    // than the counted number)
319    if (succ || mslabUsed) {
320      delta = cellSize;
321    }
322    if (sizeAddedPreOperation) {
323      delta -= cellSize;
324    }
325    long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed);
326    long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed);
327    incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
328    if (memstoreSizing != null) {
329      memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
330    }
331    getTimeRangeTracker().includeTimestamp(cellToAdd);
332    minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
333    // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call.
334    // When we use ACL CP or Visibility CP which deals with Tags during
335    // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not
336    // parse the byte[] to identify the tags length.
337    if (cellToAdd.getTagsLength() > 0) {
338      tagsPresent = true;
339    }
340  }
341
342  protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) {
343    updateMetaInfo(cellToAdd, succ, (getMemStoreLAB() != null), memstoreSizing, false);
344  }
345
346  /**
347   * @return The increase in heap size because of this cell addition. This includes this cell POJO's
348   *         heap size itself and additional overhead because of addition on to CSLM.
349   */
350  protected long heapSizeChange(Cell cell, boolean allocated) {
351    long res = 0;
352    if (allocated) {
353      boolean onHeap = true;
354      MemStoreLAB memStoreLAB = getMemStoreLAB();
355      if (memStoreLAB != null) {
356        onHeap = memStoreLAB.isOnHeap();
357      }
358      res += indexEntryOnHeapSize(onHeap);
359      if (onHeap) {
360        res += cell.heapSize();
361      }
362      res = ClassSize.align(res);
363    }
364    return res;
365  }
366
367  protected long offHeapSizeChange(Cell cell, boolean allocated) {
368    long res = 0;
369    if (allocated) {
370      boolean offHeap = false;
371      MemStoreLAB memStoreLAB = getMemStoreLAB();
372      if (memStoreLAB != null) {
373        offHeap = memStoreLAB.isOffHeap();
374      }
375      res += indexEntryOffHeapSize(offHeap);
376      if (offHeap) {
377        res += cell.heapSize();
378      }
379      res = ClassSize.align(res);
380    }
381    return res;
382  }
383
384  protected long indexEntryOnHeapSize(boolean onHeap) {
385    // in most cases index is allocated on-heap
386    // override this method when it is not always the case, e.g., in CCM
387    return indexEntrySize();
388  }
389
390  protected long indexEntryOffHeapSize(boolean offHeap) {
391    // in most cases index is allocated on-heap
392    // override this method when it is not always the case, e.g., in CCM
393    return 0;
394  }
395
396  protected abstract long indexEntrySize();
397
398  /**
399   * Returns a subset of the segment cell set, which starts with the given cell
400   * @param firstCell a cell in the segment
401   * @return a subset of the segment cell set, which starts with the given cell
402   */
403  protected SortedSet<Cell> tailSet(Cell firstCell) {
404    return getCellSet().tailSet(firstCell);
405  }
406
407  MemStoreLAB getMemStoreLAB() {
408    return memStoreLAB;
409  }
410
411  // Debug methods
412  /**
413   * Dumps all cells of the segment into the given log
414   */
415  void dump(Logger log) {
416    for (Cell cell : getCellSet()) {
417      log.debug(Objects.toString(cell));
418    }
419  }
420
421  @Override
422  public String toString() {
423    String res = "type=" + this.getClass().getSimpleName() + ", ";
424    res += "empty=" + (isEmpty() ? "yes" : "no") + ", ";
425    res += "cellCount=" + getCellsCount() + ", ";
426    res += "cellSize=" + getDataSize() + ", ";
427    res += "totalHeapSize=" + getHeapSize() + ", ";
428    res += "min timestamp=" + timeRangeTracker.getMin() + ", ";
429    res += "max timestamp=" + timeRangeTracker.getMax();
430    return res;
431  }
432
433  private ReentrantReadWriteLock getUpdatesLock() {
434    return updatesLock;
435  }
436}