001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.wal;
019
020import static org.junit.jupiter.api.Assertions.assertArrayEquals;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.Map;
029import java.util.Set;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.testclassification.SmallTests;
032import org.apache.hadoop.hbase.util.Bytes;
033import org.junit.jupiter.api.Tag;
034import org.junit.jupiter.api.Test;
035
036@Tag(SmallTests.TAG)
037public class TestSequenceIdAccounting {
038
039  private static final byte[] ENCODED_REGION_NAME = Bytes.toBytes("r");
040  private static final byte[] FAMILY_NAME = Bytes.toBytes("cf");
041  private static final byte[] META_FAMILY = Bytes.toBytes("METAFAMILY");
042  private static final Set<byte[]> FAMILIES;
043  private static final Set<byte[]> META_FAMILY_SET;
044  static {
045    FAMILIES = new HashSet<>();
046    FAMILIES.add(FAMILY_NAME);
047    META_FAMILY_SET = new HashSet<>();
048    META_FAMILY_SET.add(META_FAMILY);
049  }
050
051  @Test
052  public void testStartCacheFlush() {
053    SequenceIdAccounting sida = new SequenceIdAccounting();
054    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
055    Map<byte[], Long> m = new HashMap<>();
056    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
057    assertEquals(HConstants.NO_SEQNUM, (long) sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES));
058    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
059    long sequenceid = 1;
060    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
061    // Only one family so should return NO_SEQNUM still.
062    assertEquals(HConstants.NO_SEQNUM, (long) sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES));
063    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
064    long currentSequenceId = sequenceid;
065    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
066    final Set<byte[]> otherFamily = new HashSet<>(1);
067    otherFamily.add(Bytes.toBytes("otherCf"));
068    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
069    // Should return oldest sequence id in the region.
070    assertEquals(currentSequenceId, (long) sida.startCacheFlush(ENCODED_REGION_NAME, otherFamily));
071    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
072  }
073
074  @Test
075  public void testAreAllLower() {
076    SequenceIdAccounting sida = new SequenceIdAccounting();
077    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
078    Map<byte[], Long> m = new HashMap<>();
079    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
080    assertTrue(sida.areAllLower(m, null));
081    long sequenceid = 1;
082    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
083    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
084    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
085    assertTrue(sida.areAllLower(m, null));
086    m.put(ENCODED_REGION_NAME, sequenceid);
087    assertFalse(sida.areAllLower(m, null));
088    ArrayList<byte[]> regions = new ArrayList<>();
089    assertFalse(sida.areAllLower(m, regions));
090    assertEquals(1, regions.size());
091    assertArrayEquals(ENCODED_REGION_NAME, regions.get(0));
092    long lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME);
093    assertEquals(1, lowest, "Lowest should be first sequence id inserted");
094    m.put(ENCODED_REGION_NAME, lowest);
095    assertFalse(sida.areAllLower(m, null));
096    // Now make sure above works when flushing.
097    sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES);
098    assertFalse(sida.areAllLower(m, null));
099    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
100    assertTrue(sida.areAllLower(m, null));
101    // Let the flush complete and if we ask if the sequenceid is lower, should be yes since no edits
102    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
103    m.put(ENCODED_REGION_NAME, sequenceid);
104    assertTrue(sida.areAllLower(m, null));
105    // Flush again but add sequenceids while we are flushing.
106    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
107    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
108    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
109    lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME);
110    m.put(ENCODED_REGION_NAME, lowest);
111    assertFalse(sida.areAllLower(m, null));
112    sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES);
113    // The cache flush will clear out all sequenceid accounting by region.
114    assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));
115    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
116    // No new edits have gone in so no sequenceid to work with.
117    assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));
118    // Make an edit behind all we'll put now into sida.
119    m.put(ENCODED_REGION_NAME, sequenceid);
120    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
121    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
122    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
123    assertTrue(sida.areAllLower(m, null));
124    m.put(ENCODED_REGION_NAME, sequenceid);
125    assertFalse(sida.areAllLower(m, null));
126
127    // Test the METAFAMILY is filtered in SequenceIdAccounting.lowestUnflushedSequenceIds
128    SequenceIdAccounting meta_sida = new SequenceIdAccounting();
129    Map<byte[], Long> meta_m = new HashMap<>();
130    meta_sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
131    meta_m.put(ENCODED_REGION_NAME, sequenceid);
132    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
133    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
134    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
135    assertTrue(meta_sida.areAllLower(meta_m, null));
136    meta_m.put(ENCODED_REGION_NAME, sequenceid);
137    assertTrue(meta_sida.areAllLower(meta_m, null));
138  }
139
140  @Test
141  public void testFindLower() {
142    SequenceIdAccounting sida = new SequenceIdAccounting();
143    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
144    Map<byte[], Long> m = new HashMap<>();
145    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
146    sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, 1, true);
147    sida.update(ENCODED_REGION_NAME, FAMILIES, 2, true);
148    sida.update(ENCODED_REGION_NAME, FAMILIES, 3, true);
149    sida.update(ENCODED_REGION_NAME, FAMILIES, 4, true);
150    m.put(ENCODED_REGION_NAME, 1L);
151    assertTrue(sida.findLower(m) == null);
152    m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME));
153    assertTrue(sida.findLower(m).size() == 1);
154    m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME) - 1);
155    assertTrue(sida.findLower(m) == null);
156  }
157}