001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.wal;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.Map;
029import java.util.Set;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.testclassification.SmallTests;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.junit.ClassRule;
035import org.junit.Test;
036import org.junit.experimental.categories.Category;
037
038@Category(SmallTests.class)
039public class TestSequenceIdAccounting {
040
041  @ClassRule
042  public static final HBaseClassTestRule CLASS_RULE =
043    HBaseClassTestRule.forClass(TestSequenceIdAccounting.class);
044
045  private static final byte[] ENCODED_REGION_NAME = Bytes.toBytes("r");
046  private static final byte[] FAMILY_NAME = Bytes.toBytes("cf");
047  private static final byte[] META_FAMILY = Bytes.toBytes("METAFAMILY");
048  private static final Set<byte[]> FAMILIES;
049  private static final Set<byte[]> META_FAMILY_SET;
050  static {
051    FAMILIES = new HashSet<>();
052    FAMILIES.add(FAMILY_NAME);
053    META_FAMILY_SET = new HashSet<>();
054    META_FAMILY_SET.add(META_FAMILY);
055  }
056
057  @Test
058  public void testStartCacheFlush() {
059    SequenceIdAccounting sida = new SequenceIdAccounting();
060    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
061    Map<byte[], Long> m = new HashMap<>();
062    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
063    assertEquals(HConstants.NO_SEQNUM, (long) sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES));
064    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
065    long sequenceid = 1;
066    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
067    // Only one family so should return NO_SEQNUM still.
068    assertEquals(HConstants.NO_SEQNUM, (long) sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES));
069    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
070    long currentSequenceId = sequenceid;
071    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
072    final Set<byte[]> otherFamily = new HashSet<>(1);
073    otherFamily.add(Bytes.toBytes("otherCf"));
074    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
075    // Should return oldest sequence id in the region.
076    assertEquals(currentSequenceId, (long) sida.startCacheFlush(ENCODED_REGION_NAME, otherFamily));
077    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
078  }
079
080  @Test
081  public void testAreAllLower() {
082    SequenceIdAccounting sida = new SequenceIdAccounting();
083    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
084    Map<byte[], Long> m = new HashMap<>();
085    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
086    assertTrue(sida.areAllLower(m, null));
087    long sequenceid = 1;
088    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
089    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
090    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
091    assertTrue(sida.areAllLower(m, null));
092    m.put(ENCODED_REGION_NAME, sequenceid);
093    assertFalse(sida.areAllLower(m, null));
094    ArrayList<byte[]> regions = new ArrayList<>();
095    assertFalse(sida.areAllLower(m, regions));
096    assertEquals(1, regions.size());
097    assertArrayEquals(ENCODED_REGION_NAME, regions.get(0));
098    long lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME);
099    assertEquals("Lowest should be first sequence id inserted", 1, lowest);
100    m.put(ENCODED_REGION_NAME, lowest);
101    assertFalse(sida.areAllLower(m, null));
102    // Now make sure above works when flushing.
103    sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES);
104    assertFalse(sida.areAllLower(m, null));
105    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
106    assertTrue(sida.areAllLower(m, null));
107    // Let the flush complete and if we ask if the sequenceid is lower, should be yes since no edits
108    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
109    m.put(ENCODED_REGION_NAME, sequenceid);
110    assertTrue(sida.areAllLower(m, null));
111    // Flush again but add sequenceids while we are flushing.
112    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
113    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
114    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
115    lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME);
116    m.put(ENCODED_REGION_NAME, lowest);
117    assertFalse(sida.areAllLower(m, null));
118    sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES);
119    // The cache flush will clear out all sequenceid accounting by region.
120    assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));
121    sida.completeCacheFlush(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
122    // No new edits have gone in so no sequenceid to work with.
123    assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));
124    // Make an edit behind all we'll put now into sida.
125    m.put(ENCODED_REGION_NAME, sequenceid);
126    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
127    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
128    sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true);
129    assertTrue(sida.areAllLower(m, null));
130    m.put(ENCODED_REGION_NAME, sequenceid);
131    assertFalse(sida.areAllLower(m, null));
132
133    // Test the METAFAMILY is filtered in SequenceIdAccounting.lowestUnflushedSequenceIds
134    SequenceIdAccounting meta_sida = new SequenceIdAccounting();
135    Map<byte[], Long> meta_m = new HashMap<>();
136    meta_sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
137    meta_m.put(ENCODED_REGION_NAME, sequenceid);
138    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
139    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
140    meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true);
141    assertTrue(meta_sida.areAllLower(meta_m, null));
142    meta_m.put(ENCODED_REGION_NAME, sequenceid);
143    assertTrue(meta_sida.areAllLower(meta_m, null));
144  }
145
146  @Test
147  public void testFindLower() {
148    SequenceIdAccounting sida = new SequenceIdAccounting();
149    sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME);
150    Map<byte[], Long> m = new HashMap<>();
151    m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM);
152    long sequenceid = 1;
153    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true);
154    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
155    sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true);
156    assertTrue(sida.findLower(m) == null);
157    m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME));
158    assertTrue(sida.findLower(m).size() == 1);
159    m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME) - 1);
160    assertTrue(sida.findLower(m) == null);
161  }
162}