001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.procedure2.store.wal;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotSame;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.util.concurrent.ThreadLocalRandom;
026import org.apache.hadoop.hbase.procedure2.Procedure;
027import org.apache.hadoop.hbase.testclassification.MasterTests;
028import org.apache.hadoop.hbase.testclassification.MediumTests;
029import org.junit.jupiter.api.Tag;
030import org.junit.jupiter.api.Test;
031import org.slf4j.Logger;
032import org.slf4j.LoggerFactory;
033
034@Tag(MasterTests.TAG)
035@Tag(MediumTests.TAG)
036public class TestProcedureStoreTracker {
037
038  private static final Logger LOG = LoggerFactory.getLogger(TestProcedureStoreTracker.class);
039
040  @Test
041  public void testSeqInsertAndDelete() {
042    ProcedureStoreTracker tracker = new ProcedureStoreTracker();
043    assertTrue(tracker.isEmpty());
044
045    final int MIN_PROC = 1;
046    final int MAX_PROC = 1 << 10;
047
048    // sequential insert
049    for (int i = MIN_PROC; i < MAX_PROC; ++i) {
050      tracker.insert(i);
051
052      // All the proc that we inserted should not be deleted
053      for (int j = MIN_PROC; j <= i; ++j) {
054        assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
055      }
056      // All the proc that are not yet inserted should be result as deleted
057      for (int j = i + 1; j < MAX_PROC; ++j) {
058        assertNotSame(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
059      }
060    }
061
062    // sequential delete
063    for (int i = MIN_PROC; i < MAX_PROC; ++i) {
064      tracker.delete(i);
065
066      // All the proc that we deleted should be deleted
067      for (int j = MIN_PROC; j <= i; ++j) {
068        assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(j));
069      }
070      // All the proc that are not yet deleted should be result as not deleted
071      for (int j = i + 1; j < MAX_PROC; ++j) {
072        assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
073      }
074    }
075    assertTrue(tracker.isEmpty());
076  }
077
078  @Test
079  public void testPartialTracker() {
080    ProcedureStoreTracker tracker = new ProcedureStoreTracker();
081    tracker.setPartialFlag(true);
082
083    // nothing in the tracker, the state is unknown
084    assertTrue(tracker.isEmpty());
085    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(1));
086    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
087
088    // Mark 1 as deleted, now that is a known state
089    tracker.setDeleted(1, true);
090    tracker.dump();
091    assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
092    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
093    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
094
095    // Mark 579 as non-deleted, now that is a known state
096    tracker.setDeleted(579, false);
097    assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
098    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
099    assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(579));
100    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(577));
101    assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(580));
102
103    tracker.setDeleted(579, true);
104    tracker.setPartialFlag(false);
105    assertTrue(tracker.isEmpty());
106  }
107
108  @Test
109  public void testBasicCRUD() {
110    ProcedureStoreTracker tracker = new ProcedureStoreTracker();
111    assertTrue(tracker.isEmpty());
112
113    long[] procs = new long[] { 1, 2, 3, 4, 5, 6 };
114
115    tracker.insert(procs[0]);
116    tracker.insert(procs[1], new long[] { procs[2], procs[3], procs[4] });
117    assertFalse(tracker.isEmpty());
118    assertTrue(tracker.isAllModified());
119
120    tracker.resetModified();
121    assertFalse(tracker.isAllModified());
122
123    for (int i = 0; i < 4; ++i) {
124      tracker.update(procs[i]);
125      assertFalse(tracker.isEmpty());
126      assertFalse(tracker.isAllModified());
127    }
128
129    tracker.update(procs[4]);
130    assertFalse(tracker.isEmpty());
131    assertTrue(tracker.isAllModified());
132
133    tracker.update(procs[5]);
134    assertFalse(tracker.isEmpty());
135    assertTrue(tracker.isAllModified());
136
137    for (int i = 0; i < 5; ++i) {
138      tracker.delete(procs[i]);
139      assertFalse(tracker.isEmpty());
140      assertTrue(tracker.isAllModified());
141    }
142    tracker.delete(procs[5]);
143    assertTrue(tracker.isEmpty());
144  }
145
146  @Test
147  public void testRandLoad() {
148    final int NPROCEDURES = 2500;
149    final int NRUNS = 5000;
150
151    final ProcedureStoreTracker tracker = new ProcedureStoreTracker();
152
153    for (int i = 0; i < NRUNS; ++i) {
154      assertTrue(tracker.isEmpty());
155
156      int count = 0;
157      while (count < NPROCEDURES) {
158        long procId = ThreadLocalRandom.current().nextLong();
159        if (procId < 1) {
160          continue;
161        }
162
163        tracker.setDeleted(procId, i % 2 == 0);
164        count++;
165      }
166
167      tracker.reset();
168    }
169  }
170
171  @Test
172  public void testLoad() {
173    final int MAX_PROCS = 1000;
174    final ProcedureStoreTracker tracker = new ProcedureStoreTracker();
175    for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) {
176      for (int start = 1; start <= numProcs; ++start) {
177        assertTrue(tracker.isEmpty());
178
179        LOG.debug("loading " + numProcs + " procs from start=" + start);
180        for (int i = start; i <= numProcs; ++i) {
181          tracker.setDeleted(i, false);
182        }
183        for (int i = 1; i < start; ++i) {
184          tracker.setDeleted(i, false);
185        }
186
187        tracker.reset();
188      }
189    }
190  }
191
192  @Test
193  public void testDelete() {
194    final ProcedureStoreTracker tracker = new ProcedureStoreTracker();
195
196    long[] procIds = new long[] { 65, 1, 193 };
197    for (int i = 0; i < procIds.length; ++i) {
198      tracker.insert(procIds[i]);
199      tracker.dump();
200    }
201
202    for (int i = 0; i < (64 * 4); ++i) {
203      boolean hasProc = false;
204      for (int j = 0; j < procIds.length; ++j) {
205        if (procIds[j] == i) {
206          hasProc = true;
207          break;
208        }
209      }
210      if (hasProc) {
211        assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(i));
212      } else {
213        assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(i), "procId=" + i);
214      }
215    }
216  }
217
218  @Test
219  public void testSetDeletedIfModified() {
220    final ProcedureStoreTracker tracker = new ProcedureStoreTracker();
221    final long[] procIds = new long[] { 1, 3, 7, 152, 512, 1024, 1025 };
222
223    // test single proc
224    for (int i = 0; i < procIds.length; ++i) {
225      tracker.insert(procIds[i]);
226    }
227    assertFalse(tracker.isEmpty());
228
229    for (int i = 0; i < procIds.length; ++i) {
230      tracker.setDeletedIfModified(procIds[i] - 1);
231      tracker.setDeletedIfModified(procIds[i]);
232      tracker.setDeletedIfModified(procIds[i] + 1);
233    }
234    assertTrue(tracker.isEmpty());
235
236    // test batch
237    tracker.reset();
238    for (int i = 0; i < procIds.length; ++i) {
239      tracker.insert(procIds[i]);
240    }
241    assertFalse(tracker.isEmpty());
242
243    tracker.setDeletedIfModified(procIds);
244    assertTrue(tracker.isEmpty());
245  }
246
247  @Test
248  public void testGetActiveProcIds() {
249    ProcedureStoreTracker tracker = new ProcedureStoreTracker();
250    for (int i = 0; i < 10000; i++) {
251      tracker.insert(i * 10);
252    }
253    for (int i = 0; i < 10000; i += 2) {
254      tracker.delete(i * 10);
255    }
256    long[] activeProcIds = tracker.getAllActiveProcIds();
257    assertEquals(5000, activeProcIds.length);
258    for (int i = 0; i < 5000; i++) {
259      assertEquals((2 * i + 1) * 10, activeProcIds[i]);
260    }
261  }
262
263  @Test
264  public void testGetActiveMinProcId() {
265    ProcedureStoreTracker tracker = new ProcedureStoreTracker();
266    assertEquals(Procedure.NO_PROC_ID, tracker.getActiveMinProcId());
267    for (int i = 100; i < 1000; i = 2 * i + 1) {
268      tracker.insert(i);
269    }
270    for (int i = 100; i < 1000; i = 2 * i + 1) {
271      assertEquals(i, tracker.getActiveMinProcId());
272      tracker.delete(i);
273    }
274    assertEquals(Procedure.NO_PROC_ID, tracker.getActiveMinProcId());
275  }
276}