Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertThrows;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.util.Collections;
027import java.util.concurrent.Executors;
028import java.util.concurrent.Future;
029import org.apache.hadoop.hbase.DoNotRetryIOException;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.MetaTableAccessor;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.RegionInfo;
034import org.apache.hadoop.hbase.client.RegionInfoBuilder;
035import org.apache.hadoop.hbase.master.RegionState.State;
036import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
037import org.apache.hadoop.hbase.procedure2.util.StringUtils;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.Test;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046@Tag(MasterTests.TAG)
047@Tag(LargeTests.TAG)
048public class TestAssignmentManager extends TestAssignmentManagerBase {
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestAssignmentManager.class);
051
052  @Test
053  public void testAssignWithGoodExec() throws Exception {
054    // collect AM metrics before test
055    collectAssignmentManagerMetrics();
056
057    testAssign(new GoodRsExecutor());
058
059    assertEquals(assignSubmittedCount + NREGIONS,
060      assignProcMetrics.getSubmittedCounter().getCount());
061    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
062  }
063
064  @Test
065  public void testAssignAndCrashBeforeResponse() throws Exception {
066    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
067    RegionInfo hri = createRegionInfo(tableName, 1);
068    rsDispatcher.setMockRsExecutor(new HangThenRSCrashExecutor());
069    TransitRegionStateProcedure proc = createAssignProcedure(hri);
070    waitOnFuture(submitProcedure(proc));
071  }
072
073  @Test
074  public void testUnassignAndCrashBeforeResponse() throws Exception {
075    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
076    RegionInfo hri = createRegionInfo(tableName, 1);
077    rsDispatcher.setMockRsExecutor(new HangOnCloseThenRSCrashExecutor());
078    for (int i = 0; i < HangOnCloseThenRSCrashExecutor.TYPES_OF_FAILURE; i++) {
079      TransitRegionStateProcedure assign = createAssignProcedure(hri);
080      waitOnFuture(submitProcedure(assign));
081      TransitRegionStateProcedure unassign = createUnassignProcedure(hri);
082      waitOnFuture(submitProcedure(unassign));
083    }
084  }
085
086  @Test
087  public void testAssignSocketTimeout() throws Exception {
088    TableName tableName = TableName.valueOf(testMethodName);
089    RegionInfo hri = createRegionInfo(tableName, 1);
090
091    // collect AM metrics before test
092    collectAssignmentManagerMetrics();
093
094    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20));
095    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
096
097    // we crashed a rs, so it is possible that there are other regions on the rs which will also be
098    // reassigned, so here we just assert greater than, not the exact number.
099    assertTrue(assignProcMetrics.getSubmittedCounter().getCount() > assignSubmittedCount);
100    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
101  }
102
103  @Test
104  public void testAssignQueueFullOnce() throws Exception {
105    TableName tableName = TableName.valueOf(testMethodName);
106    RegionInfo hri = createRegionInfo(tableName, 1);
107
108    // collect AM metrics before test
109    collectAssignmentManagerMetrics();
110
111    rsDispatcher.setMockRsExecutor(new CallQueueTooBigOnceRsExecutor());
112    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
113
114    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
115    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
116  }
117
118  @Test
119  public void testTimeoutThenQueueFull() throws Exception {
120    TableName tableName = TableName.valueOf(testMethodName);
121    RegionInfo hri = createRegionInfo(tableName, 1);
122
123    // collect AM metrics before test
124    collectAssignmentManagerMetrics();
125
126    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(10));
127    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
128    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(15));
129    waitOnFuture(submitProcedure(createUnassignProcedure(hri)));
130
131    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
132    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
133    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
134    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
135  }
136
137  private void testAssign(final MockRSExecutor executor) throws Exception {
138    testAssign(executor, NREGIONS);
139  }
140
141  private void testAssign(MockRSExecutor executor, int nRegions) throws Exception {
142    rsDispatcher.setMockRsExecutor(executor);
143
144    TransitRegionStateProcedure[] assignments = new TransitRegionStateProcedure[nRegions];
145
146    long st = EnvironmentEdgeManager.currentTime();
147    bulkSubmit(assignments);
148
149    for (int i = 0; i < assignments.length; ++i) {
150      ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), assignments[i]);
151      assertTrue(assignments[i].isSuccess(), assignments[i].toString());
152    }
153    long et = EnvironmentEdgeManager.currentTime();
154    float sec = ((et - st) / 1000.0f);
155    LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)", assignments.length,
156      StringUtils.humanTimeDiff(et - st), assignments.length / sec));
157  }
158
159  @Test
160  public void testAssignAnAssignedRegion() throws Exception {
161    final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
162    final RegionInfo hri = createRegionInfo(tableName, 1);
163
164    // collect AM metrics before test
165    collectAssignmentManagerMetrics();
166
167    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
168
169    Future<byte[]> futureA = submitProcedure(createAssignProcedure(hri));
170
171    // wait first assign
172    waitOnFuture(futureA);
173    am.getRegionStates().isRegionInState(hri, State.OPEN);
174    // Second should be a noop. We should recognize region is already OPEN internally
175    // and skip out doing nothing.
176    // wait second assign
177    Future<byte[]> futureB = submitProcedure(createAssignProcedure(hri));
178    waitOnFuture(futureB);
179    am.getRegionStates().isRegionInState(hri, State.OPEN);
180    // TODO: What else can we do to ensure just a noop.
181
182    // TODO: Though second assign is noop, it's considered success, can noop be handled in a
183    // better way?
184    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
185    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
186  }
187
188  @Test
189  public void testUnassignAnUnassignedRegion() throws Exception {
190    final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
191    final RegionInfo hri = createRegionInfo(tableName, 1);
192
193    // collect AM metrics before test
194    collectAssignmentManagerMetrics();
195
196    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
197
198    // assign the region first
199    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
200
201    final Future<byte[]> futureA = submitProcedure(createUnassignProcedure(hri));
202
203    // Wait first unassign.
204    waitOnFuture(futureA);
205    am.getRegionStates().isRegionInState(hri, State.CLOSED);
206    // Second should be a noop. We should recognize region is already CLOSED internally
207    // and skip out doing nothing.
208    final Future<byte[]> futureB = submitProcedure(createUnassignProcedure(hri));
209    waitOnFuture(futureB);
210    // Ensure we are still CLOSED.
211    am.getRegionStates().isRegionInState(hri, State.CLOSED);
212    // TODO: What else can we do to ensure just a noop.
213
214    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
215    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
216    // TODO: Though second unassign is noop, it's considered success, can noop be handled in a
217    // better way?
218    assertEquals(unassignSubmittedCount + 2, unassignProcMetrics.getSubmittedCounter().getCount());
219    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
220  }
221
222  @Test
223  public void testAssignThrowsWithASplitParent() throws Exception {
224    RegionInfo splitParent = RegionInfoBuilder.newBuilder(TableName.valueOf("test-split-split"))
225      .setSplit(true).setOffline(true).build();
226    RegionStateNode rsn = am.getRegionStates().getOrCreateRegionStateNode(splitParent);
227    rsn.setState(State.SPLIT);
228
229    assertThrows(DoNotRetryIOException.class, () -> am.assign(splitParent));
230    assertNull(am.createOneAssignProcedure(splitParent, true, false));
231  }
232
233  // Simulate the pre-fix failover scenario: regionInfo.isSplit()=true but state=OFFLINE because
234  // the old code path did not write SPLIT to info:state, causing loadMeta to fall back to OFFLINE.
235  // See HBASE-30353.
236  @Test
237  public void testAssignThrowsWithASplitParentInOfflineState() throws Exception {
238    RegionInfo splitParent = RegionInfoBuilder.newBuilder(TableName.valueOf("test-split-offline"))
239      .setSplit(true).setOffline(true).build();
240    RegionStateNode rsn = am.getRegionStates().getOrCreateRegionStateNode(splitParent);
241    rsn.setState(State.OFFLINE);
242
243    assertThrows(DoNotRetryIOException.class, () -> am.assign(splitParent));
244    assertNull(am.createOneAssignProcedure(splitParent, true, false));
245  }
246
247  /**
248   * It is possible that when AM send assign meta request to a RS successfully, but RS can not send
249   * back any response, which cause master startup hangs forever
250   */
251  @Test
252  public void testAssignMetaAndCrashBeforeResponse() throws Exception {
253    tearDown();
254    // See setUp(), start HBase until set up meta
255    util = new HBaseTestingUtil();
256    this.executor = Executors.newSingleThreadScheduledExecutor();
257    setupConfiguration(util.getConfiguration());
258    master = new MockMasterServices(util.getConfiguration());
259    rsDispatcher = new MockRSProcedureDispatcher(master);
260    master.start(NSERVERS, rsDispatcher);
261    am = master.getAssignmentManager();
262
263    // Assign meta
264    rsDispatcher.setMockRsExecutor(new HangThenRSRestartExecutor());
265    am.assign(RegionInfoBuilder.FIRST_META_REGIONINFO);
266    assertEquals(true, am.isMetaAssigned());
267
268    // set it back as default, see setUpMeta()
269    am.wakeMetaLoadedEvent();
270  }
271
272  private void assertCloseThenOpen() {
273    assertEquals(closeSubmittedCount + 1, closeProcMetrics.getSubmittedCounter().getCount());
274    assertEquals(closeFailedCount, closeProcMetrics.getFailedCounter().getCount());
275    assertEquals(openSubmittedCount + 1, openProcMetrics.getSubmittedCounter().getCount());
276    assertEquals(openFailedCount, openProcMetrics.getFailedCounter().getCount());
277  }
278
279  @Test
280  public void testMove() throws Exception {
281    TableName tableName = TableName.valueOf("testMove");
282    RegionInfo hri = createRegionInfo(tableName, 1);
283    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
284    am.assign(hri);
285
286    // collect AM metrics before test
287    collectAssignmentManagerMetrics();
288
289    am.move(hri);
290
291    assertEquals(moveSubmittedCount + 1, moveProcMetrics.getSubmittedCounter().getCount());
292    assertEquals(moveFailedCount, moveProcMetrics.getFailedCounter().getCount());
293    assertCloseThenOpen();
294  }
295
296  @Test
297  public void testReopen() throws Exception {
298    TableName tableName = TableName.valueOf("testReopen");
299    RegionInfo hri = createRegionInfo(tableName, 1);
300    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
301    am.assign(hri);
302
303    // collect AM metrics before test
304    collectAssignmentManagerMetrics();
305
306    TransitRegionStateProcedure proc =
307      TransitRegionStateProcedure.reopen(master.getMasterProcedureExecutor().getEnvironment(), hri);
308    am.getRegionStates().getRegionStateNode(hri).setProcedure(proc);
309    waitOnFuture(submitProcedure(proc));
310
311    assertEquals(reopenSubmittedCount + 1, reopenProcMetrics.getSubmittedCounter().getCount());
312    assertEquals(reopenFailedCount, reopenProcMetrics.getFailedCounter().getCount());
313    assertCloseThenOpen();
314  }
315
316  @Test
317  public void testLoadRegionFromMetaAfterRegionManuallyAdded() throws Exception {
318    try {
319      this.util.startMiniCluster();
320      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
321      final TableName tableName =
322        TableName.valueOf("testLoadRegionFromMetaAfterRegionManuallyAdded");
323      this.util.createTable(tableName, "f");
324      RegionInfo hri = createRegionInfo(tableName, 1);
325      assertNull(am.getRegionStates().getRegionState(hri), "RegionInfo was just instantiated by "
326        + "the test, but shouldn't be in AM regionStates yet.");
327      MetaTableAccessor.addRegionsToMeta(this.util.getConnection(), Collections.singletonList(hri),
328        1);
329      // TODO: is there a race here -- no other thread else will refresh the table states behind
330      // the scenes?
331      assertNull(am.getRegionStates().getRegionState(hri),
332        "RegionInfo was manually added in META, but shouldn't be in AM regionStates yet.");
333      am.populateRegionStatesFromMeta(hri.getEncodedName());
334      assertNotNull(am.getRegionInfo(hri.getRegionName()));
335      assertNotNull(am.getRegionInfo(hri.getEncodedName()));
336    } finally {
337      this.util.killMiniHBaseCluster();
338    }
339  }
340
341  @Test
342  public void testLoadRegionFromMetaRegionNotInMeta() throws Exception {
343    try {
344      this.util.startMiniCluster();
345      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
346      final TableName tableName = TableName.valueOf("testLoadRegionFromMetaRegionNotInMeta");
347      this.util.createTable(tableName, "f");
348      final RegionInfo hri = createRegionInfo(tableName, 1);
349      assertNull(am.getRegionStates().getRegionState(hri),
350        "Bogus RegionInfo discovered in RegionStates.");
351      am.populateRegionStatesFromMeta(hri.getEncodedName());
352      assertNull(am.getRegionStates().getRegionState(hri), "RegionInfo was never added in META");
353    } finally {
354      this.util.killMiniHBaseCluster();
355    }
356  }
357}