Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.concurrent.Executors;
026import java.util.concurrent.Future;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.ServerName;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.client.RetriesExhaustedException;
035import org.apache.hadoop.hbase.exceptions.UnexpectedStateException;
036import org.apache.hadoop.hbase.master.RegionState.State;
037import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
038import org.apache.hadoop.hbase.procedure2.util.StringUtils;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.junit.ClassRule;
042import org.junit.Ignore;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Category({MasterTests.class, LargeTests.class})
049public class TestAssignmentManager extends TestAssignmentManagerBase {
050  private static final Logger LOG = LoggerFactory.getLogger(TestAssignmentManager.class);
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054      HBaseClassTestRule.forClass(TestAssignmentManager.class);
055
056  @Test (expected=NullPointerException.class)
057  public void testWaitServerReportEventWithNullServer() throws UnexpectedStateException {
058    // Test what happens if we pass in null server. I'd expect it throws NPE.
059    if (this.am.waitServerReportEvent(null, null)) throw new UnexpectedStateException();
060  }
061
062  @Test
063  public void testAssignWithGoodExec() throws Exception {
064    // collect AM metrics before test
065    collectAssignmentManagerMetrics();
066
067    testAssign(new GoodRsExecutor());
068
069    assertEquals(assignSubmittedCount + NREGIONS,
070        assignProcMetrics.getSubmittedCounter().getCount());
071    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
072  }
073
074  @Test
075  public void testAssignAndCrashBeforeResponse() throws Exception {
076    final TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
077    final RegionInfo hri = createRegionInfo(tableName, 1);
078    rsDispatcher.setMockRsExecutor(new HangThenRSCrashExecutor());
079    AssignProcedure proc = am.createAssignProcedure(hri);
080    waitOnFuture(submitProcedure(proc));
081  }
082
083  @Test
084  public void testUnassignAndCrashBeforeResponse() throws Exception {
085    final TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
086    final RegionInfo hri = createRegionInfo(tableName, 1);
087    rsDispatcher.setMockRsExecutor(new HangOnCloseThenRSCrashExecutor());
088    for (int i = 0; i < HangOnCloseThenRSCrashExecutor.TYPES_OF_FAILURE; i++) {
089      AssignProcedure assign = am.createAssignProcedure(hri);
090      waitOnFuture(submitProcedure(assign));
091      UnassignProcedure unassign = am.createUnassignProcedure(hri,
092          am.getRegionStates().getRegionServerOfRegion(hri), false);
093      waitOnFuture(submitProcedure(unassign));
094    }
095  }
096
097  @Test
098  public void testAssignWithRandExec() throws Exception {
099    final TableName tableName = TableName.valueOf("testAssignWithRandExec");
100    final RegionInfo hri = createRegionInfo(tableName, 1);
101
102    rsDispatcher.setMockRsExecutor(new RandRsExecutor());
103    // Loop a bunch of times so we hit various combos of exceptions.
104    for (int i = 0; i < 10; i++) {
105      LOG.info("ROUND=" + i);
106      AssignProcedure proc = am.createAssignProcedure(hri);
107      waitOnFuture(submitProcedure(proc));
108    }
109  }
110
111  @Ignore @Test // Disabled for now. Since HBASE-18551, this mock is insufficient.
112  public void testSocketTimeout() throws Exception {
113    final TableName tableName = TableName.valueOf(this.name.getMethodName());
114    final RegionInfo hri = createRegionInfo(tableName, 1);
115
116    // collect AM metrics before test
117    collectAssignmentManagerMetrics();
118
119    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 3));
120    waitOnFuture(submitProcedure(am.createAssignProcedure(hri)));
121
122    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 1));
123    // exception.expect(ServerCrashException.class);
124    waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false)));
125
126    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
127    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
128    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
129    assertEquals(unassignFailedCount + 1, unassignProcMetrics.getFailedCounter().getCount());
130  }
131
132  @Test
133  public void testServerNotYetRunning() throws Exception {
134    testRetriesExhaustedFailure(TableName.valueOf(this.name.getMethodName()),
135      new ServerNotYetRunningRsExecutor());
136  }
137
138  private void testRetriesExhaustedFailure(final TableName tableName,
139      final MockRSExecutor executor) throws Exception {
140    final RegionInfo hri = createRegionInfo(tableName, 1);
141
142    // collect AM metrics before test
143    collectAssignmentManagerMetrics();
144
145    // Test Assign operation failure
146    rsDispatcher.setMockRsExecutor(executor);
147    try {
148      waitOnFuture(submitProcedure(am.createAssignProcedure(hri)));
149      fail("unexpected assign completion");
150    } catch (RetriesExhaustedException e) {
151      // expected exception
152      LOG.info("expected exception from assign operation: " + e.getMessage(), e);
153    }
154
155    // Assign the region (without problems)
156    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
157    waitOnFuture(submitProcedure(am.createAssignProcedure(hri)));
158
159    // TODO: Currently unassign just keeps trying until it sees a server crash.
160    // There is no count on unassign.
161    /*
162    // Test Unassign operation failure
163    rsDispatcher.setMockRsExecutor(executor);
164    waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false)));
165
166    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
167    assertEquals(assignFailedCount + 1, assignProcMetrics.getFailedCounter().getCount());
168    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
169
170    // TODO: We supposed to have 1 failed assign, 1 successful assign and a failed unassign
171    // operation. But ProcV2 framework marks aborted unassign operation as success. Fix it!
172    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
173    */
174  }
175
176
177  @Test
178  public void testIOExceptionOnAssignment() throws Exception {
179    // collect AM metrics before test
180    collectAssignmentManagerMetrics();
181
182    testFailedOpen(TableName.valueOf("testExceptionOnAssignment"),
183      new FaultyRsExecutor(new IOException("test fault")));
184
185    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
186    assertEquals(assignFailedCount + 1, assignProcMetrics.getFailedCounter().getCount());
187  }
188
189  @Test
190  public void testDoNotRetryExceptionOnAssignment() throws Exception {
191    // collect AM metrics before test
192    collectAssignmentManagerMetrics();
193
194    testFailedOpen(TableName.valueOf("testDoNotRetryExceptionOnAssignment"),
195      new FaultyRsExecutor(new DoNotRetryIOException("test do not retry fault")));
196
197    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
198    assertEquals(assignFailedCount + 1, assignProcMetrics.getFailedCounter().getCount());
199  }
200
201  private void testFailedOpen(final TableName tableName,
202      final MockRSExecutor executor) throws Exception {
203    final RegionInfo hri = createRegionInfo(tableName, 1);
204
205    // Test Assign operation failure
206    rsDispatcher.setMockRsExecutor(executor);
207    try {
208      waitOnFuture(submitProcedure(am.createAssignProcedure(hri)));
209      fail("unexpected assign completion");
210    } catch (RetriesExhaustedException e) {
211      // expected exception
212      LOG.info("REGION STATE " + am.getRegionStates().getRegionStateNode(hri));
213      LOG.info("expected exception from assign operation: " + e.getMessage(), e);
214      assertEquals(true, am.getRegionStates().getRegionState(hri).isFailedOpen());
215    }
216  }
217
218  private void testAssign(final MockRSExecutor executor) throws Exception {
219    testAssign(executor, NREGIONS);
220  }
221
222  private void testAssign(final MockRSExecutor executor, final int nregions) throws Exception {
223    rsDispatcher.setMockRsExecutor(executor);
224
225    AssignProcedure[] assignments = new AssignProcedure[nregions];
226
227    long st = System.currentTimeMillis();
228    bulkSubmit(assignments);
229
230    for (int i = 0; i < assignments.length; ++i) {
231      ProcedureTestingUtility.waitProcedure(
232        master.getMasterProcedureExecutor(), assignments[i]);
233      assertTrue(assignments[i].toString(), assignments[i].isSuccess());
234    }
235    long et = System.currentTimeMillis();
236    float sec = ((et - st) / 1000.0f);
237    LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)",
238        assignments.length, StringUtils.humanTimeDiff(et - st), assignments.length / sec));
239  }
240
241  @Test
242  public void testAssignAnAssignedRegion() throws Exception {
243    final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
244    final RegionInfo hri = createRegionInfo(tableName, 1);
245
246    // collect AM metrics before test
247    collectAssignmentManagerMetrics();
248
249    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
250
251    final Future<byte[]> futureA = submitProcedure(am.createAssignProcedure(hri));
252
253    // wait first assign
254    waitOnFuture(futureA);
255    am.getRegionStates().isRegionInState(hri, State.OPEN);
256    // Second should be a noop. We should recognize region is already OPEN internally
257    // and skip out doing nothing.
258    // wait second assign
259    final Future<byte[]> futureB = submitProcedure(am.createAssignProcedure(hri));
260    waitOnFuture(futureB);
261    am.getRegionStates().isRegionInState(hri, State.OPEN);
262    // TODO: What else can we do to ensure just a noop.
263
264    // TODO: Though second assign is noop, it's considered success, can noop be handled in a
265    // better way?
266    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
267    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
268
269  }
270
271  @Test
272  public void testUnassignAnUnassignedRegion() throws Exception {
273    final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
274    final RegionInfo hri = createRegionInfo(tableName, 1);
275
276    // collect AM metrics before test
277    collectAssignmentManagerMetrics();
278
279    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
280
281    // assign the region first
282    waitOnFuture(submitProcedure(am.createAssignProcedure(hri)));
283
284    final Future<byte[]> futureA = submitProcedure(am.createUnassignProcedure(hri, null, false));
285
286    // Wait first unassign.
287    waitOnFuture(futureA);
288    am.getRegionStates().isRegionInState(hri, State.CLOSED);
289    // Second should be a noop. We should recognize region is already CLOSED internally
290    // and skip out doing nothing.
291    final Future<byte[]> futureB =
292        submitProcedure(am.createUnassignProcedure(hri,
293            ServerName.valueOf("example.org,1234,1"), false));
294    waitOnFuture(futureB);
295    // Ensure we are still CLOSED.
296    am.getRegionStates().isRegionInState(hri, State.CLOSED);
297    // TODO: What else can we do to ensure just a noop.
298
299    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
300    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
301    // TODO: Though second unassign is noop, it's considered success, can noop be handled in a
302    // better way?
303    assertEquals(unassignSubmittedCount + 2, unassignProcMetrics.getSubmittedCounter().getCount());
304    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
305  }
306
307  /**
308   * It is possible that when AM send assign meta request to a RS successfully,
309   * but RS can not send back any response, which cause master startup hangs forever
310   */
311  @Test
312  public void testAssignMetaAndCrashBeforeResponse() throws Exception {
313    tearDown();
314    // See setUp(), start HBase until set up meta
315    UTIL = new HBaseTestingUtility();
316    this.executor = Executors.newSingleThreadScheduledExecutor();
317    setupConfiguration(UTIL.getConfiguration());
318    master = new MockMasterServices(UTIL.getConfiguration(), this.regionsToRegionServers);
319    rsDispatcher = new MockRSProcedureDispatcher(master);
320    master.start(NSERVERS, rsDispatcher);
321    am = master.getAssignmentManager();
322
323    // Assign meta
324    rsDispatcher.setMockRsExecutor(new HangThenRSRestartExecutor());
325    am.assign(RegionInfoBuilder.FIRST_META_REGIONINFO);
326    assertEquals(true, am.isMetaAssigned());
327
328    // set it back as default, see setUpMeta()
329    am.wakeMetaLoadedEvent();
330  }
331}