Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023
024import java.util.Collections;
025import java.util.concurrent.Executors;
026import java.util.concurrent.Future;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.MetaTableAccessor;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.master.RegionState.State;
034import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
035import org.apache.hadoop.hbase.procedure2.util.StringUtils;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.junit.ClassRule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045@Category({ MasterTests.class, LargeTests.class })
046public class TestAssignmentManager extends TestAssignmentManagerBase {
047
048  @ClassRule
049  public static final HBaseClassTestRule CLASS_RULE =
050    HBaseClassTestRule.forClass(TestAssignmentManager.class);
051
052  private static final Logger LOG = LoggerFactory.getLogger(TestAssignmentManager.class);
053
054  @Test
055  public void testAssignWithGoodExec() throws Exception {
056    // collect AM metrics before test
057    collectAssignmentManagerMetrics();
058
059    testAssign(new GoodRsExecutor());
060
061    assertEquals(assignSubmittedCount + NREGIONS,
062      assignProcMetrics.getSubmittedCounter().getCount());
063    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
064  }
065
066  @Test
067  public void testAssignAndCrashBeforeResponse() throws Exception {
068    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
069    RegionInfo hri = createRegionInfo(tableName, 1);
070    rsDispatcher.setMockRsExecutor(new HangThenRSCrashExecutor());
071    TransitRegionStateProcedure proc = createAssignProcedure(hri);
072    waitOnFuture(submitProcedure(proc));
073  }
074
075  @Test
076  public void testUnassignAndCrashBeforeResponse() throws Exception {
077    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
078    RegionInfo hri = createRegionInfo(tableName, 1);
079    rsDispatcher.setMockRsExecutor(new HangOnCloseThenRSCrashExecutor());
080    for (int i = 0; i < HangOnCloseThenRSCrashExecutor.TYPES_OF_FAILURE; i++) {
081      TransitRegionStateProcedure assign = createAssignProcedure(hri);
082      waitOnFuture(submitProcedure(assign));
083      TransitRegionStateProcedure unassign = createUnassignProcedure(hri);
084      waitOnFuture(submitProcedure(unassign));
085    }
086  }
087
088  @Test
089  public void testAssignSocketTimeout() throws Exception {
090    TableName tableName = TableName.valueOf(this.name.getMethodName());
091    RegionInfo hri = createRegionInfo(tableName, 1);
092
093    // collect AM metrics before test
094    collectAssignmentManagerMetrics();
095
096    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20));
097    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
098
099    // we crashed a rs, so it is possible that there are other regions on the rs which will also be
100    // reassigned, so here we just assert greater than, not the exact number.
101    assertTrue(assignProcMetrics.getSubmittedCounter().getCount() > assignSubmittedCount);
102    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
103  }
104
105  @Test
106  public void testAssignQueueFullOnce() throws Exception {
107    TableName tableName = TableName.valueOf(this.name.getMethodName());
108    RegionInfo hri = createRegionInfo(tableName, 1);
109
110    // collect AM metrics before test
111    collectAssignmentManagerMetrics();
112
113    rsDispatcher.setMockRsExecutor(new CallQueueTooBigOnceRsExecutor());
114    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
115
116    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
117    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
118  }
119
120  @Test
121  public void testTimeoutThenQueueFull() throws Exception {
122    TableName tableName = TableName.valueOf(this.name.getMethodName());
123    RegionInfo hri = createRegionInfo(tableName, 1);
124
125    // collect AM metrics before test
126    collectAssignmentManagerMetrics();
127
128    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(10));
129    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
130    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(15));
131    waitOnFuture(submitProcedure(createUnassignProcedure(hri)));
132
133    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
134    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
135    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
136    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
137  }
138
139  private void testAssign(final MockRSExecutor executor) throws Exception {
140    testAssign(executor, NREGIONS);
141  }
142
143  private void testAssign(MockRSExecutor executor, int nRegions) throws Exception {
144    rsDispatcher.setMockRsExecutor(executor);
145
146    TransitRegionStateProcedure[] assignments = new TransitRegionStateProcedure[nRegions];
147
148    long st = EnvironmentEdgeManager.currentTime();
149    bulkSubmit(assignments);
150
151    for (int i = 0; i < assignments.length; ++i) {
152      ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), assignments[i]);
153      assertTrue(assignments[i].toString(), assignments[i].isSuccess());
154    }
155    long et = EnvironmentEdgeManager.currentTime();
156    float sec = ((et - st) / 1000.0f);
157    LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)", assignments.length,
158      StringUtils.humanTimeDiff(et - st), assignments.length / sec));
159  }
160
161  @Test
162  public void testAssignAnAssignedRegion() throws Exception {
163    final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
164    final RegionInfo hri = createRegionInfo(tableName, 1);
165
166    // collect AM metrics before test
167    collectAssignmentManagerMetrics();
168
169    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
170
171    Future<byte[]> futureA = submitProcedure(createAssignProcedure(hri));
172
173    // wait first assign
174    waitOnFuture(futureA);
175    am.getRegionStates().isRegionInState(hri, State.OPEN);
176    // Second should be a noop. We should recognize region is already OPEN internally
177    // and skip out doing nothing.
178    // wait second assign
179    Future<byte[]> futureB = submitProcedure(createAssignProcedure(hri));
180    waitOnFuture(futureB);
181    am.getRegionStates().isRegionInState(hri, State.OPEN);
182    // TODO: What else can we do to ensure just a noop.
183
184    // TODO: Though second assign is noop, it's considered success, can noop be handled in a
185    // better way?
186    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
187    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
188  }
189
190  @Test
191  public void testUnassignAnUnassignedRegion() throws Exception {
192    final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
193    final RegionInfo hri = createRegionInfo(tableName, 1);
194
195    // collect AM metrics before test
196    collectAssignmentManagerMetrics();
197
198    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
199
200    // assign the region first
201    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
202
203    final Future<byte[]> futureA = submitProcedure(createUnassignProcedure(hri));
204
205    // Wait first unassign.
206    waitOnFuture(futureA);
207    am.getRegionStates().isRegionInState(hri, State.CLOSED);
208    // Second should be a noop. We should recognize region is already CLOSED internally
209    // and skip out doing nothing.
210    final Future<byte[]> futureB = submitProcedure(createUnassignProcedure(hri));
211    waitOnFuture(futureB);
212    // Ensure we are still CLOSED.
213    am.getRegionStates().isRegionInState(hri, State.CLOSED);
214    // TODO: What else can we do to ensure just a noop.
215
216    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
217    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
218    // TODO: Though second unassign is noop, it's considered success, can noop be handled in a
219    // better way?
220    assertEquals(unassignSubmittedCount + 2, unassignProcMetrics.getSubmittedCounter().getCount());
221    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
222  }
223
224  /**
225   * It is possible that when AM send assign meta request to a RS successfully, but RS can not send
226   * back any response, which cause master startup hangs forever
227   */
228  @Test
229  public void testAssignMetaAndCrashBeforeResponse() throws Exception {
230    tearDown();
231    // See setUp(), start HBase until set up meta
232    util = new HBaseTestingUtility();
233    this.executor = Executors.newSingleThreadScheduledExecutor();
234    setupConfiguration(util.getConfiguration());
235    master = new MockMasterServices(util.getConfiguration(), this.regionsToRegionServers);
236    rsDispatcher = new MockRSProcedureDispatcher(master);
237    master.start(NSERVERS, rsDispatcher);
238    am = master.getAssignmentManager();
239
240    // Assign meta
241    rsDispatcher.setMockRsExecutor(new HangThenRSRestartExecutor());
242    am.assign(RegionInfoBuilder.FIRST_META_REGIONINFO);
243    assertEquals(true, am.isMetaAssigned());
244
245    // set it back as default, see setUpMeta()
246    am.wakeMetaLoadedEvent();
247  }
248
249  private void assertCloseThenOpen() {
250    assertEquals(closeSubmittedCount + 1, closeProcMetrics.getSubmittedCounter().getCount());
251    assertEquals(closeFailedCount, closeProcMetrics.getFailedCounter().getCount());
252    assertEquals(openSubmittedCount + 1, openProcMetrics.getSubmittedCounter().getCount());
253    assertEquals(openFailedCount, openProcMetrics.getFailedCounter().getCount());
254  }
255
256  @Test
257  public void testMove() throws Exception {
258    TableName tableName = TableName.valueOf("testMove");
259    RegionInfo hri = createRegionInfo(tableName, 1);
260    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
261    am.assign(hri);
262
263    // collect AM metrics before test
264    collectAssignmentManagerMetrics();
265
266    am.move(hri);
267
268    assertEquals(moveSubmittedCount + 1, moveProcMetrics.getSubmittedCounter().getCount());
269    assertEquals(moveFailedCount, moveProcMetrics.getFailedCounter().getCount());
270    assertCloseThenOpen();
271  }
272
273  @Test
274  public void testReopen() throws Exception {
275    TableName tableName = TableName.valueOf("testReopen");
276    RegionInfo hri = createRegionInfo(tableName, 1);
277    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
278    am.assign(hri);
279
280    // collect AM metrics before test
281    collectAssignmentManagerMetrics();
282
283    TransitRegionStateProcedure proc =
284      TransitRegionStateProcedure.reopen(master.getMasterProcedureExecutor().getEnvironment(), hri);
285    am.getRegionStates().getRegionStateNode(hri).setProcedure(proc);
286    waitOnFuture(submitProcedure(proc));
287
288    assertEquals(reopenSubmittedCount + 1, reopenProcMetrics.getSubmittedCounter().getCount());
289    assertEquals(reopenFailedCount, reopenProcMetrics.getFailedCounter().getCount());
290    assertCloseThenOpen();
291  }
292
293  @Test
294  public void testLoadRegionFromMetaAfterRegionManuallyAdded() throws Exception {
295    try {
296      this.util.startMiniCluster();
297      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
298      final TableName tableName =
299        TableName.valueOf("testLoadRegionFromMetaAfterRegionManuallyAdded");
300      this.util.createTable(tableName, "f");
301      RegionInfo hri = createRegionInfo(tableName, 1);
302      assertNull("RegionInfo was just instantiated by the test, but "
303        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
304      MetaTableAccessor.addRegionsToMeta(this.util.getConnection(), Collections.singletonList(hri),
305        1);
306      assertNull(
307        "RegionInfo was manually added in META, but " + "shouldn't be in AM regionStates yet.",
308        am.getRegionStates().getRegionState(hri));
309      hri = am.loadRegionFromMeta(hri.getEncodedName());
310      assertEquals(hri.getEncodedName(),
311        am.getRegionStates().getRegionState(hri).getRegion().getEncodedName());
312    } finally {
313      this.util.killMiniHBaseCluster();
314    }
315  }
316
317  @Test
318  public void testLoadRegionFromMetaRegionNotInMeta() throws Exception {
319    try {
320      this.util.startMiniCluster();
321      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
322      final TableName tableName = TableName.valueOf("testLoadRegionFromMetaRegionNotInMeta");
323      this.util.createTable(tableName, "f");
324      final RegionInfo hri = createRegionInfo(tableName, 1);
325      assertNull("RegionInfo was just instantiated by the test, but "
326        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
327      assertNull("RegionInfo was never added in META, should had returned null.",
328        am.loadRegionFromMeta(hri.getEncodedName()));
329    } finally {
330      this.util.killMiniHBaseCluster();
331    }
332  }
333}