Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.assertTrue;
023
024import java.util.Collections;
025import java.util.concurrent.Executors;
026import java.util.concurrent.Future;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.MetaTableAccessor;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.master.RegionState.State;
034import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
035import org.apache.hadoop.hbase.procedure2.util.StringUtils;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.junit.ClassRule;
039import org.junit.Test;
040import org.junit.experimental.categories.Category;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044@Category({ MasterTests.class, LargeTests.class })
045public class TestAssignmentManager extends TestAssignmentManagerBase {
046
047  @ClassRule
048  public static final HBaseClassTestRule CLASS_RULE =
049    HBaseClassTestRule.forClass(TestAssignmentManager.class);
050
051  private static final Logger LOG = LoggerFactory.getLogger(TestAssignmentManager.class);
052
053  @Test
054  public void testAssignWithGoodExec() throws Exception {
055    // collect AM metrics before test
056    collectAssignmentManagerMetrics();
057
058    testAssign(new GoodRsExecutor());
059
060    assertEquals(assignSubmittedCount + NREGIONS,
061      assignProcMetrics.getSubmittedCounter().getCount());
062    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
063  }
064
065  @Test
066  public void testAssignAndCrashBeforeResponse() throws Exception {
067    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
068    RegionInfo hri = createRegionInfo(tableName, 1);
069    rsDispatcher.setMockRsExecutor(new HangThenRSCrashExecutor());
070    TransitRegionStateProcedure proc = createAssignProcedure(hri);
071    waitOnFuture(submitProcedure(proc));
072  }
073
074  @Test
075  public void testUnassignAndCrashBeforeResponse() throws Exception {
076    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
077    RegionInfo hri = createRegionInfo(tableName, 1);
078    rsDispatcher.setMockRsExecutor(new HangOnCloseThenRSCrashExecutor());
079    for (int i = 0; i < HangOnCloseThenRSCrashExecutor.TYPES_OF_FAILURE; i++) {
080      TransitRegionStateProcedure assign = createAssignProcedure(hri);
081      waitOnFuture(submitProcedure(assign));
082      TransitRegionStateProcedure unassign = createUnassignProcedure(hri);
083      waitOnFuture(submitProcedure(unassign));
084    }
085  }
086
087  @Test
088  public void testAssignSocketTimeout() throws Exception {
089    TableName tableName = TableName.valueOf(this.name.getMethodName());
090    RegionInfo hri = createRegionInfo(tableName, 1);
091
092    // collect AM metrics before test
093    collectAssignmentManagerMetrics();
094
095    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20));
096    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
097
098    // we crashed a rs, so it is possible that there are other regions on the rs which will also be
099    // reassigned, so here we just assert greater than, not the exact number.
100    assertTrue(assignProcMetrics.getSubmittedCounter().getCount() > assignSubmittedCount);
101    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
102  }
103
104  @Test
105  public void testAssignQueueFullOnce() throws Exception {
106    TableName tableName = TableName.valueOf(this.name.getMethodName());
107    RegionInfo hri = createRegionInfo(tableName, 1);
108
109    // collect AM metrics before test
110    collectAssignmentManagerMetrics();
111
112    rsDispatcher.setMockRsExecutor(new CallQueueTooBigOnceRsExecutor());
113    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
114
115    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
116    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
117  }
118
119  @Test
120  public void testTimeoutThenQueueFull() throws Exception {
121    TableName tableName = TableName.valueOf(this.name.getMethodName());
122    RegionInfo hri = createRegionInfo(tableName, 1);
123
124    // collect AM metrics before test
125    collectAssignmentManagerMetrics();
126
127    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(10));
128    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
129    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(15));
130    waitOnFuture(submitProcedure(createUnassignProcedure(hri)));
131
132    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
133    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
134    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
135    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
136  }
137
138  private void testAssign(final MockRSExecutor executor) throws Exception {
139    testAssign(executor, NREGIONS);
140  }
141
142  private void testAssign(MockRSExecutor executor, int nRegions) throws Exception {
143    rsDispatcher.setMockRsExecutor(executor);
144
145    TransitRegionStateProcedure[] assignments = new TransitRegionStateProcedure[nRegions];
146
147    long st = System.currentTimeMillis();
148    bulkSubmit(assignments);
149
150    for (int i = 0; i < assignments.length; ++i) {
151      ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), assignments[i]);
152      assertTrue(assignments[i].toString(), assignments[i].isSuccess());
153    }
154    long et = System.currentTimeMillis();
155    float sec = ((et - st) / 1000.0f);
156    LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)", assignments.length,
157      StringUtils.humanTimeDiff(et - st), assignments.length / sec));
158  }
159
160  @Test
161  public void testAssignAnAssignedRegion() throws Exception {
162    final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
163    final RegionInfo hri = createRegionInfo(tableName, 1);
164
165    // collect AM metrics before test
166    collectAssignmentManagerMetrics();
167
168    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
169
170    Future<byte[]> futureA = submitProcedure(createAssignProcedure(hri));
171
172    // wait first assign
173    waitOnFuture(futureA);
174    am.getRegionStates().isRegionInState(hri, State.OPEN);
175    // Second should be a noop. We should recognize region is already OPEN internally
176    // and skip out doing nothing.
177    // wait second assign
178    Future<byte[]> futureB = submitProcedure(createAssignProcedure(hri));
179    waitOnFuture(futureB);
180    am.getRegionStates().isRegionInState(hri, State.OPEN);
181    // TODO: What else can we do to ensure just a noop.
182
183    // TODO: Though second assign is noop, it's considered success, can noop be handled in a
184    // better way?
185    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
186    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
187  }
188
189  @Test
190  public void testUnassignAnUnassignedRegion() throws Exception {
191    final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
192    final RegionInfo hri = createRegionInfo(tableName, 1);
193
194    // collect AM metrics before test
195    collectAssignmentManagerMetrics();
196
197    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
198
199    // assign the region first
200    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
201
202    final Future<byte[]> futureA = submitProcedure(createUnassignProcedure(hri));
203
204    // Wait first unassign.
205    waitOnFuture(futureA);
206    am.getRegionStates().isRegionInState(hri, State.CLOSED);
207    // Second should be a noop. We should recognize region is already CLOSED internally
208    // and skip out doing nothing.
209    final Future<byte[]> futureB = submitProcedure(createUnassignProcedure(hri));
210    waitOnFuture(futureB);
211    // Ensure we are still CLOSED.
212    am.getRegionStates().isRegionInState(hri, State.CLOSED);
213    // TODO: What else can we do to ensure just a noop.
214
215    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
216    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
217    // TODO: Though second unassign is noop, it's considered success, can noop be handled in a
218    // better way?
219    assertEquals(unassignSubmittedCount + 2, unassignProcMetrics.getSubmittedCounter().getCount());
220    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
221  }
222
223  /**
224   * It is possible that when AM send assign meta request to a RS successfully, but RS can not send
225   * back any response, which cause master startup hangs forever
226   */
227  @Test
228  public void testAssignMetaAndCrashBeforeResponse() throws Exception {
229    tearDown();
230    // See setUp(), start HBase until set up meta
231    util = new HBaseTestingUtility();
232    this.executor = Executors.newSingleThreadScheduledExecutor();
233    setupConfiguration(util.getConfiguration());
234    master = new MockMasterServices(util.getConfiguration(), this.regionsToRegionServers);
235    rsDispatcher = new MockRSProcedureDispatcher(master);
236    master.start(NSERVERS, rsDispatcher);
237    am = master.getAssignmentManager();
238
239    // Assign meta
240    rsDispatcher.setMockRsExecutor(new HangThenRSRestartExecutor());
241    am.assign(RegionInfoBuilder.FIRST_META_REGIONINFO);
242    assertEquals(true, am.isMetaAssigned());
243
244    // set it back as default, see setUpMeta()
245    am.wakeMetaLoadedEvent();
246  }
247
248  private void assertCloseThenOpen() {
249    assertEquals(closeSubmittedCount + 1, closeProcMetrics.getSubmittedCounter().getCount());
250    assertEquals(closeFailedCount, closeProcMetrics.getFailedCounter().getCount());
251    assertEquals(openSubmittedCount + 1, openProcMetrics.getSubmittedCounter().getCount());
252    assertEquals(openFailedCount, openProcMetrics.getFailedCounter().getCount());
253  }
254
255  @Test
256  public void testMove() throws Exception {
257    TableName tableName = TableName.valueOf("testMove");
258    RegionInfo hri = createRegionInfo(tableName, 1);
259    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
260    am.assign(hri);
261
262    // collect AM metrics before test
263    collectAssignmentManagerMetrics();
264
265    am.move(hri);
266
267    assertEquals(moveSubmittedCount + 1, moveProcMetrics.getSubmittedCounter().getCount());
268    assertEquals(moveFailedCount, moveProcMetrics.getFailedCounter().getCount());
269    assertCloseThenOpen();
270  }
271
272  @Test
273  public void testReopen() throws Exception {
274    TableName tableName = TableName.valueOf("testReopen");
275    RegionInfo hri = createRegionInfo(tableName, 1);
276    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
277    am.assign(hri);
278
279    // collect AM metrics before test
280    collectAssignmentManagerMetrics();
281
282    TransitRegionStateProcedure proc =
283      TransitRegionStateProcedure.reopen(master.getMasterProcedureExecutor().getEnvironment(), hri);
284    am.getRegionStates().getRegionStateNode(hri).setProcedure(proc);
285    waitOnFuture(submitProcedure(proc));
286
287    assertEquals(reopenSubmittedCount + 1, reopenProcMetrics.getSubmittedCounter().getCount());
288    assertEquals(reopenFailedCount, reopenProcMetrics.getFailedCounter().getCount());
289    assertCloseThenOpen();
290  }
291
292  @Test
293  public void testLoadRegionFromMetaAfterRegionManuallyAdded() throws Exception {
294    try {
295      this.util.startMiniCluster();
296      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
297      final TableName tableName = TableName.
298        valueOf("testLoadRegionFromMetaAfterRegionManuallyAdded");
299      this.util.createTable(tableName, "f");
300      RegionInfo hri = createRegionInfo(tableName, 1);
301      assertNull("RegionInfo was just instantiated by the test, but "
302        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
303      MetaTableAccessor.addRegionsToMeta(this.util.getConnection(), Collections.singletonList(hri),
304        1);
305      assertNull("RegionInfo was manually added in META, but "
306        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
307      hri = am.loadRegionFromMeta(hri.getEncodedName());
308      assertEquals(hri.getEncodedName(),
309        am.getRegionStates().getRegionState(hri).getRegion().getEncodedName());
310    }finally {
311      this.util.killMiniHBaseCluster();
312    }
313  }
314
315  @Test
316  public void testLoadRegionFromMetaRegionNotInMeta() throws Exception {
317    try {
318      this.util.startMiniCluster();
319      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
320      final TableName tableName = TableName.valueOf("testLoadRegionFromMetaRegionNotInMeta");
321      this.util.createTable(tableName, "f");
322      final RegionInfo hri = createRegionInfo(tableName, 1);
323      assertNull("RegionInfo was just instantiated by the test, but "
324        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
325      assertNull("RegionInfo was never added in META, should had returned null.",
326        am.loadRegionFromMeta(hri.getEncodedName()));
327    }finally {
328      this.util.killMiniHBaseCluster();
329    }
330  }
331}