001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertTrue;
023import java.io.IOException;
024import java.util.Arrays;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.Optional;
029import java.util.concurrent.CountDownLatch;
030import java.util.stream.Collectors;
031import org.apache.hadoop.hbase.Coprocessor;
032import org.apache.hadoop.hbase.CoprocessorEnvironment;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
038import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
039import org.apache.hadoop.hbase.coprocessor.MasterObserver;
040import org.apache.hadoop.hbase.coprocessor.ObserverContext;
041import org.apache.hadoop.hbase.master.HMaster;
042import org.apache.hadoop.hbase.master.RegionState;
043import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
044import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
045import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
046import org.apache.hadoop.hbase.procedure2.Procedure;
047import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
048import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
049import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
050import org.apache.hadoop.hbase.regionserver.HRegionServer;
051import org.apache.hadoop.hbase.testclassification.ClientTests;
052import org.apache.hadoop.hbase.testclassification.LargeTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.hbase.util.Pair;
055import org.junit.AfterClass;
056import org.junit.Before;
057import org.junit.BeforeClass;
058import org.junit.ClassRule;
059import org.junit.Rule;
060import org.junit.Test;
061import org.junit.experimental.categories.Category;
062import org.junit.rules.TestName;
063import org.junit.runner.RunWith;
064import org.junit.runners.Parameterized;
065import org.junit.runners.Parameterized.Parameter;
066import org.junit.runners.Parameterized.Parameters;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
070
071/**
072 * Class to test HBaseHbck. Spins up the minicluster once at test start and then takes it down
073 * afterward. Add any testing of HBaseHbck functionality here.
074 */
075@RunWith(Parameterized.class)
076@Category({ LargeTests.class, ClientTests.class })
077public class TestHbck {
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHbck.class);
080
081  private static final Logger LOG = LoggerFactory.getLogger(TestHbck.class);
082  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
083
084  @Rule
085  public TestName name = new TestName();
086
087  @SuppressWarnings("checkstyle:VisibilityModifier") @Parameter
088  public boolean async;
089
090  private static final TableName TABLE_NAME = TableName.valueOf(TestHbck.class.getSimpleName());
091
092  private static ProcedureExecutor<MasterProcedureEnv> procExec;
093
094  private static AsyncConnection ASYNC_CONN;
095
096  @Parameters(name = "{index}: async={0}")
097  public static List<Object[]> params() {
098    return Arrays.asList(new Object[] { false }, new Object[] { true });
099  }
100
101  private Hbck getHbck() throws Exception {
102    if (async) {
103      return ASYNC_CONN.getHbck().get();
104    } else {
105      return TEST_UTIL.getHbck();
106    }
107  }
108
109  @BeforeClass
110  public static void setUpBeforeClass() throws Exception {
111    TEST_UTIL.startMiniCluster(3);
112    TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5);
113    procExec = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
114    ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
115    TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost().load(
116      FailingMergeAfterMetaUpdatedMasterObserver.class, Coprocessor.PRIORITY_USER,
117      TEST_UTIL.getHBaseCluster().getMaster().getConfiguration());
118    TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost().load(
119      FailingSplitAfterMetaUpdatedMasterObserver.class, Coprocessor.PRIORITY_USER,
120      TEST_UTIL.getHBaseCluster().getMaster().getConfiguration());
121  }
122
123  @AfterClass
124  public static void tearDownAfterClass() throws Exception {
125    Closeables.close(ASYNC_CONN, true);
126    TEST_UTIL.shutdownMiniCluster();
127  }
128
129  @Before
130  public void setUp() throws IOException {
131    TEST_UTIL.ensureSomeRegionServersAvailable(3);
132  }
133
134  public static class SuspendProcedure extends
135      ProcedureTestingUtility.NoopProcedure<MasterProcedureEnv> implements TableProcedureInterface {
136    public SuspendProcedure() {
137      super();
138    }
139
140    @SuppressWarnings({ "rawtypes", "unchecked" })
141    @Override
142    protected Procedure[] execute(final MasterProcedureEnv env) throws ProcedureSuspendedException {
143      // Always suspend the procedure
144      throw new ProcedureSuspendedException();
145    }
146
147    @Override
148    public TableName getTableName() {
149      return TABLE_NAME;
150    }
151
152    @Override
153    public TableOperationType getTableOperationType() {
154      return TableOperationType.READ;
155    }
156  }
157
158  @Test
159  public void testBypassProcedure() throws Exception {
160    // SuspendProcedure
161    final SuspendProcedure proc = new SuspendProcedure();
162    long procId = procExec.submitProcedure(proc);
163    Thread.sleep(500);
164
165    // bypass the procedure
166    List<Long> pids = Arrays.<Long> asList(procId);
167    List<Boolean> results = getHbck().bypassProcedure(pids, 30000, false, false);
168    assertTrue("Failed to by pass procedure!", results.get(0));
169    TEST_UTIL.waitFor(5000, () -> proc.isSuccess() && proc.isBypass());
170    LOG.info("{} finished", proc);
171  }
172
173  @Test
174  public void testSetTableStateInMeta() throws Exception {
175    Hbck hbck = getHbck();
176    // set table state to DISABLED
177    hbck.setTableStateInMeta(new TableState(TABLE_NAME, TableState.State.DISABLED));
178    // Method {@link Hbck#setTableStateInMeta()} returns previous state, which in this case
179    // will be DISABLED
180    TableState prevState =
181      hbck.setTableStateInMeta(new TableState(TABLE_NAME, TableState.State.ENABLED));
182    assertTrue("Incorrect previous state! expeced=DISABLED, found=" + prevState.getState(),
183      prevState.isDisabled());
184  }
185
186  @Test
187  public void testSetRegionStateInMeta() throws Exception {
188    Hbck hbck = getHbck();
189    Admin admin = TEST_UTIL.getAdmin();
190    final List<RegionInfo> regions = admin.getRegions(TABLE_NAME);
191    final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
192    Map<String, RegionState.State> prevStates = new HashMap<>();
193    Map<String, RegionState.State> newStates = new HashMap<>();
194    final Map<String, Pair<RegionState.State, RegionState.State>> regionsMap = new HashMap<>();
195    regions.forEach(r -> {
196      RegionState prevState = am.getRegionStates().getRegionState(r);
197      prevStates.put(r.getEncodedName(), prevState.getState());
198      newStates.put(r.getEncodedName(), RegionState.State.CLOSED);
199      regionsMap.put(r.getEncodedName(),
200        new Pair<>(prevState.getState(), RegionState.State.CLOSED));
201    });
202    final Map<String, RegionState.State> result = hbck.setRegionStateInMeta(newStates);
203    result.forEach((k, v) -> {
204      RegionState.State prevState = regionsMap.get(k).getFirst();
205      assertEquals(prevState, v);
206    });
207    regions.forEach(r -> {
208      RegionState cachedState = am.getRegionStates().getRegionState(r.getEncodedName());
209      RegionState.State newState = regionsMap.get(r.getEncodedName()).getSecond();
210      assertEquals(newState, cachedState.getState());
211    });
212    hbck.setRegionStateInMeta(prevStates);
213  }
214
215  @Test
216  public void testAssigns() throws Exception {
217    Hbck hbck = getHbck();
218    try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
219      List<RegionInfo> regions = admin.getRegions(TABLE_NAME);
220      for (RegionInfo ri : regions) {
221        RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
222          .getRegionStates().getRegionState(ri.getEncodedName());
223        LOG.info("RS: {}", rs.toString());
224      }
225      List<Long> pids =
226        hbck.unassigns(regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
227      waitOnPids(pids);
228      // Rerun the unassign. Should fail for all Regions since they already unassigned; failed
229      // unassign will manifest as all pids being -1 (ever since HBASE-24885).
230      pids =
231        hbck.unassigns(regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
232      waitOnPids(pids);
233      for (long pid: pids) {
234        assertEquals(Procedure.NO_PROC_ID, pid);
235      }
236      // If we pass override, then we should be able to unassign EVEN THOUGH Regions already
237      // unassigned.... makes for a mess but operator might want to do this at an extreme when
238      // doing fixup of broke cluster.
239      pids =
240        hbck.unassigns(regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()),
241          true);
242      waitOnPids(pids);
243      for (long pid: pids) {
244        assertNotEquals(Procedure.NO_PROC_ID, pid);
245      }
246      // Clean-up by bypassing all the unassigns we just made so tests can continue.
247      hbck.bypassProcedure(pids, 10000, true, true);
248      for (RegionInfo ri : regions) {
249        RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
250          .getRegionStates().getRegionState(ri.getEncodedName());
251        LOG.info("RS: {}", rs.toString());
252        assertTrue(rs.toString(), rs.isClosed());
253      }
254      pids =
255        hbck.assigns(regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
256      waitOnPids(pids);
257      // Rerun the assign. Should fail for all Regions since they already assigned; failed
258      // assign will manifest as all pids being -1 (ever since HBASE-24885).
259      pids =
260        hbck.assigns(regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
261      for (long pid: pids) {
262        assertEquals(Procedure.NO_PROC_ID, pid);
263      }
264      for (RegionInfo ri : regions) {
265        RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
266          .getRegionStates().getRegionState(ri.getEncodedName());
267        LOG.info("RS: {}", rs.toString());
268        assertTrue(rs.toString(), rs.isOpened());
269      }
270      // What happens if crappy region list passed?
271      pids = hbck.assigns(
272        Arrays.stream(new String[] { "a", "some rubbish name" }).collect(Collectors.toList()));
273      for (long pid : pids) {
274        assertEquals(Procedure.NO_PROC_ID, pid);
275      }
276    }
277  }
278
279  @Test
280  public void testScheduleSCP() throws Exception {
281    HRegionServer testRs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME);
282    TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(TABLE_NAME), Bytes.toBytes("family1"),
283      true);
284    ServerName serverName = testRs.getServerName();
285    Hbck hbck = getHbck();
286    List<Long> pids =
287      hbck.scheduleServerCrashProcedures(Arrays.asList(serverName));
288    assertTrue(pids.get(0) > 0);
289    LOG.info("pid is {}", pids.get(0));
290
291    List<Long> newPids =
292      hbck.scheduleServerCrashProcedures(Arrays.asList(serverName));
293    assertTrue(newPids.get(0) < 0);
294    LOG.info("pid is {}", newPids.get(0));
295    waitOnPids(pids);
296  }
297
298  @Test
299  public void testRunHbckChore() throws Exception {
300    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
301    long endTimestamp = master.getHbckChore().getCheckingEndTimestamp();
302    Hbck hbck = getHbck();
303    boolean ran = false;
304    while (!ran) {
305      ran = hbck.runHbckChore();
306      if (ran) {
307        assertTrue(master.getHbckChore().getCheckingEndTimestamp() > endTimestamp);
308      }
309    }
310  }
311
312  public static class FailingSplitAfterMetaUpdatedMasterObserver
313      implements MasterCoprocessor, MasterObserver {
314    @SuppressWarnings("checkstyle:VisibilityModifier") public volatile CountDownLatch latch;
315
316    @Override
317    public void start(CoprocessorEnvironment e) throws IOException {
318      resetLatch();
319    }
320
321    @Override
322    public Optional<MasterObserver> getMasterObserver() {
323      return Optional.of(this);
324    }
325
326    @Override
327    public void preSplitRegionAfterMETAAction(ObserverContext<MasterCoprocessorEnvironment> ctx)
328        throws IOException {
329      LOG.info("I'm here");
330      latch.countDown();
331      throw new IOException("this procedure will fail at here forever");
332    }
333
334    public void resetLatch() {
335      this.latch = new CountDownLatch(1);
336    }
337  }
338
339  public static class FailingMergeAfterMetaUpdatedMasterObserver
340      implements MasterCoprocessor, MasterObserver {
341    @SuppressWarnings("checkstyle:VisibilityModifier") public volatile CountDownLatch latch;
342
343    @Override
344    public void start(CoprocessorEnvironment e) throws IOException {
345      resetLatch();
346    }
347
348    @Override
349    public Optional<MasterObserver> getMasterObserver() {
350      return Optional.of(this);
351    }
352
353    public void resetLatch() {
354      this.latch = new CountDownLatch(1);
355    }
356
357    @Override
358    public void postMergeRegionsCommitAction(
359        final ObserverContext<MasterCoprocessorEnvironment> ctx, final RegionInfo[] regionsToMerge,
360        final RegionInfo mergedRegion) throws IOException {
361      latch.countDown();
362      throw new IOException("this procedure will fail at here forever");
363    }
364  }
365
366  private void waitOnPids(List<Long> pids) {
367    TEST_UTIL.waitFor(60000, () -> pids.stream().allMatch(procExec::isFinished));
368  }
369}