001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.MASTER_PROCEDURE_LOGDIR;
021
022import java.util.List;
023import java.util.concurrent.Executors;
024import java.util.concurrent.ScheduledExecutorService;
025import java.util.concurrent.TimeUnit;
026
027import org.apache.hadoop.fs.FileStatus;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.testclassification.MasterTests;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.util.JVMClusterUtil;
038import org.junit.AfterClass;
039import org.junit.BeforeClass;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046
047@Category({MasterTests.class, MediumTests.class})
048public class TestMetaInitIfAllProceduresLost {
049  @ClassRule
050  public static final HBaseClassTestRule CLASS_RULE =
051      HBaseClassTestRule.forClass(TestMetaInitIfAllProceduresLost.class);
052
053  private static final Logger LOG = LoggerFactory
054      .getLogger(TestMetaInitIfAllProceduresLost.class);
055
056  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
057
058  @BeforeClass
059  public static void setupCluster() throws Exception {
060    UTIL.startMiniCluster(3);
061  }
062
063  @AfterClass
064  public static void cleanupTest() throws Exception {
065    try {
066      UTIL.shutdownMiniCluster();
067    } catch (Exception e) {
068      LOG.warn("failure shutting down cluster", e);
069    }
070  }
071
072  @Test
073  public void test() throws Exception {
074    for (JVMClusterUtil.RegionServerThread rst : UTIL.getMiniHBaseCluster()
075        .getRegionServerThreads()) {
076      rst.getRegionServer().abort("killAll");
077    }
078    //wait for a while, until all dirs are changed to '-splitting'
079    UTIL.waitFor(30000, () ->
080        UTIL.getMiniHBaseCluster().getMaster().getMasterWalManager()
081          .getLiveServersFromWALDir().size() == 0);
082    Thread.sleep(1000);
083    Path procedureWals = new Path(
084        UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem()
085            .getRootDir(), MASTER_PROCEDURE_LOGDIR);
086    //Kill the master
087    UTIL.getMiniHBaseCluster().killAll();
088    //Delte all procedure log to create an anomaly
089    for (FileStatus file : UTIL.getTestFileSystem().listStatus(procedureWals)) {
090      LOG.info("removing " + file);
091      UTIL.getTestFileSystem().delete(file.getPath());
092    }
093    UTIL.getMiniHBaseCluster().startMaster();
094    UTIL.getMiniHBaseCluster().startRegionServer();
095    UTIL.getMiniHBaseCluster().startRegionServer();
096    UTIL.getMiniHBaseCluster().startRegionServer();
097    ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
098    // Schedule an assign of meta after ten seconds. Then a few seconds later, do namespace assign.
099    // The meta table needs to be online before the namespace can succeed.
100    final HMaster master = UTIL.getHBaseCluster().getMaster();
101    final AssignmentManager am = master.getAssignmentManager();
102    final AssignProcedure ap = am.createAssignProcedure(RegionInfoBuilder.FIRST_META_REGIONINFO);
103    scheduler.schedule(() -> master.getMasterProcedureExecutor().submitProcedure(ap), 10,
104        TimeUnit.SECONDS);
105    scheduler.schedule(() -> {
106      // hbase:meta should be online by the time this runs. That means we should have read the
107      // regions that make up the namespace table so below query should return results.
108      List<RegionInfo> ris = am.getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME);
109      if (ris.isEmpty()) {
110        throw new RuntimeException("No namespace regions found!");
111      }
112      for (RegionInfo ri: ris) {
113        AssignProcedure riap = am.createAssignProcedure(ri);
114        master.getMasterProcedureExecutor().submitProcedure(riap);
115      }
116    }, 20 /*Must run AFTER meta is online*/, TimeUnit.SECONDS);
117    // Master should able to finish init even if all procedures are lost
118    UTIL.waitFor(180000, () -> UTIL.getMiniHBaseCluster().getMaster() != null && UTIL
119      .getMiniHBaseCluster().getMaster().isInitialized());
120  }
121}