001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import edu.umd.cs.findbugs.annotations.NonNull;
025import java.util.List;
026import java.util.Map;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
030import org.apache.hadoop.hbase.client.Put;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.ResultScanner;
033import org.apache.hadoop.hbase.client.Scan;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination;
038import org.apache.hadoop.hbase.master.HMaster;
039import org.apache.hadoop.hbase.master.LoadBalancer;
040import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.testclassification.MiscTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.CommonFSUtils;
045import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
046import org.apache.hadoop.hbase.zookeeper.ZKUtil;
047import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
048import org.apache.zookeeper.KeeperException;
049import org.junit.jupiter.api.AfterAll;
050import org.junit.jupiter.api.AfterEach;
051import org.junit.jupiter.api.BeforeAll;
052import org.junit.jupiter.api.BeforeEach;
053import org.junit.jupiter.api.Tag;
054import org.junit.jupiter.api.Test;
055import org.junit.jupiter.api.TestInfo;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059@Tag(MiscTests.TAG)
060@Tag(MediumTests.TAG)
061public class TestZooKeeper {
062
063  private static final Logger LOG = LoggerFactory.getLogger(TestZooKeeper.class);
064
065  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
066
067  @BeforeAll
068  public static void setUpBeforeClass() throws Exception {
069    // Test we can first start the ZK cluster by itself
070    Configuration conf = TEST_UTIL.getConfiguration();
071    TEST_UTIL.startMiniDFSCluster(2);
072    TEST_UTIL.startMiniZKCluster();
073    conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
074      HConstants.ZK_CONNECTION_REGISTRY_CLASS);
075    conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
076    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
077      LoadBalancer.class);
078  }
079
080  @AfterAll
081  public static void tearDownAfterClass() throws Exception {
082    TEST_UTIL.shutdownMiniCluster();
083  }
084
085  @BeforeEach
086  public void setUp() throws Exception {
087    StartTestingClusterOption option =
088      StartTestingClusterOption.builder().numMasters(2).numRegionServers(2).build();
089    TEST_UTIL.startMiniHBaseCluster(option);
090  }
091
092  @AfterEach
093  public void after() throws Exception {
094    try {
095      TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(10000);
096      // Some regionserver could fail to delete its znode.
097      // So shutdown could hang. Let's kill them all instead.
098      TEST_UTIL.getHBaseCluster().killAll();
099
100      // Still need to clean things up
101      TEST_UTIL.shutdownMiniHBaseCluster();
102    } finally {
103      TEST_UTIL.getTestFileSystem().delete(CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()),
104        true);
105      ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
106    }
107  }
108
109  @Test
110  public void testRegionServerSessionExpired(TestInfo testInfo) throws Exception {
111    LOG.info("Starting " + testInfo.getTestMethod().get().getName());
112    TEST_UTIL.expireRegionServerSession(0);
113    testSanity(testInfo.getTestMethod().get().getName());
114  }
115
116  @Test
117  public void testMasterSessionExpired(TestInfo testInfo) throws Exception {
118    LOG.info("Starting " + testInfo.getTestMethod().get().getName());
119    TEST_UTIL.expireMasterSession();
120    testSanity(testInfo.getTestMethod().get().getName());
121  }
122
123  /**
124   * Master recovery when the znode already exists. Internally, this test differs from
125   * {@link #testMasterSessionExpired} because here the master znode will exist in ZK.
126   */
127  @Test
128  public void testMasterZKSessionRecoveryFailure(TestInfo testInfo) throws Exception {
129    LOG.info("Starting " + testInfo.getTestMethod().get().getName());
130    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
131    HMaster m = cluster.getMaster();
132    m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException());
133    assertTrue(m.isStopped()); // Master doesn't recover any more
134    testSanity(testInfo.getTestMethod().get().getName());
135  }
136
137  /**
138   * Make sure we can use the cluster
139   */
140  private void testSanity(final String testName) throws Exception {
141    String tableName = testName + "_" + EnvironmentEdgeManager.currentTime();
142    TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
143      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam")).build();
144    LOG.info("Creating table " + tableName);
145    Admin admin = TEST_UTIL.getAdmin();
146    try {
147      admin.createTable(desc);
148    } finally {
149      admin.close();
150    }
151
152    Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());
153    Put put = new Put(Bytes.toBytes("testrow"));
154    put.addColumn(Bytes.toBytes("fam"), Bytes.toBytes("col"), Bytes.toBytes("testdata"));
155    LOG.info("Putting table " + tableName);
156    table.put(put);
157    table.close();
158  }
159
160  /**
161   * Tests that the master does not call retainAssignment after recovery from expired zookeeper
162   * session. Without the HBASE-6046 fix master always tries to assign all the user regions by
163   * calling retainAssignment.
164   */
165  @Test
166  public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestInfo testInfo)
167    throws Exception {
168    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
169    cluster.startRegionServer();
170    cluster.waitForActiveAndReadyMaster(10000);
171    HMaster m = cluster.getMaster();
172    final ZKWatcher zkw = m.getZooKeeper();
173    // now the cluster is up. So assign some regions.
174    try (Admin admin = TEST_UTIL.getAdmin()) {
175      byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
176        Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
177        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
178      TableDescriptor htd = TableDescriptorBuilder
179        .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()))
180        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
181      admin.createTable(htd, SPLIT_KEYS);
182      TEST_UTIL.waitUntilNoRegionsInTransition(60000);
183      m.getZooKeeper().close();
184      MockLoadBalancer.retainAssignCalled = false;
185      final int expectedNumOfListeners = countPermanentListeners(zkw);
186      // the master could already been aborted by some background tasks but here we call abort
187      // directly to make sure this will happen
188      m.abort("Test recovery from zk session expired",
189        new KeeperException.SessionExpiredException());
190      // it is possible that our abort call above returned earlier because of someone else has
191      // already called abort, but it is possible that it has not finished the abort call yet so the
192      // isStopped flag is still false, let's wait for sometime.
193      TEST_UTIL.waitFor(5000, () -> m.isStopped()); // Master doesn't recover any more
194
195      // The recovered master should not call retainAssignment, as it is not a
196      // clean startup.
197      assertFalse(MockLoadBalancer.retainAssignCalled, "Retain assignment should not be called");
198      // number of listeners should be same as the value before master aborted
199      // wait for new master is initialized
200      cluster.waitForActiveAndReadyMaster(120000);
201      final HMaster newMaster = cluster.getMasterThread().getMaster();
202      assertEquals(expectedNumOfListeners, countPermanentListeners(newMaster.getZooKeeper()));
203    }
204  }
205
206  /**
207   * Count listeners in zkw excluding listeners, that belongs to workers or other temporary
208   * processes.
209   */
210  private int countPermanentListeners(ZKWatcher watcher) {
211    return countListeners(watcher, ZkSplitLogWorkerCoordination.class);
212  }
213
214  /**
215   * Count listeners in zkw excluding provided classes
216   */
217  private int countListeners(ZKWatcher watcher, Class<?>... exclude) {
218    int cnt = 0;
219    for (Object o : watcher.getListeners()) {
220      boolean skip = false;
221      for (Class<?> aClass : exclude) {
222        if (aClass.isAssignableFrom(o.getClass())) {
223          skip = true;
224          break;
225        }
226      }
227      if (!skip) {
228        cnt += 1;
229      }
230    }
231    return cnt;
232  }
233
234  /**
235   * Tests whether the logs are split when master recovers from a expired zookeeper session and an
236   * RS goes down.
237   */
238  @Test
239  public void testLogSplittingAfterMasterRecoveryDueToZKExpiry(TestInfo testInfo) throws Exception {
240    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
241    cluster.startRegionServer();
242    TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
243    byte[] family = Bytes.toBytes("col");
244    try (Admin admin = TEST_UTIL.getAdmin()) {
245      byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
246        Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
247      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
248        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
249      admin.createTable(htd, SPLIT_KEYS);
250    }
251    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
252    HMaster m = cluster.getMaster();
253    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
254      int numberOfPuts;
255      for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
256        Put p = new Put(Bytes.toBytes(numberOfPuts));
257        p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"),
258          Bytes.toBytes("value" + numberOfPuts));
259        table.put(p);
260      }
261      m.abort("Test recovery from zk session expired",
262        new KeeperException.SessionExpiredException());
263      assertTrue(m.isStopped()); // Master doesn't recover any more
264      cluster.killRegionServer(TEST_UTIL.getRSForFirstRegionInTable(tableName).getServerName());
265      // Without patch for HBASE-6046 this test case will always timeout
266      // with patch the test case should pass.
267      int numberOfRows = 0;
268      try (ResultScanner scanner = table.getScanner(new Scan())) {
269        while (scanner.next() != null) {
270          numberOfRows++;
271        }
272      }
273      assertEquals(numberOfPuts, numberOfRows, "Number of rows should be equal to number of puts.");
274    }
275  }
276
277  static class MockLoadBalancer extends SimpleLoadBalancer {
278    static boolean retainAssignCalled = false;
279
280    @Override
281    @NonNull
282    public Map<ServerName, List<RegionInfo>> retainAssignment(Map<RegionInfo, ServerName> regions,
283      List<ServerName> servers) throws HBaseIOException {
284      retainAssignCalled = true;
285      return super.retainAssignment(regions, servers);
286    }
287  }
288
289}