Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.util.List;
025import java.util.Map;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.client.Admin;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
029import org.apache.hadoop.hbase.client.Put;
030import org.apache.hadoop.hbase.client.RegionInfo;
031import org.apache.hadoop.hbase.client.ResultScanner;
032import org.apache.hadoop.hbase.client.Scan;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.client.TableDescriptor;
035import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
036import org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination;
037import org.apache.hadoop.hbase.master.HMaster;
038import org.apache.hadoop.hbase.master.LoadBalancer;
039import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
040import org.apache.hadoop.hbase.testclassification.MediumTests;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.CommonFSUtils;
044import org.apache.hadoop.hbase.zookeeper.ZKUtil;
045import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
046import org.apache.zookeeper.KeeperException;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059@Category({MiscTests.class, MediumTests.class})
060public class TestZooKeeper {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestZooKeeper.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestZooKeeper.class);
067
068  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
069
070  @Rule
071  public TestName name = new TestName();
072
073  @BeforeClass
074  public static void setUpBeforeClass() throws Exception {
075    // Test we can first start the ZK cluster by itself
076    Configuration conf = TEST_UTIL.getConfiguration();
077    // A couple of tests rely on master expiring ZK session, hence killing the only master. So it
078    // makes sense only for ZK registry. Enforcing it.
079    conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
080        HConstants.ZK_CONNECTION_REGISTRY_CLASS);
081    TEST_UTIL.startMiniDFSCluster(2);
082    TEST_UTIL.startMiniZKCluster();
083    conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
084    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
085        LoadBalancer.class);
086    TEST_UTIL.startMiniDFSCluster(2);
087  }
088
089  @AfterClass
090  public static void tearDownAfterClass() throws Exception {
091    TEST_UTIL.shutdownMiniCluster();
092  }
093
094  @Before
095  public void setUp() throws Exception {
096    StartMiniClusterOption option = StartMiniClusterOption.builder()
097        .numMasters(2).numRegionServers(2).build();
098    TEST_UTIL.startMiniHBaseCluster(option);
099  }
100
101  @After
102  public void after() throws Exception {
103    try {
104      TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(10000);
105      // Some regionserver could fail to delete its znode.
106      // So shutdown could hang. Let's kill them all instead.
107      TEST_UTIL.getHBaseCluster().killAll();
108
109      // Still need to clean things up
110      TEST_UTIL.shutdownMiniHBaseCluster();
111    } finally {
112      TEST_UTIL.getTestFileSystem().delete(CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()),
113        true);
114      ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
115    }
116  }
117
118  @Test
119  public void testRegionServerSessionExpired() throws Exception {
120    LOG.info("Starting " + name.getMethodName());
121    TEST_UTIL.expireRegionServerSession(0);
122    testSanity(name.getMethodName());
123  }
124
125  @Test
126  public void testMasterSessionExpired() throws Exception {
127    LOG.info("Starting " + name.getMethodName());
128    TEST_UTIL.expireMasterSession();
129    testSanity(name.getMethodName());
130  }
131
132  /**
133   * Master recovery when the znode already exists. Internally, this
134   *  test differs from {@link #testMasterSessionExpired} because here
135   *  the master znode will exist in ZK.
136   */
137  @Test
138  public void testMasterZKSessionRecoveryFailure() throws Exception {
139    LOG.info("Starting " + name.getMethodName());
140    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
141    HMaster m = cluster.getMaster();
142    m.abort("Test recovery from zk session expired",
143        new KeeperException.SessionExpiredException());
144    assertTrue(m.isStopped()); // Master doesn't recover any more
145    testSanity(name.getMethodName());
146  }
147
148  /**
149   * Make sure we can use the cluster
150   */
151  private void testSanity(final String testName) throws Exception {
152    String tableName = testName + "_" + System.currentTimeMillis();
153    TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
154        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam")).build();
155    LOG.info("Creating table " + tableName);
156    Admin admin = TEST_UTIL.getAdmin();
157    try {
158      admin.createTable(desc);
159    } finally {
160      admin.close();
161    }
162
163    Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());
164    Put put = new Put(Bytes.toBytes("testrow"));
165    put.addColumn(Bytes.toBytes("fam"), Bytes.toBytes("col"), Bytes.toBytes("testdata"));
166    LOG.info("Putting table " + tableName);
167    table.put(put);
168    table.close();
169  }
170
171  /**
172   * Tests that the master does not call retainAssignment after recovery from expired zookeeper
173   * session. Without the HBASE-6046 fix master always tries to assign all the user regions by
174   * calling retainAssignment.
175   */
176  @Test
177  public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
178    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
179    cluster.startRegionServer();
180    cluster.waitForActiveAndReadyMaster(10000);
181    HMaster m = cluster.getMaster();
182    final ZKWatcher zkw = m.getZooKeeper();
183    // now the cluster is up. So assign some regions.
184    try (Admin admin = TEST_UTIL.getAdmin()) {
185      byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
186          Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
187          Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
188      TableDescriptor htd =
189          TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
190              .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
191      admin.createTable(htd, SPLIT_KEYS);
192      TEST_UTIL.waitUntilNoRegionsInTransition(60000);
193      m.getZooKeeper().close();
194      MockLoadBalancer.retainAssignCalled = false;
195      final int expectedNumOfListeners = countPermanentListeners(zkw);
196      m.abort("Test recovery from zk session expired",
197          new KeeperException.SessionExpiredException());
198      assertTrue(m.isStopped()); // Master doesn't recover any more
199      // The recovered master should not call retainAssignment, as it is not a
200      // clean startup.
201      assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
202      // number of listeners should be same as the value before master aborted
203      // wait for new master is initialized
204      cluster.waitForActiveAndReadyMaster(120000);
205      final HMaster newMaster = cluster.getMasterThread().getMaster();
206      assertEquals(expectedNumOfListeners, countPermanentListeners(newMaster.getZooKeeper()));
207    }
208  }
209
210  /**
211   * Count listeners in zkw excluding listeners, that belongs to workers or other
212   * temporary processes.
213   */
214  private int countPermanentListeners(ZKWatcher watcher) {
215    return countListeners(watcher, ZkSplitLogWorkerCoordination.class);
216  }
217
218  /**
219   * Count listeners in zkw excluding provided classes
220   */
221  private int countListeners(ZKWatcher watcher, Class<?>... exclude) {
222    int cnt = 0;
223    for (Object o : watcher.getListeners()) {
224      boolean skip = false;
225      for (Class<?> aClass : exclude) {
226        if (aClass.isAssignableFrom(o.getClass())) {
227          skip = true;
228          break;
229        }
230      }
231      if (!skip) {
232        cnt += 1;
233      }
234    }
235    return cnt;
236  }
237
238  /**
239   * Tests whether the logs are split when master recovers from a expired zookeeper session and an
240   * RS goes down.
241   */
242  @Test
243  public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws Exception {
244    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
245    cluster.startRegionServer();
246    TableName tableName = TableName.valueOf(name.getMethodName());
247    byte[] family = Bytes.toBytes("col");
248    try (Admin admin = TEST_UTIL.getAdmin()) {
249      byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
250        Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
251      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
252          .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
253      admin.createTable(htd, SPLIT_KEYS);
254    }
255    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
256    HMaster m = cluster.getMaster();
257    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
258      int numberOfPuts;
259      for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
260        Put p = new Put(Bytes.toBytes(numberOfPuts));
261        p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"),
262          Bytes.toBytes("value" + numberOfPuts));
263        table.put(p);
264      }
265      m.abort("Test recovery from zk session expired",
266        new KeeperException.SessionExpiredException());
267      assertTrue(m.isStopped()); // Master doesn't recover any more
268      cluster.killRegionServer(TEST_UTIL.getRSForFirstRegionInTable(tableName).getServerName());
269      // Without patch for HBASE-6046 this test case will always timeout
270      // with patch the test case should pass.
271      int numberOfRows = 0;
272      try (ResultScanner scanner = table.getScanner(new Scan())) {
273        while (scanner.next() != null) {
274          numberOfRows++;
275        }
276      }
277      assertEquals("Number of rows should be equal to number of puts.", numberOfPuts, numberOfRows);
278    }
279  }
280
281  static class MockLoadBalancer extends SimpleLoadBalancer {
282    static boolean retainAssignCalled = false;
283
284    @Override
285    public Map<ServerName, List<RegionInfo>> retainAssignment(
286        Map<RegionInfo, ServerName> regions, List<ServerName> servers) throws HBaseIOException {
287      retainAssignCalled = true;
288      return super.retainAssignment(regions, servers);
289    }
290  }
291
292}
293