001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.util.List; 025import java.util.Map; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.client.Admin; 028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 029import org.apache.hadoop.hbase.client.Put; 030import org.apache.hadoop.hbase.client.RegionInfo; 031import org.apache.hadoop.hbase.client.ResultScanner; 032import org.apache.hadoop.hbase.client.Scan; 033import org.apache.hadoop.hbase.client.Table; 034import org.apache.hadoop.hbase.client.TableDescriptor; 035import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 036import org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination; 037import org.apache.hadoop.hbase.master.HMaster; 038import org.apache.hadoop.hbase.master.LoadBalancer; 039import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer; 040import org.apache.hadoop.hbase.testclassification.LargeTests; 041import org.apache.hadoop.hbase.testclassification.MiscTests; 042import org.apache.hadoop.hbase.util.Bytes; 043import org.apache.hadoop.hbase.util.FSUtils; 044import org.apache.hadoop.hbase.zookeeper.ZKUtil; 045import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 046import org.apache.zookeeper.KeeperException; 047import org.junit.After; 048import org.junit.AfterClass; 049import org.junit.Before; 050import org.junit.BeforeClass; 051import org.junit.ClassRule; 052import org.junit.Rule; 053import org.junit.Test; 054import org.junit.experimental.categories.Category; 055import org.junit.rules.TestName; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058 059@Category({MiscTests.class, LargeTests.class}) 060public class TestZooKeeper { 061 062 @ClassRule 063 public static final HBaseClassTestRule CLASS_RULE = 064 HBaseClassTestRule.forClass(TestZooKeeper.class); 065 066 private static final Logger LOG = LoggerFactory.getLogger(TestZooKeeper.class); 067 068 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 069 070 @Rule 071 public TestName name = new TestName(); 072 073 @BeforeClass 074 public static void setUpBeforeClass() throws Exception { 075 // Test we can first start the ZK cluster by itself 076 Configuration conf = TEST_UTIL.getConfiguration(); 077 TEST_UTIL.startMiniDFSCluster(2); 078 TEST_UTIL.startMiniZKCluster(); 079 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000); 080 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class, 081 LoadBalancer.class); 082 } 083 084 @AfterClass 085 public static void tearDownAfterClass() throws Exception { 086 TEST_UTIL.shutdownMiniCluster(); 087 } 088 089 @Before 090 public void setUp() throws Exception { 091 TEST_UTIL.startMiniHBaseCluster(2, 2); 092 } 093 094 @After 095 public void after() throws Exception { 096 try { 097 TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(10000); 098 // Some regionserver could fail to delete its znode. 099 // So shutdown could hang. Let's kill them all instead. 100 TEST_UTIL.getHBaseCluster().killAll(); 101 102 // Still need to clean things up 103 TEST_UTIL.shutdownMiniHBaseCluster(); 104 } finally { 105 TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true); 106 ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase"); 107 } 108 } 109 110 @Test 111 public void testRegionServerSessionExpired() throws Exception { 112 LOG.info("Starting " + name.getMethodName()); 113 TEST_UTIL.expireRegionServerSession(0); 114 testSanity(name.getMethodName()); 115 } 116 117 @Test 118 public void testMasterSessionExpired() throws Exception { 119 LOG.info("Starting " + name.getMethodName()); 120 TEST_UTIL.expireMasterSession(); 121 testSanity(name.getMethodName()); 122 } 123 124 /** 125 * Master recovery when the znode already exists. Internally, this 126 * test differs from {@link #testMasterSessionExpired} because here 127 * the master znode will exist in ZK. 128 */ 129 @Test 130 public void testMasterZKSessionRecoveryFailure() throws Exception { 131 LOG.info("Starting " + name.getMethodName()); 132 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 133 HMaster m = cluster.getMaster(); 134 m.abort("Test recovery from zk session expired", 135 new KeeperException.SessionExpiredException()); 136 assertTrue(m.isStopped()); // Master doesn't recover any more 137 testSanity(name.getMethodName()); 138 } 139 140 /** 141 * Make sure we can use the cluster 142 */ 143 private void testSanity(final String testName) throws Exception { 144 String tableName = testName + "_" + System.currentTimeMillis(); 145 TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) 146 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam")).build(); 147 LOG.info("Creating table " + tableName); 148 Admin admin = TEST_UTIL.getAdmin(); 149 try { 150 admin.createTable(desc); 151 } finally { 152 admin.close(); 153 } 154 155 Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); 156 Put put = new Put(Bytes.toBytes("testrow")); 157 put.addColumn(Bytes.toBytes("fam"), Bytes.toBytes("col"), Bytes.toBytes("testdata")); 158 LOG.info("Putting table " + tableName); 159 table.put(put); 160 table.close(); 161 } 162 163 /** 164 * Tests that the master does not call retainAssignment after recovery from expired zookeeper 165 * session. Without the HBASE-6046 fix master always tries to assign all the user regions by 166 * calling retainAssignment. 167 */ 168 @Test 169 public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception { 170 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 171 cluster.startRegionServer(); 172 cluster.waitForActiveAndReadyMaster(10000); 173 HMaster m = cluster.getMaster(); 174 final ZKWatcher zkw = m.getZooKeeper(); 175 // now the cluster is up. So assign some regions. 176 try (Admin admin = TEST_UTIL.getAdmin()) { 177 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), 178 Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), 179 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") }; 180 TableDescriptor htd = 181 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 182 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); 183 admin.createTable(htd, SPLIT_KEYS); 184 TEST_UTIL.waitUntilNoRegionsInTransition(60000); 185 m.getZooKeeper().close(); 186 MockLoadBalancer.retainAssignCalled = false; 187 final int expectedNumOfListeners = countPermanentListeners(zkw); 188 m.abort("Test recovery from zk session expired", 189 new KeeperException.SessionExpiredException()); 190 assertTrue(m.isStopped()); // Master doesn't recover any more 191 // The recovered master should not call retainAssignment, as it is not a 192 // clean startup. 193 assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled); 194 // number of listeners should be same as the value before master aborted 195 // wait for new master is initialized 196 cluster.waitForActiveAndReadyMaster(120000); 197 final HMaster newMaster = cluster.getMasterThread().getMaster(); 198 assertEquals(expectedNumOfListeners, countPermanentListeners(newMaster.getZooKeeper())); 199 } 200 } 201 202 /** 203 * Count listeners in zkw excluding listeners, that belongs to workers or other 204 * temporary processes. 205 */ 206 private int countPermanentListeners(ZKWatcher watcher) { 207 return countListeners(watcher, ZkSplitLogWorkerCoordination.class); 208 } 209 210 /** 211 * Count listeners in zkw excluding provided classes 212 */ 213 private int countListeners(ZKWatcher watcher, Class<?>... exclude) { 214 int cnt = 0; 215 for (Object o : watcher.getListeners()) { 216 boolean skip = false; 217 for (Class<?> aClass : exclude) { 218 if (aClass.isAssignableFrom(o.getClass())) { 219 skip = true; 220 break; 221 } 222 } 223 if (!skip) { 224 cnt += 1; 225 } 226 } 227 return cnt; 228 } 229 230 /** 231 * Tests whether the logs are split when master recovers from a expired zookeeper session and an 232 * RS goes down. 233 */ 234 @Test 235 public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws Exception { 236 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 237 cluster.startRegionServer(); 238 TableName tableName = TableName.valueOf(name.getMethodName()); 239 byte[] family = Bytes.toBytes("col"); 240 try (Admin admin = TEST_UTIL.getAdmin()) { 241 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"), 242 Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") }; 243 TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 244 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); 245 admin.createTable(htd, SPLIT_KEYS); 246 } 247 TEST_UTIL.waitUntilNoRegionsInTransition(60000); 248 HMaster m = cluster.getMaster(); 249 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { 250 int numberOfPuts; 251 for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) { 252 Put p = new Put(Bytes.toBytes(numberOfPuts)); 253 p.addColumn(Bytes.toBytes("col"), Bytes.toBytes("ql"), 254 Bytes.toBytes("value" + numberOfPuts)); 255 table.put(p); 256 } 257 m.abort("Test recovery from zk session expired", 258 new KeeperException.SessionExpiredException()); 259 assertTrue(m.isStopped()); // Master doesn't recover any more 260 cluster.killRegionServer(TEST_UTIL.getRSForFirstRegionInTable(tableName).getServerName()); 261 // Without patch for HBASE-6046 this test case will always timeout 262 // with patch the test case should pass. 263 int numberOfRows = 0; 264 try (ResultScanner scanner = table.getScanner(new Scan())) { 265 while (scanner.next() != null) { 266 numberOfRows++; 267 } 268 } 269 assertEquals("Number of rows should be equal to number of puts.", numberOfPuts, numberOfRows); 270 } 271 } 272 273 static class MockLoadBalancer extends SimpleLoadBalancer { 274 static boolean retainAssignCalled = false; 275 276 @Override 277 public Map<ServerName, List<RegionInfo>> retainAssignment( 278 Map<RegionInfo, ServerName> regions, List<ServerName> servers) throws HBaseIOException { 279 retainAssignCalled = true; 280 return super.retainAssignment(regions, servers); 281 } 282 } 283 284} 285