001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotEquals; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024 025import java.io.IOException; 026import java.nio.charset.StandardCharsets; 027import java.util.ArrayList; 028import java.util.Collection; 029import java.util.EnumSet; 030import java.util.HashMap; 031import java.util.HashSet; 032import java.util.List; 033import java.util.Map; 034import java.util.Set; 035import java.util.concurrent.atomic.AtomicInteger; 036 037import org.apache.hadoop.conf.Configuration; 038import org.apache.hadoop.hbase.ClusterMetrics.Option; 039import org.apache.hadoop.hbase.HBaseClassTestRule; 040import org.apache.hadoop.hbase.HBaseTestingUtility; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.HRegionLocation; 043import org.apache.hadoop.hbase.MetaTableAccessor; 044import org.apache.hadoop.hbase.MetaTableAccessor.Visitor; 045import org.apache.hadoop.hbase.RegionLocations; 046import org.apache.hadoop.hbase.ServerName; 047import org.apache.hadoop.hbase.StartMiniClusterOption; 048import org.apache.hadoop.hbase.TableName; 049import org.apache.hadoop.hbase.client.Admin; 050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 051import org.apache.hadoop.hbase.client.Connection; 052import org.apache.hadoop.hbase.client.ConnectionFactory; 053import org.apache.hadoop.hbase.client.Delete; 054import org.apache.hadoop.hbase.client.RegionInfo; 055import org.apache.hadoop.hbase.client.RegionReplicaUtil; 056import org.apache.hadoop.hbase.client.Result; 057import org.apache.hadoop.hbase.client.Table; 058import org.apache.hadoop.hbase.client.TableDescriptor; 059import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 060import org.apache.hadoop.hbase.testclassification.MasterTests; 061import org.apache.hadoop.hbase.testclassification.MediumTests; 062import org.apache.hadoop.hbase.util.Bytes; 063import org.apache.hadoop.hbase.util.JVMClusterUtil; 064import org.junit.AfterClass; 065import org.junit.BeforeClass; 066import org.junit.ClassRule; 067import org.junit.Ignore; 068import org.junit.Rule; 069import org.junit.Test; 070import org.junit.experimental.categories.Category; 071import org.junit.rules.TestName; 072import org.slf4j.Logger; 073import org.slf4j.LoggerFactory; 074 075import org.apache.hbase.thirdparty.com.google.common.io.Closeables; 076 077@Category({ MasterTests.class, MediumTests.class }) 078public class TestMasterOperationsForRegionReplicas { 079 080 @ClassRule 081 public static final HBaseClassTestRule CLASS_RULE = 082 HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class); 083 084 private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class); 085 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 086 private static Connection CONNECTION = null; 087 private static Admin ADMIN; 088 private static int numSlaves = 2; 089 private static Configuration conf; 090 091 @Rule 092 public TestName name = new TestName(); 093 094 @BeforeClass 095 public static void setupBeforeClass() throws Exception { 096 conf = TEST_UTIL.getConfiguration(); 097 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); 098 TEST_UTIL.startMiniCluster(numSlaves); 099 TEST_UTIL.getAdmin().balancerSwitch(false, true); 100 CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); 101 ADMIN = CONNECTION.getAdmin(); 102 while (ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics() 103 .size() < numSlaves) { 104 Thread.sleep(100); 105 } 106 } 107 108 @AfterClass 109 public static void tearDownAfterClass() throws Exception { 110 Closeables.close(ADMIN, true); 111 Closeables.close(CONNECTION, true); 112 TEST_UTIL.shutdownMiniCluster(); 113 } 114 115 @Test 116 public void testCreateTableWithSingleReplica() throws Exception { 117 final int numRegions = 3; 118 final int numReplica = 1; 119 final TableName tableName = TableName.valueOf(name.getMethodName()); 120 try { 121 TableDescriptor desc = 122 TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica) 123 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); 124 ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions); 125 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 126 TEST_UTIL.waitUntilNoRegionsInTransition(); 127 128 validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection()); 129 List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName); 130 assertEquals(numRegions * numReplica, hris.size()); 131 } finally { 132 ADMIN.disableTable(tableName); 133 ADMIN.deleteTable(tableName); 134 } 135 } 136 137 @Test 138 public void testCreateTableWithMultipleReplicas() throws Exception { 139 final TableName tableName = TableName.valueOf(name.getMethodName()); 140 final int numRegions = 3; 141 final int numReplica = 2; 142 try { 143 TableDescriptor desc = 144 TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica) 145 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); 146 ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions); 147 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 148 TEST_UTIL.waitUntilNoRegionsInTransition(); 149 validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection()); 150 151 List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName); 152 assertEquals(numRegions * numReplica, hris.size()); 153 assertRegionStateNotNull(hris, numRegions, numReplica); 154 155 List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection()); 156 int numRows = 0; 157 for (Result result : metaRows) { 158 RegionLocations locations = MetaTableAccessor.getRegionLocations(result); 159 RegionInfo hri = locations.getRegionLocation().getRegion(); 160 if (!hri.getTable().equals(tableName)) continue; 161 numRows += 1; 162 HRegionLocation[] servers = locations.getRegionLocations(); 163 // have two locations for the replicas of a region, and the locations should be different 164 assertEquals(2, servers.length); 165 assertNotEquals(servers[1], servers[0]); 166 } 167 assertEquals(numRegions, numRows); 168 169 // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta 170 // class 171 validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, 172 ADMIN.getConnection()); 173 174 // Now kill the master, restart it and see if the assignments are kept 175 ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName(); 176 TEST_UTIL.getHBaseClusterInterface().stopMaster(master); 177 TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000); 178 TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort()); 179 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster(); 180 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 181 TEST_UTIL.waitUntilNoRegionsInTransition(); 182 assertRegionStateNotNull(hris, numRegions, numReplica); 183 validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, 184 ADMIN.getConnection()); 185 186 // Now shut the whole cluster down, and verify the assignments are kept so that the 187 // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each 188 // restart. This messes with our being able to test that we retain locality. Therefore, 189 // figure current cluster ports and pass them in on next cluster start so new cluster comes 190 // up at same coordinates -- and the assignment retention logic has a chance to cut in. 191 List<Integer> rsports = new ArrayList<>(); 192 for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster() 193 .getLiveRegionServerThreads()) { 194 rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort()); 195 } 196 TEST_UTIL.shutdownMiniHBaseCluster(); 197 StartMiniClusterOption option = 198 StartMiniClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build(); 199 TEST_UTIL.startMiniHBaseCluster(option); 200 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 201 TEST_UTIL.waitUntilNoRegionsInTransition(); 202 validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, 203 ADMIN.getConnection()); 204 205 // Now shut the whole cluster down, and verify regions are assigned even if there is only 206 // one server running 207 TEST_UTIL.shutdownMiniHBaseCluster(); 208 TEST_UTIL.startMiniHBaseCluster(); 209 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 210 TEST_UTIL.waitUntilNoRegionsInTransition(); 211 validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica); 212 for (int i = 1; i < numSlaves; i++) { // restore the cluster 213 TEST_UTIL.getMiniHBaseCluster().startRegionServer(); 214 } 215 216 // Check on alter table 217 ADMIN.disableTable(tableName); 218 assertTrue(ADMIN.isTableDisabled(tableName)); 219 // increase the replica 220 ADMIN.modifyTable( 221 TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build()); 222 ADMIN.enableTable(tableName); 223 LOG.info(ADMIN.getDescriptor(tableName).toString()); 224 assertTrue(ADMIN.isTableEnabled(tableName)); 225 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 226 TEST_UTIL.waitUntilNoRegionsInTransition(); 227 List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() 228 .getRegionStates().getRegionsOfTable(tableName); 229 assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" + 230 numReplica, regions.size() == numRegions * (numReplica + 1)); 231 232 // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1) 233 ADMIN.disableTable(tableName); 234 ADMIN.modifyTable( 235 TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build()); 236 ADMIN.enableTable(tableName); 237 assertTrue(ADMIN.isTableEnabled(tableName)); 238 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 239 TEST_UTIL.waitUntilNoRegionsInTransition(); 240 regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() 241 .getRegionsOfTable(tableName); 242 assertEquals(numRegions * numReplica, regions.size()); 243 // also make sure the meta table has the replica locations removed 244 hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName); 245 assertEquals(numRegions * numReplica, hris.size()); 246 // just check that the number of default replica regions in the meta table are the same 247 // as the number of regions the table was created with, and the count of the 248 // replicas is numReplica for each region 249 Map<RegionInfo, Integer> defaultReplicas = new HashMap<>(); 250 for (RegionInfo hri : hris) { 251 RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri); 252 Integer i = defaultReplicas.get(regionReplica0); 253 defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1); 254 } 255 assertEquals(numRegions, defaultReplicas.size()); 256 Collection<Integer> counts = new HashSet<>(defaultReplicas.values()); 257 assertEquals(1, counts.size()); 258 assertTrue(counts.contains(numReplica)); 259 } finally { 260 ADMIN.disableTable(tableName); 261 ADMIN.deleteTable(tableName); 262 } 263 } 264 265 private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) { 266 // check that the master created expected number of RegionState objects 267 for (int i = 0; i < numRegions; i++) { 268 for (int j = 0; j < numReplica; j++) { 269 RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j); 270 RegionState state = 271 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates() 272 .getRegionState(replica); 273 assertNotNull(state); 274 } 275 } 276 } 277 278 @Test 279 @Ignore("Enable when we have support for alter_table- HBASE-10361") 280 public void testIncompleteMetaTableReplicaInformation() throws Exception { 281 final TableName tableName = TableName.valueOf(name.getMethodName()); 282 final int numRegions = 3; 283 final int numReplica = 2; 284 try { 285 // Create a table and let the meta table be updated with the location of the 286 // region locations. 287 TableDescriptor desc = 288 TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica) 289 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); 290 ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions); 291 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 292 TEST_UTIL.waitUntilNoRegionsInTransition(); 293 Set<byte[]> tableRows = new HashSet<>(); 294 List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName); 295 for (RegionInfo hri : hris) { 296 tableRows.add(hri.getRegionName()); 297 } 298 ADMIN.disableTable(tableName); 299 // now delete one replica info from all the rows 300 // this is to make the meta appear to be only partially updated 301 Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME); 302 for (byte[] row : tableRows) { 303 Delete deleteOneReplicaLocation = new Delete(row); 304 deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, 305 MetaTableAccessor.getServerColumn(1)); 306 deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, 307 MetaTableAccessor.getSeqNumColumn(1)); 308 deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY, 309 MetaTableAccessor.getStartCodeColumn(1)); 310 metaTable.delete(deleteOneReplicaLocation); 311 } 312 metaTable.close(); 313 // even if the meta table is partly updated, when we re-enable the table, we should 314 // get back the desired number of replicas for the regions 315 ADMIN.enableTable(tableName); 316 assertTrue(ADMIN.isTableEnabled(tableName)); 317 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 318 TEST_UTIL.waitUntilNoRegionsInTransition(); 319 List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() 320 .getRegionStates().getRegionsOfTable(tableName); 321 assertEquals(numRegions * numReplica, regions.size()); 322 } finally { 323 ADMIN.disableTable(tableName); 324 ADMIN.deleteTable(tableName); 325 } 326 } 327 328 private void validateNumberOfRowsInMeta(final TableName table, int numRegions, 329 Connection connection) throws IOException { 330 assert (ADMIN.tableExists(table)); 331 final AtomicInteger count = new AtomicInteger(); 332 Visitor visitor = new Visitor() { 333 @Override 334 public boolean visit(Result r) throws IOException { 335 if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet(); 336 return true; 337 } 338 }; 339 MetaTableAccessor.fullScanRegions(connection, visitor); 340 assertEquals(numRegions, count.get()); 341 } 342 343 private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table, 344 int numRegions, int numReplica, Connection connection) throws IOException { 345 SnapshotOfRegionAssignmentFromMeta snapshot = 346 new SnapshotOfRegionAssignmentFromMeta(connection); 347 snapshot.initialize(); 348 Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap(); 349 assert (regionToServerMap.size() == numRegions * numReplica + 1); // '1' for the namespace 350 Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap(); 351 for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) { 352 if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) { 353 continue; 354 } 355 List<RegionInfo> regions = entry.getValue(); 356 Set<byte[]> setOfStartKeys = new HashSet<>(); 357 for (RegionInfo region : regions) { 358 byte[] startKey = region.getStartKey(); 359 if (region.getTable().equals(table)) { 360 setOfStartKeys.add(startKey); // ignore other tables 361 LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8)); 362 } 363 } 364 // the number of startkeys will be equal to the number of regions hosted in each server 365 // (each server will be hosting one replica of a region) 366 assertEquals(numRegions, setOfStartKeys.size()); 367 } 368 } 369 370 private void validateSingleRegionServerAssignment(Connection connection, int numRegions, 371 int numReplica) throws IOException { 372 SnapshotOfRegionAssignmentFromMeta snapshot = 373 new SnapshotOfRegionAssignmentFromMeta(connection); 374 snapshot.initialize(); 375 Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap(); 376 assertEquals(regionToServerMap.size(), numRegions * numReplica + 1); 377 Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap(); 378 assertEquals("One Region Only", 1, serverToRegionMap.keySet().size()); 379 for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) { 380 if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) { 381 continue; 382 } 383 assertEquals(entry.getValue().size(), numRegions * numReplica + 1); 384 } 385 } 386}