001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.ClusterMetrics.Option;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HRegionLocation;
043import org.apache.hadoop.hbase.MetaTableAccessor;
044import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
045import org.apache.hadoop.hbase.RegionLocations;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.StartMiniClusterOption;
048import org.apache.hadoop.hbase.TableName;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
051import org.apache.hadoop.hbase.client.Connection;
052import org.apache.hadoop.hbase.client.ConnectionFactory;
053import org.apache.hadoop.hbase.client.Delete;
054import org.apache.hadoop.hbase.client.RegionInfo;
055import org.apache.hadoop.hbase.client.RegionReplicaUtil;
056import org.apache.hadoop.hbase.client.Result;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
060import org.apache.hadoop.hbase.testclassification.MasterTests;
061import org.apache.hadoop.hbase.testclassification.MediumTests;
062import org.apache.hadoop.hbase.util.Bytes;
063import org.apache.hadoop.hbase.util.JVMClusterUtil;
064import org.junit.AfterClass;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Ignore;
068import org.junit.Rule;
069import org.junit.Test;
070import org.junit.experimental.categories.Category;
071import org.junit.rules.TestName;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
076
077@Category({ MasterTests.class, MediumTests.class })
078public class TestMasterOperationsForRegionReplicas {
079
080  @ClassRule
081  public static final HBaseClassTestRule CLASS_RULE =
082    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
083
084  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
085  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
086  private static Connection CONNECTION = null;
087  private static Admin ADMIN;
088  private static int numSlaves = 2;
089  private static Configuration conf;
090
091  @Rule
092  public TestName name = new TestName();
093
094  @BeforeClass
095  public static void setupBeforeClass() throws Exception {
096    conf = TEST_UTIL.getConfiguration();
097    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
098    TEST_UTIL.startMiniCluster(numSlaves);
099    TEST_UTIL.getAdmin().balancerSwitch(false, true);
100    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
101    ADMIN = CONNECTION.getAdmin();
102    while (ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics()
103      .size() < numSlaves) {
104      Thread.sleep(100);
105    }
106  }
107
108  @AfterClass
109  public static void tearDownAfterClass() throws Exception {
110    Closeables.close(ADMIN, true);
111    Closeables.close(CONNECTION, true);
112    TEST_UTIL.shutdownMiniCluster();
113  }
114
115  @Test
116  public void testCreateTableWithSingleReplica() throws Exception {
117    final int numRegions = 3;
118    final int numReplica = 1;
119    final TableName tableName = TableName.valueOf(name.getMethodName());
120    try {
121      TableDescriptor desc =
122        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
123          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
124      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
125      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
126      TEST_UTIL.waitUntilNoRegionsInTransition();
127
128      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
129      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
130      assertEquals(numRegions * numReplica, hris.size());
131    } finally {
132      ADMIN.disableTable(tableName);
133      ADMIN.deleteTable(tableName);
134    }
135  }
136
137  @Test
138  public void testCreateTableWithMultipleReplicas() throws Exception {
139    final TableName tableName = TableName.valueOf(name.getMethodName());
140    final int numRegions = 3;
141    final int numReplica = 2;
142    try {
143      TableDescriptor desc =
144        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
145          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
146      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
147      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
148      TEST_UTIL.waitUntilNoRegionsInTransition();
149      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
150
151      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
152      assertEquals(numRegions * numReplica, hris.size());
153      assertRegionStateNotNull(hris, numRegions, numReplica);
154
155      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
156      int numRows = 0;
157      for (Result result : metaRows) {
158        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
159        RegionInfo hri = locations.getRegionLocation().getRegion();
160        if (!hri.getTable().equals(tableName)) continue;
161        numRows += 1;
162        HRegionLocation[] servers = locations.getRegionLocations();
163        // have two locations for the replicas of a region, and the locations should be different
164        assertEquals(2, servers.length);
165        assertNotEquals(servers[1], servers[0]);
166      }
167      assertEquals(numRegions, numRows);
168
169      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
170      // class
171      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
172        ADMIN.getConnection());
173
174      // Now kill the master, restart it and see if the assignments are kept
175      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
176      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
177      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
178      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
179      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
180      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
181      TEST_UTIL.waitUntilNoRegionsInTransition();
182      assertRegionStateNotNull(hris, numRegions, numReplica);
183      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
184        ADMIN.getConnection());
185
186      // Now shut the whole cluster down, and verify the assignments are kept so that the
187      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
188      // restart. This messes with our being able to test that we retain locality. Therefore,
189      // figure current cluster ports and pass them in on next cluster start so new cluster comes
190      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
191      List<Integer> rsports = new ArrayList<>();
192      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
193        .getLiveRegionServerThreads()) {
194        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
195      }
196      TEST_UTIL.shutdownMiniHBaseCluster();
197      StartMiniClusterOption option =
198        StartMiniClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
199      TEST_UTIL.startMiniHBaseCluster(option);
200      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
201      TEST_UTIL.waitUntilNoRegionsInTransition();
202      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
203        ADMIN.getConnection());
204
205      // Now shut the whole cluster down, and verify regions are assigned even if there is only
206      // one server running
207      TEST_UTIL.shutdownMiniHBaseCluster();
208      TEST_UTIL.startMiniHBaseCluster();
209      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
210      TEST_UTIL.waitUntilNoRegionsInTransition();
211      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
212      for (int i = 1; i < numSlaves; i++) { // restore the cluster
213        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
214      }
215
216      // Check on alter table
217      ADMIN.disableTable(tableName);
218      assertTrue(ADMIN.isTableDisabled(tableName));
219      // increase the replica
220      ADMIN.modifyTable(
221        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
222      ADMIN.enableTable(tableName);
223      LOG.info(ADMIN.getDescriptor(tableName).toString());
224      assertTrue(ADMIN.isTableEnabled(tableName));
225      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
226      TEST_UTIL.waitUntilNoRegionsInTransition();
227      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
228        .getRegionStates().getRegionsOfTable(tableName);
229      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" +
230        numReplica, regions.size() == numRegions * (numReplica + 1));
231
232      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
233      ADMIN.disableTable(tableName);
234      ADMIN.modifyTable(
235        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
236      ADMIN.enableTable(tableName);
237      assertTrue(ADMIN.isTableEnabled(tableName));
238      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
239      TEST_UTIL.waitUntilNoRegionsInTransition();
240      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
241        .getRegionsOfTable(tableName);
242      assertEquals(numRegions * numReplica, regions.size());
243      // also make sure the meta table has the replica locations removed
244      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
245      assertEquals(numRegions * numReplica, hris.size());
246      // just check that the number of default replica regions in the meta table are the same
247      // as the number of regions the table was created with, and the count of the
248      // replicas is numReplica for each region
249      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
250      for (RegionInfo hri : hris) {
251        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
252        Integer i = defaultReplicas.get(regionReplica0);
253        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
254      }
255      assertEquals(numRegions, defaultReplicas.size());
256      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
257      assertEquals(1, counts.size());
258      assertTrue(counts.contains(numReplica));
259    } finally {
260      ADMIN.disableTable(tableName);
261      ADMIN.deleteTable(tableName);
262    }
263  }
264
265  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
266    // check that the master created expected number of RegionState objects
267    for (int i = 0; i < numRegions; i++) {
268      for (int j = 0; j < numReplica; j++) {
269        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
270        RegionState state =
271            TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
272                .getRegionState(replica);
273        assertNotNull(state);
274      }
275    }
276  }
277
278  @Test
279  @Ignore("Enable when we have support for alter_table- HBASE-10361")
280  public void testIncompleteMetaTableReplicaInformation() throws Exception {
281    final TableName tableName = TableName.valueOf(name.getMethodName());
282    final int numRegions = 3;
283    final int numReplica = 2;
284    try {
285      // Create a table and let the meta table be updated with the location of the
286      // region locations.
287      TableDescriptor desc =
288        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
289          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
290      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
291      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
292      TEST_UTIL.waitUntilNoRegionsInTransition();
293      Set<byte[]> tableRows = new HashSet<>();
294      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
295      for (RegionInfo hri : hris) {
296        tableRows.add(hri.getRegionName());
297      }
298      ADMIN.disableTable(tableName);
299      // now delete one replica info from all the rows
300      // this is to make the meta appear to be only partially updated
301      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
302      for (byte[] row : tableRows) {
303        Delete deleteOneReplicaLocation = new Delete(row);
304        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
305          MetaTableAccessor.getServerColumn(1));
306        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
307          MetaTableAccessor.getSeqNumColumn(1));
308        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
309          MetaTableAccessor.getStartCodeColumn(1));
310        metaTable.delete(deleteOneReplicaLocation);
311      }
312      metaTable.close();
313      // even if the meta table is partly updated, when we re-enable the table, we should
314      // get back the desired number of replicas for the regions
315      ADMIN.enableTable(tableName);
316      assertTrue(ADMIN.isTableEnabled(tableName));
317      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
318      TEST_UTIL.waitUntilNoRegionsInTransition();
319      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
320        .getRegionStates().getRegionsOfTable(tableName);
321      assertEquals(numRegions * numReplica, regions.size());
322    } finally {
323      ADMIN.disableTable(tableName);
324      ADMIN.deleteTable(tableName);
325    }
326  }
327
328  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
329      Connection connection) throws IOException {
330    assert (ADMIN.tableExists(table));
331    final AtomicInteger count = new AtomicInteger();
332    Visitor visitor = new Visitor() {
333      @Override
334      public boolean visit(Result r) throws IOException {
335        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
336        return true;
337      }
338    };
339    MetaTableAccessor.fullScanRegions(connection, visitor);
340    assertEquals(numRegions, count.get());
341  }
342
343  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
344      int numRegions, int numReplica, Connection connection) throws IOException {
345    SnapshotOfRegionAssignmentFromMeta snapshot =
346      new SnapshotOfRegionAssignmentFromMeta(connection);
347    snapshot.initialize();
348    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
349    assert (regionToServerMap.size() == numRegions * numReplica);
350    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
351    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
352      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
353        continue;
354      }
355      List<RegionInfo> regions = entry.getValue();
356      Set<byte[]> setOfStartKeys = new HashSet<>();
357      for (RegionInfo region : regions) {
358        byte[] startKey = region.getStartKey();
359        if (region.getTable().equals(table)) {
360          setOfStartKeys.add(startKey); // ignore other tables
361          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
362        }
363      }
364      // the number of startkeys will be equal to the number of regions hosted in each server
365      // (each server will be hosting one replica of a region)
366      assertEquals(numRegions, setOfStartKeys.size());
367    }
368  }
369
370  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
371      int numReplica) throws IOException {
372    SnapshotOfRegionAssignmentFromMeta snapshot =
373      new SnapshotOfRegionAssignmentFromMeta(connection);
374    snapshot.initialize();
375    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
376    assertEquals(regionToServerMap.size(), numRegions * numReplica);
377    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
378    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
379    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
380      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
381        continue;
382      }
383      assertEquals(entry.getValue().size(), numRegions * numReplica);
384    }
385  }
386}