Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036
037import org.apache.commons.io.IOUtils;
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HBaseTestingUtility;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.HRegionLocation;
044import org.apache.hadoop.hbase.MetaTableAccessor;
045import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
046import org.apache.hadoop.hbase.RegionLocations;
047import org.apache.hadoop.hbase.ServerName;
048import org.apache.hadoop.hbase.StartMiniClusterOption;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.client.Admin;
051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
052import org.apache.hadoop.hbase.client.Connection;
053import org.apache.hadoop.hbase.client.ConnectionFactory;
054import org.apache.hadoop.hbase.client.Delete;
055import org.apache.hadoop.hbase.client.RegionInfo;
056import org.apache.hadoop.hbase.client.RegionReplicaUtil;
057import org.apache.hadoop.hbase.client.Result;
058import org.apache.hadoop.hbase.client.Table;
059import org.apache.hadoop.hbase.client.TableDescriptor;
060import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
061import org.apache.hadoop.hbase.testclassification.MasterTests;
062import org.apache.hadoop.hbase.testclassification.MediumTests;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.apache.hadoop.hbase.util.JVMClusterUtil;
065import org.junit.AfterClass;
066import org.junit.BeforeClass;
067import org.junit.ClassRule;
068import org.junit.Rule;
069import org.junit.Test;
070import org.junit.experimental.categories.Category;
071import org.junit.rules.TestName;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
076
077@Category({ MasterTests.class, MediumTests.class })
078public class TestMasterOperationsForRegionReplicas {
079
080  @ClassRule
081  public static final HBaseClassTestRule CLASS_RULE =
082    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
083
084  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
085  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
086  private static Connection CONNECTION = null;
087  private static Admin ADMIN;
088  private static int numSlaves = 2;
089  private final static StartMiniClusterOption option = StartMiniClusterOption.builder().
090      numRegionServers(numSlaves).numMasters(1).numAlwaysStandByMasters(1).build();
091  private static Configuration conf;
092
093  @Rule
094  public TestName name = new TestName();
095
096  @BeforeClass
097  public static void setupBeforeClass() throws Exception {
098    conf = TEST_UTIL.getConfiguration();
099    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
100    TEST_UTIL.startMiniCluster(option);
101    TEST_UTIL.getAdmin().balancerSwitch(false, true);
102    resetConnections();
103    while (ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics()
104      .size() < numSlaves) {
105      Thread.sleep(100);
106    }
107  }
108
109  private static void resetConnections() throws IOException {
110    IOUtils.closeQuietly(ADMIN, CONNECTION);
111    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
112    ADMIN = CONNECTION.getAdmin();
113  }
114
115  @AfterClass
116  public static void tearDownAfterClass() throws Exception {
117    Closeables.close(ADMIN, true);
118    Closeables.close(CONNECTION, true);
119    TEST_UTIL.shutdownMiniCluster();
120  }
121
122  @Test
123  public void testCreateTableWithSingleReplica() throws Exception {
124    final int numRegions = 3;
125    final int numReplica = 1;
126    final TableName tableName = TableName.valueOf(name.getMethodName());
127    try {
128      TableDescriptor desc =
129        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
130          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
131      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
132      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
133      TEST_UTIL.waitUntilNoRegionsInTransition();
134
135      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
136      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
137      assertEquals(numRegions * numReplica, hris.size());
138    } finally {
139      ADMIN.disableTable(tableName);
140      ADMIN.deleteTable(tableName);
141    }
142  }
143
144  @Test
145  public void testCreateTableWithMultipleReplicas() throws Exception {
146    final TableName tableName = TableName.valueOf(name.getMethodName());
147    final int numRegions = 3;
148    final int numReplica = 2;
149    try {
150      TableDescriptor desc =
151        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
152          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
153      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
154      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
155      TEST_UTIL.waitUntilNoRegionsInTransition();
156      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
157
158      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
159      assertEquals(numRegions * numReplica, hris.size());
160      assertRegionStateNotNull(hris, numRegions, numReplica);
161
162      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
163      int numRows = 0;
164      for (Result result : metaRows) {
165        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
166        RegionInfo hri = locations.getRegionLocation().getRegion();
167        if (!hri.getTable().equals(tableName)) continue;
168        numRows += 1;
169        HRegionLocation[] servers = locations.getRegionLocations();
170        // have two locations for the replicas of a region, and the locations should be different
171        assertEquals(2, servers.length);
172        assertNotEquals(servers[1], servers[0]);
173      }
174      assertEquals(numRegions, numRows);
175
176      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
177      // class
178      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
179        ADMIN.getConnection());
180
181      // Now kill the master, restart it and see if the assignments are kept
182      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
183      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
184      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
185      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
186      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
187      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
188      TEST_UTIL.waitUntilNoRegionsInTransition();
189      assertRegionStateNotNull(hris, numRegions, numReplica);
190      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
191        ADMIN.getConnection());
192
193      // Now shut the whole cluster down, and verify the assignments are kept so that the
194      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
195      // restart. This messes with our being able to test that we retain locality. Therefore,
196      // figure current cluster ports and pass them in on next cluster start so new cluster comes
197      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
198      List<Integer> rsports = new ArrayList<>();
199      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
200        .getLiveRegionServerThreads()) {
201        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
202      }
203      TEST_UTIL.shutdownMiniHBaseCluster();
204      StartMiniClusterOption option =
205        StartMiniClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
206      TEST_UTIL.startMiniHBaseCluster(option);
207      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
208      TEST_UTIL.waitUntilNoRegionsInTransition();
209      resetConnections();
210      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
211        ADMIN.getConnection());
212
213      // Now shut the whole cluster down, and verify regions are assigned even if there is only
214      // one server running
215      TEST_UTIL.shutdownMiniHBaseCluster();
216      TEST_UTIL.startMiniHBaseCluster();
217      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
218      TEST_UTIL.waitUntilNoRegionsInTransition();
219      resetConnections();
220      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
221      for (int i = 1; i < numSlaves; i++) { // restore the cluster
222        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
223      }
224
225      // Check on alter table
226      ADMIN.disableTable(tableName);
227      assertTrue(ADMIN.isTableDisabled(tableName));
228      // increase the replica
229      ADMIN.modifyTable(
230        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
231      ADMIN.enableTable(tableName);
232      LOG.info(ADMIN.getDescriptor(tableName).toString());
233      assertTrue(ADMIN.isTableEnabled(tableName));
234      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
235      TEST_UTIL.waitUntilNoRegionsInTransition();
236      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
237        .getRegionStates().getRegionsOfTable(tableName);
238      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" +
239        numReplica, regions.size() == numRegions * (numReplica + 1));
240
241      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
242      ADMIN.disableTable(tableName);
243      ADMIN.modifyTable(
244        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
245      ADMIN.enableTable(tableName);
246      assertTrue(ADMIN.isTableEnabled(tableName));
247      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
248      TEST_UTIL.waitUntilNoRegionsInTransition();
249      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
250        .getRegionsOfTable(tableName);
251      assertEquals(numRegions * numReplica, regions.size());
252      // also make sure the meta table has the replica locations removed
253      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
254      assertEquals(numRegions * numReplica, hris.size());
255      // just check that the number of default replica regions in the meta table are the same
256      // as the number of regions the table was created with, and the count of the
257      // replicas is numReplica for each region
258      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
259      for (RegionInfo hri : hris) {
260        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
261        Integer i = defaultReplicas.get(regionReplica0);
262        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
263      }
264      assertEquals(numRegions, defaultReplicas.size());
265      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
266      assertEquals(1, counts.size());
267      assertTrue(counts.contains(numReplica));
268    } finally {
269      ADMIN.disableTable(tableName);
270      ADMIN.deleteTable(tableName);
271    }
272  }
273
274  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
275    // check that the master created expected number of RegionState objects
276    for (int i = 0; i < numRegions; i++) {
277      for (int j = 0; j < numReplica; j++) {
278        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
279        RegionState state =
280            TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
281                .getRegionState(replica);
282        assertNotNull(state);
283      }
284    }
285  }
286
287  @Test
288  public void testIncompleteMetaTableReplicaInformation() throws Exception {
289    final TableName tableName = TableName.valueOf(name.getMethodName());
290    final int numRegions = 3;
291    final int numReplica = 2;
292    try {
293      // Create a table and let the meta table be updated with the location of the
294      // region locations.
295      TableDescriptor desc =
296        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
297          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
298      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
299      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
300      TEST_UTIL.waitUntilNoRegionsInTransition();
301      Set<byte[]> tableRows = new HashSet<>();
302      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
303      for (RegionInfo hri : hris) {
304        tableRows.add(hri.getRegionName());
305      }
306      ADMIN.disableTable(tableName);
307      // now delete one replica info from all the rows
308      // this is to make the meta appear to be only partially updated
309      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
310      for (byte[] row : tableRows) {
311        Delete deleteOneReplicaLocation = new Delete(row);
312        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
313          MetaTableAccessor.getServerColumn(1));
314        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
315          MetaTableAccessor.getSeqNumColumn(1));
316        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
317          MetaTableAccessor.getStartCodeColumn(1));
318        metaTable.delete(deleteOneReplicaLocation);
319      }
320      metaTable.close();
321      // even if the meta table is partly updated, when we re-enable the table, we should
322      // get back the desired number of replicas for the regions
323      ADMIN.enableTable(tableName);
324      assertTrue(ADMIN.isTableEnabled(tableName));
325      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
326      TEST_UTIL.waitUntilNoRegionsInTransition();
327      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
328        .getRegionStates().getRegionsOfTable(tableName);
329      assertEquals(numRegions * numReplica, regions.size());
330    } finally {
331      ADMIN.disableTable(tableName);
332      ADMIN.deleteTable(tableName);
333    }
334  }
335
336  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
337      Connection connection) throws IOException {
338    assert (ADMIN.tableExists(table));
339    final AtomicInteger count = new AtomicInteger();
340    Visitor visitor = new Visitor() {
341      @Override
342      public boolean visit(Result r) throws IOException {
343        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
344        return true;
345      }
346    };
347    MetaTableAccessor.fullScanRegions(connection, visitor);
348    assertEquals(numRegions, count.get());
349  }
350
351  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
352      int numRegions, int numReplica, Connection connection) throws IOException {
353    SnapshotOfRegionAssignmentFromMeta snapshot =
354      new SnapshotOfRegionAssignmentFromMeta(connection);
355    snapshot.initialize();
356    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
357    assert (regionToServerMap.size() == numRegions * numReplica + 1); // '1' for the namespace
358    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
359    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
360      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
361        continue;
362      }
363      List<RegionInfo> regions = entry.getValue();
364      Set<byte[]> setOfStartKeys = new HashSet<>();
365      for (RegionInfo region : regions) {
366        byte[] startKey = region.getStartKey();
367        if (region.getTable().equals(table)) {
368          setOfStartKeys.add(startKey); // ignore other tables
369          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
370        }
371      }
372      // the number of startkeys will be equal to the number of regions hosted in each server
373      // (each server will be hosting one replica of a region)
374      assertEquals(numRegions, setOfStartKeys.size());
375    }
376  }
377
378  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
379      int numReplica) throws IOException {
380    SnapshotOfRegionAssignmentFromMeta snapshot =
381      new SnapshotOfRegionAssignmentFromMeta(connection);
382    snapshot.initialize();
383    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
384    assertEquals(regionToServerMap.size(), numRegions * numReplica + 1);
385    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
386    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
387    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
388      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
389        continue;
390      }
391      assertEquals(entry.getValue().size(), numRegions * numReplica + 1);
392    }
393  }
394}