001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotEquals;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CatalogFamilyFormat;
038import org.apache.hadoop.hbase.ClientMetaTableAccessor;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HRegionLocation;
043import org.apache.hadoop.hbase.MetaTableAccessor;
044import org.apache.hadoop.hbase.RegionLocations;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.StartTestingClusterOption;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.ConnectionFactory;
052import org.apache.hadoop.hbase.client.Delete;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionReplicaUtil;
055import org.apache.hadoop.hbase.client.Result;
056import org.apache.hadoop.hbase.client.Table;
057import org.apache.hadoop.hbase.client.TableDescriptor;
058import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
059import org.apache.hadoop.hbase.testclassification.MasterTests;
060import org.apache.hadoop.hbase.testclassification.MediumTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.JVMClusterUtil;
063import org.junit.jupiter.api.AfterAll;
064import org.junit.jupiter.api.BeforeAll;
065import org.junit.jupiter.api.BeforeEach;
066import org.junit.jupiter.api.Tag;
067import org.junit.jupiter.api.Test;
068import org.junit.jupiter.api.TestInfo;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
073
074@Tag(MasterTests.TAG)
075@Tag(MediumTests.TAG)
076public class TestMasterOperationsForRegionReplicas {
077
078  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
079  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
080  private static Connection CONNECTION = null;
081  private static Admin ADMIN;
082  private static int numSlaves = 2;
083  private final static StartTestingClusterOption option = StartTestingClusterOption.builder()
084    .numRegionServers(numSlaves).numMasters(1).numAlwaysStandByMasters(1).build();
085  private static Configuration conf;
086  private String testMethodName;
087
088  @BeforeEach
089  public void setTestMethod(TestInfo testInfo) {
090    testMethodName = testInfo.getTestMethod().get().getName();
091  }
092
093  @BeforeAll
094  public static void setupBeforeClass() throws Exception {
095    conf = TEST_UTIL.getConfiguration();
096    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
097    TEST_UTIL.startMiniCluster(option);
098    TEST_UTIL.getAdmin().balancerSwitch(false, true);
099    resetConnections();
100    while (
101      ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size()
102          < numSlaves
103    ) {
104      Thread.sleep(100);
105    }
106  }
107
108  private static void resetConnections() throws IOException {
109    Closeables.close(ADMIN, true);
110    Closeables.close(CONNECTION, true);
111    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
112    ADMIN = CONNECTION.getAdmin();
113  }
114
115  @AfterAll
116  public static void tearDownAfterClass() throws Exception {
117    Closeables.close(ADMIN, true);
118    Closeables.close(CONNECTION, true);
119    TEST_UTIL.shutdownMiniCluster();
120  }
121
122  @Test
123  public void testCreateTableWithSingleReplica() throws Exception {
124    final int numRegions = 3;
125    final int numReplica = 1;
126    final TableName tableName = TableName.valueOf(testMethodName);
127    try {
128      TableDescriptor desc =
129        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
130          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
131      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
132      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
133      TEST_UTIL.waitUntilNoRegionsInTransition();
134
135      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
136      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
137      assertEquals(numRegions * numReplica, hris.size());
138    } finally {
139      ADMIN.disableTable(tableName);
140      ADMIN.deleteTable(tableName);
141    }
142  }
143
144  @Test
145  public void testCreateTableWithMultipleReplicas() throws Exception {
146    final TableName tableName = TableName.valueOf(testMethodName);
147    final int numRegions = 3;
148    final int numReplica = 2;
149    try {
150      TableDescriptor desc =
151        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
152          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
153      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
154      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
155      TEST_UTIL.waitUntilNoRegionsInTransition();
156      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
157
158      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
159      assertEquals(numRegions * numReplica, hris.size());
160      assertRegionStateNotNull(hris, numRegions, numReplica);
161
162      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
163      int numRows = 0;
164      for (Result result : metaRows) {
165        RegionLocations locations = CatalogFamilyFormat.getRegionLocations(result);
166        RegionInfo hri = locations.getRegionLocation().getRegion();
167        if (!hri.getTable().equals(tableName)) continue;
168        numRows += 1;
169        HRegionLocation[] servers = locations.getRegionLocations();
170        // have two locations for the replicas of a region, and the locations should be different
171        assertEquals(2, servers.length);
172        assertNotEquals(servers[1], servers[0]);
173      }
174      assertEquals(numRegions, numRows);
175
176      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
177      // class
178      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
179        ADMIN.getConnection());
180
181      // Now kill the master, restart it and see if the assignments are kept
182      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
183      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
184      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
185      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
186      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
187      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
188      TEST_UTIL.waitUntilNoRegionsInTransition();
189      assertRegionStateNotNull(hris, numRegions, numReplica);
190      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
191        ADMIN.getConnection());
192
193      // Now shut the whole cluster down, and verify the assignments are kept so that the
194      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
195      // restart. This messes with our being able to test that we retain locality. Therefore,
196      // figure current cluster ports and pass them in on next cluster start so new cluster comes
197      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
198      List<Integer> rsports = new ArrayList<>();
199      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
200        .getLiveRegionServerThreads()) {
201        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
202      }
203      TEST_UTIL.shutdownMiniHBaseCluster();
204      StartTestingClusterOption option =
205        StartTestingClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
206      TEST_UTIL.startMiniHBaseCluster(option);
207      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
208      TEST_UTIL.waitUntilNoRegionsInTransition();
209      resetConnections();
210      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
211        ADMIN.getConnection());
212
213      // Now shut the whole cluster down, and verify regions are assigned even if there is only
214      // one server running
215      TEST_UTIL.shutdownMiniHBaseCluster();
216      TEST_UTIL.startMiniHBaseCluster();
217      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
218      TEST_UTIL.waitUntilNoRegionsInTransition();
219      resetConnections();
220      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
221      for (int i = 1; i < numSlaves; i++) { // restore the cluster
222        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
223      }
224
225      // Check on alter table
226      ADMIN.disableTable(tableName);
227      assertTrue(ADMIN.isTableDisabled(tableName));
228      // increase the replica
229      ADMIN.modifyTable(
230        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
231      ADMIN.enableTable(tableName);
232      LOG.info(ADMIN.getDescriptor(tableName).toString());
233      assertTrue(ADMIN.isTableEnabled(tableName));
234      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
235      TEST_UTIL.waitUntilNoRegionsInTransition();
236      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
237        .getRegionStates().getRegionsOfTable(tableName);
238      assertTrue(regions.size() == numRegions * (numReplica + 1), "regions.size=" + regions.size()
239        + ", numRegions=" + numRegions + ", numReplica=" + numReplica);
240
241      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
242      ADMIN.disableTable(tableName);
243      ADMIN.modifyTable(
244        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
245      ADMIN.enableTable(tableName);
246      assertTrue(ADMIN.isTableEnabled(tableName));
247      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
248      TEST_UTIL.waitUntilNoRegionsInTransition();
249      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
250        .getRegionsOfTable(tableName);
251      assertEquals(numRegions * numReplica, regions.size());
252      // also make sure the meta table has the replica locations removed
253      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
254      assertEquals(numRegions * numReplica, hris.size());
255      // just check that the number of default replica regions in the meta table are the same
256      // as the number of regions the table was created with, and the count of the
257      // replicas is numReplica for each region
258      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
259      for (RegionInfo hri : hris) {
260        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
261        Integer i = defaultReplicas.get(regionReplica0);
262        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
263      }
264      assertEquals(numRegions, defaultReplicas.size());
265      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
266      assertEquals(1, counts.size());
267      assertTrue(counts.contains(numReplica));
268    } finally {
269      ADMIN.disableTable(tableName);
270      ADMIN.deleteTable(tableName);
271    }
272  }
273
274  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
275    // check that the master created expected number of RegionState objects
276    for (int i = 0; i < numRegions; i++) {
277      for (int j = 0; j < numReplica; j++) {
278        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
279        RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
280          .getRegionStates().getRegionState(replica);
281        assertNotNull(state);
282      }
283    }
284  }
285
286  @Test
287  public void testIncompleteMetaTableReplicaInformation() throws Exception {
288    final TableName tableName = TableName.valueOf(testMethodName);
289    final int numRegions = 3;
290    final int numReplica = 2;
291    try {
292      // Create a table and let the meta table be updated with the location of the
293      // region locations.
294      TableDescriptor desc =
295        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
296          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
297      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
298      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
299      TEST_UTIL.waitUntilNoRegionsInTransition();
300      Set<byte[]> tableRows = new HashSet<>();
301      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
302      for (RegionInfo hri : hris) {
303        tableRows.add(hri.getRegionName());
304      }
305      ADMIN.disableTable(tableName);
306      // now delete one replica info from all the rows
307      // this is to make the meta appear to be only partially updated
308      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
309      for (byte[] row : tableRows) {
310        Delete deleteOneReplicaLocation = new Delete(row);
311        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
312          CatalogFamilyFormat.getServerColumn(1));
313        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
314          CatalogFamilyFormat.getSeqNumColumn(1));
315        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
316          CatalogFamilyFormat.getStartCodeColumn(1));
317        metaTable.delete(deleteOneReplicaLocation);
318      }
319      metaTable.close();
320      // even if the meta table is partly updated, when we re-enable the table, we should
321      // get back the desired number of replicas for the regions
322      ADMIN.enableTable(tableName);
323      assertTrue(ADMIN.isTableEnabled(tableName));
324      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
325      TEST_UTIL.waitUntilNoRegionsInTransition();
326      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
327        .getRegionStates().getRegionsOfTable(tableName);
328      assertEquals(numRegions * numReplica, regions.size());
329    } finally {
330      ADMIN.disableTable(tableName);
331      ADMIN.deleteTable(tableName);
332    }
333  }
334
335  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
336    Connection connection) throws IOException {
337    assert (ADMIN.tableExists(table));
338    final AtomicInteger count = new AtomicInteger();
339    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
340      @Override
341      public boolean visit(Result r) throws IOException {
342        if (CatalogFamilyFormat.getRegionInfo(r).getTable().equals(table)) {
343          count.incrementAndGet();
344        }
345        return true;
346      }
347    };
348    MetaTableAccessor.fullScanRegions(connection, visitor);
349    assertEquals(numRegions, count.get());
350  }
351
352  private void validateFromSnapshotFromMeta(HBaseTestingUtil util, TableName table, int numRegions,
353    int numReplica, Connection connection) throws IOException {
354    SnapshotOfRegionAssignmentFromMeta snapshot =
355      new SnapshotOfRegionAssignmentFromMeta(connection);
356    snapshot.initialize();
357    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
358    assert (regionToServerMap.size() == numRegions * numReplica);
359    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
360    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
361      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
362        continue;
363      }
364      List<RegionInfo> regions = entry.getValue();
365      Set<byte[]> setOfStartKeys = new HashSet<>();
366      for (RegionInfo region : regions) {
367        byte[] startKey = region.getStartKey();
368        if (region.getTable().equals(table)) {
369          setOfStartKeys.add(startKey); // ignore other tables
370          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
371        }
372      }
373      // the number of startkeys will be equal to the number of regions hosted in each server
374      // (each server will be hosting one replica of a region)
375      assertEquals(numRegions, setOfStartKeys.size());
376    }
377  }
378
379  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
380    int numReplica) throws IOException {
381    SnapshotOfRegionAssignmentFromMeta snapshot =
382      new SnapshotOfRegionAssignmentFromMeta(connection);
383    snapshot.initialize();
384    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
385    assertEquals(regionToServerMap.size(), numRegions * numReplica);
386    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
387    assertEquals(1, serverToRegionMap.keySet().size(), "One Region Only");
388    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
389      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
390        continue;
391      }
392      assertEquals(entry.getValue().size(), numRegions * numReplica);
393    }
394  }
395}