001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotEquals;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CatalogFamilyFormat;
038import org.apache.hadoop.hbase.ClientMetaTableAccessor;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HRegionLocation;
043import org.apache.hadoop.hbase.MetaTableAccessor;
044import org.apache.hadoop.hbase.RegionLocations;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.StartTestingClusterOption;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.ConnectionFactory;
052import org.apache.hadoop.hbase.client.Delete;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionReplicaUtil;
055import org.apache.hadoop.hbase.client.Result;
056import org.apache.hadoop.hbase.client.Table;
057import org.apache.hadoop.hbase.client.TableDescriptor;
058import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
059import org.apache.hadoop.hbase.testclassification.MasterTests;
060import org.apache.hadoop.hbase.testclassification.MediumTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.JVMClusterUtil;
063import org.junit.jupiter.api.AfterAll;
064import org.junit.jupiter.api.BeforeAll;
065import org.junit.jupiter.api.BeforeEach;
066import org.junit.jupiter.api.Tag;
067import org.junit.jupiter.api.Test;
068import org.junit.jupiter.api.TestInfo;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
073
074@Tag(MasterTests.TAG)
075@Tag(MediumTests.TAG)
076public class TestMasterOperationsForRegionReplicas {
077
078  private static final Logger LOG =
079    LoggerFactory.getLogger(TestMasterOperationsForRegionReplicas.class);
080  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
081  private static Connection CONNECTION = null;
082  private static Admin ADMIN;
083  private static int numSlaves = 2;
084  private final static StartTestingClusterOption option = StartTestingClusterOption.builder()
085    .numRegionServers(numSlaves).numMasters(1).numAlwaysStandByMasters(1).build();
086  private static Configuration conf;
087  private String testMethodName;
088
089  @BeforeEach
090  public void setTestMethod(TestInfo testInfo) {
091    testMethodName = testInfo.getTestMethod().get().getName();
092  }
093
094  @BeforeAll
095  public static void setupBeforeClass() throws Exception {
096    conf = TEST_UTIL.getConfiguration();
097    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
098    TEST_UTIL.startMiniCluster(option);
099    TEST_UTIL.getAdmin().balancerSwitch(false, true);
100    resetConnections();
101    while (
102      ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size()
103          < numSlaves
104    ) {
105      Thread.sleep(100);
106    }
107  }
108
109  private static void resetConnections() throws IOException {
110    Closeables.close(ADMIN, true);
111    Closeables.close(CONNECTION, true);
112    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
113    ADMIN = CONNECTION.getAdmin();
114  }
115
116  @AfterAll
117  public static void tearDownAfterClass() throws Exception {
118    Closeables.close(ADMIN, true);
119    Closeables.close(CONNECTION, true);
120    TEST_UTIL.shutdownMiniCluster();
121  }
122
123  @Test
124  public void testCreateTableWithSingleReplica() throws Exception {
125    final int numRegions = 3;
126    final int numReplica = 1;
127    final TableName tableName = TableName.valueOf(testMethodName);
128    try {
129      TableDescriptor desc =
130        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
131          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
132      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
133      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
134      TEST_UTIL.waitUntilNoRegionsInTransition();
135
136      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
137      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
138      assertEquals(numRegions * numReplica, hris.size());
139    } finally {
140      ADMIN.disableTable(tableName);
141      ADMIN.deleteTable(tableName);
142    }
143  }
144
145  @Test
146  public void testCreateTableWithMultipleReplicas() throws Exception {
147    final TableName tableName = TableName.valueOf(testMethodName);
148    final int numRegions = 3;
149    final int numReplica = 2;
150    try {
151      TableDescriptor desc =
152        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
153          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
154      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
155      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
156      TEST_UTIL.waitUntilNoRegionsInTransition();
157      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
158
159      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
160      assertEquals(numRegions * numReplica, hris.size());
161      assertRegionStateNotNull(hris, numRegions, numReplica);
162
163      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
164      int numRows = 0;
165      for (Result result : metaRows) {
166        RegionLocations locations = CatalogFamilyFormat.getRegionLocations(result);
167        RegionInfo hri = locations.getRegionLocation().getRegion();
168        if (!hri.getTable().equals(tableName)) continue;
169        numRows += 1;
170        HRegionLocation[] servers = locations.getRegionLocations();
171        // have two locations for the replicas of a region, and the locations should be different
172        assertEquals(2, servers.length);
173        assertNotEquals(servers[1], servers[0]);
174      }
175      assertEquals(numRegions, numRows);
176
177      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
178      // class
179      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
180        ADMIN.getConnection());
181
182      // Now kill the master, restart it and see if the assignments are kept
183      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
184      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
185      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
186      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
187      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
188      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
189      TEST_UTIL.waitUntilNoRegionsInTransition();
190      assertRegionStateNotNull(hris, numRegions, numReplica);
191      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
192        ADMIN.getConnection());
193
194      // Now shut the whole cluster down, and verify the assignments are kept so that the
195      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
196      // restart. This messes with our being able to test that we retain locality. Therefore,
197      // figure current cluster ports and pass them in on next cluster start so new cluster comes
198      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
199      List<Integer> rsports = new ArrayList<>();
200      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
201        .getLiveRegionServerThreads()) {
202        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
203      }
204      TEST_UTIL.shutdownMiniHBaseCluster();
205      StartTestingClusterOption option =
206        StartTestingClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
207      TEST_UTIL.startMiniHBaseCluster(option);
208      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
209      TEST_UTIL.waitUntilNoRegionsInTransition();
210      resetConnections();
211      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
212        ADMIN.getConnection());
213
214      // Now shut the whole cluster down, and verify regions are assigned even if there is only
215      // one server running
216      TEST_UTIL.shutdownMiniHBaseCluster();
217      TEST_UTIL.startMiniHBaseCluster();
218      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
219      TEST_UTIL.waitUntilNoRegionsInTransition();
220      resetConnections();
221      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
222      for (int i = 1; i < numSlaves; i++) { // restore the cluster
223        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
224      }
225
226      // Check on alter table
227      ADMIN.disableTable(tableName);
228      assertTrue(ADMIN.isTableDisabled(tableName));
229      // increase the replica
230      ADMIN.modifyTable(
231        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
232      ADMIN.enableTable(tableName);
233      LOG.info(ADMIN.getDescriptor(tableName).toString());
234      assertTrue(ADMIN.isTableEnabled(tableName));
235      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
236      TEST_UTIL.waitUntilNoRegionsInTransition();
237      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
238        .getRegionStates().getRegionsOfTable(tableName);
239      assertTrue(regions.size() == numRegions * (numReplica + 1), "regions.size=" + regions.size()
240        + ", numRegions=" + numRegions + ", numReplica=" + numReplica);
241
242      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
243      ADMIN.disableTable(tableName);
244      ADMIN.modifyTable(
245        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
246      ADMIN.enableTable(tableName);
247      assertTrue(ADMIN.isTableEnabled(tableName));
248      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
249      TEST_UTIL.waitUntilNoRegionsInTransition();
250      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
251        .getRegionsOfTable(tableName);
252      assertEquals(numRegions * numReplica, regions.size());
253      // also make sure the meta table has the replica locations removed
254      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
255      assertEquals(numRegions * numReplica, hris.size());
256      // just check that the number of default replica regions in the meta table are the same
257      // as the number of regions the table was created with, and the count of the
258      // replicas is numReplica for each region
259      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
260      for (RegionInfo hri : hris) {
261        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
262        Integer i = defaultReplicas.get(regionReplica0);
263        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
264      }
265      assertEquals(numRegions, defaultReplicas.size());
266      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
267      assertEquals(1, counts.size());
268      assertTrue(counts.contains(numReplica));
269    } finally {
270      ADMIN.disableTable(tableName);
271      ADMIN.deleteTable(tableName);
272    }
273  }
274
275  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
276    // check that the master created expected number of RegionState objects
277    for (int i = 0; i < numRegions; i++) {
278      for (int j = 0; j < numReplica; j++) {
279        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
280        RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
281          .getRegionStates().getRegionState(replica);
282        assertNotNull(state);
283      }
284    }
285  }
286
287  @Test
288  public void testIncompleteMetaTableReplicaInformation() throws Exception {
289    final TableName tableName = TableName.valueOf(testMethodName);
290    final int numRegions = 3;
291    final int numReplica = 2;
292    try {
293      // Create a table and let the meta table be updated with the location of the
294      // region locations.
295      TableDescriptor desc =
296        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
297          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
298      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
299      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
300      TEST_UTIL.waitUntilNoRegionsInTransition();
301      Set<byte[]> tableRows = new HashSet<>();
302      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
303      for (RegionInfo hri : hris) {
304        tableRows.add(hri.getRegionName());
305      }
306      ADMIN.disableTable(tableName);
307      // now delete one replica info from all the rows
308      // this is to make the meta appear to be only partially updated
309      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
310      for (byte[] row : tableRows) {
311        Delete deleteOneReplicaLocation = new Delete(row);
312        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
313          CatalogFamilyFormat.getServerColumn(1));
314        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
315          CatalogFamilyFormat.getSeqNumColumn(1));
316        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
317          CatalogFamilyFormat.getStartCodeColumn(1));
318        metaTable.delete(deleteOneReplicaLocation);
319      }
320      metaTable.close();
321      // even if the meta table is partly updated, when we re-enable the table, we should
322      // get back the desired number of replicas for the regions
323      ADMIN.enableTable(tableName);
324      assertTrue(ADMIN.isTableEnabled(tableName));
325      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
326      TEST_UTIL.waitUntilNoRegionsInTransition();
327      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
328        .getRegionStates().getRegionsOfTable(tableName);
329      assertEquals(numRegions * numReplica, regions.size());
330    } finally {
331      ADMIN.disableTable(tableName);
332      ADMIN.deleteTable(tableName);
333    }
334  }
335
336  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
337    Connection connection) throws IOException {
338    assert (ADMIN.tableExists(table));
339    final AtomicInteger count = new AtomicInteger();
340    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
341      @Override
342      public boolean visit(Result r) throws IOException {
343        if (CatalogFamilyFormat.getRegionInfo(r).getTable().equals(table)) {
344          count.incrementAndGet();
345        }
346        return true;
347      }
348    };
349    MetaTableAccessor.fullScanRegions(connection, visitor);
350    assertEquals(numRegions, count.get());
351  }
352
353  private void validateFromSnapshotFromMeta(HBaseTestingUtil util, TableName table, int numRegions,
354    int numReplica, Connection connection) throws IOException {
355    SnapshotOfRegionAssignmentFromMeta snapshot =
356      new SnapshotOfRegionAssignmentFromMeta(connection);
357    snapshot.initialize();
358    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
359    assert (regionToServerMap.size() == numRegions * numReplica);
360    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
361    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
362      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
363        continue;
364      }
365      List<RegionInfo> regions = entry.getValue();
366      Set<byte[]> setOfStartKeys = new HashSet<>();
367      for (RegionInfo region : regions) {
368        byte[] startKey = region.getStartKey();
369        if (region.getTable().equals(table)) {
370          setOfStartKeys.add(startKey); // ignore other tables
371          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
372        }
373      }
374      // the number of startkeys will be equal to the number of regions hosted in each server
375      // (each server will be hosting one replica of a region)
376      assertEquals(numRegions, setOfStartKeys.size());
377    }
378  }
379
380  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
381    int numReplica) throws IOException {
382    SnapshotOfRegionAssignmentFromMeta snapshot =
383      new SnapshotOfRegionAssignmentFromMeta(connection);
384    snapshot.initialize();
385    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
386    assertEquals(regionToServerMap.size(), numRegions * numReplica);
387    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
388    assertEquals(1, serverToRegionMap.keySet().size(), "One Region Only");
389    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
390      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
391        continue;
392      }
393      assertEquals(entry.getValue().size(), numRegions * numReplica);
394    }
395  }
396}