001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.ClusterMetrics.Option;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.HRegionLocation;
042import org.apache.hadoop.hbase.MetaTableAccessor;
043import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
044import org.apache.hadoop.hbase.RegionLocations;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.StartMiniClusterOption;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.ConnectionFactory;
052import org.apache.hadoop.hbase.client.Delete;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionReplicaUtil;
055import org.apache.hadoop.hbase.client.Result;
056import org.apache.hadoop.hbase.client.Table;
057import org.apache.hadoop.hbase.client.TableDescriptor;
058import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
059import org.apache.hadoop.hbase.testclassification.MasterTests;
060import org.apache.hadoop.hbase.testclassification.MediumTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.JVMClusterUtil;
063import org.junit.AfterClass;
064import org.junit.BeforeClass;
065import org.junit.ClassRule;
066import org.junit.Ignore;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
075
076@Category({ MasterTests.class, MediumTests.class })
077public class TestMasterOperationsForRegionReplicas {
078
079  @ClassRule
080  public static final HBaseClassTestRule CLASS_RULE =
081    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
082
083  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
084  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
085  private static Connection CONNECTION = null;
086  private static Admin ADMIN;
087  private static int numSlaves = 2;
088  private static Configuration conf;
089
090  @Rule
091  public TestName name = new TestName();
092
093  @BeforeClass
094  public static void setupBeforeClass() throws Exception {
095    conf = TEST_UTIL.getConfiguration();
096    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
097    TEST_UTIL.startMiniCluster(numSlaves);
098    TEST_UTIL.getAdmin().balancerSwitch(false, true);
099    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
100    ADMIN = CONNECTION.getAdmin();
101    while (ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics()
102      .size() < numSlaves) {
103      Thread.sleep(100);
104    }
105  }
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    Closeables.close(ADMIN, true);
110    Closeables.close(CONNECTION, true);
111    TEST_UTIL.shutdownMiniCluster();
112  }
113
114  @Test
115  public void testCreateTableWithSingleReplica() throws Exception {
116    final int numRegions = 3;
117    final int numReplica = 1;
118    final TableName tableName = TableName.valueOf(name.getMethodName());
119    try {
120      TableDescriptor desc =
121        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
122          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
123      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
124      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
125      TEST_UTIL.waitUntilNoRegionsInTransition();
126
127      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
128      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
129      assertEquals(numRegions * numReplica, hris.size());
130    } finally {
131      ADMIN.disableTable(tableName);
132      ADMIN.deleteTable(tableName);
133    }
134  }
135
136  @Test
137  public void testCreateTableWithMultipleReplicas() throws Exception {
138    final TableName tableName = TableName.valueOf(name.getMethodName());
139    final int numRegions = 3;
140    final int numReplica = 2;
141    try {
142      TableDescriptor desc =
143        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
144          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
145      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
146      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
147      TEST_UTIL.waitUntilNoRegionsInTransition();
148      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
149
150      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
151      assertEquals(numRegions * numReplica, hris.size());
152      // check that the master created expected number of RegionState objects
153      for (int i = 0; i < numRegions; i++) {
154        for (int j = 0; j < numReplica; j++) {
155          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
156          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
157            .getRegionStates().getRegionState(replica);
158          assertNotNull(state);
159        }
160      }
161
162      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
163      int numRows = 0;
164      for (Result result : metaRows) {
165        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
166        RegionInfo hri = locations.getRegionLocation().getRegion();
167        if (!hri.getTable().equals(tableName)) continue;
168        numRows += 1;
169        HRegionLocation[] servers = locations.getRegionLocations();
170        // have two locations for the replicas of a region, and the locations should be different
171        assertEquals(2, servers.length);
172        assertNotEquals(servers[1], servers[0]);
173      }
174      assertEquals(numRegions, numRows);
175
176      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
177      // class
178      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
179        ADMIN.getConnection());
180
181      // Now kill the master, restart it and see if the assignments are kept
182      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
183      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
184      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
185      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
186      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
187      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
188      TEST_UTIL.waitUntilNoRegionsInTransition();
189      for (int i = 0; i < numRegions; i++) {
190        for (int j = 0; j < numReplica; j++) {
191          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
192          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
193            .getRegionStates().getRegionState(replica);
194          assertNotNull(state);
195        }
196      }
197      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
198        ADMIN.getConnection());
199      // Now shut the whole cluster down, and verify the assignments are kept so that the
200      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
201      // restart. This messes with our being able to test that we retain locality. Therefore,
202      // figure current cluster ports and pass them in on next cluster start so new cluster comes
203      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
204      List<Integer> rsports = new ArrayList<>();
205      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
206        .getLiveRegionServerThreads()) {
207        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
208      }
209      TEST_UTIL.shutdownMiniHBaseCluster();
210      StartMiniClusterOption option =
211        StartMiniClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
212      TEST_UTIL.startMiniHBaseCluster(option);
213      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
214      TEST_UTIL.waitUntilNoRegionsInTransition();
215      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
216        ADMIN.getConnection());
217
218      // Now shut the whole cluster down, and verify regions are assigned even if there is only
219      // one server running
220      TEST_UTIL.shutdownMiniHBaseCluster();
221      TEST_UTIL.startMiniHBaseCluster();
222      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
223      TEST_UTIL.waitUntilNoRegionsInTransition();
224      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
225      for (int i = 1; i < numSlaves; i++) { // restore the cluster
226        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
227      }
228
229      // Check on alter table
230      ADMIN.disableTable(tableName);
231      assertTrue(ADMIN.isTableDisabled(tableName));
232      // increase the replica
233      ADMIN.modifyTable(
234        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
235      ADMIN.enableTable(tableName);
236      LOG.info(ADMIN.getDescriptor(tableName).toString());
237      assertTrue(ADMIN.isTableEnabled(tableName));
238      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
239      TEST_UTIL.waitUntilNoRegionsInTransition();
240      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
241        .getRegionStates().getRegionsOfTable(tableName);
242      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" +
243        numReplica, regions.size() == numRegions * (numReplica + 1));
244
245      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
246      ADMIN.disableTable(tableName);
247      ADMIN.modifyTable(
248        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
249      ADMIN.enableTable(tableName);
250      assertTrue(ADMIN.isTableEnabled(tableName));
251      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
252      TEST_UTIL.waitUntilNoRegionsInTransition();
253      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
254        .getRegionsOfTable(tableName);
255      assertEquals(numRegions * numReplica, regions.size());
256      // also make sure the meta table has the replica locations removed
257      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
258      assertEquals(numRegions * numReplica, hris.size());
259      // just check that the number of default replica regions in the meta table are the same
260      // as the number of regions the table was created with, and the count of the
261      // replicas is numReplica for each region
262      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
263      for (RegionInfo hri : hris) {
264        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
265        Integer i = defaultReplicas.get(regionReplica0);
266        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
267      }
268      assertEquals(numRegions, defaultReplicas.size());
269      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
270      assertEquals(1, counts.size());
271      assertTrue(counts.contains(numReplica));
272    } finally {
273      ADMIN.disableTable(tableName);
274      ADMIN.deleteTable(tableName);
275    }
276  }
277
278  @Test
279  @Ignore("Enable when we have support for alter_table- HBASE-10361")
280  public void testIncompleteMetaTableReplicaInformation() throws Exception {
281    final TableName tableName = TableName.valueOf(name.getMethodName());
282    final int numRegions = 3;
283    final int numReplica = 2;
284    try {
285      // Create a table and let the meta table be updated with the location of the
286      // region locations.
287      TableDescriptor desc =
288        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
289          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
290      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
291      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
292      TEST_UTIL.waitUntilNoRegionsInTransition();
293      Set<byte[]> tableRows = new HashSet<>();
294      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
295      for (RegionInfo hri : hris) {
296        tableRows.add(hri.getRegionName());
297      }
298      ADMIN.disableTable(tableName);
299      // now delete one replica info from all the rows
300      // this is to make the meta appear to be only partially updated
301      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
302      for (byte[] row : tableRows) {
303        Delete deleteOneReplicaLocation = new Delete(row);
304        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
305          MetaTableAccessor.getServerColumn(1));
306        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
307          MetaTableAccessor.getSeqNumColumn(1));
308        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
309          MetaTableAccessor.getStartCodeColumn(1));
310        metaTable.delete(deleteOneReplicaLocation);
311      }
312      metaTable.close();
313      // even if the meta table is partly updated, when we re-enable the table, we should
314      // get back the desired number of replicas for the regions
315      ADMIN.enableTable(tableName);
316      assertTrue(ADMIN.isTableEnabled(tableName));
317      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
318      TEST_UTIL.waitUntilNoRegionsInTransition();
319      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
320        .getRegionStates().getRegionsOfTable(tableName);
321      assertEquals(numRegions * numReplica, regions.size());
322    } finally {
323      ADMIN.disableTable(tableName);
324      ADMIN.deleteTable(tableName);
325    }
326  }
327
328  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
329      Connection connection) throws IOException {
330    assert (ADMIN.tableExists(table));
331    final AtomicInteger count = new AtomicInteger();
332    Visitor visitor = new Visitor() {
333      @Override
334      public boolean visit(Result r) throws IOException {
335        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
336        return true;
337      }
338    };
339    MetaTableAccessor.fullScanRegions(connection, visitor);
340    assertEquals(numRegions, count.get());
341  }
342
343  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
344      int numRegions, int numReplica, Connection connection) throws IOException {
345    SnapshotOfRegionAssignmentFromMeta snapshot =
346      new SnapshotOfRegionAssignmentFromMeta(connection);
347    snapshot.initialize();
348    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
349    assert (regionToServerMap.size() == numRegions * numReplica);
350    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
351    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
352      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
353        continue;
354      }
355      List<RegionInfo> regions = entry.getValue();
356      Set<byte[]> setOfStartKeys = new HashSet<>();
357      for (RegionInfo region : regions) {
358        byte[] startKey = region.getStartKey();
359        if (region.getTable().equals(table)) {
360          setOfStartKeys.add(startKey); // ignore other tables
361          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
362        }
363      }
364      // the number of startkeys will be equal to the number of regions hosted in each server
365      // (each server will be hosting one replica of a region)
366      assertEquals(numRegions, setOfStartKeys.size());
367    }
368  }
369
370  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
371      int numReplica) throws IOException {
372    SnapshotOfRegionAssignmentFromMeta snapshot =
373      new SnapshotOfRegionAssignmentFromMeta(connection);
374    snapshot.initialize();
375    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
376    assertEquals(regionToServerMap.size(), numRegions * numReplica);
377    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
378    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
379    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
380      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
381        continue;
382      }
383      assertEquals(entry.getValue().size(), numRegions * numReplica);
384    }
385  }
386}