Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.Collection;
028import java.util.EnumSet;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Map;
033import java.util.Set;
034import java.util.concurrent.atomic.AtomicInteger;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.ClusterMetrics.Option;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.HRegionLocation;
041import org.apache.hadoop.hbase.MetaTableAccessor;
042import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
043import org.apache.hadoop.hbase.RegionLocations;
044import org.apache.hadoop.hbase.ServerName;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.client.Admin;
047import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
048import org.apache.hadoop.hbase.client.Connection;
049import org.apache.hadoop.hbase.client.ConnectionFactory;
050import org.apache.hadoop.hbase.client.Delete;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.RegionReplicaUtil;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.Table;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.testclassification.MasterTests;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.junit.AfterClass;
061import org.junit.BeforeClass;
062import org.junit.ClassRule;
063import org.junit.Ignore;
064import org.junit.Rule;
065import org.junit.Test;
066import org.junit.experimental.categories.Category;
067import org.junit.rules.TestName;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
072
073@Category({ MasterTests.class, MediumTests.class })
074public class TestMasterOperationsForRegionReplicas {
075
076  @ClassRule
077  public static final HBaseClassTestRule CLASS_RULE =
078    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
079
080  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
081  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
082  private static Connection CONNECTION = null;
083  private static Admin ADMIN;
084  private static int numSlaves = 2;
085  private static Configuration conf;
086
087  @Rule
088  public TestName name = new TestName();
089
090  @BeforeClass
091  public static void setupBeforeClass() throws Exception {
092    conf = TEST_UTIL.getConfiguration();
093    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
094    TEST_UTIL.startMiniCluster(numSlaves);
095    TEST_UTIL.getAdmin().balancerSwitch(false, true);
096    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
097    ADMIN = CONNECTION.getAdmin();
098    while (ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics()
099      .size() < numSlaves) {
100      Thread.sleep(100);
101    }
102  }
103
104  @AfterClass
105  public static void tearDownAfterClass() throws Exception {
106    Closeables.close(ADMIN, true);
107    Closeables.close(CONNECTION, true);
108    TEST_UTIL.shutdownMiniCluster();
109  }
110
111  @Test
112  public void testCreateTableWithSingleReplica() throws Exception {
113    final int numRegions = 3;
114    final int numReplica = 1;
115    final TableName tableName = TableName.valueOf(name.getMethodName());
116    try {
117      TableDescriptor desc =
118        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
119          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
120      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
121      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
122      TEST_UTIL.waitUntilNoRegionsInTransition();
123
124      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
125      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
126      assertEquals(numRegions * numReplica, hris.size());
127    } finally {
128      ADMIN.disableTable(tableName);
129      ADMIN.deleteTable(tableName);
130    }
131  }
132
133  @Test
134  public void testCreateTableWithMultipleReplicas() throws Exception {
135    final TableName tableName = TableName.valueOf(name.getMethodName());
136    final int numRegions = 3;
137    final int numReplica = 2;
138    try {
139      TableDescriptor desc =
140        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
141          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
142      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
143      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
144      TEST_UTIL.waitUntilNoRegionsInTransition();
145      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
146
147      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
148      assertEquals(numRegions * numReplica, hris.size());
149      // check that the master created expected number of RegionState objects
150      for (int i = 0; i < numRegions; i++) {
151        for (int j = 0; j < numReplica; j++) {
152          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
153          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
154            .getRegionStates().getRegionState(replica);
155          assertNotNull(state);
156        }
157      }
158
159      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
160      int numRows = 0;
161      for (Result result : metaRows) {
162        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
163        RegionInfo hri = locations.getRegionLocation().getRegion();
164        if (!hri.getTable().equals(tableName)) continue;
165        numRows += 1;
166        HRegionLocation[] servers = locations.getRegionLocations();
167        // have two locations for the replicas of a region, and the locations should be different
168        assertEquals(2, servers.length);
169        assertNotEquals(servers[1], servers[0]);
170      }
171      assertEquals(numRegions, numRows);
172
173      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
174      // class
175      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
176        ADMIN.getConnection());
177
178      // Now kill the master, restart it and see if the assignments are kept
179      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
180      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
181      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
182      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
183      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
184      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
185      TEST_UTIL.waitUntilNoRegionsInTransition();
186      for (int i = 0; i < numRegions; i++) {
187        for (int j = 0; j < numReplica; j++) {
188          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
189          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
190            .getRegionStates().getRegionState(replica);
191          assertNotNull(state);
192        }
193      }
194      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
195        ADMIN.getConnection());
196
197      // Now shut the whole cluster down, and verify regions are assigned even if there is only
198      // one server running
199      TEST_UTIL.shutdownMiniHBaseCluster();
200      TEST_UTIL.startMiniHBaseCluster(1, 1);
201      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
202      TEST_UTIL.waitUntilNoRegionsInTransition();
203      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
204      for (int i = 1; i < numSlaves; i++) { // restore the cluster
205        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
206      }
207
208      // Check on alter table
209      ADMIN.disableTable(tableName);
210      assertTrue(ADMIN.isTableDisabled(tableName));
211      // increase the replica
212      ADMIN.modifyTable(
213        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
214      ADMIN.enableTable(tableName);
215      LOG.info(ADMIN.getDescriptor(tableName).toString());
216      assertTrue(ADMIN.isTableEnabled(tableName));
217      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
218      TEST_UTIL.waitUntilNoRegionsInTransition();
219      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
220        .getRegionStates().getRegionsOfTable(tableName);
221      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" +
222        numReplica, regions.size() == numRegions * (numReplica + 1));
223
224      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
225      ADMIN.disableTable(tableName);
226      ADMIN.modifyTable(
227        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
228      ADMIN.enableTable(tableName);
229      assertTrue(ADMIN.isTableEnabled(tableName));
230      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
231      TEST_UTIL.waitUntilNoRegionsInTransition();
232      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
233        .getRegionsOfTable(tableName);
234      assertEquals(numRegions * numReplica, regions.size());
235      // also make sure the meta table has the replica locations removed
236      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
237      assertEquals(numRegions * numReplica, hris.size());
238      // just check that the number of default replica regions in the meta table are the same
239      // as the number of regions the table was created with, and the count of the
240      // replicas is numReplica for each region
241      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
242      for (RegionInfo hri : hris) {
243        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
244        Integer i = defaultReplicas.get(regionReplica0);
245        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
246      }
247      assertEquals(numRegions, defaultReplicas.size());
248      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
249      assertEquals(1, counts.size());
250      assertTrue(counts.contains(numReplica));
251    } finally {
252      ADMIN.disableTable(tableName);
253      ADMIN.deleteTable(tableName);
254    }
255  }
256
257  @Test
258  @Ignore("Enable when we have support for alter_table- HBASE-10361")
259  public void testIncompleteMetaTableReplicaInformation() throws Exception {
260    final TableName tableName = TableName.valueOf(name.getMethodName());
261    final int numRegions = 3;
262    final int numReplica = 2;
263    try {
264      // Create a table and let the meta table be updated with the location of the
265      // region locations.
266      TableDescriptor desc =
267        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
268          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
269      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
270      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
271      TEST_UTIL.waitUntilNoRegionsInTransition();
272      Set<byte[]> tableRows = new HashSet<>();
273      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
274      for (RegionInfo hri : hris) {
275        tableRows.add(hri.getRegionName());
276      }
277      ADMIN.disableTable(tableName);
278      // now delete one replica info from all the rows
279      // this is to make the meta appear to be only partially updated
280      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
281      for (byte[] row : tableRows) {
282        Delete deleteOneReplicaLocation = new Delete(row);
283        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
284          MetaTableAccessor.getServerColumn(1));
285        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
286          MetaTableAccessor.getSeqNumColumn(1));
287        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
288          MetaTableAccessor.getStartCodeColumn(1));
289        metaTable.delete(deleteOneReplicaLocation);
290      }
291      metaTable.close();
292      // even if the meta table is partly updated, when we re-enable the table, we should
293      // get back the desired number of replicas for the regions
294      ADMIN.enableTable(tableName);
295      assertTrue(ADMIN.isTableEnabled(tableName));
296      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
297      TEST_UTIL.waitUntilNoRegionsInTransition();
298      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
299        .getRegionStates().getRegionsOfTable(tableName);
300      assertEquals(numRegions * numReplica, regions.size());
301    } finally {
302      ADMIN.disableTable(tableName);
303      ADMIN.deleteTable(tableName);
304    }
305  }
306
307  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
308      Connection connection) throws IOException {
309    assert (ADMIN.tableExists(table));
310    final AtomicInteger count = new AtomicInteger();
311    Visitor visitor = new Visitor() {
312      @Override
313      public boolean visit(Result r) throws IOException {
314        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
315        return true;
316      }
317    };
318    MetaTableAccessor.fullScanRegions(connection, visitor);
319    assertEquals(numRegions, count.get());
320  }
321
322  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
323      int numRegions, int numReplica, Connection connection) throws IOException {
324    SnapshotOfRegionAssignmentFromMeta snapshot =
325      new SnapshotOfRegionAssignmentFromMeta(connection);
326    snapshot.initialize();
327    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
328    assert (regionToServerMap.size() == numRegions * numReplica + 1); // '1' for the namespace
329    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
330    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
331      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
332        continue;
333      }
334      List<RegionInfo> regions = entry.getValue();
335      Set<byte[]> setOfStartKeys = new HashSet<>();
336      for (RegionInfo region : regions) {
337        byte[] startKey = region.getStartKey();
338        if (region.getTable().equals(table)) {
339          setOfStartKeys.add(startKey); // ignore other tables
340          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
341        }
342      }
343      // the number of startkeys will be equal to the number of regions hosted in each server
344      // (each server will be hosting one replica of a region)
345      assertEquals(numRegions, setOfStartKeys.size());
346    }
347  }
348
349  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
350      int numReplica) throws IOException {
351    SnapshotOfRegionAssignmentFromMeta snapshot =
352      new SnapshotOfRegionAssignmentFromMeta(connection);
353    snapshot.initialize();
354    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
355    assertEquals(regionToServerMap.size(), numRegions * numReplica + 1);
356    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
357    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
358    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
359      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
360        continue;
361      }
362      assertEquals(entry.getValue().size(), numRegions * numReplica + 1);
363    }
364  }
365}