Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.nio.charset.StandardCharsets;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.EnumSet;
028import java.util.HashMap;
029import java.util.HashSet;
030import java.util.List;
031import java.util.Map;
032import java.util.Set;
033import java.util.concurrent.atomic.AtomicInteger;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics.Option;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HColumnDescriptor;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.HRegionLocation;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.MetaTableAccessor;
043import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
044import org.apache.hadoop.hbase.RegionLocations;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.Admin;
048import org.apache.hadoop.hbase.client.Connection;
049import org.apache.hadoop.hbase.client.ConnectionFactory;
050import org.apache.hadoop.hbase.client.Delete;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.RegionReplicaUtil;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.Table;
055import org.apache.hadoop.hbase.testclassification.MasterTests;
056import org.apache.hadoop.hbase.testclassification.MediumTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.JVMClusterUtil;
059import org.junit.AfterClass;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Ignore;
063import org.junit.Rule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.junit.rules.TestName;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070@Category({MasterTests.class, MediumTests.class})
071public class TestMasterOperationsForRegionReplicas {
072
073  @ClassRule
074  public static final HBaseClassTestRule CLASS_RULE =
075      HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
076
077  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
078  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
079  private static Connection CONNECTION = null;
080  private static Admin ADMIN;
081  private static int numSlaves = 2;
082  private static Configuration conf;
083
084  @Rule
085  public TestName name = new TestName();
086
087  @BeforeClass
088  public static void setupBeforeClass() throws Exception {
089    conf = TEST_UTIL.getConfiguration();
090    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
091    TEST_UTIL.startMiniCluster(numSlaves);
092    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
093    ADMIN = CONNECTION.getAdmin();
094    while(ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
095               .getLiveServerMetrics().size() < numSlaves) {
096      Thread.sleep(100);
097    }
098  }
099
100  @AfterClass
101  public static void tearDownAfterClass() throws Exception {
102    if (ADMIN != null) ADMIN.close();
103    if (CONNECTION != null && !CONNECTION.isClosed()) CONNECTION.close();
104    TEST_UTIL.shutdownMiniCluster();
105  }
106
107  @Test
108  public void testCreateTableWithSingleReplica() throws Exception {
109    final int numRegions = 3;
110    final int numReplica = 1;
111    final TableName tableName = TableName.valueOf(name.getMethodName());
112    try {
113      HTableDescriptor desc = new HTableDescriptor(tableName);
114      desc.setRegionReplication(numReplica);
115      desc.addFamily(new HColumnDescriptor("family"));
116      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
117
118      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
119      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(
120        ADMIN.getConnection(), tableName);
121      assert(hris.size() == numRegions * numReplica);
122    } finally {
123      ADMIN.disableTable(tableName);
124      ADMIN.deleteTable(tableName);
125    }
126  }
127
128  @Test
129  public void testCreateTableWithMultipleReplicas() throws Exception {
130    final TableName tableName = TableName.valueOf(name.getMethodName());
131    final int numRegions = 3;
132    final int numReplica = 2;
133    try {
134      HTableDescriptor desc = new HTableDescriptor(tableName);
135      desc.setRegionReplication(numReplica);
136      desc.addFamily(new HColumnDescriptor("family"));
137      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
138      TEST_UTIL.waitTableEnabled(tableName);
139      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
140
141      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
142      assert(hris.size() == numRegions * numReplica);
143      // check that the master created expected number of RegionState objects
144      for (int i = 0; i < numRegions; i++) {
145        for (int j = 0; j < numReplica; j++) {
146          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
147          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
148              .getRegionStates().getRegionState(replica);
149          assert (state != null);
150        }
151      }
152
153      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
154      int numRows = 0;
155      for (Result result : metaRows) {
156        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
157        RegionInfo hri = locations.getRegionLocation().getRegionInfo();
158        if (!hri.getTable().equals(tableName)) continue;
159        numRows += 1;
160        HRegionLocation[] servers = locations.getRegionLocations();
161        // have two locations for the replicas of a region, and the locations should be different
162        assert(servers.length == 2);
163        assert(!servers[0].equals(servers[1]));
164      }
165      assert(numRows == numRegions);
166
167      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
168      // class
169      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
170        ADMIN.getConnection());
171
172      // Now kill the master, restart it and see if the assignments are kept
173      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
174      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
175      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
176      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
177      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
178      for (int i = 0; i < numRegions; i++) {
179        for (int j = 0; j < numReplica; j++) {
180          RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
181          RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
182              .getRegionStates().getRegionState(replica);
183          assert (state != null);
184        }
185      }
186      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
187        ADMIN.getConnection());
188      // Now shut the whole cluster down, and verify the assignments are kept so that the
189      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
190      // restart. This messes with our being able to test that we retain locality. Therefore,
191      // figure current cluster ports and pass them in on next cluster start so new cluster comes
192      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
193      List<Integer> rsports = new ArrayList<>();
194      for (JVMClusterUtil.RegionServerThread rst:
195          TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads()) {
196        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
197      }
198      TEST_UTIL.shutdownMiniHBaseCluster();
199      TEST_UTIL.startMiniHBaseCluster(1, numSlaves, rsports);
200      TEST_UTIL.waitTableEnabled(tableName);
201      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
202        ADMIN.getConnection());
203
204      // Now shut the whole cluster down, and verify regions are assigned even if there is only
205      // one server running
206      TEST_UTIL.shutdownMiniHBaseCluster();
207      TEST_UTIL.startMiniHBaseCluster(1, 1);
208      TEST_UTIL.waitTableEnabled(tableName);
209      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
210      for (int i = 1; i < numSlaves; i++) { //restore the cluster
211        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
212      }
213
214      // Check on alter table
215      ADMIN.disableTable(tableName);
216      assert(ADMIN.isTableDisabled(tableName));
217      //increase the replica
218      desc.setRegionReplication(numReplica + 1);
219      ADMIN.modifyTable(tableName, desc);
220      ADMIN.enableTable(tableName);
221      LOG.info(ADMIN.getTableDescriptor(tableName).toString());
222      assert(ADMIN.isTableEnabled(tableName));
223      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().
224          getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
225      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions +
226          ", numReplica=" + numReplica, regions.size() == numRegions * (numReplica + 1));
227
228      //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
229      ADMIN.disableTable(tableName);
230      desc.setRegionReplication(numReplica);
231      ADMIN.modifyTable(tableName, desc);
232      ADMIN.enableTable(tableName);
233      assert(ADMIN.isTableEnabled(tableName));
234      regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
235          .getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
236      assert(regions.size() == numRegions * numReplica);
237      //also make sure the meta table has the replica locations removed
238      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
239      assert(hris.size() == numRegions * numReplica);
240      //just check that the number of default replica regions in the meta table are the same
241      //as the number of regions the table was created with, and the count of the
242      //replicas is numReplica for each region
243      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
244      for (RegionInfo hri : hris) {
245        Integer i;
246        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
247        defaultReplicas.put(regionReplica0,
248            (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1);
249      }
250      assert(defaultReplicas.size() == numRegions);
251      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
252      assert(counts.size() == 1 && counts.contains(numReplica));
253    } finally {
254      ADMIN.disableTable(tableName);
255      ADMIN.deleteTable(tableName);
256    }
257  }
258
259  @Test @Ignore("Enable when we have support for alter_table- HBASE-10361")
260  public void testIncompleteMetaTableReplicaInformation() throws Exception {
261    final TableName tableName = TableName.valueOf(name.getMethodName());
262    final int numRegions = 3;
263    final int numReplica = 2;
264    try {
265      // Create a table and let the meta table be updated with the location of the
266      // region locations.
267      HTableDescriptor desc = new HTableDescriptor(tableName);
268      desc.setRegionReplication(numReplica);
269      desc.addFamily(new HColumnDescriptor("family"));
270      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
271      TEST_UTIL.waitTableEnabled(tableName);
272      Set<byte[]> tableRows = new HashSet<>();
273      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
274      for (RegionInfo hri : hris) {
275        tableRows.add(hri.getRegionName());
276      }
277      ADMIN.disableTable(tableName);
278      // now delete one replica info from all the rows
279      // this is to make the meta appear to be only partially updated
280      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
281      for (byte[] row : tableRows) {
282        Delete deleteOneReplicaLocation = new Delete(row);
283        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
284          MetaTableAccessor.getServerColumn(1));
285        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
286          MetaTableAccessor.getSeqNumColumn(1));
287        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
288          MetaTableAccessor.getStartCodeColumn(1));
289        metaTable.delete(deleteOneReplicaLocation);
290      }
291      metaTable.close();
292      // even if the meta table is partly updated, when we re-enable the table, we should
293      // get back the desired number of replicas for the regions
294      ADMIN.enableTable(tableName);
295      assert(ADMIN.isTableEnabled(tableName));
296      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster()
297          .getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
298      assert(regions.size() == numRegions * numReplica);
299    } finally {
300      ADMIN.disableTable(tableName);
301      ADMIN.deleteTable(tableName);
302    }
303  }
304
305  private String printRegions(List<RegionInfo> regions) {
306    StringBuilder strBuf = new StringBuilder();
307    for (RegionInfo r : regions) {
308      strBuf.append(" ____ " + r.toString());
309    }
310    return strBuf.toString();
311  }
312
313  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
314      Connection connection) throws IOException {
315    assert(ADMIN.tableExists(table));
316    final AtomicInteger count = new AtomicInteger();
317    Visitor visitor = new Visitor() {
318      @Override
319      public boolean visit(Result r) throws IOException {
320        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
321        return true;
322      }
323    };
324    MetaTableAccessor.fullScanRegions(connection, visitor);
325    assert(count.get() == numRegions);
326  }
327
328  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
329      int numRegions, int numReplica, Connection connection) throws IOException {
330    SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
331      connection);
332    snapshot.initialize();
333    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
334    assert(regionToServerMap.size() == numRegions * numReplica + 1); //'1' for the namespace
335    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
336    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
337      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
338        continue;
339      }
340      List<RegionInfo> regions = entry.getValue();
341      Set<byte[]> setOfStartKeys = new HashSet<>();
342      for (RegionInfo region : regions) {
343        byte[] startKey = region.getStartKey();
344        if (region.getTable().equals(table)) {
345          setOfStartKeys.add(startKey); //ignore other tables
346          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
347        }
348      }
349      // the number of startkeys will be equal to the number of regions hosted in each server
350      // (each server will be hosting one replica of a region)
351      assertEquals(numRegions, setOfStartKeys.size());
352    }
353  }
354
355  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
356      int numReplica) throws IOException {
357    SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
358      connection);
359    snapshot.initialize();
360    Map<RegionInfo, ServerName>  regionToServerMap = snapshot.getRegionToRegionServerMap();
361    assertEquals(regionToServerMap.size(), numRegions * numReplica + 1);
362    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
363    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
364    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
365      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
366        continue;
367      }
368      assertEquals(entry.getValue().size(), numRegions * numReplica + 1);
369    }
370  }
371}