Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.util.ArrayList;
021import java.util.List;
022import java.util.concurrent.TimeUnit;
023import java.util.stream.Collectors;
024import org.apache.hadoop.hbase.HBaseClassTestRule;
025import org.apache.hadoop.hbase.HBaseTestingUtility;
026import org.apache.hadoop.hbase.HRegionLocation;
027import org.apache.hadoop.hbase.MiniHBaseCluster;
028import org.apache.hadoop.hbase.ServerName;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.RegionState;
038import org.apache.hadoop.hbase.regionserver.HRegion;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
043import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
044import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
045import org.junit.After;
046import org.junit.AfterClass;
047import org.junit.Assert;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
060 * exclude functionality useful for rack decommissioning
061 */
062@Category({ MiscTests.class, LargeTests.class })
063public class TestRegionMover2 {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestRegionMover2.class);
068
069  @Rule
070  public TestName name = new TestName();
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
073
074  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
075
076  @BeforeClass
077  public static void setUpBeforeClass() throws Exception {
078    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 4);
079    TEST_UTIL.startMiniCluster(3);
080    TEST_UTIL.getAdmin().balancerSwitch(false, true);
081  }
082
083  @AfterClass
084  public static void tearDownAfterClass() throws Exception {
085    TEST_UTIL.shutdownMiniCluster();
086  }
087
088  @Before
089  public void setUp() throws Exception {
090    final TableName tableName = TableName.valueOf(name.getMethodName());
091    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
092      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
093    int startKey = 0;
094    int endKey = 80000;
095    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
096  }
097
098  @After
099  public void tearDown() throws Exception {
100    final TableName tableName = TableName.valueOf(name.getMethodName());
101    TEST_UTIL.getAdmin().disableTable(tableName);
102    TEST_UTIL.getAdmin().deleteTable(tableName);
103  }
104
105  @Test
106  public void testWithSplitRegions() throws Exception {
107    final TableName tableName = TableName.valueOf(name.getMethodName());
108    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
109    Admin admin = TEST_UTIL.getAdmin();
110    Table table = TEST_UTIL.getConnection().getTable(tableName);
111    List<Put> puts = new ArrayList<>();
112    for (int i = 10; i < 50000; i++) {
113      puts.add(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
114        Bytes.toBytes("val_" + i)));
115    }
116    table.put(puts);
117    admin.flush(tableName);
118    admin.compact(tableName);
119    HRegionServer regionServer = cluster.getRegionServer(0);
120    String rsName = regionServer.getServerName().getAddress().toString();
121    int numRegions = regionServer.getNumberOfOnlineRegions();
122    List<HRegion> hRegions = regionServer.getRegions().stream()
123      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
124      .collect(Collectors.toList());
125
126    RegionMover.RegionMoverBuilder rmBuilder =
127      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
128        .maxthreads(8);
129    try (RegionMover rm = rmBuilder.build()) {
130      LOG.debug("Unloading {}", regionServer.getServerName());
131      rm.unload();
132      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
133      LOG.debug("Successfully Unloaded, now Loading");
134      HRegion hRegion = hRegions.get(1);
135      if (hRegion.getRegionInfo().getStartKey().length == 0) {
136        hRegion = hRegions.get(0);
137      }
138      int startKey = 0;
139      int endKey = Integer.MAX_VALUE;
140      if (hRegion.getRegionInfo().getStartKey().length > 0) {
141        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
142      }
143      if (hRegion.getRegionInfo().getEndKey().length > 0) {
144        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
145      }
146      int midKey = startKey + (endKey - startKey) / 2;
147      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
148        TimeUnit.SECONDS);
149      Assert.assertTrue(rm.load());
150      Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
151    }
152  }
153
154  @Test
155  public void testFailedRegionMove() throws Exception {
156    final TableName tableName = TableName.valueOf(name.getMethodName());
157    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
158    Admin admin = TEST_UTIL.getAdmin();
159    Table table = TEST_UTIL.getConnection().getTable(tableName);
160    List<Put> puts = new ArrayList<>();
161    for (int i = 0; i < 1000; i++) {
162      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"),
163        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
164    }
165    table.put(puts);
166    admin.flush(tableName);
167    HRegionServer regionServer = cluster.getRegionServer(0);
168    String rsName = regionServer.getServerName().getAddress().toString();
169    List<HRegion> hRegions = regionServer.getRegions().stream()
170      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
171      .collect(Collectors.toList());
172    RegionMover.RegionMoverBuilder rmBuilder =
173      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
174        .maxthreads(8);
175    try (RegionMover rm = rmBuilder.build()) {
176      LOG.debug("Unloading {}", regionServer.getServerName());
177      rm.unload();
178      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
179      LOG.debug("Successfully Unloaded, now Loading");
180      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
181      // loading regions will fail because of offline region
182      Assert.assertFalse(rm.load());
183    }
184  }
185
186  public void loadDummyDataInTable(TableName tableName) throws Exception {
187    Admin admin = TEST_UTIL.getAdmin();
188    Table table = TEST_UTIL.getConnection().getTable(tableName);
189    List<Put> puts = new ArrayList<>();
190    for (int i = 0; i < 1000; i++) {
191      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"),
192        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
193    }
194    table.put(puts);
195    admin.flush(tableName);
196  }
197
198  @Test
199  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
200    final TableName tableName = TableName.valueOf(name.getMethodName());
201    loadDummyDataInTable(tableName);
202    ServerName sourceServerName = findSourceServerName(tableName);
203    // Isolating 1 region on the same region server.
204    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
205  }
206
207  @Test
208  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
209    final TableName tableName = TableName.valueOf(name.getMethodName());
210    loadDummyDataInTable(tableName);
211    ServerName sourceServerName = findSourceServerName(tableName);
212    ServerName destinationServerName = findDestinationServerName(sourceServerName);
213    // Isolating 1 region on the different region server.
214    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
215  }
216
217  @Test
218  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
219    final TableName tableName = TableName.valueOf(name.getMethodName());
220    loadDummyDataInTable(tableName);
221    ServerName sourceServerName = findSourceServerName(tableName);
222    // Isolating 2 regions on the same region server.
223    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
224  }
225
226  @Test
227  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
228    final TableName tableName = TableName.valueOf(name.getMethodName());
229    loadDummyDataInTable(tableName);
230    // Isolating 2 regions on the different region server.
231    ServerName sourceServerName = findSourceServerName(tableName);
232    ServerName destinationServerName = findDestinationServerName(sourceServerName);
233    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
234  }
235
236  @Test
237  public void testIsolateMetaOnTheSameSever() throws Exception {
238    ServerName metaServerSource = findMetaRSLocation();
239    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
240  }
241
242  @Test
243  public void testIsolateMetaOnTheDifferentServer() throws Exception {
244    ServerName metaServerSource = findMetaRSLocation();
245    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
246    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
247  }
248
249  @Test
250  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
251    final TableName tableName = TableName.valueOf(name.getMethodName());
252    loadDummyDataInTable(tableName);
253    ServerName metaServerSource = findMetaRSLocation();
254    ServerName randomSeverRegion = findSourceServerName(tableName);
255    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
256  }
257
258  @Test
259  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
260    final TableName tableName = TableName.valueOf(name.getMethodName());
261    loadDummyDataInTable(tableName);
262    ServerName randomSeverRegion = findSourceServerName(tableName);
263    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
264  }
265
266  public ServerName findMetaRSLocation() throws Exception {
267    ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null);
268    List<HRegionLocation> result = new ArrayList<>();
269    for (String znode : zkWatcher.getMetaReplicaNodes()) {
270      String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode);
271      int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path);
272      RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId);
273      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
274    }
275    return result.get(0).getServerName();
276  }
277
278  public ServerName findSourceServerName(TableName tableName) throws Exception {
279    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
280    int numOfRS = cluster.getNumLiveRegionServers();
281    ServerName sourceServer = null;
282    for (int i = 0; i < numOfRS; i++) {
283      HRegionServer regionServer = cluster.getRegionServer(i);
284      List<HRegion> hRegions = regionServer.getRegions().stream()
285        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
286        .collect(Collectors.toList());
287      if (hRegions.size() >= 2) {
288        sourceServer = regionServer.getServerName();
289        break;
290      }
291    }
292    if (sourceServer == null) {
293      throw new Exception(
294        "This shouln't happen, No RS found with more than 2 regions of table : " + tableName);
295    }
296    return sourceServer;
297  }
298
299  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
300    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
301    ServerName destinationServerName = null;
302    int numOfRS = cluster.getNumLiveRegionServers();
303    for (int i = 0; i < numOfRS; i++) {
304      destinationServerName = cluster.getRegionServer(i).getServerName();
305      if (!destinationServerName.equals(sourceServerName)) {
306        break;
307      }
308    }
309    if (destinationServerName == null) {
310      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
311    }
312    return destinationServerName;
313  }
314
315  public void regionIsolationOperation(ServerName sourceServerName,
316    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
317    throws Exception {
318    final TableName tableName = TableName.valueOf(name.getMethodName());
319    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
320    Admin admin = TEST_UTIL.getAdmin();
321    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
322    List<HRegion> hRegions = sourceRS.getRegions().stream()
323      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
324      .collect(Collectors.toList());
325    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
326    for (int i = 0; i < numRegionsToIsolate; i++) {
327      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
328    }
329
330    if (isolateMetaAlso) {
331      listOfRegionIDsToIsolate.remove(0);
332      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
333    }
334
335    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
336    String destinationRSName = destinationRS.getServerName().getAddress().toString();
337    RegionMover.RegionMoverBuilder rmBuilder =
338      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
339        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
340    try (RegionMover rm = rmBuilder.build()) {
341      LOG.debug("Unloading {} except regions : {}", destinationRS.getServerName(),
342        listOfRegionIDsToIsolate);
343      rm.isolateRegions();
344      Assert.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
345      List<HRegion> onlineRegions = destinationRS.getRegions();
346      for (int i = 0; i < numRegionsToIsolate; i++) {
347        Assert.assertTrue(
348          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
349      }
350      LOG.debug("Successfully Isolated " + listOfRegionIDsToIsolate.size() + " regions : "
351        + listOfRegionIDsToIsolate + " on " + destinationRS.getServerName());
352    } finally {
353      admin.recommissionRegionServer(destinationRS.getServerName(), null);
354    }
355  }
356}