001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.util.ArrayList;
021import java.util.List;
022import java.util.concurrent.TimeUnit;
023import java.util.stream.Collectors;
024import org.apache.hadoop.hbase.HBaseClassTestRule;
025import org.apache.hadoop.hbase.HBaseTestingUtil;
026import org.apache.hadoop.hbase.HRegionLocation;
027import org.apache.hadoop.hbase.ServerName;
028import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.RegionState;
038import org.apache.hadoop.hbase.regionserver.HRegion;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
043import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
044import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
045import org.junit.After;
046import org.junit.AfterClass;
047import org.junit.Assert;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
060 * exclude functionality useful for rack decommissioning
061 */
062@Category({ MiscTests.class, LargeTests.class })
063public class TestRegionMover2 {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestRegionMover2.class);
068
069  @Rule
070  public TestName name = new TestName();
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
073
074  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
075
076  @BeforeClass
077  public static void setUpBeforeClass() throws Exception {
078    TEST_UTIL.startMiniCluster(3);
079    TEST_UTIL.getAdmin().balancerSwitch(false, true);
080  }
081
082  @AfterClass
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @Before
088  public void setUp() throws Exception {
089    final TableName tableName = TableName.valueOf(name.getMethodName());
090    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
091      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
092    int startKey = 0;
093    int endKey = 80000;
094    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
095  }
096
097  @After
098  public void tearDown() throws Exception {
099    final TableName tableName = TableName.valueOf(name.getMethodName());
100    TEST_UTIL.getAdmin().disableTable(tableName);
101    TEST_UTIL.getAdmin().deleteTable(tableName);
102  }
103
104  @Test
105  public void testWithMergedRegions() throws Exception {
106    final TableName tableName = TableName.valueOf(name.getMethodName());
107    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
108    Admin admin = TEST_UTIL.getAdmin();
109    Table table = TEST_UTIL.getConnection().getTable(tableName);
110    List<Put> puts = new ArrayList<>();
111    for (int i = 0; i < 10000; i++) {
112      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"),
113        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
114    }
115    table.put(puts);
116    admin.flush(tableName);
117    HRegionServer regionServer = cluster.getRegionServer(0);
118    String rsName = regionServer.getServerName().getAddress().toString();
119    int numRegions = regionServer.getNumberOfOnlineRegions();
120    List<HRegion> hRegions = regionServer.getRegions().stream()
121      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
122      .collect(Collectors.toList());
123    RegionMover.RegionMoverBuilder rmBuilder =
124      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
125        .maxthreads(8);
126    try (RegionMover rm = rmBuilder.build()) {
127      LOG.debug("Unloading {}", regionServer.getServerName());
128      rm.unload();
129      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
130      LOG.debug("Successfully Unloaded, now Loading");
131      admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(),
132        hRegions.get(1).getRegionInfo().getRegionName() }, true).get(5, TimeUnit.SECONDS);
133      Assert.assertTrue(rm.load());
134      Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions());
135    }
136  }
137
138  @Test
139  public void testWithSplitRegions() throws Exception {
140    final TableName tableName = TableName.valueOf(name.getMethodName());
141    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
142    Admin admin = TEST_UTIL.getAdmin();
143    Table table = TEST_UTIL.getConnection().getTable(tableName);
144    List<Put> puts = new ArrayList<>();
145    for (int i = 10; i < 50000; i++) {
146      puts.add(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
147        Bytes.toBytes("val_" + i)));
148    }
149    table.put(puts);
150    admin.flush(tableName);
151    admin.compact(tableName);
152    HRegionServer regionServer = cluster.getRegionServer(0);
153    String rsName = regionServer.getServerName().getAddress().toString();
154    int numRegions = regionServer.getNumberOfOnlineRegions();
155    List<HRegion> hRegions = regionServer.getRegions().stream()
156      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
157      .collect(Collectors.toList());
158
159    RegionMover.RegionMoverBuilder rmBuilder =
160      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
161        .maxthreads(8);
162    try (RegionMover rm = rmBuilder.build()) {
163      LOG.debug("Unloading {}", regionServer.getServerName());
164      rm.unload();
165      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
166      LOG.debug("Successfully Unloaded, now Loading");
167      HRegion hRegion = hRegions.get(1);
168      if (hRegion.getRegionInfo().getStartKey().length == 0) {
169        hRegion = hRegions.get(0);
170      }
171      int startKey = 0;
172      int endKey = Integer.MAX_VALUE;
173      if (hRegion.getRegionInfo().getStartKey().length > 0) {
174        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
175      }
176      if (hRegion.getRegionInfo().getEndKey().length > 0) {
177        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
178      }
179      int midKey = startKey + (endKey - startKey) / 2;
180      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
181        TimeUnit.SECONDS);
182      Assert.assertTrue(rm.load());
183      Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
184    }
185  }
186
187  @Test
188  public void testFailedRegionMove() throws Exception {
189    final TableName tableName = TableName.valueOf(name.getMethodName());
190    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
191    Admin admin = TEST_UTIL.getAdmin();
192    Table table = TEST_UTIL.getConnection().getTable(tableName);
193    List<Put> puts = new ArrayList<>();
194    for (int i = 0; i < 1000; i++) {
195      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"),
196        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
197    }
198    table.put(puts);
199    admin.flush(tableName);
200    HRegionServer regionServer = cluster.getRegionServer(0);
201    String rsName = regionServer.getServerName().getAddress().toString();
202    int numRegions = regionServer.getNumberOfOnlineRegions();
203    List<HRegion> hRegions = regionServer.getRegions().stream()
204      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
205      .collect(Collectors.toList());
206    RegionMover.RegionMoverBuilder rmBuilder =
207      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
208        .maxthreads(8);
209    try (RegionMover rm = rmBuilder.build()) {
210      LOG.debug("Unloading {}", regionServer.getServerName());
211      rm.unload();
212      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
213      LOG.debug("Successfully Unloaded, now Loading");
214      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
215      // loading regions will fail because of offline region
216      Assert.assertFalse(rm.load());
217    }
218  }
219
220  public void loadDummyDataInTable(TableName tableName) throws Exception {
221    Admin admin = TEST_UTIL.getAdmin();
222    Table table = TEST_UTIL.getConnection().getTable(tableName);
223    List<Put> puts = new ArrayList<>();
224    for (int i = 0; i < 1000; i++) {
225      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"),
226        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
227    }
228    table.put(puts);
229    admin.flush(tableName);
230  }
231
232  @Test
233  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
234    final TableName tableName = TableName.valueOf(name.getMethodName());
235    loadDummyDataInTable(tableName);
236    ServerName sourceServerName = findSourceServerName(tableName);
237    // Isolating 1 region on the same region server.
238    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
239  }
240
241  @Test
242  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
243    final TableName tableName = TableName.valueOf(name.getMethodName());
244    loadDummyDataInTable(tableName);
245    ServerName sourceServerName = findSourceServerName(tableName);
246    ServerName destinationServerName = findDestinationServerName(sourceServerName);
247    // Isolating 1 region on the different region server.
248    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
249  }
250
251  @Test
252  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
253    final TableName tableName = TableName.valueOf(name.getMethodName());
254    loadDummyDataInTable(tableName);
255    ServerName sourceServerName = findSourceServerName(tableName);
256    // Isolating 2 regions on the same region server.
257    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
258  }
259
260  @Test
261  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
262    final TableName tableName = TableName.valueOf(name.getMethodName());
263    loadDummyDataInTable(tableName);
264    // Isolating 2 regions on the different region server.
265    ServerName sourceServerName = findSourceServerName(tableName);
266    ServerName destinationServerName = findDestinationServerName(sourceServerName);
267    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
268  }
269
270  @Test
271  public void testIsolateMetaOnTheSameSever() throws Exception {
272    ServerName metaServerSource = findMetaRSLocation();
273    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
274  }
275
276  @Test
277  public void testIsolateMetaOnTheDifferentServer() throws Exception {
278    ServerName metaServerSource = findMetaRSLocation();
279    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
280    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
281  }
282
283  @Test
284  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
285    final TableName tableName = TableName.valueOf(name.getMethodName());
286    loadDummyDataInTable(tableName);
287    ServerName metaServerSource = findMetaRSLocation();
288    ServerName randomSeverRegion = findSourceServerName(tableName);
289    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
290  }
291
292  @Test
293  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
294    final TableName tableName = TableName.valueOf(name.getMethodName());
295    loadDummyDataInTable(tableName);
296    ServerName randomSeverRegion = findSourceServerName(tableName);
297    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
298  }
299
300  public ServerName findMetaRSLocation() throws Exception {
301    ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null);
302    List<HRegionLocation> result = new ArrayList<>();
303    for (String znode : zkWatcher.getMetaReplicaNodes()) {
304      String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode);
305      int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path);
306      RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId);
307      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
308    }
309    return result.get(0).getServerName();
310  }
311
312  public ServerName findSourceServerName(TableName tableName) throws Exception {
313    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
314    int numOfRS = cluster.getNumLiveRegionServers();
315    ServerName sourceServer = null;
316    for (int i = 0; i < numOfRS; i++) {
317      HRegionServer regionServer = cluster.getRegionServer(i);
318      List<HRegion> hRegions = regionServer.getRegions().stream()
319        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
320        .collect(Collectors.toList());
321      if (hRegions.size() >= 2) {
322        sourceServer = regionServer.getServerName();
323        break;
324      }
325    }
326    if (sourceServer == null) {
327      throw new Exception(
328        "This shouln't happen, No RS found with more than 2 regions of table : " + tableName);
329    }
330    return sourceServer;
331  }
332
333  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
334    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
335    ServerName destinationServerName = null;
336    int numOfRS = cluster.getNumLiveRegionServers();
337    for (int i = 0; i < numOfRS; i++) {
338      destinationServerName = cluster.getRegionServer(i).getServerName();
339      if (!destinationServerName.equals(sourceServerName)) {
340        break;
341      }
342    }
343    if (destinationServerName == null) {
344      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
345    }
346    return destinationServerName;
347  }
348
349  public void regionIsolationOperation(ServerName sourceServerName,
350    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
351    throws Exception {
352    final TableName tableName = TableName.valueOf(name.getMethodName());
353    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
354    Admin admin = TEST_UTIL.getAdmin();
355    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
356    List<HRegion> hRegions = sourceRS.getRegions().stream()
357      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
358      .collect(Collectors.toList());
359    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
360    for (int i = 0; i < numRegionsToIsolate; i++) {
361      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
362    }
363
364    if (isolateMetaAlso) {
365      listOfRegionIDsToIsolate.remove(0);
366      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
367    }
368
369    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
370    String destinationRSName = destinationRS.getServerName().getAddress().toString();
371    RegionMover.RegionMoverBuilder rmBuilder =
372      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
373        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
374    try (RegionMover rm = rmBuilder.build()) {
375      LOG.debug("Unloading {} except regions : {}", destinationRS.getServerName(),
376        listOfRegionIDsToIsolate);
377      rm.isolateRegions();
378      Assert.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
379      List<HRegion> onlineRegions = destinationRS.getRegions();
380      for (int i = 0; i < numRegionsToIsolate; i++) {
381        Assert.assertTrue(
382          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
383      }
384      LOG.debug("Successfully Isolated " + listOfRegionIDsToIsolate.size() + " regions : "
385        + listOfRegionIDsToIsolate + " on " + destinationRS.getServerName());
386    } finally {
387      admin.recommissionRegionServer(destinationRS.getServerName(), null);
388    }
389  }
390}