001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import java.util.concurrent.TimeUnit;
024import java.util.stream.Collectors;
025import org.apache.hadoop.hbase.HBaseTestingUtil;
026import org.apache.hadoop.hbase.HRegionLocation;
027import org.apache.hadoop.hbase.ServerName;
028import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.RegionState;
038import org.apache.hadoop.hbase.regionserver.HRegion;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
043import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
044import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.AfterEach;
047import org.junit.jupiter.api.Assertions;
048import org.junit.jupiter.api.BeforeAll;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052import org.junit.jupiter.api.TestInfo;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
058 * exclude functionality useful for rack decommissioning
059 */
060@Tag(MiscTests.TAG)
061@Tag(LargeTests.TAG)
062public class TestRegionMover2 {
063
064  private static final String CF = "fam1";
065
066  private String testMethodName;
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
069  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
070
071  @BeforeAll
072  public static void setUpBeforeClass() throws Exception {
073    TEST_UTIL.startMiniCluster(3);
074    TEST_UTIL.getAdmin().balancerSwitch(false, true);
075  }
076
077  @AfterAll
078  public static void tearDownAfterClass() throws Exception {
079    TEST_UTIL.shutdownMiniCluster();
080  }
081
082  @BeforeEach
083  public void setUp(TestInfo testInfo) throws Exception {
084    testMethodName = testInfo.getTestMethod().get().getName();
085    createTable(testMethodName);
086  }
087
088  @AfterEach
089  public void tearDown() throws Exception {
090    final TableName tableName = TableName.valueOf(testMethodName);
091    TEST_UTIL.getAdmin().disableTable(tableName);
092    TEST_UTIL.getAdmin().deleteTable(tableName);
093  }
094
095  private TableName createTable(String name) throws IOException {
096    final TableName tableName = TableName.valueOf(name);
097    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
098      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
099    int startKey = 0;
100    int endKey = 80000;
101    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
102    return tableName;
103  }
104
105  @Test
106  public void testWithMergedRegions() throws Exception {
107    final TableName tableName = TableName.valueOf(testMethodName);
108    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
109    Admin admin = TEST_UTIL.getAdmin();
110    Table table = TEST_UTIL.getConnection().getTable(tableName);
111    List<Put> puts = createPuts(10000);
112    table.put(puts);
113    admin.flush(tableName);
114    HRegionServer regionServer = cluster.getRegionServer(0);
115    String rsName = regionServer.getServerName().getAddress().toString();
116    int numRegions = regionServer.getNumberOfOnlineRegions();
117    List<HRegion> hRegions = regionServer.getRegions().stream()
118      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
119      .collect(Collectors.toList());
120    RegionMover.RegionMoverBuilder rmBuilder =
121      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
122        .maxthreads(8);
123    try (RegionMover rm = rmBuilder.build()) {
124      LOG.debug("Unloading {}", regionServer.getServerName());
125      rm.unload();
126      Assertions.assertEquals(0, regionServer.getNumberOfOnlineRegions());
127      LOG.debug("Successfully Unloaded, now Loading");
128      admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(),
129        hRegions.get(1).getRegionInfo().getRegionName() }, true).get(5, TimeUnit.SECONDS);
130      Assertions.assertTrue(rm.load());
131      Assertions.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions());
132    }
133  }
134
135  @Test
136  public void testWithSplitRegions() throws Exception {
137    final TableName tableName = TableName.valueOf(testMethodName);
138    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
139    Admin admin = TEST_UTIL.getAdmin();
140    Table table = TEST_UTIL.getConnection().getTable(tableName);
141    List<Put> puts = createPuts(50000);
142    table.put(puts);
143    admin.flush(tableName);
144    admin.compact(tableName);
145    HRegionServer regionServer = cluster.getRegionServer(0);
146    String rsName = regionServer.getServerName().getAddress().toString();
147    int numRegions = regionServer.getNumberOfOnlineRegions();
148    List<HRegion> hRegions = regionServer.getRegions().stream()
149      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
150      .collect(Collectors.toList());
151
152    RegionMover.RegionMoverBuilder rmBuilder =
153      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
154        .maxthreads(8);
155    try (RegionMover rm = rmBuilder.build()) {
156      LOG.debug("Unloading {}", regionServer.getServerName());
157      rm.unload();
158      Assertions.assertEquals(0, regionServer.getNumberOfOnlineRegions());
159      LOG.debug("Successfully Unloaded, now Loading");
160      HRegion hRegion = hRegions.get(1);
161      if (hRegion.getRegionInfo().getStartKey().length == 0) {
162        hRegion = hRegions.get(0);
163      }
164      int startKey = 0;
165      int endKey = Integer.MAX_VALUE;
166      if (hRegion.getRegionInfo().getStartKey().length > 0) {
167        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
168      }
169      if (hRegion.getRegionInfo().getEndKey().length > 0) {
170        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
171      }
172      int midKey = startKey + (endKey - startKey) / 2;
173      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
174        TimeUnit.SECONDS);
175      Assertions.assertTrue(rm.load());
176      Assertions.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
177    }
178  }
179
180  @Test
181  public void testFailedRegionMove() throws Exception {
182    final TableName tableName = TableName.valueOf(testMethodName);
183    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
184    Admin admin = TEST_UTIL.getAdmin();
185    Table table = TEST_UTIL.getConnection().getTable(tableName);
186    List<Put> puts = createPuts(1000);
187    table.put(puts);
188    admin.flush(tableName);
189    HRegionServer regionServer = cluster.getRegionServer(0);
190    String rsName = regionServer.getServerName().getAddress().toString();
191    List<HRegion> hRegions = regionServer.getRegions().stream()
192      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
193      .collect(Collectors.toList());
194    RegionMover.RegionMoverBuilder rmBuilder =
195      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
196        .maxthreads(8);
197    try (RegionMover rm = rmBuilder.build()) {
198      LOG.debug("Unloading {}", regionServer.getServerName());
199      rm.unload();
200      Assertions.assertEquals(0, regionServer.getNumberOfOnlineRegions());
201      LOG.debug("Successfully Unloaded, now Loading");
202      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
203      // loading regions will fail because of offline region
204      Assertions.assertFalse(rm.load());
205    }
206  }
207
208  @Test
209  public void testDeletedTable() throws Exception {
210    TableName tableNameToDelete = createTable(testMethodName + "ToDelete");
211    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
212    HRegionServer regionServer = cluster.getRegionServer(0);
213    String rsName = regionServer.getServerName().getAddress().toString();
214    RegionMover.RegionMoverBuilder rmBuilder =
215      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
216        .maxthreads(8);
217    try (Admin admin = TEST_UTIL.getAdmin(); RegionMover rm = rmBuilder.build()) {
218      LOG.debug("Unloading {}", regionServer.getServerName());
219      rm.unload();
220      Assertions.assertEquals(0, regionServer.getNumberOfOnlineRegions());
221      LOG.debug("Successfully Unloaded, now delete table");
222      admin.disableTable(tableNameToDelete);
223      admin.deleteTable(tableNameToDelete);
224      Assertions.assertTrue(rm.load());
225    }
226  }
227
228  public void loadDummyDataInTable(TableName tableName) throws Exception {
229    Admin admin = TEST_UTIL.getAdmin();
230    Table table = TEST_UTIL.getConnection().getTable(tableName);
231    List<Put> puts = createPuts(1000);
232    table.put(puts);
233    admin.flush(tableName);
234  }
235
236  @Test
237  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
238    final TableName tableName = TableName.valueOf(testMethodName);
239    loadDummyDataInTable(tableName);
240    ServerName sourceServerName = findSourceServerName(tableName);
241    // Isolating 1 region on the same region server.
242    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
243  }
244
245  @Test
246  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
247    final TableName tableName = TableName.valueOf(testMethodName);
248    loadDummyDataInTable(tableName);
249    ServerName sourceServerName = findSourceServerName(tableName);
250    ServerName destinationServerName = findDestinationServerName(sourceServerName);
251    // Isolating 1 region on the different region server.
252    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
253  }
254
255  @Test
256  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
257    final TableName tableName = TableName.valueOf(testMethodName);
258    loadDummyDataInTable(tableName);
259    ServerName sourceServerName = findSourceServerName(tableName);
260    // Isolating 2 regions on the same region server.
261    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
262  }
263
264  @Test
265  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
266    final TableName tableName = TableName.valueOf(testMethodName);
267    loadDummyDataInTable(tableName);
268    // Isolating 2 regions on the different region server.
269    ServerName sourceServerName = findSourceServerName(tableName);
270    ServerName destinationServerName = findDestinationServerName(sourceServerName);
271    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
272  }
273
274  @Test
275  public void testIsolateMetaOnTheSameSever() throws Exception {
276    ServerName metaServerSource = findMetaRSLocation();
277    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
278  }
279
280  @Test
281  public void testIsolateMetaOnTheDifferentServer() throws Exception {
282    ServerName metaServerSource = findMetaRSLocation();
283    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
284    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
285  }
286
287  @Test
288  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
289    final TableName tableName = TableName.valueOf(testMethodName);
290    loadDummyDataInTable(tableName);
291    ServerName metaServerSource = findMetaRSLocation();
292    ServerName randomSeverRegion = findSourceServerName(tableName);
293    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
294  }
295
296  @Test
297  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
298    final TableName tableName = TableName.valueOf(testMethodName);
299    loadDummyDataInTable(tableName);
300    ServerName randomSeverRegion = findSourceServerName(tableName);
301    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
302  }
303
304  private List<Put> createPuts(int count) {
305    List<Put> puts = new ArrayList<>();
306    for (int i = 0; i < count; i++) {
307      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes(CF),
308        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
309    }
310    return puts;
311  }
312
313  public ServerName findMetaRSLocation() throws Exception {
314    ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null);
315    List<HRegionLocation> result = new ArrayList<>();
316    for (String znode : zkWatcher.getMetaReplicaNodes()) {
317      String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode);
318      int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path);
319      RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId);
320      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
321    }
322    return result.get(0).getServerName();
323  }
324
325  public ServerName findSourceServerName(TableName tableName) throws Exception {
326    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
327    int numOfRS = cluster.getNumLiveRegionServers();
328    ServerName sourceServer = null;
329    for (int i = 0; i < numOfRS; i++) {
330      HRegionServer regionServer = cluster.getRegionServer(i);
331      List<HRegion> hRegions = regionServer.getRegions().stream()
332        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
333        .collect(Collectors.toList());
334      if (hRegions.size() >= 2) {
335        sourceServer = regionServer.getServerName();
336        break;
337      }
338    }
339    if (sourceServer == null) {
340      throw new Exception(
341        "This shouldn't happen, No RS found with more than 2 regions of table : " + tableName);
342    }
343    return sourceServer;
344  }
345
346  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
347    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
348    ServerName destinationServerName = null;
349    int numOfRS = cluster.getNumLiveRegionServers();
350    for (int i = 0; i < numOfRS; i++) {
351      destinationServerName = cluster.getRegionServer(i).getServerName();
352      if (!destinationServerName.equals(sourceServerName)) {
353        break;
354      }
355    }
356    if (destinationServerName == null) {
357      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
358    }
359    return destinationServerName;
360  }
361
362  public void regionIsolationOperation(ServerName sourceServerName,
363    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
364    throws Exception {
365    final TableName tableName = TableName.valueOf(testMethodName);
366    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
367    Admin admin = TEST_UTIL.getAdmin();
368    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
369    List<HRegion> hRegions = sourceRS.getRegions().stream()
370      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
371      .collect(Collectors.toList());
372    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
373    for (int i = 0; i < numRegionsToIsolate; i++) {
374      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
375    }
376
377    if (isolateMetaAlso) {
378      listOfRegionIDsToIsolate.remove(0);
379      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
380    }
381
382    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
383    String destinationRSName = destinationRS.getServerName().getAddress().toString();
384    RegionMover.RegionMoverBuilder rmBuilder =
385      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
386        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
387    try (RegionMover rm = rmBuilder.build()) {
388      LOG.debug("Unloading {} except regions: {}", destinationRS.getServerName(),
389        listOfRegionIDsToIsolate);
390      rm.isolateRegions();
391      Assertions.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
392      List<HRegion> onlineRegions = destinationRS.getRegions();
393      for (int i = 0; i < numRegionsToIsolate; i++) {
394        Assertions.assertTrue(
395          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
396      }
397      LOG.debug("Successfully Isolated {} regions: {} on {}", listOfRegionIDsToIsolate.size(),
398        listOfRegionIDsToIsolate, destinationRS.getServerName());
399    } finally {
400      admin.recommissionRegionServer(destinationRS.getServerName(), null);
401    }
402  }
403}