Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.concurrent.TimeUnit;
029import java.util.stream.Collectors;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HRegionLocation;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
044import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
045import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
046import org.apache.hadoop.hbase.regionserver.HRegion;
047import org.apache.hadoop.hbase.regionserver.HRegionServer;
048import org.apache.hadoop.hbase.testclassification.LargeTests;
049import org.apache.hadoop.hbase.testclassification.MiscTests;
050import org.junit.jupiter.api.AfterAll;
051import org.junit.jupiter.api.AfterEach;
052import org.junit.jupiter.api.BeforeAll;
053import org.junit.jupiter.api.BeforeEach;
054import org.junit.jupiter.api.Tag;
055import org.junit.jupiter.api.Test;
056import org.junit.jupiter.api.TestInfo;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060/**
061 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
062 * exclude functionality useful for rack decommissioning
063 */
064@Tag(MiscTests.TAG)
065@Tag(LargeTests.TAG)
066public class TestRegionMover2 {
067
068  private static final String CF = "fam1";
069
070  private String testMethodName;
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
073  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
074
075  @BeforeAll
076  public static void setUpBeforeClass() throws Exception {
077    TEST_UTIL.startMiniCluster(3);
078    TEST_UTIL.getAdmin().balancerSwitch(false, true);
079  }
080
081  @AfterAll
082  public static void tearDownAfterClass() throws Exception {
083    TEST_UTIL.shutdownMiniCluster();
084  }
085
086  @BeforeEach
087  public void setUp(TestInfo testInfo) throws Exception {
088    testMethodName = testInfo.getTestMethod().get().getName();
089    createTable(testMethodName);
090  }
091
092  @AfterEach
093  public void tearDown() throws Exception {
094    final TableName tableName = TableName.valueOf(testMethodName);
095    TEST_UTIL.getAdmin().disableTable(tableName);
096    TEST_UTIL.getAdmin().deleteTable(tableName);
097  }
098
099  private TableName createTable(String name) throws IOException {
100    final TableName tableName = TableName.valueOf(name);
101    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
102      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
103    int startKey = 0;
104    int endKey = 80000;
105    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
106    return tableName;
107  }
108
109  @Test
110  public void testWithMergedRegions() throws Exception {
111    final TableName tableName = TableName.valueOf(testMethodName);
112    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
113    Admin admin = TEST_UTIL.getAdmin();
114    Table table = TEST_UTIL.getConnection().getTable(tableName);
115    List<Put> puts = createPuts(10000);
116    table.put(puts);
117    admin.flush(tableName);
118    HRegionServer regionServer = null;
119    RegionInfo regionA = null;
120    RegionInfo regionB = null;
121    for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) {
122      HRegionServer rs = rst.getRegionServer();
123      for (HRegion region : rs.getRegions(tableName)) {
124        if (!region.getRegionInfo().isLast()) {
125          regionServer = rs;
126          regionA = region.getRegionInfo();
127          regionB = admin.getRegions(tableName).stream()
128            .filter(ri -> Bytes.equals(ri.getStartKey(), region.getRegionInfo().getEndKey()))
129            .findFirst().get();
130          break;
131        }
132      }
133    }
134    assertNotNull(regionServer);
135    // if regionB is not this regionServer, move it
136    AssignmentManager am = cluster.getMaster().getAssignmentManager();
137    while (
138      !regionServer.getServerName()
139        .equals(am.getRegionStates().getRegionState(regionB).getServerName())
140    ) {
141      TransitRegionStateProcedure trsp =
142        am.createMoveRegionProcedure(regionB, regionServer.getServerName());
143      ProcedureSyncWait.submitAndWaitProcedure(cluster.getMaster().getMasterProcedureExecutor(),
144        trsp);
145    }
146
147    String rsName = regionServer.getServerName().getAddress().toString();
148    int numRegions = regionServer.getNumberOfOnlineRegions();
149    RegionMover.RegionMoverBuilder rmBuilder =
150      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
151        .maxthreads(8);
152    try (RegionMover rm = rmBuilder.build()) {
153      LOG.debug("Unloading {}", regionServer.getServerName());
154      rm.unload();
155      assertEquals(0, regionServer.getNumberOfOnlineRegions());
156      LOG.debug("Successfully Unloaded, now Loading");
157      admin
158        .mergeRegionsAsync(new byte[][] { regionA.getRegionName(), regionB.getRegionName() }, true)
159        .get(15, TimeUnit.SECONDS);
160      assertTrue(rm.load());
161      assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions());
162    }
163  }
164
165  @Test
166  public void testWithSplitRegions() throws Exception {
167    final TableName tableName = TableName.valueOf(testMethodName);
168    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
169    Admin admin = TEST_UTIL.getAdmin();
170    Table table = TEST_UTIL.getConnection().getTable(tableName);
171    List<Put> puts = createPuts(50000);
172    table.put(puts);
173    admin.flush(tableName);
174    admin.compact(tableName);
175    HRegionServer regionServer = cluster.getRegionServer(0);
176    String rsName = regionServer.getServerName().getAddress().toString();
177    int numRegions = regionServer.getNumberOfOnlineRegions();
178    List<HRegion> hRegions = regionServer.getRegions().stream()
179      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
180      .collect(Collectors.toList());
181
182    RegionMover.RegionMoverBuilder rmBuilder =
183      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
184        .maxthreads(8);
185    try (RegionMover rm = rmBuilder.build()) {
186      LOG.debug("Unloading {}", regionServer.getServerName());
187      rm.unload();
188      assertEquals(0, regionServer.getNumberOfOnlineRegions());
189      LOG.debug("Successfully Unloaded, now Loading");
190      HRegion hRegion = hRegions.get(1);
191      if (hRegion.getRegionInfo().getStartKey().length == 0) {
192        hRegion = hRegions.get(0);
193      }
194      int startKey = 0;
195      int endKey = Integer.MAX_VALUE;
196      if (hRegion.getRegionInfo().getStartKey().length > 0) {
197        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
198      }
199      if (hRegion.getRegionInfo().getEndKey().length > 0) {
200        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
201      }
202      int midKey = startKey + (endKey - startKey) / 2;
203      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
204        TimeUnit.SECONDS);
205      assertTrue(rm.load());
206      assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
207    }
208  }
209
210  @Test
211  public void testFailedRegionMove() throws Exception {
212    final TableName tableName = TableName.valueOf(testMethodName);
213    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
214    Admin admin = TEST_UTIL.getAdmin();
215    Table table = TEST_UTIL.getConnection().getTable(tableName);
216    List<Put> puts = createPuts(1000);
217    table.put(puts);
218    admin.flush(tableName);
219    HRegionServer regionServer = cluster.getRegionServer(0);
220    String rsName = regionServer.getServerName().getAddress().toString();
221    List<HRegion> hRegions = regionServer.getRegions().stream()
222      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
223      .collect(Collectors.toList());
224    RegionMover.RegionMoverBuilder rmBuilder =
225      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
226        .maxthreads(8);
227    try (RegionMover rm = rmBuilder.build()) {
228      LOG.debug("Unloading {}", regionServer.getServerName());
229      rm.unload();
230      assertEquals(0, regionServer.getNumberOfOnlineRegions());
231      LOG.debug("Successfully Unloaded, now Loading");
232      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
233      // loading regions will fail because of offline region
234      assertFalse(rm.load());
235    }
236  }
237
238  @Test
239  public void testDeletedTable() throws Exception {
240    TableName tableNameToDelete = createTable(testMethodName + "ToDelete");
241    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
242    HRegionServer regionServer = cluster.getRegionServer(0);
243    String rsName = regionServer.getServerName().getAddress().toString();
244    RegionMover.RegionMoverBuilder rmBuilder =
245      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
246        .maxthreads(8);
247    try (Admin admin = TEST_UTIL.getAdmin(); RegionMover rm = rmBuilder.build()) {
248      LOG.debug("Unloading {}", regionServer.getServerName());
249      rm.unload();
250      assertEquals(0, regionServer.getNumberOfOnlineRegions());
251      LOG.debug("Successfully Unloaded, now delete table");
252      admin.disableTable(tableNameToDelete);
253      admin.deleteTable(tableNameToDelete);
254      assertTrue(rm.load());
255    }
256  }
257
258  public void loadDummyDataInTable(TableName tableName) throws Exception {
259    Admin admin = TEST_UTIL.getAdmin();
260    Table table = TEST_UTIL.getConnection().getTable(tableName);
261    List<Put> puts = createPuts(1000);
262    table.put(puts);
263    admin.flush(tableName);
264  }
265
266  @Test
267  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
268    final TableName tableName = TableName.valueOf(testMethodName);
269    loadDummyDataInTable(tableName);
270    ServerName sourceServerName = findSourceServerName(tableName);
271    // Isolating 1 region on the same region server.
272    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
273  }
274
275  @Test
276  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
277    final TableName tableName = TableName.valueOf(testMethodName);
278    loadDummyDataInTable(tableName);
279    ServerName sourceServerName = findSourceServerName(tableName);
280    ServerName destinationServerName = findDestinationServerName(sourceServerName);
281    // Isolating 1 region on the different region server.
282    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
283  }
284
285  @Test
286  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
287    final TableName tableName = TableName.valueOf(testMethodName);
288    loadDummyDataInTable(tableName);
289    ServerName sourceServerName = findSourceServerName(tableName);
290    // Isolating 2 regions on the same region server.
291    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
292  }
293
294  @Test
295  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
296    final TableName tableName = TableName.valueOf(testMethodName);
297    loadDummyDataInTable(tableName);
298    // Isolating 2 regions on the different region server.
299    ServerName sourceServerName = findSourceServerName(tableName);
300    ServerName destinationServerName = findDestinationServerName(sourceServerName);
301    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
302  }
303
304  @Test
305  public void testIsolateMetaOnTheSameSever() throws Exception {
306    ServerName metaServerSource = findMetaRSLocation();
307    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
308  }
309
310  @Test
311  public void testIsolateMetaOnTheDifferentServer() throws Exception {
312    ServerName metaServerSource = findMetaRSLocation();
313    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
314    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
315  }
316
317  @Test
318  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
319    final TableName tableName = TableName.valueOf(testMethodName);
320    loadDummyDataInTable(tableName);
321    ServerName metaServerSource = findMetaRSLocation();
322    ServerName randomSeverRegion = findSourceServerName(tableName);
323    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
324  }
325
326  @Test
327  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
328    final TableName tableName = TableName.valueOf(testMethodName);
329    loadDummyDataInTable(tableName);
330    ServerName randomSeverRegion = findSourceServerName(tableName);
331    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
332  }
333
334  private List<Put> createPuts(int count) {
335    List<Put> puts = new ArrayList<>();
336    for (int i = 0; i < count; i++) {
337      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes(CF),
338        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
339    }
340    return puts;
341  }
342
343  public ServerName findMetaRSLocation() throws Exception {
344    List<HRegionLocation> result =
345      TEST_UTIL.getConnection().getRegionLocator(TableName.META_TABLE_NAME).getAllRegionLocations();
346    return result.get(0).getServerName();
347  }
348
349  public ServerName findSourceServerName(TableName tableName) throws Exception {
350    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
351    int numOfRS = cluster.getNumLiveRegionServers();
352    ServerName sourceServer = null;
353    for (int i = 0; i < numOfRS; i++) {
354      HRegionServer regionServer = cluster.getRegionServer(i);
355      List<HRegion> hRegions = regionServer.getRegions().stream()
356        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
357        .collect(Collectors.toList());
358      if (hRegions.size() >= 2) {
359        sourceServer = regionServer.getServerName();
360        break;
361      }
362    }
363    if (sourceServer == null) {
364      throw new Exception(
365        "This shouldn't happen, No RS found with more than 2 regions of table : " + tableName);
366    }
367    return sourceServer;
368  }
369
370  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
371    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
372    ServerName destinationServerName = null;
373    int numOfRS = cluster.getNumLiveRegionServers();
374    for (int i = 0; i < numOfRS; i++) {
375      destinationServerName = cluster.getRegionServer(i).getServerName();
376      if (!destinationServerName.equals(sourceServerName)) {
377        break;
378      }
379    }
380    if (destinationServerName == null) {
381      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
382    }
383    return destinationServerName;
384  }
385
386  public void regionIsolationOperation(ServerName sourceServerName,
387    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
388    throws Exception {
389    final TableName tableName = TableName.valueOf(testMethodName);
390    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
391    Admin admin = TEST_UTIL.getAdmin();
392    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
393    List<HRegion> hRegions = sourceRS.getRegions().stream()
394      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
395      .collect(Collectors.toList());
396    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
397    for (int i = 0; i < numRegionsToIsolate; i++) {
398      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
399    }
400
401    if (isolateMetaAlso) {
402      listOfRegionIDsToIsolate.remove(0);
403      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
404    }
405
406    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
407    String destinationRSName = destinationRS.getServerName().getAddress().toString();
408    RegionMover.RegionMoverBuilder rmBuilder =
409      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
410        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
411    try (RegionMover rm = rmBuilder.build()) {
412      LOG.debug("Unloading {} except regions: {}", destinationRS.getServerName(),
413        listOfRegionIDsToIsolate);
414      rm.isolateRegions();
415      assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
416      List<HRegion> onlineRegions = destinationRS.getRegions();
417      for (int i = 0; i < numRegionsToIsolate; i++) {
418        assertTrue(
419          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
420      }
421      LOG.debug("Successfully Isolated {} regions: {} on {}", listOfRegionIDsToIsolate.size(),
422        listOfRegionIDsToIsolate, destinationRS.getServerName());
423    } finally {
424      admin.recommissionRegionServer(destinationRS.getServerName(), null);
425    }
426  }
427}