001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.concurrent.TimeUnit;
029import java.util.stream.Collectors;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HRegionLocation;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.master.RegionState;
044import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
045import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
046import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
047import org.apache.hadoop.hbase.regionserver.HRegion;
048import org.apache.hadoop.hbase.regionserver.HRegionServer;
049import org.apache.hadoop.hbase.testclassification.LargeTests;
050import org.apache.hadoop.hbase.testclassification.MiscTests;
051import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
052import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
053import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
054import org.junit.jupiter.api.AfterAll;
055import org.junit.jupiter.api.AfterEach;
056import org.junit.jupiter.api.BeforeAll;
057import org.junit.jupiter.api.BeforeEach;
058import org.junit.jupiter.api.Tag;
059import org.junit.jupiter.api.Test;
060import org.junit.jupiter.api.TestInfo;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064/**
065 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
066 * exclude functionality useful for rack decommissioning
067 */
068@Tag(MiscTests.TAG)
069@Tag(LargeTests.TAG)
070public class TestRegionMover2 {
071
072  private static final String CF = "fam1";
073
074  private String testMethodName;
075
076  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
077  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
078
079  @BeforeAll
080  public static void setUpBeforeClass() throws Exception {
081    TEST_UTIL.startMiniCluster(3);
082    TEST_UTIL.getAdmin().balancerSwitch(false, true);
083  }
084
085  @AfterAll
086  public static void tearDownAfterClass() throws Exception {
087    TEST_UTIL.shutdownMiniCluster();
088  }
089
090  @BeforeEach
091  public void setUp(TestInfo testInfo) throws Exception {
092    testMethodName = testInfo.getTestMethod().get().getName();
093    createTable(testMethodName);
094  }
095
096  @AfterEach
097  public void tearDown() throws Exception {
098    final TableName tableName = TableName.valueOf(testMethodName);
099    TEST_UTIL.getAdmin().disableTable(tableName);
100    TEST_UTIL.getAdmin().deleteTable(tableName);
101  }
102
103  private TableName createTable(String name) throws IOException {
104    final TableName tableName = TableName.valueOf(name);
105    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
106      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
107    int startKey = 0;
108    int endKey = 80000;
109    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
110    return tableName;
111  }
112
113  @Test
114  public void testWithMergedRegions() throws Exception {
115    final TableName tableName = TableName.valueOf(testMethodName);
116    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
117    Admin admin = TEST_UTIL.getAdmin();
118    Table table = TEST_UTIL.getConnection().getTable(tableName);
119    List<Put> puts = createPuts(10000);
120    table.put(puts);
121    admin.flush(tableName);
122    HRegionServer regionServer = null;
123    RegionInfo regionA = null;
124    RegionInfo regionB = null;
125    for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) {
126      HRegionServer rs = rst.getRegionServer();
127      for (HRegion region : rs.getRegions(tableName)) {
128        if (!region.getRegionInfo().isLast()) {
129          regionServer = rs;
130          regionA = region.getRegionInfo();
131          regionB = admin.getRegions(tableName).stream()
132            .filter(ri -> Bytes.equals(ri.getStartKey(), region.getRegionInfo().getEndKey()))
133            .findFirst().get();
134          break;
135        }
136      }
137    }
138    assertNotNull(regionServer);
139    // if regionB is not this regionServer, move it
140    AssignmentManager am = cluster.getMaster().getAssignmentManager();
141    while (
142      !regionServer.getServerName()
143        .equals(am.getRegionStates().getRegionState(regionB).getServerName())
144    ) {
145      TransitRegionStateProcedure trsp =
146        am.createMoveRegionProcedure(regionB, regionServer.getServerName());
147      ProcedureSyncWait.submitAndWaitProcedure(cluster.getMaster().getMasterProcedureExecutor(),
148        trsp);
149    }
150
151    String rsName = regionServer.getServerName().getAddress().toString();
152    int numRegions = regionServer.getNumberOfOnlineRegions();
153    RegionMover.RegionMoverBuilder rmBuilder =
154      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
155        .maxthreads(8);
156    try (RegionMover rm = rmBuilder.build()) {
157      LOG.debug("Unloading {}", regionServer.getServerName());
158      rm.unload();
159      assertEquals(0, regionServer.getNumberOfOnlineRegions());
160      LOG.debug("Successfully Unloaded, now Loading");
161      admin
162        .mergeRegionsAsync(new byte[][] { regionA.getRegionName(), regionB.getRegionName() }, true)
163        .get(15, TimeUnit.SECONDS);
164      assertTrue(rm.load());
165      assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions());
166    }
167  }
168
169  @Test
170  public void testWithSplitRegions() throws Exception {
171    final TableName tableName = TableName.valueOf(testMethodName);
172    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
173    Admin admin = TEST_UTIL.getAdmin();
174    Table table = TEST_UTIL.getConnection().getTable(tableName);
175    List<Put> puts = createPuts(50000);
176    table.put(puts);
177    admin.flush(tableName);
178    admin.compact(tableName);
179    HRegionServer regionServer = cluster.getRegionServer(0);
180    String rsName = regionServer.getServerName().getAddress().toString();
181    int numRegions = regionServer.getNumberOfOnlineRegions();
182    List<HRegion> hRegions = regionServer.getRegions().stream()
183      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
184      .collect(Collectors.toList());
185
186    RegionMover.RegionMoverBuilder rmBuilder =
187      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
188        .maxthreads(8);
189    try (RegionMover rm = rmBuilder.build()) {
190      LOG.debug("Unloading {}", regionServer.getServerName());
191      rm.unload();
192      assertEquals(0, regionServer.getNumberOfOnlineRegions());
193      LOG.debug("Successfully Unloaded, now Loading");
194      HRegion hRegion = hRegions.get(1);
195      if (hRegion.getRegionInfo().getStartKey().length == 0) {
196        hRegion = hRegions.get(0);
197      }
198      int startKey = 0;
199      int endKey = Integer.MAX_VALUE;
200      if (hRegion.getRegionInfo().getStartKey().length > 0) {
201        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
202      }
203      if (hRegion.getRegionInfo().getEndKey().length > 0) {
204        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
205      }
206      int midKey = startKey + (endKey - startKey) / 2;
207      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
208        TimeUnit.SECONDS);
209      assertTrue(rm.load());
210      assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
211    }
212  }
213
214  @Test
215  public void testFailedRegionMove() throws Exception {
216    final TableName tableName = TableName.valueOf(testMethodName);
217    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
218    Admin admin = TEST_UTIL.getAdmin();
219    Table table = TEST_UTIL.getConnection().getTable(tableName);
220    List<Put> puts = createPuts(1000);
221    table.put(puts);
222    admin.flush(tableName);
223    HRegionServer regionServer = cluster.getRegionServer(0);
224    String rsName = regionServer.getServerName().getAddress().toString();
225    List<HRegion> hRegions = regionServer.getRegions().stream()
226      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
227      .collect(Collectors.toList());
228    RegionMover.RegionMoverBuilder rmBuilder =
229      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
230        .maxthreads(8);
231    try (RegionMover rm = rmBuilder.build()) {
232      LOG.debug("Unloading {}", regionServer.getServerName());
233      rm.unload();
234      assertEquals(0, regionServer.getNumberOfOnlineRegions());
235      LOG.debug("Successfully Unloaded, now Loading");
236      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
237      // loading regions will fail because of offline region
238      assertFalse(rm.load());
239    }
240  }
241
242  @Test
243  public void testDeletedTable() throws Exception {
244    TableName tableNameToDelete = createTable(testMethodName + "ToDelete");
245    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
246    HRegionServer regionServer = cluster.getRegionServer(0);
247    String rsName = regionServer.getServerName().getAddress().toString();
248    RegionMover.RegionMoverBuilder rmBuilder =
249      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
250        .maxthreads(8);
251    try (Admin admin = TEST_UTIL.getAdmin(); RegionMover rm = rmBuilder.build()) {
252      LOG.debug("Unloading {}", regionServer.getServerName());
253      rm.unload();
254      assertEquals(0, regionServer.getNumberOfOnlineRegions());
255      LOG.debug("Successfully Unloaded, now delete table");
256      admin.disableTable(tableNameToDelete);
257      admin.deleteTable(tableNameToDelete);
258      assertTrue(rm.load());
259    }
260  }
261
262  public void loadDummyDataInTable(TableName tableName) throws Exception {
263    Admin admin = TEST_UTIL.getAdmin();
264    Table table = TEST_UTIL.getConnection().getTable(tableName);
265    List<Put> puts = createPuts(1000);
266    table.put(puts);
267    admin.flush(tableName);
268  }
269
270  @Test
271  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
272    final TableName tableName = TableName.valueOf(testMethodName);
273    loadDummyDataInTable(tableName);
274    ServerName sourceServerName = findSourceServerName(tableName);
275    // Isolating 1 region on the same region server.
276    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
277  }
278
279  @Test
280  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
281    final TableName tableName = TableName.valueOf(testMethodName);
282    loadDummyDataInTable(tableName);
283    ServerName sourceServerName = findSourceServerName(tableName);
284    ServerName destinationServerName = findDestinationServerName(sourceServerName);
285    // Isolating 1 region on the different region server.
286    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
287  }
288
289  @Test
290  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
291    final TableName tableName = TableName.valueOf(testMethodName);
292    loadDummyDataInTable(tableName);
293    ServerName sourceServerName = findSourceServerName(tableName);
294    // Isolating 2 regions on the same region server.
295    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
296  }
297
298  @Test
299  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
300    final TableName tableName = TableName.valueOf(testMethodName);
301    loadDummyDataInTable(tableName);
302    // Isolating 2 regions on the different region server.
303    ServerName sourceServerName = findSourceServerName(tableName);
304    ServerName destinationServerName = findDestinationServerName(sourceServerName);
305    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
306  }
307
308  @Test
309  public void testIsolateMetaOnTheSameSever() throws Exception {
310    ServerName metaServerSource = findMetaRSLocation();
311    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
312  }
313
314  @Test
315  public void testIsolateMetaOnTheDifferentServer() throws Exception {
316    ServerName metaServerSource = findMetaRSLocation();
317    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
318    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
319  }
320
321  @Test
322  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
323    final TableName tableName = TableName.valueOf(testMethodName);
324    loadDummyDataInTable(tableName);
325    ServerName metaServerSource = findMetaRSLocation();
326    ServerName randomSeverRegion = findSourceServerName(tableName);
327    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
328  }
329
330  @Test
331  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
332    final TableName tableName = TableName.valueOf(testMethodName);
333    loadDummyDataInTable(tableName);
334    ServerName randomSeverRegion = findSourceServerName(tableName);
335    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
336  }
337
338  private List<Put> createPuts(int count) {
339    List<Put> puts = new ArrayList<>();
340    for (int i = 0; i < count; i++) {
341      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes(CF),
342        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
343    }
344    return puts;
345  }
346
347  public ServerName findMetaRSLocation() throws Exception {
348    ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null);
349    List<HRegionLocation> result = new ArrayList<>();
350    for (String znode : zkWatcher.getMetaReplicaNodes()) {
351      String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode);
352      int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path);
353      RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId);
354      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
355    }
356    return result.get(0).getServerName();
357  }
358
359  public ServerName findSourceServerName(TableName tableName) throws Exception {
360    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
361    int numOfRS = cluster.getNumLiveRegionServers();
362    ServerName sourceServer = null;
363    for (int i = 0; i < numOfRS; i++) {
364      HRegionServer regionServer = cluster.getRegionServer(i);
365      List<HRegion> hRegions = regionServer.getRegions().stream()
366        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
367        .collect(Collectors.toList());
368      if (hRegions.size() >= 2) {
369        sourceServer = regionServer.getServerName();
370        break;
371      }
372    }
373    if (sourceServer == null) {
374      throw new Exception(
375        "This shouldn't happen, No RS found with more than 2 regions of table : " + tableName);
376    }
377    return sourceServer;
378  }
379
380  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
381    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
382    ServerName destinationServerName = null;
383    int numOfRS = cluster.getNumLiveRegionServers();
384    for (int i = 0; i < numOfRS; i++) {
385      destinationServerName = cluster.getRegionServer(i).getServerName();
386      if (!destinationServerName.equals(sourceServerName)) {
387        break;
388      }
389    }
390    if (destinationServerName == null) {
391      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
392    }
393    return destinationServerName;
394  }
395
396  public void regionIsolationOperation(ServerName sourceServerName,
397    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
398    throws Exception {
399    final TableName tableName = TableName.valueOf(testMethodName);
400    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
401    Admin admin = TEST_UTIL.getAdmin();
402    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
403    List<HRegion> hRegions = sourceRS.getRegions().stream()
404      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
405      .collect(Collectors.toList());
406    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
407    for (int i = 0; i < numRegionsToIsolate; i++) {
408      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
409    }
410
411    if (isolateMetaAlso) {
412      listOfRegionIDsToIsolate.remove(0);
413      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
414    }
415
416    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
417    String destinationRSName = destinationRS.getServerName().getAddress().toString();
418    RegionMover.RegionMoverBuilder rmBuilder =
419      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
420        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
421    try (RegionMover rm = rmBuilder.build()) {
422      LOG.debug("Unloading {} except regions: {}", destinationRS.getServerName(),
423        listOfRegionIDsToIsolate);
424      rm.isolateRegions();
425      assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
426      List<HRegion> onlineRegions = destinationRS.getRegions();
427      for (int i = 0; i < numRegionsToIsolate; i++) {
428        assertTrue(
429          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
430      }
431      LOG.debug("Successfully Isolated {} regions: {} on {}", listOfRegionIDsToIsolate.size(),
432        listOfRegionIDsToIsolate, destinationRS.getServerName());
433    } finally {
434      admin.recommissionRegionServer(destinationRS.getServerName(), null);
435    }
436  }
437}