Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022
023import java.io.File;
024import java.io.FileWriter;
025import java.io.IOException;
026import java.util.Collections;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.Waiter.Predicate;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.regionserver.HRegion;
040import org.apache.hadoop.hbase.regionserver.HRegionServer;
041import org.apache.hadoop.hbase.testclassification.LargeTests;
042import org.apache.hadoop.hbase.testclassification.MiscTests;
043import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder;
044import org.junit.jupiter.api.AfterAll;
045import org.junit.jupiter.api.AfterEach;
046import org.junit.jupiter.api.Assertions;
047import org.junit.jupiter.api.BeforeAll;
048import org.junit.jupiter.api.BeforeEach;
049import org.junit.jupiter.api.Tag;
050import org.junit.jupiter.api.Test;
051import org.junit.jupiter.api.TestInfo;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
057 * exclude functionality useful for rack decommissioning
058 */
059@Tag(MiscTests.TAG)
060@Tag(LargeTests.TAG)
061public class TestRegionMover1 {
062
063  private String testMethodName;
064
065  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover1.class);
066
067  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
068
069  @BeforeAll
070  public static void setUpBeforeClass() throws Exception {
071    TEST_UTIL.startMiniCluster(3);
072    TEST_UTIL.getAdmin().balancerSwitch(false, true);
073  }
074
075  @AfterAll
076  public static void tearDownAfterClass() throws Exception {
077    TEST_UTIL.shutdownMiniCluster();
078  }
079
080  @BeforeEach
081  public void setUp(TestInfo testInfo) throws Exception {
082    testMethodName = testInfo.getTestMethod().get().getName();
083    final TableName tableName = TableName.valueOf(testMethodName);
084    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
085      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
086    String startKey = "a";
087    String endKey = "z";
088    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
089  }
090
091  @AfterEach
092  public void tearDown() throws Exception {
093    final TableName tableName = TableName.valueOf(testMethodName);
094    TEST_UTIL.getAdmin().disableTable(tableName);
095    TEST_UTIL.getAdmin().deleteTable(tableName);
096  }
097
098  @Test
099  public void testWithAck() throws Exception {
100    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
101    HRegionServer regionServer = cluster.getRegionServer(0);
102    String rsName = regionServer.getServerName().getAddress().toString();
103    int numRegions = regionServer.getNumberOfOnlineRegions();
104    RegionMoverBuilder rmBuilder =
105      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8);
106    try (RegionMover rm = rmBuilder.build()) {
107      LOG.info("Unloading " + regionServer.getServerName());
108      rm.unload();
109      assertEquals(0, regionServer.getNumberOfOnlineRegions());
110      LOG.info("Successfully Unloaded\nNow Loading");
111      rm.load();
112      assertEquals(numRegions, regionServer.getNumberOfOnlineRegions());
113      // Repeat the same load. It should be very fast because all regions are already moved.
114      rm.load();
115    }
116  }
117
118  /**
119   * Test to unload a regionserver first and then load it using no Ack mode.
120   */
121  @Test
122  public void testWithoutAck() throws Exception {
123    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
124    HRegionServer regionServer = cluster.getRegionServer(0);
125    String rsName = regionServer.getServerName().getAddress().toString();
126    int numRegions = regionServer.getNumberOfOnlineRegions();
127    RegionMoverBuilder rmBuilder =
128      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false);
129    try (RegionMover rm = rmBuilder.build()) {
130      LOG.info("Unloading " + regionServer.getServerName());
131      rm.unload();
132      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
133        @Override
134        public boolean evaluate() throws Exception {
135          return regionServer.getNumberOfOnlineRegions() == 0;
136        }
137      });
138      LOG.info("Successfully Unloaded\nNow Loading");
139      rm.load();
140      // In UT we only have 10 regions so it is not likely to fail, so here we check for all
141      // regions, in the real production this may not be true.
142      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
143        @Override
144        public boolean evaluate() throws Exception {
145          return regionServer.getNumberOfOnlineRegions() == numRegions;
146        }
147      });
148    }
149  }
150
151  /**
152   * To test that we successfully exclude a server from the unloading process We test for the number
153   * of regions on Excluded server and also test that regions are unloaded successfully
154   */
155  @Test
156  public void testExclude() throws Exception {
157    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
158    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
159    FileWriter fos = new FileWriter(excludeFile);
160    HRegionServer excludeServer = cluster.getRegionServer(1);
161    String excludeHostname = excludeServer.getServerName().getHostname();
162    int excludeServerPort = excludeServer.getServerName().getPort();
163    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
164    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
165    fos.write(excludeServerName);
166    fos.close();
167    HRegionServer regionServer = cluster.getRegionServer(0);
168    String rsName = regionServer.getServerName().getHostname();
169    int port = regionServer.getServerName().getPort();
170    String rs = rsName + ":" + Integer.toString(port);
171    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
172      .ack(true).excludeFile(excludeFile.getCanonicalPath());
173    try (RegionMover rm = rmBuilder.build()) {
174      rm.unload();
175      LOG.info("Unloading " + rs);
176      assertEquals(0, regionServer.getNumberOfOnlineRegions());
177      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
178      LOG.info("Before:" + regionsExcludeServer + " After:"
179        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
180    }
181  }
182
183  @Test
184  public void testDesignatedFile() throws Exception {
185    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
186    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file");
187    HRegionServer designatedServer = cluster.getRegionServer(0);
188    try (FileWriter fos = new FileWriter(designatedFile)) {
189      String designatedHostname = designatedServer.getServerName().getHostname();
190      int designatedServerPort = designatedServer.getServerName().getPort();
191      String excludeServerName = designatedHostname + ":" + designatedServerPort;
192      fos.write(excludeServerName);
193    }
194    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
195    HRegionServer regionServer = cluster.getRegionServer(1);
196    String rsName = regionServer.getServerName().getHostname();
197    int port = regionServer.getServerName().getPort();
198    String rs = rsName + ":" + port;
199    int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
200    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
201      .designatedFile(designatedFile.getCanonicalPath());
202    try (RegionMover rm = rmBuilder.build()) {
203      LOG.debug("Unloading {} regions", rs);
204      rm.unload();
205      assertEquals(0, regionServer.getNumberOfOnlineRegions());
206      assertEquals(regionsInDesignatedServer + regionsInRegionServer,
207        designatedServer.getNumberOfOnlineRegions());
208      LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
209        designatedServer.getNumberOfOnlineRegions());
210    }
211  }
212
213  @Test
214  public void testExcludeAndDesignated() throws Exception {
215    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
216    // create designated file
217    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file");
218    HRegionServer designatedServer = cluster.getRegionServer(0);
219    try (FileWriter fos = new FileWriter(designatedFile)) {
220      String designatedHostname = designatedServer.getServerName().getHostname();
221      int designatedServerPort = designatedServer.getServerName().getPort();
222      String excludeServerName = designatedHostname + ":" + designatedServerPort;
223      fos.write(excludeServerName);
224    }
225    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
226    // create exclude file
227    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
228    HRegionServer excludeServer = cluster.getRegionServer(1);
229    try (FileWriter fos = new FileWriter(excludeFile)) {
230      String excludeHostname = excludeServer.getServerName().getHostname();
231      int excludeServerPort = excludeServer.getServerName().getPort();
232      String excludeServerName = excludeHostname + ":" + excludeServerPort;
233      fos.write(excludeServerName);
234    }
235    int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
236
237    HRegionServer targetRegionServer = cluster.getRegionServer(2);
238    String rsName = targetRegionServer.getServerName().getHostname();
239    int port = targetRegionServer.getServerName().getPort();
240    String rs = rsName + ":" + port;
241    int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
242
243    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
244      .designatedFile(designatedFile.getCanonicalPath())
245      .excludeFile(excludeFile.getCanonicalPath());
246    try (RegionMover rm = rmBuilder.build()) {
247      LOG.debug("Unloading {}", rs);
248      rm.unload();
249      assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
250      assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
251        designatedServer.getNumberOfOnlineRegions());
252      LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
253        designatedServer.getNumberOfOnlineRegions());
254      assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
255      LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
256        excludeServer.getNumberOfOnlineRegions());
257    }
258  }
259
260  @Test
261  public void testRegionServerPort() throws Exception {
262    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
263    HRegionServer regionServer = cluster.getRegionServer(0);
264    String rsName = regionServer.getServerName().getHostname();
265
266    final int PORT = 16021;
267    Configuration conf = TEST_UTIL.getConfiguration();
268    String originalPort = conf.get(HConstants.REGIONSERVER_PORT);
269    conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT));
270    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf);
271    assertEquals(PORT, rmBuilder.port);
272    if (originalPort != null) {
273      conf.set(HConstants.REGIONSERVER_PORT, originalPort);
274    }
275  }
276
277  /**
278   * UT for HBASE-21746
279   */
280  @Test
281  public void testLoadMetaRegion() throws Exception {
282    HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
283      .map(t -> t.getRegionServer())
284      .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get();
285    int onlineRegions = rsWithMeta.getNumberOfOnlineRegions();
286    String rsName = rsWithMeta.getServerName().getAddress().toString();
287    try (RegionMover rm =
288      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) {
289      LOG.info("Unloading " + rsWithMeta.getServerName());
290      rm.unload();
291      assertEquals(0, rsWithMeta.getNumberOfOnlineRegions());
292      LOG.info("Loading " + rsWithMeta.getServerName());
293      rm.load();
294      assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions());
295    }
296  }
297
298  /**
299   * UT for HBASE-21746
300   */
301  @Test
302  public void testTargetServerDeadWhenLoading() throws Exception {
303    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
304    String rsName = rs.getServerName().getAddress().toString();
305    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
306    // wait 5 seconds at most
307    conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5);
308    String filename =
309      new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString();
310    // unload the region server
311    try (
312      RegionMover rm = new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) {
313      LOG.info("Unloading " + rs.getServerName());
314      rm.unload();
315      assertEquals(0, rs.getNumberOfOnlineRegions());
316    }
317    String inexistRsName = "whatever:123";
318    try (RegionMover rm =
319      new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) {
320      // load the regions to an inexist region server, which should fail and return false
321      LOG.info("Loading to an inexist region server {}", inexistRsName);
322      assertFalse(rm.load());
323    }
324  }
325
326  @Test
327  public void testDecomServerExclusionWithAck() throws Exception {
328    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
329    HRegionServer excludeServer = cluster.getRegionServer(1);
330    List<HRegion> regions = excludeServer.getRegions();
331    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
332    TEST_UTIL.getAdmin()
333      .decommissionRegionServers(Collections.singletonList(excludeServer.getServerName()), false);
334
335    waitForServerDecom(excludeServer);
336
337    HRegionServer regionServer = cluster.getRegionServer(0);
338    String rsName = regionServer.getServerName().getHostname();
339    int port = regionServer.getServerName().getPort();
340    String hostname = rsName + ":" + Integer.toString(port);
341    RegionMoverBuilder rmBuilder =
342      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(true);
343
344    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
345    int sourceServerRegions = regionServer.getRegions().size();
346
347    try (RegionMover regionMover = rmBuilder.build()) {
348      Assertions.assertTrue(regionMover.unload());
349      LOG.info("Unloading {}", hostname);
350      assertEquals(0, regionServer.getNumberOfOnlineRegions());
351      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
352      LOG.info("Before:" + regionsExcludeServer + " After:"
353        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
354      List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
355      int index = 0;
356      for (HRegion hRegion : regionList) {
357        Assertions.assertEquals(hRegion, regions.get(index++));
358      }
359      Assertions.assertEquals(targetServerRegions + sourceServerRegions,
360        cluster.getRegionServer(2).getNumberOfOnlineRegions());
361      Assertions.assertTrue(regionMover.load());
362    }
363
364    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
365      Collections.emptyList());
366  }
367
368  private void waitForServerDecom(HRegionServer excludeServer) {
369
370    TEST_UTIL.waitFor(3000, () -> {
371      try {
372        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
373        return decomServers.size() == 1
374          && decomServers.get(0).equals(excludeServer.getServerName());
375      } catch (IOException e) {
376        throw new RuntimeException(e);
377      }
378    });
379  }
380
381  @Test
382  public void testDecomServerExclusion() throws Exception {
383    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
384    HRegionServer excludeServer = cluster.getRegionServer(0);
385    List<HRegion> regions = excludeServer.getRegions();
386    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
387    TEST_UTIL.getAdmin()
388      .decommissionRegionServers(Collections.singletonList(excludeServer.getServerName()), false);
389
390    waitForServerDecom(excludeServer);
391
392    HRegionServer sourceRegionServer = cluster.getRegionServer(1);
393    String rsName = sourceRegionServer.getServerName().getHostname();
394    int port = sourceRegionServer.getServerName().getPort();
395    String hostname = rsName + ":" + Integer.toString(port);
396    RegionMoverBuilder rmBuilder =
397      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);
398
399    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
400    int sourceServerRegions = sourceRegionServer.getRegions().size();
401
402    try (RegionMover regionMover = rmBuilder.build()) {
403      Assertions.assertTrue(regionMover.unload());
404      LOG.info("Unloading {}", hostname);
405      assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
406      assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
407      LOG.info("Before:" + regionsExcludeServer + " After:"
408        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
409      List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
410      int index = 0;
411      for (HRegion hRegion : regionList) {
412        Assertions.assertEquals(hRegion, regions.get(index++));
413      }
414      Assertions.assertEquals(targetServerRegions + sourceServerRegions,
415        cluster.getRegionServer(2).getNumberOfOnlineRegions());
416      Assertions.assertTrue(regionMover.load());
417    }
418
419    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
420      Collections.emptyList());
421  }
422
423  @Test
424  public void testExcludeAndDecomServers() throws Exception {
425    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
426    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
427    FileWriter fos = new FileWriter(excludeFile);
428    HRegionServer excludeServer = cluster.getRegionServer(1);
429    String excludeHostname = excludeServer.getServerName().getHostname();
430    int excludeServerPort = excludeServer.getServerName().getPort();
431    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
432    fos.write(excludeServerName);
433    fos.close();
434
435    HRegionServer decomServer = cluster.getRegionServer(2);
436    TEST_UTIL.getAdmin()
437      .decommissionRegionServers(Collections.singletonList(decomServer.getServerName()), false);
438
439    waitForServerDecom(decomServer);
440
441    HRegionServer regionServer = cluster.getRegionServer(0);
442    String rsName = regionServer.getServerName().getHostname();
443    int port = regionServer.getServerName().getPort();
444    String sourceServer = rsName + ":" + Integer.toString(port);
445    RegionMoverBuilder rmBuilder =
446      new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration()).ack(true)
447        .excludeFile(excludeFile.getCanonicalPath());
448    try (RegionMover regionMover = rmBuilder.build()) {
449      Assertions.assertFalse(regionMover.unload());
450    }
451
452    TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(),
453      Collections.emptyList());
454  }
455
456}