001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022
023import java.io.File;
024import java.io.FileWriter;
025import java.io.IOException;
026import java.util.Collections;
027import java.util.List;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MiniHBaseCluster;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.Waiter.Predicate;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.HRegionServer;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MiscTests;
046import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder;
047import org.junit.AfterClass;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057/**
058 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
059 * exclude functionality useful for rack decommissioning
060 */
061@Category({MiscTests.class, LargeTests.class})
062public class TestRegionMover {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestRegionMover.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover.class);
069
070  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
071
072  @BeforeClass
073  public static void setUpBeforeClass() throws Exception {
074    TEST_UTIL.startMiniCluster(3);
075    TEST_UTIL.getAdmin().balancerSwitch(false, true);
076  }
077
078  @AfterClass
079  public static void tearDownAfterClass() throws Exception {
080    TEST_UTIL.shutdownMiniCluster();
081  }
082
083  @Before
084  public void setUp() throws Exception {
085    // Create a pre-split table just to populate some regions
086    TableName tableName = TableName.valueOf("testRegionMover");
087    Admin admin = TEST_UTIL.getAdmin();
088    if (admin.tableExists(tableName)) {
089      TEST_UTIL.deleteTable(tableName);
090    }
091    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
092      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
093    String startKey = "a";
094    String endKey = "z";
095    admin.createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
096  }
097
098  @Test
099  public void testWithAck() throws Exception {
100    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
101    HRegionServer regionServer = cluster.getRegionServer(0);
102    String rsName = regionServer.getServerName().getAddress().toString();
103    int numRegions = regionServer.getNumberOfOnlineRegions();
104    RegionMoverBuilder rmBuilder =
105      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8);
106    try (RegionMover rm = rmBuilder.build()) {
107      LOG.info("Unloading " + regionServer.getServerName());
108      rm.unload();
109      assertEquals(0, regionServer.getNumberOfOnlineRegions());
110      LOG.info("Successfully Unloaded\nNow Loading");
111      rm.load();
112      assertEquals(numRegions, regionServer.getNumberOfOnlineRegions());
113      // Repeat the same load. It should be very fast because all regions are already moved.
114      rm.load();
115    }
116  }
117
118  /**
119   * Test to unload a regionserver first and then load it using no Ack mode.
120   */
121  @Test
122  public void testWithoutAck() throws Exception {
123    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
124    HRegionServer regionServer = cluster.getRegionServer(0);
125    String rsName = regionServer.getServerName().getAddress().toString();
126    int numRegions = regionServer.getNumberOfOnlineRegions();
127    RegionMoverBuilder rmBuilder =
128      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false);
129    try (RegionMover rm = rmBuilder.build()) {
130      LOG.info("Unloading " + regionServer.getServerName());
131      rm.unload();
132      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
133        @Override
134        public boolean evaluate() throws Exception {
135          return regionServer.getNumberOfOnlineRegions() == 0;
136        }
137      });
138      LOG.info("Successfully Unloaded\nNow Loading");
139      rm.load();
140      // In UT we only have 10 regions so it is not likely to fail, so here we check for all
141      // regions, in the real production this may not be true.
142      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
143        @Override
144        public boolean evaluate() throws Exception {
145          return regionServer.getNumberOfOnlineRegions() == numRegions;
146        }
147      });
148    }
149  }
150
151  /**
152   * To test that we successfully exclude a server from the unloading process We test for the number
153   * of regions on Excluded server and also test that regions are unloaded successfully
154   */
155  @Test
156  public void testExclude() throws Exception {
157    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
158    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
159    FileWriter fos = new FileWriter(excludeFile);
160    HRegionServer excludeServer = cluster.getRegionServer(1);
161    String excludeHostname = excludeServer.getServerName().getHostname();
162    int excludeServerPort = excludeServer.getServerName().getPort();
163    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
164    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
165    fos.write(excludeServerName);
166    fos.close();
167    HRegionServer regionServer = cluster.getRegionServer(0);
168    String rsName = regionServer.getServerName().getHostname();
169    int port = regionServer.getServerName().getPort();
170    String rs = rsName + ":" + Integer.toString(port);
171    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
172      .ack(true).excludeFile(excludeFile.getCanonicalPath());
173    try (RegionMover rm = rmBuilder.build()) {
174      rm.unload();
175      LOG.info("Unloading " + rs);
176      assertEquals(0, regionServer.getNumberOfOnlineRegions());
177      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
178      LOG.info("Before:" + regionsExcludeServer + " After:" +
179        cluster.getRegionServer(1).getNumberOfOnlineRegions());
180    }
181  }
182
183  @Test
184  public void testDesignatedFile() throws Exception{
185    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
186    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
187      "designated_file");
188    HRegionServer designatedServer = cluster.getRegionServer(0);
189    try(FileWriter fos = new FileWriter(designatedFile)) {
190      String designatedHostname = designatedServer.getServerName().getHostname();
191      int designatedServerPort = designatedServer.getServerName().getPort();
192      String excludeServerName = designatedHostname + ":" + designatedServerPort;
193      fos.write(excludeServerName);
194    }
195    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
196    HRegionServer regionServer = cluster.getRegionServer(1);
197    String rsName = regionServer.getServerName().getHostname();
198    int port = regionServer.getServerName().getPort();
199    String rs = rsName + ":" + port;
200    int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
201    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
202      .designatedFile(designatedFile.getCanonicalPath());
203    try (RegionMover rm = rmBuilder.build()) {
204      LOG.debug("Unloading {} regions", rs);
205      rm.unload();
206      assertEquals(0, regionServer.getNumberOfOnlineRegions());
207      assertEquals(regionsInDesignatedServer + regionsInRegionServer,
208        designatedServer.getNumberOfOnlineRegions());
209      LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
210        designatedServer.getNumberOfOnlineRegions());
211    }
212  }
213
214  @Test
215  public void testExcludeAndDesignated() throws Exception{
216    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
217    // create designated file
218    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
219      "designated_file");
220    HRegionServer designatedServer = cluster.getRegionServer(0);
221    try(FileWriter fos = new FileWriter(designatedFile)) {
222      String designatedHostname = designatedServer.getServerName().getHostname();
223      int designatedServerPort = designatedServer.getServerName().getPort();
224      String excludeServerName = designatedHostname + ":" + designatedServerPort;
225      fos.write(excludeServerName);
226    }
227    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
228    // create exclude file
229    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
230    HRegionServer excludeServer = cluster.getRegionServer(1);
231    try(FileWriter fos = new FileWriter(excludeFile)) {
232      String excludeHostname = excludeServer.getServerName().getHostname();
233      int excludeServerPort = excludeServer.getServerName().getPort();
234      String excludeServerName = excludeHostname + ":" + excludeServerPort;
235      fos.write(excludeServerName);
236    }
237    int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
238
239    HRegionServer targetRegionServer = cluster.getRegionServer(2);
240    String rsName = targetRegionServer.getServerName().getHostname();
241    int port = targetRegionServer.getServerName().getPort();
242    String rs = rsName + ":" + port;
243    int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
244
245    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
246      .designatedFile(designatedFile.getCanonicalPath())
247      .excludeFile(excludeFile.getCanonicalPath());
248    try (RegionMover rm = rmBuilder.build()) {
249      LOG.debug("Unloading {}", rs);
250      rm.unload();
251      assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
252      assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
253        designatedServer.getNumberOfOnlineRegions());
254      LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
255        designatedServer.getNumberOfOnlineRegions());
256      assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
257      LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
258        excludeServer.getNumberOfOnlineRegions());
259    }
260  }
261
262  @Test
263  public void testRegionServerPort() {
264    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
265    HRegionServer regionServer = cluster.getRegionServer(0);
266    String rsName = regionServer.getServerName().getHostname();
267
268    final int PORT = 16021;
269    Configuration conf = TEST_UTIL.getConfiguration();
270    String originalPort = conf.get(HConstants.REGIONSERVER_PORT);
271    conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT));
272    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf);
273    assertEquals(PORT, rmBuilder.port);
274    if (originalPort != null) {
275      conf.set(HConstants.REGIONSERVER_PORT, originalPort);
276    }
277  }
278
279  /**
280   * UT for HBASE-21746
281   */
282  @Test
283  public void testLoadMetaRegion() throws Exception {
284    HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
285      .map(t -> t.getRegionServer())
286      .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get();
287    int onlineRegions = rsWithMeta.getNumberOfOnlineRegions();
288    String rsName = rsWithMeta.getServerName().getAddress().toString();
289    try (RegionMover rm =
290      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) {
291      LOG.info("Unloading " + rsWithMeta.getServerName());
292      rm.unload();
293      assertEquals(0, rsWithMeta.getNumberOfOnlineRegions());
294      LOG.info("Loading " + rsWithMeta.getServerName());
295      rm.load();
296      assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions());
297    }
298  }
299
300  /**
301   * UT for HBASE-21746
302   */
303  @Test
304  public void testTargetServerDeadWhenLoading() throws Exception {
305    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
306    String rsName = rs.getServerName().getAddress().toString();
307    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
308    // wait 5 seconds at most
309    conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5);
310    String filename =
311      new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString();
312    // unload the region server
313    try (RegionMover rm =
314      new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) {
315      LOG.info("Unloading " + rs.getServerName());
316      rm.unload();
317      assertEquals(0, rs.getNumberOfOnlineRegions());
318    }
319    String inexistRsName = "whatever:123";
320    try (RegionMover rm =
321      new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) {
322      // load the regions to an inexist region server, which should fail and return false
323      LOG.info("Loading to an inexist region server {}", inexistRsName);
324      assertFalse(rm.load());
325    }
326  }
327
328  @Test
329  public void testDecomServerExclusionWithAck() throws Exception {
330    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
331    HRegionServer excludeServer = cluster.getRegionServer(1);
332    List<HRegion> regions = excludeServer.getRegions();
333    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
334    TEST_UTIL.getAdmin().decommissionRegionServers(
335      Collections.singletonList(excludeServer.getServerName()), false);
336
337    waitForServerDecom(excludeServer);
338
339    HRegionServer regionServer = cluster.getRegionServer(0);
340    String rsName = regionServer.getServerName().getHostname();
341    int port = regionServer.getServerName().getPort();
342    String hostname = rsName + ":" + Integer.toString(port);
343    RegionMoverBuilder rmBuilder =
344      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration())
345        .ack(true);
346
347    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
348    int sourceServerRegions = regionServer.getRegions().size();
349
350    try (RegionMover regionMover = rmBuilder.build()) {
351      Assert.assertTrue(regionMover.unload());
352      LOG.info("Unloading {}", hostname);
353      assertEquals(0, regionServer.getNumberOfOnlineRegions());
354      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
355      LOG.info("Before:" + regionsExcludeServer + " After:" +
356        cluster.getRegionServer(1).getNumberOfOnlineRegions());
357      List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
358      int index = 0;
359      for (HRegion hRegion : regionList) {
360        Assert.assertEquals(hRegion, regions.get(index++));
361      }
362      Assert.assertEquals(targetServerRegions + sourceServerRegions,
363        cluster.getRegionServer(2).getNumberOfOnlineRegions());
364      Assert.assertTrue(regionMover.load());
365    }
366
367    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
368      Collections.emptyList());
369  }
370
371  private void waitForServerDecom(HRegionServer excludeServer) {
372
373    TEST_UTIL.waitFor(3000, () -> {
374      try {
375        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
376        return decomServers.size() == 1
377          && decomServers.get(0).equals(excludeServer.getServerName());
378      } catch (IOException e) {
379        throw new RuntimeException(e);
380      }
381    });
382  }
383
384  @Test
385  public void testDecomServerExclusion() throws Exception {
386    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
387    HRegionServer excludeServer = cluster.getRegionServer(0);
388    List<HRegion> regions = excludeServer.getRegions();
389    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
390    TEST_UTIL.getAdmin().decommissionRegionServers(
391      Collections.singletonList(excludeServer.getServerName()), false);
392
393    waitForServerDecom(excludeServer);
394
395    HRegionServer sourceRegionServer = cluster.getRegionServer(1);
396    String rsName = sourceRegionServer.getServerName().getHostname();
397    int port = sourceRegionServer.getServerName().getPort();
398    String hostname = rsName + ":" + Integer.toString(port);
399    RegionMoverBuilder rmBuilder =
400      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);
401
402    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
403    int sourceServerRegions = sourceRegionServer.getRegions().size();
404
405    try (RegionMover regionMover = rmBuilder.build()) {
406      Assert.assertTrue(regionMover.unload());
407      LOG.info("Unloading {}", hostname);
408      assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
409      assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
410      LOG.info("Before:" + regionsExcludeServer + " After:" +
411        cluster.getRegionServer(1).getNumberOfOnlineRegions());
412      List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
413      int index = 0;
414      for (HRegion hRegion : regionList) {
415        Assert.assertEquals(hRegion, regions.get(index++));
416      }
417      Assert.assertEquals(targetServerRegions + sourceServerRegions,
418        cluster.getRegionServer(2).getNumberOfOnlineRegions());
419      Assert.assertTrue(regionMover.load());
420    }
421
422    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
423      Collections.emptyList());
424  }
425
426  @Test
427  public void testExcludeAndDecomServers() throws Exception {
428    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
429    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
430    FileWriter fos = new FileWriter(excludeFile);
431    HRegionServer excludeServer = cluster.getRegionServer(1);
432    String excludeHostname = excludeServer.getServerName().getHostname();
433    int excludeServerPort = excludeServer.getServerName().getPort();
434    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
435    fos.write(excludeServerName);
436    fos.close();
437
438    HRegionServer decomServer = cluster.getRegionServer(2);
439    TEST_UTIL.getAdmin().decommissionRegionServers(
440      Collections.singletonList(decomServer.getServerName()), false);
441
442    waitForServerDecom(decomServer);
443
444    HRegionServer regionServer = cluster.getRegionServer(0);
445    String rsName = regionServer.getServerName().getHostname();
446    int port = regionServer.getServerName().getPort();
447    String sourceServer = rsName + ":" + Integer.toString(port);
448    RegionMoverBuilder rmBuilder =
449      new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration())
450        .ack(true)
451        .excludeFile(excludeFile.getCanonicalPath());
452    try (RegionMover regionMover = rmBuilder.build()) {
453      Assert.assertFalse(regionMover.unload());
454    }
455
456    TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(),
457      Collections.emptyList());
458  }
459
460}