001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022
023import java.io.File;
024import java.io.FileWriter;
025import java.io.IOException;
026import java.util.Collections;
027import java.util.List;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MiniHBaseCluster;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.Waiter.Predicate;
038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.regionserver.HRegion;
042import org.apache.hadoop.hbase.regionserver.HRegionServer;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.testclassification.MiscTests;
045import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059/**
060 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
061 * exclude functionality useful for rack decommissioning
062 */
063@Category({MiscTests.class, LargeTests.class})
064public class TestRegionMover1 {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068    HBaseClassTestRule.forClass(TestRegionMover1.class);
069
070  @Rule
071  public TestName name = new TestName();
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover1.class);
074
075  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
076
077  @BeforeClass
078  public static void setUpBeforeClass() throws Exception {
079    TEST_UTIL.startMiniCluster(3);
080    TEST_UTIL.getAdmin().balancerSwitch(false, true);
081  }
082
083  @AfterClass
084  public static void tearDownAfterClass() throws Exception {
085    TEST_UTIL.shutdownMiniCluster();
086  }
087
088  @Before
089  public void setUp() throws Exception {
090    final TableName tableName = TableName.valueOf(name.getMethodName());
091    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
092      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
093    String startKey = "a";
094    String endKey = "z";
095    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
096  }
097
098  @After
099  public void tearDown() throws Exception {
100    final TableName tableName = TableName.valueOf(name.getMethodName());
101    TEST_UTIL.getAdmin().disableTable(tableName);
102    TEST_UTIL.getAdmin().deleteTable(tableName);
103  }
104
105  @Test
106  public void testWithAck() throws Exception {
107    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
108    HRegionServer regionServer = cluster.getRegionServer(0);
109    String rsName = regionServer.getServerName().getAddress().toString();
110    int numRegions = regionServer.getNumberOfOnlineRegions();
111    RegionMoverBuilder rmBuilder =
112      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8);
113    try (RegionMover rm = rmBuilder.build()) {
114      LOG.info("Unloading " + regionServer.getServerName());
115      rm.unload();
116      assertEquals(0, regionServer.getNumberOfOnlineRegions());
117      LOG.info("Successfully Unloaded\nNow Loading");
118      rm.load();
119      assertEquals(numRegions, regionServer.getNumberOfOnlineRegions());
120      // Repeat the same load. It should be very fast because all regions are already moved.
121      rm.load();
122    }
123  }
124
125  /**
126   * Test to unload a regionserver first and then load it using no Ack mode.
127   */
128  @Test
129  public void testWithoutAck() throws Exception {
130    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
131    HRegionServer regionServer = cluster.getRegionServer(0);
132    String rsName = regionServer.getServerName().getAddress().toString();
133    int numRegions = regionServer.getNumberOfOnlineRegions();
134    RegionMoverBuilder rmBuilder =
135      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false);
136    try (RegionMover rm = rmBuilder.build()) {
137      LOG.info("Unloading " + regionServer.getServerName());
138      rm.unload();
139      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
140        @Override
141        public boolean evaluate() throws Exception {
142          return regionServer.getNumberOfOnlineRegions() == 0;
143        }
144      });
145      LOG.info("Successfully Unloaded\nNow Loading");
146      rm.load();
147      // In UT we only have 10 regions so it is not likely to fail, so here we check for all
148      // regions, in the real production this may not be true.
149      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
150        @Override
151        public boolean evaluate() throws Exception {
152          return regionServer.getNumberOfOnlineRegions() == numRegions;
153        }
154      });
155    }
156  }
157
158  /**
159   * To test that we successfully exclude a server from the unloading process We test for the number
160   * of regions on Excluded server and also test that regions are unloaded successfully
161   */
162  @Test
163  public void testExclude() throws Exception {
164    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
165    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
166    FileWriter fos = new FileWriter(excludeFile);
167    HRegionServer excludeServer = cluster.getRegionServer(1);
168    String excludeHostname = excludeServer.getServerName().getHostname();
169    int excludeServerPort = excludeServer.getServerName().getPort();
170    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
171    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
172    fos.write(excludeServerName);
173    fos.close();
174    HRegionServer regionServer = cluster.getRegionServer(0);
175    String rsName = regionServer.getServerName().getHostname();
176    int port = regionServer.getServerName().getPort();
177    String rs = rsName + ":" + Integer.toString(port);
178    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
179      .ack(true).excludeFile(excludeFile.getCanonicalPath());
180    try (RegionMover rm = rmBuilder.build()) {
181      rm.unload();
182      LOG.info("Unloading " + rs);
183      assertEquals(0, regionServer.getNumberOfOnlineRegions());
184      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
185      LOG.info("Before:" + regionsExcludeServer + " After:" +
186        cluster.getRegionServer(1).getNumberOfOnlineRegions());
187    }
188  }
189
190  @Test
191  public void testDesignatedFile() throws Exception{
192    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
193    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
194      "designated_file");
195    HRegionServer designatedServer = cluster.getRegionServer(0);
196    try(FileWriter fos = new FileWriter(designatedFile)) {
197      String designatedHostname = designatedServer.getServerName().getHostname();
198      int designatedServerPort = designatedServer.getServerName().getPort();
199      String excludeServerName = designatedHostname + ":" + designatedServerPort;
200      fos.write(excludeServerName);
201    }
202    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
203    HRegionServer regionServer = cluster.getRegionServer(1);
204    String rsName = regionServer.getServerName().getHostname();
205    int port = regionServer.getServerName().getPort();
206    String rs = rsName + ":" + port;
207    int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
208    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
209      .designatedFile(designatedFile.getCanonicalPath());
210    try (RegionMover rm = rmBuilder.build()) {
211      LOG.debug("Unloading {} regions", rs);
212      rm.unload();
213      assertEquals(0, regionServer.getNumberOfOnlineRegions());
214      assertEquals(regionsInDesignatedServer + regionsInRegionServer,
215        designatedServer.getNumberOfOnlineRegions());
216      LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
217        designatedServer.getNumberOfOnlineRegions());
218    }
219  }
220
221  @Test
222  public void testExcludeAndDesignated() throws Exception{
223    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
224    // create designated file
225    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
226      "designated_file");
227    HRegionServer designatedServer = cluster.getRegionServer(0);
228    try(FileWriter fos = new FileWriter(designatedFile)) {
229      String designatedHostname = designatedServer.getServerName().getHostname();
230      int designatedServerPort = designatedServer.getServerName().getPort();
231      String excludeServerName = designatedHostname + ":" + designatedServerPort;
232      fos.write(excludeServerName);
233    }
234    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
235    // create exclude file
236    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
237    HRegionServer excludeServer = cluster.getRegionServer(1);
238    try(FileWriter fos = new FileWriter(excludeFile)) {
239      String excludeHostname = excludeServer.getServerName().getHostname();
240      int excludeServerPort = excludeServer.getServerName().getPort();
241      String excludeServerName = excludeHostname + ":" + excludeServerPort;
242      fos.write(excludeServerName);
243    }
244    int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
245
246    HRegionServer targetRegionServer = cluster.getRegionServer(2);
247    String rsName = targetRegionServer.getServerName().getHostname();
248    int port = targetRegionServer.getServerName().getPort();
249    String rs = rsName + ":" + port;
250    int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
251
252    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
253      .designatedFile(designatedFile.getCanonicalPath())
254      .excludeFile(excludeFile.getCanonicalPath());
255    try (RegionMover rm = rmBuilder.build()) {
256      LOG.debug("Unloading {}", rs);
257      rm.unload();
258      assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
259      assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
260        designatedServer.getNumberOfOnlineRegions());
261      LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
262        designatedServer.getNumberOfOnlineRegions());
263      assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
264      LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
265        excludeServer.getNumberOfOnlineRegions());
266    }
267  }
268
269  @Test
270  public void testRegionServerPort() throws Exception {
271    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
272    HRegionServer regionServer = cluster.getRegionServer(0);
273    String rsName = regionServer.getServerName().getHostname();
274
275    final int PORT = 16021;
276    Configuration conf = TEST_UTIL.getConfiguration();
277    String originalPort = conf.get(HConstants.REGIONSERVER_PORT);
278    conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT));
279    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf);
280    assertEquals(PORT, rmBuilder.port);
281    if (originalPort != null) {
282      conf.set(HConstants.REGIONSERVER_PORT, originalPort);
283    }
284  }
285
286  /**
287   * UT for HBASE-21746
288   */
289  @Test
290  public void testLoadMetaRegion() throws Exception {
291    HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
292      .map(t -> t.getRegionServer())
293      .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get();
294    int onlineRegions = rsWithMeta.getNumberOfOnlineRegions();
295    String rsName = rsWithMeta.getServerName().getAddress().toString();
296    try (RegionMover rm =
297      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) {
298      LOG.info("Unloading " + rsWithMeta.getServerName());
299      rm.unload();
300      assertEquals(0, rsWithMeta.getNumberOfOnlineRegions());
301      LOG.info("Loading " + rsWithMeta.getServerName());
302      rm.load();
303      assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions());
304    }
305  }
306
307  /**
308   * UT for HBASE-21746
309   */
310  @Test
311  public void testTargetServerDeadWhenLoading() throws Exception {
312    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
313    String rsName = rs.getServerName().getAddress().toString();
314    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
315    // wait 5 seconds at most
316    conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5);
317    String filename =
318      new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString();
319    // unload the region server
320    try (RegionMover rm =
321      new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) {
322      LOG.info("Unloading " + rs.getServerName());
323      rm.unload();
324      assertEquals(0, rs.getNumberOfOnlineRegions());
325    }
326    String inexistRsName = "whatever:123";
327    try (RegionMover rm =
328      new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) {
329      // load the regions to an inexist region server, which should fail and return false
330      LOG.info("Loading to an inexist region server {}", inexistRsName);
331      assertFalse(rm.load());
332    }
333  }
334
335  @Test
336  public void testDecomServerExclusionWithAck() throws Exception {
337    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
338    HRegionServer excludeServer = cluster.getRegionServer(1);
339    List<HRegion> regions = excludeServer.getRegions();
340    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
341    TEST_UTIL.getAdmin().decommissionRegionServers(
342      Collections.singletonList(excludeServer.getServerName()), false);
343
344    waitForServerDecom(excludeServer);
345
346    HRegionServer regionServer = cluster.getRegionServer(0);
347    String rsName = regionServer.getServerName().getHostname();
348    int port = regionServer.getServerName().getPort();
349    String hostname = rsName + ":" + Integer.toString(port);
350    RegionMoverBuilder rmBuilder =
351      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration())
352        .ack(true);
353
354    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
355    int sourceServerRegions = regionServer.getRegions().size();
356
357    try (RegionMover regionMover = rmBuilder.build()) {
358      Assert.assertTrue(regionMover.unload());
359      LOG.info("Unloading {}", hostname);
360      assertEquals(0, regionServer.getNumberOfOnlineRegions());
361      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
362      LOG.info("Before:" + regionsExcludeServer + " After:" +
363        cluster.getRegionServer(1).getNumberOfOnlineRegions());
364      List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
365      int index = 0;
366      for (HRegion hRegion : regionList) {
367        Assert.assertEquals(hRegion, regions.get(index++));
368      }
369      Assert.assertEquals(targetServerRegions + sourceServerRegions,
370        cluster.getRegionServer(2).getNumberOfOnlineRegions());
371      Assert.assertTrue(regionMover.load());
372    }
373
374    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
375      Collections.emptyList());
376  }
377
378  private void waitForServerDecom(HRegionServer excludeServer) {
379
380    TEST_UTIL.waitFor(3000, () -> {
381      try {
382        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
383        return decomServers.size() == 1
384          && decomServers.get(0).equals(excludeServer.getServerName());
385      } catch (IOException e) {
386        throw new RuntimeException(e);
387      }
388    });
389  }
390
391  @Test
392  public void testDecomServerExclusion() throws Exception {
393    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
394    HRegionServer excludeServer = cluster.getRegionServer(0);
395    List<HRegion> regions = excludeServer.getRegions();
396    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
397    TEST_UTIL.getAdmin().decommissionRegionServers(
398      Collections.singletonList(excludeServer.getServerName()), false);
399
400    waitForServerDecom(excludeServer);
401
402    HRegionServer sourceRegionServer = cluster.getRegionServer(1);
403    String rsName = sourceRegionServer.getServerName().getHostname();
404    int port = sourceRegionServer.getServerName().getPort();
405    String hostname = rsName + ":" + Integer.toString(port);
406    RegionMoverBuilder rmBuilder =
407      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);
408
409    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
410    int sourceServerRegions = sourceRegionServer.getRegions().size();
411
412    try (RegionMover regionMover = rmBuilder.build()) {
413      Assert.assertTrue(regionMover.unload());
414      LOG.info("Unloading {}", hostname);
415      assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
416      assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
417      LOG.info("Before:" + regionsExcludeServer + " After:" +
418        cluster.getRegionServer(1).getNumberOfOnlineRegions());
419      List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
420      int index = 0;
421      for (HRegion hRegion : regionList) {
422        Assert.assertEquals(hRegion, regions.get(index++));
423      }
424      Assert.assertEquals(targetServerRegions + sourceServerRegions,
425        cluster.getRegionServer(2).getNumberOfOnlineRegions());
426      Assert.assertTrue(regionMover.load());
427    }
428
429    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
430      Collections.emptyList());
431  }
432
433  @Test
434  public void testExcludeAndDecomServers() throws Exception {
435    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
436    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
437    FileWriter fos = new FileWriter(excludeFile);
438    HRegionServer excludeServer = cluster.getRegionServer(1);
439    String excludeHostname = excludeServer.getServerName().getHostname();
440    int excludeServerPort = excludeServer.getServerName().getPort();
441    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
442    fos.write(excludeServerName);
443    fos.close();
444
445    HRegionServer decomServer = cluster.getRegionServer(2);
446    TEST_UTIL.getAdmin().decommissionRegionServers(
447      Collections.singletonList(decomServer.getServerName()), false);
448
449    waitForServerDecom(decomServer);
450
451    HRegionServer regionServer = cluster.getRegionServer(0);
452    String rsName = regionServer.getServerName().getHostname();
453    int port = regionServer.getServerName().getPort();
454    String sourceServer = rsName + ":" + Integer.toString(port);
455    RegionMoverBuilder rmBuilder =
456      new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration())
457        .ack(true)
458        .excludeFile(excludeFile.getCanonicalPath());
459    try (RegionMover regionMover = rmBuilder.build()) {
460      Assert.assertFalse(regionMover.unload());
461    }
462
463    TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(),
464      Collections.emptyList());
465  }
466
467}