Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022
023import java.io.File;
024import java.io.FileWriter;
025import java.io.IOException;
026import java.util.Collections;
027import java.util.List;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MiniHBaseCluster;
035import org.apache.hadoop.hbase.ServerName;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.Waiter.Predicate;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.HRegionServer;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MiscTests;
046import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder;
047import org.junit.AfterClass;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057/**
058 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
059 * exclude functionality useful for rack decommissioning
060 */
061@Category({MiscTests.class, LargeTests.class})
062public class TestRegionMover {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestRegionMover.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover.class);
069
070  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
071
072  @BeforeClass
073  public static void setUpBeforeClass() throws Exception {
074    TEST_UTIL.startMiniCluster(3);
075    TEST_UTIL.getAdmin().balancerSwitch(false, true);
076  }
077
078  @AfterClass
079  public static void tearDownAfterClass() throws Exception {
080    TEST_UTIL.shutdownMiniCluster();
081  }
082
083  @Before
084  public void setUp() throws Exception {
085    // Create a pre-split table just to populate some regions
086    TableName tableName = TableName.valueOf("testRegionMover");
087    Admin admin = TEST_UTIL.getAdmin();
088    if (admin.tableExists(tableName)) {
089      TEST_UTIL.deleteTable(tableName);
090    }
091    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
092      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
093    String startKey = "a";
094    String endKey = "z";
095    admin.createTable(tableDesc, startKey.getBytes(), endKey.getBytes(), 9);
096  }
097
098  @Test
099  public void testWithAck() throws Exception {
100    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
101    HRegionServer regionServer = cluster.getRegionServer(0);
102    String rsName = regionServer.getServerName().getAddress().toString();
103    int numRegions = regionServer.getNumberOfOnlineRegions();
104    RegionMoverBuilder rmBuilder =
105      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8);
106    try (RegionMover rm = rmBuilder.build()) {
107      LOG.info("Unloading " + regionServer.getServerName());
108      rm.unload();
109      assertEquals(0, regionServer.getNumberOfOnlineRegions());
110      LOG.info("Successfully Unloaded\nNow Loading");
111      rm.load();
112      assertEquals(numRegions, regionServer.getNumberOfOnlineRegions());
113      // Repeat the same load. It should be very fast because all regions are already moved.
114      rm.load();
115    }
116  }
117
118  /**
119   * Test to unload a regionserver first and then load it using no Ack mode.
120   */
121  @Test
122  public void testWithoutAck() throws Exception {
123    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
124    HRegionServer regionServer = cluster.getRegionServer(0);
125    String rsName = regionServer.getServerName().getAddress().toString();
126    int numRegions = regionServer.getNumberOfOnlineRegions();
127    RegionMoverBuilder rmBuilder =
128      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false);
129    try (RegionMover rm = rmBuilder.build()) {
130      LOG.info("Unloading " + regionServer.getServerName());
131      rm.unload();
132      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
133        @Override
134        public boolean evaluate() throws Exception {
135          return regionServer.getNumberOfOnlineRegions() == 0;
136        }
137      });
138      LOG.info("Successfully Unloaded\nNow Loading");
139      rm.load();
140      // In UT we only have 10 regions so it is not likely to fail, so here we check for all
141      // regions, in the real production this may not be true.
142      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
143        @Override
144        public boolean evaluate() throws Exception {
145          return regionServer.getNumberOfOnlineRegions() == numRegions;
146        }
147      });
148    }
149  }
150
151  /**
152   * To test that we successfully exclude a server from the unloading process We test for the number
153   * of regions on Excluded server and also test that regions are unloaded successfully
154   */
155  @Test
156  public void testExclude() throws Exception {
157    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
158    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
159    FileWriter fos = new FileWriter(excludeFile);
160    HRegionServer excludeServer = cluster.getRegionServer(1);
161    String excludeHostname = excludeServer.getServerName().getHostname();
162    int excludeServerPort = excludeServer.getServerName().getPort();
163    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
164    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
165    fos.write(excludeServerName);
166    fos.close();
167    HRegionServer regionServer = cluster.getRegionServer(0);
168    String rsName = regionServer.getServerName().getHostname();
169    int port = regionServer.getServerName().getPort();
170    String rs = rsName + ":" + Integer.toString(port);
171    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
172      .ack(true).excludeFile(excludeFile.getCanonicalPath());
173    try (RegionMover rm = rmBuilder.build()) {
174      rm.unload();
175      LOG.info("Unloading " + rs);
176      assertEquals(0, regionServer.getNumberOfOnlineRegions());
177      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
178      LOG.info("Before:" + regionsExcludeServer + " After:" +
179        cluster.getRegionServer(1).getNumberOfOnlineRegions());
180    }
181  }
182
183  @Test
184  public void testRegionServerPort() {
185    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
186    HRegionServer regionServer = cluster.getRegionServer(0);
187    String rsName = regionServer.getServerName().getHostname();
188
189    final int PORT = 16021;
190    Configuration conf = TEST_UTIL.getConfiguration();
191    String originalPort = conf.get(HConstants.REGIONSERVER_PORT);
192    conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT));
193    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf);
194    assertEquals(PORT, rmBuilder.port);
195    if (originalPort != null) {
196      conf.set(HConstants.REGIONSERVER_PORT, originalPort);
197    }
198  }
199
200  /**
201   * UT for HBASE-21746
202   */
203  @Test
204  public void testLoadMetaRegion() throws Exception {
205    HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
206      .map(t -> t.getRegionServer())
207      .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get();
208    int onlineRegions = rsWithMeta.getNumberOfOnlineRegions();
209    String rsName = rsWithMeta.getServerName().getAddress().toString();
210    try (RegionMover rm =
211      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) {
212      LOG.info("Unloading " + rsWithMeta.getServerName());
213      rm.unload();
214      assertEquals(0, rsWithMeta.getNumberOfOnlineRegions());
215      LOG.info("Loading " + rsWithMeta.getServerName());
216      rm.load();
217      assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions());
218    }
219  }
220
221  /**
222   * UT for HBASE-21746
223   */
224  @Test
225  public void testTargetServerDeadWhenLoading() throws Exception {
226    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
227    String rsName = rs.getServerName().getAddress().toString();
228    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
229    // wait 5 seconds at most
230    conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5);
231    String filename =
232      new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString();
233    // unload the region server
234    try (RegionMover rm =
235      new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) {
236      LOG.info("Unloading " + rs.getServerName());
237      rm.unload();
238      assertEquals(0, rs.getNumberOfOnlineRegions());
239    }
240    String inexistRsName = "whatever:123";
241    try (RegionMover rm =
242      new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) {
243      // load the regions to an inexist region server, which should fail and return false
244      LOG.info("Loading to an inexist region server {}", inexistRsName);
245      assertFalse(rm.load());
246    }
247  }
248
249  @Test
250  public void testDecomServerExclusionWithAck() throws Exception {
251    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
252    HRegionServer excludeServer = cluster.getRegionServer(1);
253    List<HRegion> regions = excludeServer.getRegions();
254    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
255    TEST_UTIL.getAdmin().decommissionRegionServers(
256      Collections.singletonList(excludeServer.getServerName()), false);
257
258    waitForServerDecom(excludeServer);
259
260    HRegionServer regionServer = cluster.getRegionServer(0);
261    String rsName = regionServer.getServerName().getHostname();
262    int port = regionServer.getServerName().getPort();
263    String hostname = rsName + ":" + Integer.toString(port);
264    RegionMoverBuilder rmBuilder =
265      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration())
266        .ack(true);
267
268    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
269    int sourceServerRegions = regionServer.getRegions().size();
270
271    try (RegionMover regionMover = rmBuilder.build()) {
272      Assert.assertTrue(regionMover.unload());
273      LOG.info("Unloading {}", hostname);
274      assertEquals(0, regionServer.getNumberOfOnlineRegions());
275      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
276      LOG.info("Before:" + regionsExcludeServer + " After:" +
277        cluster.getRegionServer(1).getNumberOfOnlineRegions());
278      List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
279      int index = 0;
280      for (HRegion hRegion : regionList) {
281        Assert.assertEquals(hRegion, regions.get(index++));
282      }
283      Assert.assertEquals(targetServerRegions + sourceServerRegions,
284        cluster.getRegionServer(2).getNumberOfOnlineRegions());
285      Assert.assertTrue(regionMover.load());
286    }
287
288    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
289      Collections.emptyList());
290  }
291
292  private void waitForServerDecom(HRegionServer excludeServer) {
293
294    TEST_UTIL.waitFor(3000, () -> {
295      try {
296        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
297        return decomServers.size() == 1
298          && decomServers.get(0).equals(excludeServer.getServerName());
299      } catch (IOException e) {
300        throw new RuntimeException(e);
301      }
302    });
303  }
304
305  @Test
306  public void testDecomServerExclusion() throws Exception {
307    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
308    HRegionServer excludeServer = cluster.getRegionServer(0);
309    List<HRegion> regions = excludeServer.getRegions();
310    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
311    TEST_UTIL.getAdmin().decommissionRegionServers(
312      Collections.singletonList(excludeServer.getServerName()), false);
313
314    waitForServerDecom(excludeServer);
315
316    HRegionServer sourceRegionServer = cluster.getRegionServer(1);
317    String rsName = sourceRegionServer.getServerName().getHostname();
318    int port = sourceRegionServer.getServerName().getPort();
319    String hostname = rsName + ":" + Integer.toString(port);
320    RegionMoverBuilder rmBuilder =
321      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);
322
323    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
324    int sourceServerRegions = sourceRegionServer.getRegions().size();
325
326    try (RegionMover regionMover = rmBuilder.build()) {
327      Assert.assertTrue(regionMover.unload());
328      LOG.info("Unloading {}", hostname);
329      assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
330      assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
331      LOG.info("Before:" + regionsExcludeServer + " After:" +
332        cluster.getRegionServer(1).getNumberOfOnlineRegions());
333      List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
334      int index = 0;
335      for (HRegion hRegion : regionList) {
336        Assert.assertEquals(hRegion, regions.get(index++));
337      }
338      Assert.assertEquals(targetServerRegions + sourceServerRegions,
339        cluster.getRegionServer(2).getNumberOfOnlineRegions());
340      Assert.assertTrue(regionMover.load());
341    }
342
343    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
344      Collections.emptyList());
345  }
346
347  @Test
348  public void testExcludeAndDecomServers() throws Exception {
349    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
350    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
351    FileWriter fos = new FileWriter(excludeFile);
352    HRegionServer excludeServer = cluster.getRegionServer(1);
353    String excludeHostname = excludeServer.getServerName().getHostname();
354    int excludeServerPort = excludeServer.getServerName().getPort();
355    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
356    fos.write(excludeServerName);
357    fos.close();
358
359    HRegionServer decomServer = cluster.getRegionServer(2);
360    TEST_UTIL.getAdmin().decommissionRegionServers(
361      Collections.singletonList(decomServer.getServerName()), false);
362
363    waitForServerDecom(decomServer);
364
365    HRegionServer regionServer = cluster.getRegionServer(0);
366    String rsName = regionServer.getServerName().getHostname();
367    int port = regionServer.getServerName().getPort();
368    String sourceServer = rsName + ":" + Integer.toString(port);
369    RegionMoverBuilder rmBuilder =
370      new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration())
371        .ack(true)
372        .excludeFile(excludeFile.getCanonicalPath());
373    try (RegionMover regionMover = rmBuilder.build()) {
374      Assert.assertFalse(regionMover.unload());
375    }
376
377    TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(),
378      Collections.emptyList());
379  }
380
381}