Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.hamcrest.CoreMatchers.instanceOf;
022import static org.hamcrest.MatcherAssert.assertThat;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertFalse;
025import static org.junit.jupiter.api.Assertions.assertNotEquals;
026import static org.junit.jupiter.api.Assertions.assertNotNull;
027import static org.junit.jupiter.api.Assertions.assertTrue;
028import static org.junit.jupiter.api.Assertions.fail;
029
030import java.io.IOException;
031import java.util.ArrayList;
032import java.util.Collections;
033import java.util.List;
034import java.util.concurrent.ExecutionException;
035import java.util.function.Supplier;
036import java.util.stream.Collectors;
037import org.apache.hadoop.hbase.ClientMetaTableAccessor;
038import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.HRegionLocation;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.master.HMaster;
043import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
044import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
045import org.apache.hadoop.hbase.testclassification.ClientTests;
046import org.apache.hadoop.hbase.testclassification.LargeTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.Threads;
049import org.junit.jupiter.api.AfterAll;
050import org.junit.jupiter.api.BeforeAll;
051import org.junit.jupiter.api.Disabled;
052import org.junit.jupiter.api.Tag;
053import org.junit.jupiter.api.TestTemplate;
054
055/**
056 * Class to test asynchronous region admin operations.
057 * @see TestAsyncRegionAdminApi This test and it used to be joined it was taking longer than our ten
058 *      minute timeout so they were split.
059 */
060@Tag(LargeTests.TAG)
061@Tag(ClientTests.TAG)
062@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
063public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase {
064
065  public TestAsyncRegionAdminApi2(Supplier<AsyncAdmin> admin) {
066    super(admin);
067  }
068
069  @BeforeAll
070  public static void setUpBeforeClass() throws Exception {
071    TestAsyncAdminBase.setUpBeforeClass();
072  }
073
074  @AfterAll
075  public static void tearDownAfterClass() throws Exception {
076    TestAsyncAdminBase.tearDownAfterClass();
077  }
078
079  @TestTemplate
080  public void testGetRegionLocation() throws Exception {
081    RawAsyncHBaseAdmin rawAdmin = (RawAsyncHBaseAdmin) ASYNC_CONN.getAdmin();
082    TEST_UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
083    AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(tableName);
084    HRegionLocation regionLocation = locator.getRegionLocation(Bytes.toBytes("mmm")).get();
085    RegionInfo region = regionLocation.getRegion();
086    byte[] regionName = regionLocation.getRegion().getRegionName();
087    HRegionLocation location = rawAdmin.getRegionLocation(regionName).get();
088    assertTrue(Bytes.equals(regionName, location.getRegion().getRegionName()));
089    location = rawAdmin.getRegionLocation(region.getEncodedNameAsBytes()).get();
090    assertTrue(Bytes.equals(regionName, location.getRegion().getRegionName()));
091  }
092
093  @TestTemplate
094  public void testSplitSwitch() throws Exception {
095    createTableWithDefaultConf(tableName);
096    byte[][] families = { FAMILY };
097    final int rows = 10000;
098    TestAsyncRegionAdminApi.loadData(tableName, families, rows);
099
100    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
101    List<HRegionLocation> regionLocations =
102      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
103    int originalCount = regionLocations.size();
104
105    initSplitMergeSwitch();
106    assertTrue(admin.splitSwitch(false).get());
107    try {
108      admin.split(tableName, Bytes.toBytes(rows / 2)).join();
109    } catch (Exception e) {
110      // Expected
111    }
112    int count = admin.getRegions(tableName).get().size();
113    assertTrue(originalCount == count);
114
115    assertFalse(admin.splitSwitch(true).get());
116    admin.split(tableName).join();
117    while ((count = admin.getRegions(tableName).get().size()) == originalCount) {
118      Threads.sleep(100);
119    }
120    assertTrue(originalCount < count);
121  }
122
123  @TestTemplate
124  @Disabled
125  // It was ignored in TestSplitOrMergeStatus, too
126  public void testMergeSwitch() throws Exception {
127    createTableWithDefaultConf(tableName);
128    byte[][] families = { FAMILY };
129    TestAsyncRegionAdminApi.loadData(tableName, families, 1000);
130
131    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
132    List<HRegionLocation> regionLocations =
133      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
134    int originalCount = regionLocations.size();
135
136    initSplitMergeSwitch();
137    admin.split(tableName).join();
138    int postSplitCount = originalCount;
139    while ((postSplitCount = admin.getRegions(tableName).get().size()) == originalCount) {
140      Threads.sleep(100);
141    }
142    assertTrue(originalCount != postSplitCount,
143      "originalCount=" + originalCount + ", postSplitCount=" + postSplitCount);
144
145    // Merge switch is off so merge should NOT succeed.
146    assertTrue(admin.mergeSwitch(false).get());
147    List<RegionInfo> regions = admin.getRegions(tableName).get();
148    assertTrue(regions.size() > 1);
149    admin.mergeRegions(regions.get(0).getRegionName(), regions.get(1).getRegionName(), true).join();
150    int count = admin.getRegions(tableName).get().size();
151    assertTrue(postSplitCount == count, "postSplitCount=" + postSplitCount + ", count=" + count);
152
153    // Merge switch is on so merge should succeed.
154    assertFalse(admin.mergeSwitch(true).get());
155    admin.mergeRegions(regions.get(0).getRegionName(), regions.get(1).getRegionName(), true).join();
156    count = admin.getRegions(tableName).get().size();
157    assertTrue((postSplitCount / 2) == count);
158  }
159
160  private void initSplitMergeSwitch() throws Exception {
161    if (!admin.isSplitEnabled().get()) {
162      admin.splitSwitch(true).get();
163    }
164    if (!admin.isMergeEnabled().get()) {
165      admin.mergeSwitch(true).get();
166    }
167    assertTrue(admin.isSplitEnabled().get());
168    assertTrue(admin.isMergeEnabled().get());
169  }
170
171  @TestTemplate
172  public void testMergeRegions() throws Exception {
173    byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
174    createTableWithDefaultConf(tableName, splitRows);
175
176    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
177    List<HRegionLocation> regionLocations =
178      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
179    RegionInfo regionA;
180    RegionInfo regionB;
181    RegionInfo regionC;
182    RegionInfo mergedChildRegion = null;
183
184    // merge with full name
185    assertEquals(3, regionLocations.size());
186    regionA = regionLocations.get(0).getRegion();
187    regionB = regionLocations.get(1).getRegion();
188    regionC = regionLocations.get(2).getRegion();
189    admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
190
191    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
192
193    assertEquals(2, regionLocations.size());
194    for (HRegionLocation rl : regionLocations) {
195      if (regionC.compareTo(rl.getRegion()) != 0) {
196        mergedChildRegion = rl.getRegion();
197        break;
198      }
199    }
200
201    assertNotNull(mergedChildRegion);
202    // Need to wait GC for merged child region is done.
203    HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
204    CatalogJanitor cj = services.getCatalogJanitor();
205    assertTrue(cj.scan() > 0);
206    // Wait until all procedures settled down
207    while (!services.getMasterProcedureExecutor().getActiveProcIds().isEmpty()) {
208      Thread.sleep(200);
209    }
210    // merge with encoded name
211    admin.mergeRegions(regionC.getRegionName(), mergedChildRegion.getRegionName(), false).get();
212
213    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
214    assertEquals(1, regionLocations.size());
215  }
216
217  @TestTemplate
218  public void testMergeRegionsInvalidRegionCount() throws Exception {
219    byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
220    createTableWithDefaultConf(tableName, splitRows);
221    List<RegionInfo> regions = admin.getRegions(tableName).join();
222    // 0
223    try {
224      admin.mergeRegions(Collections.emptyList(), false).get();
225      fail();
226    } catch (ExecutionException e) {
227      // expected
228      assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
229    }
230    // 1
231    try {
232      admin.mergeRegions(regions.stream().limit(1).map(RegionInfo::getEncodedNameAsBytes)
233        .collect(Collectors.toList()), false).get();
234      fail();
235    } catch (ExecutionException e) {
236      // expected
237      assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
238    }
239  }
240
241  @TestTemplate
242  public void testSplitTable() throws Exception {
243    initSplitMergeSwitch();
244    splitTest(TableName.valueOf("testSplitTable"), 3000, false, null);
245    splitTest(TableName.valueOf("testSplitTableWithSplitPoint"), 3000, false, Bytes.toBytes("3"));
246    splitTest(TableName.valueOf("testSplitTableRegion"), 3000, true, null);
247    splitTest(TableName.valueOf("testSplitTableRegionWithSplitPoint2"), 3000, true,
248      Bytes.toBytes("3"));
249  }
250
251  private void splitTest(TableName tableName, int rowCount, boolean isSplitRegion,
252    byte[] splitPoint) throws Exception {
253    // create table
254    createTableWithDefaultConf(tableName);
255
256    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
257    List<HRegionLocation> regionLocations =
258      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
259    assertEquals(1, regionLocations.size());
260
261    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
262    List<Put> puts = new ArrayList<>();
263    for (int i = 0; i < rowCount; i++) {
264      Put put = new Put(Bytes.toBytes(i));
265      put.addColumn(FAMILY, null, Bytes.toBytes("value" + i));
266      puts.add(put);
267    }
268    table.putAll(puts).join();
269
270    if (isSplitRegion) {
271      if (splitPoint == null) {
272        admin.splitRegion(regionLocations.get(0).getRegion().getRegionName()).get();
273      } else {
274        admin.splitRegion(regionLocations.get(0).getRegion().getRegionName(), splitPoint).get();
275      }
276    } else {
277      if (splitPoint == null) {
278        admin.split(tableName).get();
279      } else {
280        admin.split(tableName, splitPoint).get();
281      }
282    }
283
284    int count = 0;
285    for (int i = 0; i < 45; i++) {
286      try {
287        regionLocations =
288          ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
289        count = regionLocations.size();
290        if (count >= 2) {
291          break;
292        }
293        Thread.sleep(1000L);
294      } catch (Exception e) {
295        LOG.error(e.toString(), e);
296      }
297    }
298    assertEquals(2, count);
299  }
300
301  @TestTemplate
302  public void testTruncateRegion() throws Exception {
303    // Arrange - Create table, insert data, identify region to truncate.
304    final byte[][] splitKeys =
305      new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
306    String family1 = "f1";
307    String family2 = "f2";
308
309    final String[] sFamilies = new String[] { family1, family2 };
310    final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
311    createTableWithDefaultConf(tableName, splitKeys, bFamilies);
312
313    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
314    List<HRegionLocation> regionLocations =
315      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
316    RegionInfo regionToBeTruncated = regionLocations.get(0).getRegion();
317
318    assertEquals(4, regionLocations.size());
319
320    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies);
321    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies);
322    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies);
323    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies);
324    int rowCountBeforeTruncate = TEST_UTIL.countRows(tableName);
325
326    // Act - Truncate the first region
327    admin.truncateRegion(regionToBeTruncated.getRegionName()).get();
328
329    // Assert
330    int rowCountAfterTruncate = TEST_UTIL.countRows(tableName);
331    assertNotEquals(rowCountBeforeTruncate, rowCountAfterTruncate);
332    int expectedRowCount = rowCountBeforeTruncate - 2;// Since region with 2 rows was truncated.
333    assertEquals(expectedRowCount, rowCountAfterTruncate);
334  }
335
336  @TestTemplate
337  public void testTruncateReplicaRegionNotAllowed() throws Exception {
338    // Arrange - Create table, insert data, identify region to truncate.
339    final byte[][] splitKeys =
340      new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
341    String family1 = "f1";
342    String family2 = "f2";
343
344    final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
345    createTableWithDefaultConf(tableName, 2, splitKeys, bFamilies);
346
347    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
348    List<HRegionLocation> regionLocations =
349      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
350    RegionInfo primaryRegion = regionLocations.get(0).getRegion();
351
352    RegionInfo firstReplica = RegionReplicaUtil.getRegionInfoForReplica(primaryRegion, 1);
353
354    // Act - Truncate the first region
355    try {
356      admin.truncateRegion(firstReplica.getRegionName()).get();
357    } catch (Exception e) {
358      // Assert
359      assertEquals(
360        "Can't truncate replicas directly.Replicas are auto-truncated "
361          + "when their primary is truncated.",
362        e.getCause().getMessage(), "Expected message is different");
363    }
364  }
365
366  @TestTemplate
367  public void testTruncateRegionsMetaTableRegionsNotAllowed() throws Exception {
368    AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(META_TABLE_NAME);
369    List<HRegionLocation> regionLocations = locator.getAllRegionLocations().get();
370    HRegionLocation regionToBeTruncated = regionLocations.get(0);
371    // 1
372    try {
373      admin.truncateRegion(regionToBeTruncated.getRegion().getRegionName()).get();
374      fail();
375    } catch (ExecutionException e) {
376      // expected
377      assertThat(e.getCause(), instanceOf(IOException.class));
378      assertEquals("Can't truncate region in catalog tables", e.getCause().getMessage());
379    }
380  }
381}