001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.normalizer;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.Collections;
024import java.util.Comparator;
025import java.util.List;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HBaseTestingUtility;
028import org.apache.hadoop.hbase.HConstants;
029import org.apache.hadoop.hbase.HTableDescriptor;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.MiniHBaseCluster;
032import org.apache.hadoop.hbase.NamespaceDescriptor;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.master.HMaster;
039import org.apache.hadoop.hbase.master.TableNamespaceManager;
040import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
041import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor;
042import org.apache.hadoop.hbase.quotas.QuotaUtil;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.Region;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
049import org.junit.AfterClass;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059/**
060 * Testing {@link SimpleRegionNormalizer} on minicluster.
061 */
062@Category({MasterTests.class, MediumTests.class})
063public class TestSimpleRegionNormalizerOnCluster {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067      HBaseClassTestRule.forClass(TestSimpleRegionNormalizerOnCluster.class);
068
069  private static final Logger LOG =
070      LoggerFactory.getLogger(TestSimpleRegionNormalizerOnCluster.class);
071  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
072  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
073  private static Admin admin;
074
075  @Rule
076  public TestName name = new TestName();
077
078  @BeforeClass
079  public static void beforeAllTests() throws Exception {
080    // we will retry operations when PleaseHoldException is thrown
081    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
082    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
083
084    // Start a cluster of two regionservers.
085    TEST_UTIL.startMiniCluster(1);
086    TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL);
087    admin = TEST_UTIL.getAdmin();
088  }
089
090  @AfterClass
091  public static void afterAllTests() throws Exception {
092    TEST_UTIL.shutdownMiniCluster();
093  }
094
095  @Test
096  @SuppressWarnings("deprecation")
097  public void testRegionNormalizationSplitOnCluster() throws Exception {
098    testRegionNormalizationSplitOnCluster(false);
099    testRegionNormalizationSplitOnCluster(true);
100  }
101
102  void testRegionNormalizationSplitOnCluster(boolean limitedByQuota) throws Exception {
103    TableName TABLENAME;
104    if (limitedByQuota) {
105      String nsp = "np2";
106      NamespaceDescriptor nspDesc =
107          NamespaceDescriptor.create(nsp)
108          .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
109          .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
110      admin.createNamespace(nspDesc);
111      TABLENAME = TableName.valueOf(nsp +
112        TableName.NAMESPACE_DELIM + name.getMethodName());
113    } else {
114      TABLENAME = TableName.valueOf(name.getMethodName());
115    }
116    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
117    HMaster m = cluster.getMaster();
118
119    try (Table ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
120      // Need to get sorted list of regions here
121      List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
122      Collections.sort(generatedRegions, Comparator.comparing(HRegion::getRegionInfo, RegionInfo.COMPARATOR));
123      HRegion region = generatedRegions.get(0);
124      generateTestData(region, 1);
125      region.flush(true);
126
127      region = generatedRegions.get(1);
128      generateTestData(region, 1);
129      region.flush(true);
130
131      region = generatedRegions.get(2);
132      generateTestData(region, 2);
133      region.flush(true);
134
135      region = generatedRegions.get(3);
136      generateTestData(region, 2);
137      region.flush(true);
138
139      region = generatedRegions.get(4);
140      generateTestData(region, 5);
141      region.flush(true);
142    }
143
144    HTableDescriptor htd = new HTableDescriptor(admin.getTableDescriptor(TABLENAME));
145    htd.setNormalizationEnabled(true);
146    admin.modifyTable(TABLENAME, htd);
147
148    admin.flush(TABLENAME);
149
150    assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
151
152    // Now trigger a split and stop when the split is in progress
153    Thread.sleep(5000); // to let region load to update
154    m.normalizeRegions();
155    if (limitedByQuota) {
156      long skippedSplitcnt = 0;
157      do {
158        skippedSplitcnt = m.getRegionNormalizer().getSkippedCount(PlanType.SPLIT);
159        Thread.sleep(100);
160      } while (skippedSplitcnt == 0L);
161      assert(skippedSplitcnt > 0);
162    } else {
163      while (true) {
164        List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
165        int cnt = 0;
166        for (HRegion region : regions) {
167          String regionName = region.getRegionInfo().getRegionNameAsString();
168          if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
169            cnt++;
170          }
171        }
172        if (cnt >= 2) {
173          break;
174        }
175      }
176    }
177
178    admin.disableTable(TABLENAME);
179    admin.deleteTable(TABLENAME);
180  }
181
182  @Test
183  @SuppressWarnings("deprecation")
184  public void testRegionNormalizationMergeOnCluster() throws Exception {
185    final TableName tableName = TableName.valueOf(name.getMethodName());
186    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
187    HMaster m = cluster.getMaster();
188
189    // create 5 regions with sizes to trigger merge of small regions
190    try (Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILYNAME, 5)) {
191      // Need to get sorted list of regions here
192      List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
193      Collections.sort(generatedRegions, Comparator.comparing(HRegion::getRegionInfo, RegionInfo.COMPARATOR));
194
195      HRegion region = generatedRegions.get(0);
196      generateTestData(region, 1);
197      region.flush(true);
198
199      region = generatedRegions.get(1);
200      generateTestData(region, 1);
201      region.flush(true);
202
203      region = generatedRegions.get(2);
204      generateTestData(region, 3);
205      region.flush(true);
206
207      region = generatedRegions.get(3);
208      generateTestData(region, 3);
209      region.flush(true);
210
211      region = generatedRegions.get(4);
212      generateTestData(region, 5);
213      region.flush(true);
214    }
215
216    HTableDescriptor htd = new HTableDescriptor(admin.getTableDescriptor(tableName));
217    htd.setNormalizationEnabled(true);
218    admin.modifyTable(tableName, htd);
219
220    admin.flush(tableName);
221
222    assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
223
224    // Now trigger a merge and stop when the merge is in progress
225    Thread.sleep(5000); // to let region load to update
226    m.normalizeRegions();
227
228    while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName) > 4) {
229      LOG.info("Waiting for normalization merge to complete");
230      Thread.sleep(100);
231    }
232
233    assertEquals(4, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
234
235    admin.disableTable(tableName);
236    admin.deleteTable(tableName);
237  }
238
239  private void generateTestData(Region region, int numRows) throws IOException {
240    // generating 1Mb values
241    LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024);
242    for (int i = 0; i < numRows; ++i) {
243      byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i));
244      for (int j = 0; j < 1; ++j) {
245        Put put = new Put(key);
246        byte[] col = Bytes.toBytes(String.valueOf(j));
247        byte[] value = dataGenerator.generateRandomSizeValue(key, col);
248        put.addColumn(FAMILYNAME, col, value);
249        region.put(put);
250      }
251    }
252  }
253}