001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.util.List;
023import java.util.concurrent.Callable;
024import java.util.concurrent.ExecutorCompletionService;
025import java.util.concurrent.Future;
026import java.util.concurrent.ThreadPoolExecutor;
027import java.util.concurrent.TimeUnit;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.RegionInfo;
031import org.apache.hadoop.hbase.client.RegionInfoBuilder;
032import org.apache.hadoop.hbase.procedure2.util.StringUtils;
033import org.apache.hadoop.hbase.testclassification.MasterTests;
034import org.apache.hadoop.hbase.testclassification.MediumTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
037import org.apache.hadoop.hbase.util.Threads;
038import org.junit.jupiter.api.AfterAll;
039import org.junit.jupiter.api.AfterEach;
040import org.junit.jupiter.api.BeforeAll;
041import org.junit.jupiter.api.BeforeEach;
042import org.junit.jupiter.api.Tag;
043import org.junit.jupiter.api.Test;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
048
049@Tag(MasterTests.TAG)
050@Tag(MediumTests.TAG)
051public class TestRegionStates {
052
053  private static final Logger LOG = LoggerFactory.getLogger(TestRegionStates.class);
054
055  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
056
057  private static ThreadPoolExecutor threadPool;
058  private static ExecutorCompletionService<Object> executorService;
059
060  @BeforeAll
061  public static void setUp() throws Exception {
062    threadPool = Threads.getBoundedCachedThreadPool(32, 60L, TimeUnit.SECONDS,
063      new ThreadFactoryBuilder().setNameFormat("ProcedureDispatcher-pool-%d").setDaemon(true)
064        .setUncaughtExceptionHandler((t, e) -> LOG.warn("Failed thread " + t.getName(), e))
065        .build());
066    executorService = new ExecutorCompletionService<>(threadPool);
067  }
068
069  @AfterAll
070  public static void tearDown() throws Exception {
071    threadPool.shutdown();
072  }
073
074  @BeforeEach
075  public void testSetup() {
076  }
077
078  @AfterEach
079  public void testTearDown() throws Exception {
080    while (true) {
081      Future<Object> f = executorService.poll();
082      if (f == null) break;
083      f.get();
084    }
085  }
086
087  private static void waitExecutorService(final int count) throws Exception {
088    for (int i = 0; i < count; ++i) {
089      executorService.take().get();
090    }
091  }
092
093  // ==========================================================================
094  // Regions related
095  // ==========================================================================
096
097  @Test
098  public void testRegionDoubleCreation() throws Exception {
099    // NOTE: RegionInfo sort by table first, so we are relying on that
100    final TableName TABLE_NAME_A = TableName.valueOf("testOrderedByTableA");
101    final TableName TABLE_NAME_B = TableName.valueOf("testOrderedByTableB");
102    final TableName TABLE_NAME_C = TableName.valueOf("testOrderedByTableC");
103    final RegionStates stateMap = new RegionStates();
104    final int NRUNS = 1000;
105    final int NSMALL_RUNS = 3;
106
107    // add some regions for table B
108    for (int i = 0; i < NRUNS; ++i) {
109      addRegionNode(stateMap, TABLE_NAME_B, i);
110    }
111    // re-add the regions for table B
112    for (int i = 0; i < NRUNS; ++i) {
113      addRegionNode(stateMap, TABLE_NAME_B, i);
114    }
115    waitExecutorService(NRUNS * 2);
116
117    // add two other tables A and C that will be placed before and after table B (sort order)
118    for (int i = 0; i < NSMALL_RUNS; ++i) {
119      addRegionNode(stateMap, TABLE_NAME_A, i);
120      addRegionNode(stateMap, TABLE_NAME_C, i);
121    }
122    waitExecutorService(NSMALL_RUNS * 2);
123    // check for the list of regions of the 3 tables
124    checkTableRegions(stateMap, TABLE_NAME_A, NSMALL_RUNS);
125    checkTableRegions(stateMap, TABLE_NAME_B, NRUNS);
126    checkTableRegions(stateMap, TABLE_NAME_C, NSMALL_RUNS);
127  }
128
129  private void checkTableRegions(final RegionStates stateMap, final TableName tableName,
130    final int nregions) {
131    List<RegionStateNode> rns = stateMap.getTableRegionStateNodes(tableName);
132    assertEquals(nregions, rns.size());
133    for (int i = 1; i < rns.size(); ++i) {
134      long a = Bytes.toLong(rns.get(i - 1).getRegionInfo().getStartKey());
135      long b = Bytes.toLong(rns.get(i + 0).getRegionInfo().getStartKey());
136      assertEquals(b, a + 1);
137    }
138  }
139
140  private void addRegionNode(final RegionStates stateMap, final TableName tableName,
141    final long regionId) {
142    executorService.submit(new Callable<Object>() {
143      @Override
144      public Object call() {
145        return stateMap.getOrCreateRegionStateNode(
146          RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes(regionId))
147            .setEndKey(Bytes.toBytes(regionId + 1)).setSplit(false).setRegionId(0).build());
148      }
149    });
150  }
151
152  private RegionInfo createRegionInfo(final TableName tableName, final long regionId) {
153    return RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes(regionId))
154      .setEndKey(Bytes.toBytes(regionId + 1)).setSplit(false).setRegionId(0).build();
155  }
156
157  @Test
158  public void testPerf() throws Exception {
159    final TableName TABLE_NAME = TableName.valueOf("testPerf");
160    final int NRUNS = 1000000; // 1M
161    final RegionStates stateMap = new RegionStates();
162
163    long st = EnvironmentEdgeManager.currentTime();
164    for (int i = 0; i < NRUNS; ++i) {
165      final int regionId = i;
166      executorService.submit(new Callable<Object>() {
167        @Override
168        public Object call() {
169          RegionInfo hri = createRegionInfo(TABLE_NAME, regionId);
170          return stateMap.getOrCreateRegionStateNode(hri);
171        }
172      });
173    }
174    waitExecutorService(NRUNS);
175    long et = EnvironmentEdgeManager.currentTime();
176    LOG.info(String.format("PERF STATEMAP INSERT: %s %s/sec", StringUtils.humanTimeDiff(et - st),
177      StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
178
179    st = EnvironmentEdgeManager.currentTime();
180    for (int i = 0; i < NRUNS; ++i) {
181      final int regionId = i;
182      executorService.submit(new Callable<Object>() {
183        @Override
184        public Object call() {
185          RegionInfo hri = createRegionInfo(TABLE_NAME, regionId);
186          return stateMap.getRegionState(hri);
187        }
188      });
189    }
190
191    waitExecutorService(NRUNS);
192    et = EnvironmentEdgeManager.currentTime();
193    LOG.info(String.format("PERF STATEMAP GET: %s %s/sec", StringUtils.humanTimeDiff(et - st),
194      StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
195  }
196
197  @Test
198  public void testPerfSingleThread() {
199    final TableName TABLE_NAME = TableName.valueOf("testPerf");
200    final int NRUNS = 1 * 1000000; // 1M
201
202    final RegionStates stateMap = new RegionStates();
203    long st = EnvironmentEdgeManager.currentTime();
204    for (int i = 0; i < NRUNS; ++i) {
205      stateMap.createRegionStateNode(createRegionInfo(TABLE_NAME, i));
206    }
207    long et = EnvironmentEdgeManager.currentTime();
208    LOG.info(String.format("PERF SingleThread: %s %s/sec", StringUtils.humanTimeDiff(et - st),
209      StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
210  }
211}