001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.PRIMARY;
021import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.SECONDARY;
022import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.TERTIARY;
023import static org.junit.Assert.assertArrayEquals;
024import static org.junit.Assert.assertEquals;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028
029import java.io.IOException;
030import java.net.InetSocketAddress;
031import java.util.List;
032import java.util.Map;
033import java.util.concurrent.TimeUnit;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.HRegionInfo;
039import org.apache.hadoop.hbase.HRegionLocation;
040import org.apache.hadoop.hbase.MetaTableAccessor;
041import org.apache.hadoop.hbase.NamespaceDescriptor;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.Waiter;
045import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
046import org.apache.hadoop.hbase.favored.FavoredNodesManager;
047import org.apache.hadoop.hbase.master.LoadBalancer;
048import org.apache.hadoop.hbase.master.ServerManager;
049import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
050import org.apache.hadoop.hbase.master.balancer.LoadOnlyFavoredStochasticBalancer;
051import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
052import org.apache.hadoop.hbase.regionserver.HRegionServer;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.MediumTests;
055import org.apache.hadoop.hbase.util.Bytes;
056import org.apache.hadoop.hbase.util.JVMClusterUtil;
057import org.apache.hadoop.hbase.util.Threads;
058import org.junit.AfterClass;
059import org.junit.Before;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Rule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.junit.rules.TestName;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
070import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
071
072@Category({ClientTests.class, MediumTests.class})
073public class TestTableFavoredNodes {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077      HBaseClassTestRule.forClass(TestTableFavoredNodes.class);
078
079  private static final Logger LOG = LoggerFactory.getLogger(TestTableFavoredNodes.class);
080
081  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
082  private final static int WAIT_TIMEOUT = 60000;
083  private final static int SLAVES = 8;
084  private FavoredNodesManager fnm;
085  private Admin admin;
086
087  private final byte[][] splitKeys = new byte[][] {Bytes.toBytes(1), Bytes.toBytes(9)};
088  private final int NUM_REGIONS = splitKeys.length + 1;
089
090  @Rule
091  public TestName name = new TestName();
092
093  @BeforeClass
094  public static void setupBeforeClass() throws Exception {
095    Configuration conf = TEST_UTIL.getConfiguration();
096    // Setting FavoredNodeBalancer will enable favored nodes
097    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
098        LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class);
099    conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "" + SLAVES);
100
101    // This helps test if RS get the appropriate FN updates.
102    conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none");
103    TEST_UTIL.startMiniCluster(SLAVES);
104    TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(WAIT_TIMEOUT);
105  }
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    TEST_UTIL.shutdownMiniCluster();
110    TEST_UTIL.cleanupTestDir();
111  }
112
113  @Before
114  public void setup() throws IOException {
115    fnm = TEST_UTIL.getMiniHBaseCluster().getMaster().getFavoredNodesManager();
116    admin = TEST_UTIL.getAdmin();
117    admin.setBalancerRunning(false, true);
118    admin.enableCatalogJanitor(false);
119  }
120
121  /*
122   * Create a table with FN enabled and check if all its regions have favored nodes set.
123   */
124  @Test
125  public void testCreateTable() throws Exception {
126    final TableName tableName = TableName.valueOf(name.getMethodName());
127    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
128    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
129
130    // All regions should have favored nodes
131    checkIfFavoredNodeInformationIsCorrect(tableName);
132
133    List<HRegionInfo> regions = admin.getTableRegions(tableName);
134
135    TEST_UTIL.deleteTable(tableName);
136
137    checkNoFNForDeletedTable(regions);
138  }
139
140  /*
141   * Checks if favored node information is removed on table truncation.
142   */
143  @Test
144  public void testTruncateTable() throws Exception {
145    final TableName tableName = TableName.valueOf(name.getMethodName());
146    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
147    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
148
149    // All regions should have favored nodes
150    checkIfFavoredNodeInformationIsCorrect(tableName);
151
152    List<HRegionInfo> regions = admin.getTableRegions(tableName);
153    TEST_UTIL.truncateTable(tableName, true);
154
155    checkNoFNForDeletedTable(regions);
156    checkIfFavoredNodeInformationIsCorrect(tableName);
157
158    regions = admin.getTableRegions(tableName);
159    TEST_UTIL.truncateTable(tableName, false);
160    checkNoFNForDeletedTable(regions);
161
162    TEST_UTIL.deleteTable(tableName);
163  }
164
165  /*
166   * Check if daughters inherit at-least 2 FN from parent after region split.
167   */
168  @Test
169  public void testSplitTable() throws Exception {
170    final TableName tableName = TableName.valueOf(name.getMethodName());
171    Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
172    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
173    final int numberOfRegions = admin.getTableRegions(t.getName()).size();
174
175    checkIfFavoredNodeInformationIsCorrect(tableName);
176
177    byte[] splitPoint = Bytes.toBytes(0);
178    RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
179    HRegionInfo parent = locator.getRegionLocation(splitPoint).getRegionInfo();
180    List<ServerName> parentFN = fnm.getFavoredNodes(parent);
181    assertNotNull("FN should not be null for region: " + parent, parentFN);
182
183    LOG.info("SPLITTING TABLE");
184    admin.split(tableName, splitPoint);
185
186    TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT);
187    LOG.info("FINISHED WAITING ON RIT");
188    waitUntilTableRegionCountReached(tableName, numberOfRegions + 1);
189
190    // All regions should have favored nodes    checkIfFavoredNodeInformationIsCorrect(tableName);
191
192    // Get the daughters of parent.
193    HRegionInfo daughter1 = locator.getRegionLocation(parent.getStartKey(), true).getRegionInfo();
194    List<ServerName> daughter1FN = fnm.getFavoredNodes(daughter1);
195
196    HRegionInfo daughter2 = locator.getRegionLocation(splitPoint, true).getRegionInfo();
197    List<ServerName> daughter2FN = fnm.getFavoredNodes(daughter2);
198
199    checkIfDaughterInherits2FN(parentFN, daughter1FN);
200    checkIfDaughterInherits2FN(parentFN, daughter2FN);
201
202    assertEquals("Daughter's PRIMARY FN should be PRIMARY of parent",
203        parentFN.get(PRIMARY.ordinal()), daughter1FN.get(PRIMARY.ordinal()));
204    assertEquals("Daughter's SECONDARY FN should be SECONDARY of parent",
205        parentFN.get(SECONDARY.ordinal()), daughter1FN.get(SECONDARY.ordinal()));
206
207    assertEquals("Daughter's PRIMARY FN should be PRIMARY of parent",
208        parentFN.get(PRIMARY.ordinal()), daughter2FN.get(PRIMARY.ordinal()));
209    assertEquals("Daughter's SECONDARY FN should be TERTIARY of parent",
210        parentFN.get(TERTIARY.ordinal()), daughter2FN.get(SECONDARY.ordinal()));
211
212    // Major compact table and run catalog janitor. Parent's FN should be removed
213    TEST_UTIL.getMiniHBaseCluster().compact(tableName, true);
214    admin.runCatalogScan();
215    // Catalog cleanup is async. Wait on procedure to finish up.
216    ProcedureTestingUtility.waitAllProcedures(
217        TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
218    // assertEquals("Parent region should have been cleaned", 1, admin.runCatalogScan());
219    assertNull("Parent FN should be null", fnm.getFavoredNodes(parent));
220
221    List<HRegionInfo> regions = admin.getTableRegions(tableName);
222    // Split and Table Disable interfere with each other around region replicas
223    // TODO. Meantime pause a few seconds.
224    Threads.sleep(2000);
225    LOG.info("STARTING DELETE");
226    TEST_UTIL.deleteTable(tableName);
227
228    checkNoFNForDeletedTable(regions);
229  }
230
231  /*
232   * Check if merged region inherits FN from one of its regions.
233   */
234  @Test
235  public void testMergeTable() throws Exception {
236    final TableName tableName = TableName.valueOf(name.getMethodName());
237    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
238    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
239
240    checkIfFavoredNodeInformationIsCorrect(tableName);
241
242    RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
243    HRegionInfo regionA = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
244    HRegionInfo regionB = locator.getRegionLocation(splitKeys[0]).getRegionInfo();
245
246    List<ServerName> regionAFN = fnm.getFavoredNodes(regionA);
247    LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA));
248    LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB));
249
250    int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName);
251    admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(),
252        regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS);
253
254    TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT);
255    waitUntilTableRegionCountReached(tableName, countOfRegions - 1);
256
257    // All regions should have favored nodes
258    checkIfFavoredNodeInformationIsCorrect(tableName);
259
260    HRegionInfo mergedRegion =
261      locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
262    List<ServerName> mergedFN = fnm.getFavoredNodes(mergedRegion);
263
264    assertArrayEquals("Merged region doesn't match regionA's FN",
265        regionAFN.toArray(), mergedFN.toArray());
266
267    // Major compact table and run catalog janitor. Parent FN should be removed
268    TEST_UTIL.getMiniHBaseCluster().compact(tableName, true);
269    assertEquals("Merge parents should have been cleaned", 1, admin.runCatalogScan());
270    // Catalog cleanup is async. Wait on procedure to finish up.
271    ProcedureTestingUtility.waitAllProcedures(
272        TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
273    assertNull("Parent FN should be null", fnm.getFavoredNodes(regionA));
274    assertNull("Parent FN should be null", fnm.getFavoredNodes(regionB));
275
276    List<HRegionInfo> regions = admin.getTableRegions(tableName);
277
278    TEST_UTIL.deleteTable(tableName);
279
280    checkNoFNForDeletedTable(regions);
281  }
282
283  private void checkNoFNForDeletedTable(List<HRegionInfo> regions) {
284    for (HRegionInfo region : regions) {
285      LOG.info("Testing if FN data for " + region);
286      assertNull("FN not null for deleted table's region: " + region, fnm.getFavoredNodes(region));
287    }
288  }
289
290  /*
291   * This checks the following:
292   *
293   * 1. Do all regions of the table have favored nodes updated in master?
294   * 2. Is the number of favored nodes correct for a region? Is the start code -1?
295   * 3. Is the FN information consistent between Master and the respective RegionServer?
296   */
297  private void checkIfFavoredNodeInformationIsCorrect(TableName tableName) throws Exception {
298
299    /*
300     * Since we need HRegionServer to check for consistency of FN between Master and RS,
301     * lets construct a map for each serverName lookup. Makes it easy later.
302     */
303    Map<ServerName, HRegionServer> snRSMap = Maps.newHashMap();
304    for (JVMClusterUtil.RegionServerThread rst :
305      TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) {
306      snRSMap.put(rst.getRegionServer().getServerName(), rst.getRegionServer());
307    }
308    // Also include master, since it can also host user regions.
309    for (JVMClusterUtil.MasterThread rst :
310      TEST_UTIL.getMiniHBaseCluster().getLiveMasterThreads()) {
311      snRSMap.put(rst.getMaster().getServerName(), rst.getMaster());
312    }
313
314    int dnPort = fnm.getDataNodePort();
315    RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableName);
316    for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) {
317
318      HRegionInfo regionInfo = regionLocation.getRegionInfo();
319      List<ServerName> fnList = fnm.getFavoredNodes(regionInfo);
320
321      // 1. Does each region have favored node?
322      assertNotNull("Favored nodes should not be null for region:" + regionInfo, fnList);
323
324      // 2. Do we have the right number of favored nodes? Is start code -1?
325      assertEquals("Incorrect favored nodes for region:" + regionInfo + " fnlist: " + fnList,
326        FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, fnList.size());
327      for (ServerName sn : fnList) {
328        assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode());
329      }
330
331      // 3. Check if the regionServers have all the FN updated and in sync with Master
332      HRegionServer regionServer = snRSMap.get(regionLocation.getServerName());
333      assertNotNull("RS should not be null for regionLocation: " + regionLocation, regionServer);
334
335      InetSocketAddress[] rsFavNodes =
336        regionServer.getFavoredNodesForRegion(regionInfo.getEncodedName());
337      assertNotNull("RS " + regionLocation.getServerName()
338        + " does not have FN for region: " + regionInfo, rsFavNodes);
339      assertEquals("Incorrect FN for region:" + regionInfo.getEncodedName() + " on server:" +
340        regionLocation.getServerName(), FavoredNodeAssignmentHelper.FAVORED_NODES_NUM,
341        rsFavNodes.length);
342
343      // 4. Does DN port match all FN node list?
344      for (ServerName sn : fnm.getFavoredNodesWithDNPort(regionInfo)) {
345        assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode());
346        assertEquals("FN port should belong to DN port, fnlist:" + fnList, dnPort, sn.getPort());
347      }
348    }
349  }
350
351  /*
352   * Check favored nodes for system tables
353   */
354  @Test
355  public void testSystemTables() throws Exception {
356    final TableName tableName = TableName.valueOf(name.getMethodName());
357    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys);
358    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
359
360    // All regions should have favored nodes
361    checkIfFavoredNodeInformationIsCorrect(tableName);
362
363    for (TableName sysTable :
364        admin.listTableNamesByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
365      List<HRegionInfo> regions = admin.getTableRegions(sysTable);
366      for (HRegionInfo region : regions) {
367        assertNull("FN should be null for sys region", fnm.getFavoredNodes(region));
368      }
369    }
370
371    TEST_UTIL.deleteTable(tableName);
372  }
373
374  private void checkIfDaughterInherits2FN(List<ServerName> parentFN, List<ServerName> daughterFN) {
375
376    assertNotNull(parentFN);
377    assertNotNull(daughterFN);
378
379    List<ServerName> favoredNodes = Lists.newArrayList(daughterFN);
380    favoredNodes.removeAll(parentFN);
381
382    /*
383     * With a small cluster its likely some FN might accidentally get shared. Its likely the
384     * 3rd FN the balancer chooses might still belong to the parent in which case favoredNodes
385     * size would be 0.
386     */
387    assertTrue("Daughter FN:" + daughterFN + " should have inherited 2 FN from parent FN:"
388      + parentFN, favoredNodes.size() <= 1);
389  }
390
391  private void waitUntilTableRegionCountReached(final TableName tableName, final int numRegions)
392      throws Exception {
393    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
394      @Override
395      public boolean evaluate() throws Exception {
396        return MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName) == numRegions;
397      }
398    });
399  }
400}