001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.net.InetSocketAddress;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.Random;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CatalogFamilyFormat;
038import org.apache.hadoop.hbase.ClientMetaTableAccessor;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HRegionLocation;
043import org.apache.hadoop.hbase.MetaTableAccessor;
044import org.apache.hadoop.hbase.NamespaceDescriptor;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.RegionLocator;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
057import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer;
058import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
059import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
060import org.apache.hadoop.hbase.regionserver.HRegion;
061import org.apache.hadoop.hbase.regionserver.HRegionServer;
062import org.apache.hadoop.hbase.regionserver.Region;
063import org.apache.hadoop.hbase.testclassification.MasterTests;
064import org.apache.hadoop.hbase.testclassification.MediumTests;
065import org.apache.hadoop.hbase.util.Bytes;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.Pair;
068import org.apache.zookeeper.KeeperException;
069import org.junit.AfterClass;
070import org.junit.BeforeClass;
071import org.junit.ClassRule;
072import org.junit.Ignore;
073import org.junit.Test;
074import org.junit.experimental.categories.Category;
075import org.slf4j.Logger;
076import org.slf4j.LoggerFactory;
077
078@Category({MasterTests.class, MediumTests.class})
079public class TestRegionPlacement {
080
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083      HBaseClassTestRule.forClass(TestRegionPlacement.class);
084
085  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
086  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
087  private final static int SLAVES = 10;
088  private static Connection CONNECTION;
089  private static Admin admin;
090  private static RegionPlacementMaintainer rp;
091  private static Position[] positions = Position.values();
092  private int lastRegionOnPrimaryRSCount = 0;
093  private int REGION_NUM = 10;
094  private Map<RegionInfo, ServerName[]> favoredNodesAssignmentPlan = new HashMap<>();
095
096  @BeforeClass
097  public static void setupBeforeClass() throws Exception {
098    Configuration conf = TEST_UTIL.getConfiguration();
099    // Enable the favored nodes based load balancer
100    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
101        FavoredNodeLoadBalancer.class, LoadBalancer.class);
102    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
103    TEST_UTIL.startMiniCluster(SLAVES);
104    CONNECTION = TEST_UTIL.getConnection();
105    admin = CONNECTION.getAdmin();
106    rp = new RegionPlacementMaintainer(conf);
107  }
108
109  @AfterClass
110  public static void tearDownAfterClass() throws Exception {
111    TEST_UTIL.shutdownMiniCluster();
112  }
113
114  @Ignore ("Test for unfinished feature") @Test
115  public void testRegionPlacement() throws Exception {
116    String tableStr = "testRegionAssignment";
117    TableName table = TableName.valueOf(tableStr);
118    // Create a table with REGION_NUM regions.
119    createTable(table, REGION_NUM);
120
121    TEST_UTIL.waitTableAvailable(table);
122
123    // Verify all the user regions are assigned to the primary region server
124    // based on the plan
125    verifyRegionOnPrimaryRS(REGION_NUM);
126
127    FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
128    // Verify all the region server are update with the latest favored nodes
129    verifyRegionServerUpdated(currentPlan);
130    // Test Case 2: To verify whether the region placement tools can
131    // correctly update the new assignment plan to hbase:meta and Region Server.
132    // The new assignment plan is generated by shuffle the existing assignment
133    // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
134    // Shuffle the plan by switching the secondary region server with
135    // the tertiary.
136
137    // Shuffle the secondary with tertiary favored nodes
138    FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
139        FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
140    // Let the region placement update the hbase:meta and Region Servers
141    rp.updateAssignmentPlan(shuffledPlan);
142
143    // Verify the region assignment. There are supposed to no region reassignment
144    // All the regions are still on the primary region server
145    verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
146
147    // Shuffle the plan by switching the primary with secondary and
148    // verify the region reassignment is consistent with the plan.
149    shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
150        FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
151
152    // Let the region placement update the hbase:meta and Region Servers
153    rp.updateAssignmentPlan(shuffledPlan);
154
155    verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
156
157    // also verify that the AssignmentVerificationReport has the correct information
158    RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
159    // we are interested in only one table (and hence one report)
160    rp.setTargetTableName(new String[]{tableStr});
161    List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
162    AssignmentVerificationReport report = reports.get(0);
163    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
164    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
165    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
166    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
167    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
168    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
169    assertTrue(report.getUnassignedRegions().isEmpty());
170
171    // Check when a RS stops, the regions get assigned to their secondary/tertiary
172    killRandomServerAndVerifyAssignment();
173
174    // also verify that the AssignmentVerificationReport has the correct information
175    reports = rp.verifyRegionPlacement(false);
176    report = reports.get(0);
177    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
178    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
179    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
180    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
181    assertTrue("secondary " +
182    report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
183        + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
184        (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
185        || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
186    assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
187        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
188        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
189    RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
190  }
191
192  private void killRandomServerAndVerifyAssignment()
193      throws IOException, InterruptedException, KeeperException {
194    ServerName serverToKill = null;
195    int killIndex = 0;
196    Random random = new Random(EnvironmentEdgeManager.currentTime());
197    ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
198    LOG.debug("Server holding meta " + metaServer);
199    boolean isNamespaceServer = false;
200    do {
201      // kill a random non-meta server carrying at least one region
202      killIndex = random.nextInt(SLAVES);
203      serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
204      Collection<HRegion> regs =
205          TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
206      isNamespaceServer = false;
207      for (HRegion r : regs) {
208        if (r.getRegionInfo().getTable().getNamespaceAsString()
209            .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
210          isNamespaceServer = true;
211          break;
212        }
213      }
214    } while (ServerName.isSameAddress(metaServer, serverToKill) || isNamespaceServer ||
215        TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
216    LOG.debug("Stopping RS " + serverToKill);
217    Map<RegionInfo, Pair<ServerName, ServerName>> regionsToVerify = new HashMap<>();
218    // mark the regions to track
219    for (Map.Entry<RegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
220      ServerName s = entry.getValue()[0];
221      if (ServerName.isSameAddress(s, serverToKill)) {
222        regionsToVerify.put(entry.getKey(), new Pair<>(entry.getValue()[1], entry.getValue()[2]));
223        LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
224            entry.getValue()[1] + " " + entry.getValue()[2]);
225      }
226    }
227    int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
228    TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
229    TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
230    int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
231    while (curr - orig < regionsToVerify.size()) {
232      LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
233          " Current #regions " + curr + " Original #regions " + orig);
234      Thread.sleep(200);
235      curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
236    }
237    // now verify
238    for (Map.Entry<RegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
239      ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
240          .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
241      Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
242      LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
243          " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
244          + "/" + secondaryTertiaryServers.getSecond());
245      if (!(ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getFirst())||
246          ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getSecond()))){
247        fail("Region " + entry.getKey() + " not present on any of the expected servers");
248      }
249    }
250    // start(reinstate) region server since we killed one before
251    TEST_UTIL.getHBaseCluster().startRegionServer();
252  }
253
254  /**
255   * Used to test the correctness of this class.
256   */
257  @Ignore ("Test for unfinished feature") @Test
258  public void testRandomizedMatrix() {
259    int rows = 100;
260    int cols = 100;
261    float[][] matrix = new float[rows][cols];
262    Random random = new Random();
263    for (int i = 0; i < rows; i++) {
264      for (int j = 0; j < cols; j++) {
265        matrix[i][j] = random.nextFloat();
266      }
267    }
268
269    // Test that inverting a transformed matrix gives the original matrix.
270    RegionPlacementMaintainer.RandomizedMatrix rm =
271      new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
272    float[][] transformed = rm.transform(matrix);
273    float[][] invertedTransformed = rm.invert(transformed);
274    for (int i = 0; i < rows; i++) {
275      for (int j = 0; j < cols; j++) {
276        if (matrix[i][j] != invertedTransformed[i][j]) {
277          throw new RuntimeException();
278        }
279      }
280    }
281
282    // Test that the indices on a transformed matrix can be inverted to give
283    // the same values on the original matrix.
284    int[] transformedIndices = new int[rows];
285    for (int i = 0; i < rows; i++) {
286      transformedIndices[i] = random.nextInt(cols);
287    }
288    int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
289    float[] transformedValues = new float[rows];
290    float[] invertedTransformedValues = new float[rows];
291    for (int i = 0; i < rows; i++) {
292      transformedValues[i] = transformed[i][transformedIndices[i]];
293      invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
294    }
295    Arrays.sort(transformedValues);
296    Arrays.sort(invertedTransformedValues);
297    if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
298      throw new RuntimeException();
299    }
300  }
301
302  /**
303   * Shuffle the assignment plan by switching two favored node positions.
304   * @param plan The assignment plan
305   * @param p1 The first switch position
306   * @param p2 The second switch position
307   * @return the shuffled assignment plan
308   */
309  private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
310      FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
311    FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
312
313    Map<String, RegionInfo> regionToHRegion =
314        rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
315    for (Map.Entry<String, List<ServerName>> entry :
316      plan.getAssignmentMap().entrySet()) {
317
318      // copy the server list from the original plan
319      List<ServerName> shuffledServerList = new ArrayList<>();
320      shuffledServerList.addAll(entry.getValue());
321
322      // start to shuffle
323      shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
324      shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
325
326      // update the plan
327      shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
328    }
329    return shuffledPlan;
330  }
331
332  /**
333   * To verify the region assignment status.
334   * It will check the assignment plan consistency between hbase:meta and
335   * region servers.
336   * Also it will verify weather the number of region movement and
337   * the number regions on the primary region server are expected
338   *
339   * @param plan
340   * @param regionMovementNum
341   * @param numRegionsOnPrimaryRS
342   * @throws InterruptedException
343   * @throws IOException
344   */
345  private void verifyRegionAssignment(FavoredNodesPlan plan,
346      int regionMovementNum, int numRegionsOnPrimaryRS)
347  throws InterruptedException, IOException {
348    // Verify the assignment plan in hbase:meta is consistent with the expected plan.
349    verifyMETAUpdated(plan);
350
351    // Verify the number of region movement is expected
352    verifyRegionMovementNum(regionMovementNum);
353
354    // Verify the number of regions is assigned to the primary region server
355    // based on the plan is expected
356    verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
357
358    // Verify all the online region server are updated with the assignment plan
359    verifyRegionServerUpdated(plan);
360  }
361
362  /**
363   * Verify the meta has updated to the latest assignment plan
364   * @param expectedPlan the region assignment plan
365   * @throws IOException if an IO problem is encountered
366   */
367  private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
368  throws IOException {
369    FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
370    assertTrue("The assignment plan is NOT consistent with the expected plan ",
371        planFromMETA.equals(expectedPlan));
372  }
373
374  /**
375   * Verify the number of region movement is expected
376   */
377  private void verifyRegionMovementNum(int expected)
378      throws InterruptedException, IOException {
379    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
380    HMaster m = cluster.getMaster();
381    int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
382    // get the assignments start to execute
383    m.balance();
384
385    int retry = 10;
386    long sleep = 3000;
387    int attempt = 0;
388    int currentRegionOpened, regionMovement;
389    do {
390      currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
391      regionMovement= currentRegionOpened - lastRegionOpenedCount;
392      LOG.debug("There are " + regionMovement + "/" + expected +
393          " regions moved after " + attempt + " attempts");
394      Thread.sleep((++attempt) * sleep);
395    } while (regionMovement != expected && attempt <= retry);
396
397    // update the lastRegionOpenedCount
398    lastRegionOpenedCount = currentRegionOpened;
399
400    assertEquals("There are only " + regionMovement + " instead of "
401          + expected + " region movement for " + attempt + " attempts", expected, regionMovement);
402  }
403
404  /**
405   * Verify the number of user regions is assigned to the primary
406   * region server based on the plan is expected
407   * @param expectedNum the expected number of assigned regions
408   * @throws IOException
409   */
410  private void verifyRegionOnPrimaryRS(int expectedNum)
411      throws IOException {
412    lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
413    assertEquals("Only " +  expectedNum + " of user regions running " +
414        "on the primary region server", expectedNum ,
415        lastRegionOnPrimaryRSCount);
416  }
417
418  /**
419   * Verify all the online region servers has been updated to the
420   * latest assignment plan
421   * @param plan
422   * @throws IOException
423   */
424  private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
425    // Verify all region servers contain the correct favored nodes information
426    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
427    for (int i = 0; i < SLAVES; i++) {
428      HRegionServer rs = cluster.getRegionServer(i);
429      for (Region region: rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
430        InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
431            region.getRegionInfo().getEncodedName());
432        String regionName = region.getRegionInfo().getRegionNameAsString();
433        List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);
434
435        // All regions are supposed to have favored nodes,
436        // except for hbase:meta and ROOT
437        if (favoredServerList == null) {
438          TableDescriptor desc = region.getTableDescriptor();
439          // Verify they are ROOT and hbase:meta regions since no favored nodes
440          assertNull(favoredSocketAddress);
441          assertTrue("User region " +
442              region.getTableDescriptor().getTableName() +
443              " should have favored nodes", desc.isMetaRegion());
444        } else {
445          // For user region, the favored nodes in the region server should be
446          // identical to favored nodes in the assignmentPlan
447          assertTrue(favoredSocketAddress.length == favoredServerList.size());
448          assertTrue(favoredServerList.size() > 0);
449          for (int j = 0; j < favoredServerList.size(); j++) {
450            InetSocketAddress addrFromRS = favoredSocketAddress[j];
451            InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
452                favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
453
454            assertNotNull(addrFromRS);
455            assertNotNull(addrFromPlan);
456            assertTrue("Region server " + rs.getServerName().getAddress()
457                + " has the " + positions[j] +
458                " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
459                addrFromRS + " which is inconsistent with the plan "
460                + addrFromPlan, addrFromRS.equals(addrFromPlan));
461          }
462        }
463      }
464    }
465  }
466
467  /**
468   * Check whether regions are assigned to servers consistent with the explicit
469   * hints that are persisted in the hbase:meta table.
470   * Also keep track of the number of the regions are assigned to the
471   * primary region server.
472   * @return the number of regions are assigned to the primary region server
473   * @throws IOException
474   */
475  private int getNumRegionisOnPrimaryRS() throws IOException {
476    final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
477    final AtomicInteger totalRegionNum = new AtomicInteger(0);
478    LOG.info("The start of region placement verification");
479    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
480      @Override
481      public boolean visit(Result result) throws IOException {
482        try {
483          @SuppressWarnings("deprecation")
484          RegionInfo info = CatalogFamilyFormat.getRegionInfo(result);
485          if(info.getTable().getNamespaceAsString()
486              .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
487            return true;
488          }
489          byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
490              HConstants.SERVER_QUALIFIER);
491          byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
492              FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
493          // Add the favored nodes into assignment plan
494          ServerName[] favoredServerList =
495              FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
496          favoredNodesAssignmentPlan.put(info, favoredServerList);
497
498          Position[] positions = Position.values();
499          if (info != null) {
500            totalRegionNum.incrementAndGet();
501            if (server != null) {
502              ServerName serverName =
503                  ServerName.valueOf(Bytes.toString(server), -1);
504              if (favoredNodes != null) {
505                String placement = "[NOT FAVORED NODE]";
506                for (int i = 0; i < favoredServerList.length; i++) {
507                  if (favoredServerList[i].equals(serverName)) {
508                    placement = positions[i].toString();
509                    if (i == Position.PRIMARY.ordinal()) {
510                      regionOnPrimaryNum.incrementAndGet();
511                    }
512                    break;
513                  }
514                }
515                LOG.info(info.getRegionNameAsString() + " on " +
516                    serverName + " " + placement);
517              } else {
518                LOG.info(info.getRegionNameAsString() + " running on " +
519                    serverName + " but there is no favored region server");
520              }
521            } else {
522              LOG.info(info.getRegionNameAsString() +
523                  " not assigned to any server");
524            }
525          }
526          return true;
527        } catch (RuntimeException e) {
528          LOG.error("Result=" + result);
529          throw e;
530        }
531      }
532    };
533    MetaTableAccessor.fullScanRegions(CONNECTION, visitor);
534    LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
535        totalRegionNum.intValue() + " regions running on the primary" +
536        " region servers" );
537    return regionOnPrimaryNum.intValue() ;
538  }
539
540  /**
541   * Create a table with specified table name and region number.
542   * @param tableName the name of the table to be created
543   * @param regionNum number of regions to create
544   * @throws IOException
545   */
546  private static void createTable(TableName tableName, int regionNum)
547      throws IOException {
548    int expectedRegions = regionNum;
549    byte[][] splitKeys = new byte[expectedRegions - 1][];
550    for (int i = 1; i < expectedRegions; i++) {
551      byte splitKey = (byte) i;
552      splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
553    }
554
555    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
556      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
557    admin.createTable(tableDescriptor, splitKeys);
558
559    try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
560      List<HRegionLocation> regions = r.getAllRegionLocations();
561      assertEquals("Tried to create " + expectedRegions + " regions "
562          + "but only found " + regions.size(), expectedRegions, regions.size());
563    }
564  }
565}