001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.net.InetSocketAddress;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.Random;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HColumnDescriptor;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.HRegionLocation;
042import org.apache.hadoop.hbase.HTableDescriptor;
043import org.apache.hadoop.hbase.MetaTableAccessor;
044import org.apache.hadoop.hbase.MiniHBaseCluster;
045import org.apache.hadoop.hbase.NamespaceDescriptor;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.Connection;
050import org.apache.hadoop.hbase.client.RegionInfo;
051import org.apache.hadoop.hbase.client.RegionLocator;
052import org.apache.hadoop.hbase.client.Result;
053import org.apache.hadoop.hbase.client.TableDescriptor;
054import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
055import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer;
056import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
057import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
058import org.apache.hadoop.hbase.regionserver.HRegion;
059import org.apache.hadoop.hbase.regionserver.HRegionServer;
060import org.apache.hadoop.hbase.regionserver.Region;
061import org.apache.hadoop.hbase.testclassification.MasterTests;
062import org.apache.hadoop.hbase.testclassification.MediumTests;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.apache.hadoop.hbase.util.Pair;
065import org.apache.zookeeper.KeeperException;
066import org.junit.AfterClass;
067import org.junit.BeforeClass;
068import org.junit.ClassRule;
069import org.junit.Ignore;
070import org.junit.Test;
071import org.junit.experimental.categories.Category;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075@Category({MasterTests.class, MediumTests.class})
076public class TestRegionPlacement {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE =
080      HBaseClassTestRule.forClass(TestRegionPlacement.class);
081
082  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
083  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
084  private final static int SLAVES = 10;
085  private static Connection CONNECTION;
086  private static Admin admin;
087  private static RegionPlacementMaintainer rp;
088  private static Position[] positions = Position.values();
089  private int lastRegionOnPrimaryRSCount = 0;
090  private int REGION_NUM = 10;
091  private Map<RegionInfo, ServerName[]> favoredNodesAssignmentPlan = new HashMap<>();
092
093  @BeforeClass
094  public static void setupBeforeClass() throws Exception {
095    Configuration conf = TEST_UTIL.getConfiguration();
096    // Enable the favored nodes based load balancer
097    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
098        FavoredNodeLoadBalancer.class, LoadBalancer.class);
099    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
100    TEST_UTIL.startMiniCluster(SLAVES);
101    CONNECTION = TEST_UTIL.getConnection();
102    admin = CONNECTION.getAdmin();
103    rp = new RegionPlacementMaintainer(conf);
104  }
105
106  @AfterClass
107  public static void tearDownAfterClass() throws Exception {
108    TEST_UTIL.shutdownMiniCluster();
109  }
110
111  @Ignore ("Test for unfinished feature") @Test
112  public void testRegionPlacement() throws Exception {
113    String tableStr = "testRegionAssignment";
114    TableName table = TableName.valueOf(tableStr);
115    // Create a table with REGION_NUM regions.
116    createTable(table, REGION_NUM);
117
118    TEST_UTIL.waitTableAvailable(table);
119
120    // Verify all the user regions are assigned to the primary region server
121    // based on the plan
122    verifyRegionOnPrimaryRS(REGION_NUM);
123
124    FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
125    // Verify all the region server are update with the latest favored nodes
126    verifyRegionServerUpdated(currentPlan);
127    // Test Case 2: To verify whether the region placement tools can
128    // correctly update the new assignment plan to hbase:meta and Region Server.
129    // The new assignment plan is generated by shuffle the existing assignment
130    // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
131    // Shuffle the plan by switching the secondary region server with
132    // the tertiary.
133
134    // Shuffle the secondary with tertiary favored nodes
135    FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
136        FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
137    // Let the region placement update the hbase:meta and Region Servers
138    rp.updateAssignmentPlan(shuffledPlan);
139
140    // Verify the region assignment. There are supposed to no region reassignment
141    // All the regions are still on the primary region server
142    verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
143
144    // Shuffle the plan by switching the primary with secondary and
145    // verify the region reassignment is consistent with the plan.
146    shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
147        FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
148
149    // Let the region placement update the hbase:meta and Region Servers
150    rp.updateAssignmentPlan(shuffledPlan);
151
152    verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
153
154    // also verify that the AssignmentVerificationReport has the correct information
155    RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
156    // we are interested in only one table (and hence one report)
157    rp.setTargetTableName(new String[]{tableStr});
158    List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
159    AssignmentVerificationReport report = reports.get(0);
160    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
161    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
162    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
163    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
164    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
165    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
166    assertTrue(report.getUnassignedRegions().isEmpty());
167
168    // Check when a RS stops, the regions get assigned to their secondary/tertiary
169    killRandomServerAndVerifyAssignment();
170
171    // also verify that the AssignmentVerificationReport has the correct information
172    reports = rp.verifyRegionPlacement(false);
173    report = reports.get(0);
174    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
175    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
176    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
177    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
178    assertTrue("secondary " +
179    report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
180        + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
181        (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
182        || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
183    assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
184        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
185        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
186    RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
187  }
188
189  private void killRandomServerAndVerifyAssignment()
190      throws IOException, InterruptedException, KeeperException {
191    ServerName serverToKill = null;
192    int killIndex = 0;
193    Random random = new Random(System.currentTimeMillis());
194    ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
195    LOG.debug("Server holding meta " + metaServer);
196    boolean isNamespaceServer = false;
197    do {
198      // kill a random non-meta server carrying at least one region
199      killIndex = random.nextInt(SLAVES);
200      serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
201      Collection<HRegion> regs =
202          TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
203      isNamespaceServer = false;
204      for (HRegion r : regs) {
205        if (r.getRegionInfo().getTable().getNamespaceAsString()
206            .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
207          isNamespaceServer = true;
208          break;
209        }
210      }
211    } while (ServerName.isSameAddress(metaServer, serverToKill) || isNamespaceServer ||
212        TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
213    LOG.debug("Stopping RS " + serverToKill);
214    Map<RegionInfo, Pair<ServerName, ServerName>> regionsToVerify = new HashMap<>();
215    // mark the regions to track
216    for (Map.Entry<RegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
217      ServerName s = entry.getValue()[0];
218      if (ServerName.isSameAddress(s, serverToKill)) {
219        regionsToVerify.put(entry.getKey(), new Pair<>(entry.getValue()[1], entry.getValue()[2]));
220        LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
221            entry.getValue()[1] + " " + entry.getValue()[2]);
222      }
223    }
224    int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
225    TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
226    TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
227    int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
228    while (curr - orig < regionsToVerify.size()) {
229      LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
230          " Current #regions " + curr + " Original #regions " + orig);
231      Thread.sleep(200);
232      curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
233    }
234    // now verify
235    for (Map.Entry<RegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
236      ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
237          .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
238      Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
239      LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
240          " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
241          + "/" + secondaryTertiaryServers.getSecond());
242      if (!(ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getFirst())||
243          ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getSecond()))){
244        fail("Region " + entry.getKey() + " not present on any of the expected servers");
245      }
246    }
247    // start(reinstate) region server since we killed one before
248    TEST_UTIL.getHBaseCluster().startRegionServer();
249  }
250
251  /**
252   * Used to test the correctness of this class.
253   */
254  @Ignore ("Test for unfinished feature") @Test
255  public void testRandomizedMatrix() {
256    int rows = 100;
257    int cols = 100;
258    float[][] matrix = new float[rows][cols];
259    Random random = new Random();
260    for (int i = 0; i < rows; i++) {
261      for (int j = 0; j < cols; j++) {
262        matrix[i][j] = random.nextFloat();
263      }
264    }
265
266    // Test that inverting a transformed matrix gives the original matrix.
267    RegionPlacementMaintainer.RandomizedMatrix rm =
268      new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
269    float[][] transformed = rm.transform(matrix);
270    float[][] invertedTransformed = rm.invert(transformed);
271    for (int i = 0; i < rows; i++) {
272      for (int j = 0; j < cols; j++) {
273        if (matrix[i][j] != invertedTransformed[i][j]) {
274          throw new RuntimeException();
275        }
276      }
277    }
278
279    // Test that the indices on a transformed matrix can be inverted to give
280    // the same values on the original matrix.
281    int[] transformedIndices = new int[rows];
282    for (int i = 0; i < rows; i++) {
283      transformedIndices[i] = random.nextInt(cols);
284    }
285    int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
286    float[] transformedValues = new float[rows];
287    float[] invertedTransformedValues = new float[rows];
288    for (int i = 0; i < rows; i++) {
289      transformedValues[i] = transformed[i][transformedIndices[i]];
290      invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
291    }
292    Arrays.sort(transformedValues);
293    Arrays.sort(invertedTransformedValues);
294    if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
295      throw new RuntimeException();
296    }
297  }
298
299  /**
300   * Shuffle the assignment plan by switching two favored node positions.
301   * @param plan The assignment plan
302   * @param p1 The first switch position
303   * @param p2 The second switch position
304   * @return the shuffled assignment plan
305   */
306  private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
307      FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
308    FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
309
310    Map<String, RegionInfo> regionToHRegion =
311        rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
312    for (Map.Entry<String, List<ServerName>> entry :
313      plan.getAssignmentMap().entrySet()) {
314
315      // copy the server list from the original plan
316      List<ServerName> shuffledServerList = new ArrayList<>();
317      shuffledServerList.addAll(entry.getValue());
318
319      // start to shuffle
320      shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
321      shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
322
323      // update the plan
324      shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
325    }
326    return shuffledPlan;
327  }
328
329  /**
330   * To verify the region assignment status.
331   * It will check the assignment plan consistency between hbase:meta and
332   * region servers.
333   * Also it will verify weather the number of region movement and
334   * the number regions on the primary region server are expected
335   *
336   * @param plan
337   * @param regionMovementNum
338   * @param numRegionsOnPrimaryRS
339   * @throws InterruptedException
340   * @throws IOException
341   */
342  private void verifyRegionAssignment(FavoredNodesPlan plan,
343      int regionMovementNum, int numRegionsOnPrimaryRS)
344  throws InterruptedException, IOException {
345    // Verify the assignment plan in hbase:meta is consistent with the expected plan.
346    verifyMETAUpdated(plan);
347
348    // Verify the number of region movement is expected
349    verifyRegionMovementNum(regionMovementNum);
350
351    // Verify the number of regions is assigned to the primary region server
352    // based on the plan is expected
353    verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
354
355    // Verify all the online region server are updated with the assignment plan
356    verifyRegionServerUpdated(plan);
357  }
358
359  /**
360   * Verify the meta has updated to the latest assignment plan
361   * @param expectedPlan the region assignment plan
362   * @throws IOException if an IO problem is encountered
363   */
364  private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
365  throws IOException {
366    FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
367    assertTrue("The assignment plan is NOT consistent with the expected plan ",
368        planFromMETA.equals(expectedPlan));
369  }
370
371  /**
372   * Verify the number of region movement is expected
373   */
374  private void verifyRegionMovementNum(int expected)
375      throws InterruptedException, IOException {
376    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
377    HMaster m = cluster.getMaster();
378    int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
379    // get the assignments start to execute
380    m.balance();
381
382    int retry = 10;
383    long sleep = 3000;
384    int attempt = 0;
385    int currentRegionOpened, regionMovement;
386    do {
387      currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
388      regionMovement= currentRegionOpened - lastRegionOpenedCount;
389      LOG.debug("There are " + regionMovement + "/" + expected +
390          " regions moved after " + attempt + " attempts");
391      Thread.sleep((++attempt) * sleep);
392    } while (regionMovement != expected && attempt <= retry);
393
394    // update the lastRegionOpenedCount
395    lastRegionOpenedCount = currentRegionOpened;
396
397    assertEquals("There are only " + regionMovement + " instead of "
398          + expected + " region movement for " + attempt + " attempts", expected, regionMovement);
399  }
400
401  /**
402   * Verify the number of user regions is assigned to the primary
403   * region server based on the plan is expected
404   * @param expectedNum.
405   * @throws IOException
406   */
407  private void verifyRegionOnPrimaryRS(int expectedNum)
408      throws IOException {
409    lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
410    assertEquals("Only " +  expectedNum + " of user regions running " +
411        "on the primary region server", expectedNum ,
412        lastRegionOnPrimaryRSCount);
413  }
414
415  /**
416   * Verify all the online region servers has been updated to the
417   * latest assignment plan
418   * @param plan
419   * @throws IOException
420   */
421  private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
422    // Verify all region servers contain the correct favored nodes information
423    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
424    for (int i = 0; i < SLAVES; i++) {
425      HRegionServer rs = cluster.getRegionServer(i);
426      for (Region region: rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
427        InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
428            region.getRegionInfo().getEncodedName());
429        String regionName = region.getRegionInfo().getRegionNameAsString();
430        List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);
431
432        // All regions are supposed to have favored nodes,
433        // except for hbase:meta and ROOT
434        if (favoredServerList == null) {
435          TableDescriptor desc = region.getTableDescriptor();
436          // Verify they are ROOT and hbase:meta regions since no favored nodes
437          assertNull(favoredSocketAddress);
438          assertTrue("User region " +
439              region.getTableDescriptor().getTableName() +
440              " should have favored nodes", desc.isMetaRegion());
441        } else {
442          // For user region, the favored nodes in the region server should be
443          // identical to favored nodes in the assignmentPlan
444          assertTrue(favoredSocketAddress.length == favoredServerList.size());
445          assertTrue(favoredServerList.size() > 0);
446          for (int j = 0; j < favoredServerList.size(); j++) {
447            InetSocketAddress addrFromRS = favoredSocketAddress[j];
448            InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
449                favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
450
451            assertNotNull(addrFromRS);
452            assertNotNull(addrFromPlan);
453            assertTrue("Region server " + rs.getServerName().getHostAndPort()
454                + " has the " + positions[j] +
455                " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
456                addrFromRS + " which is inconsistent with the plan "
457                + addrFromPlan, addrFromRS.equals(addrFromPlan));
458          }
459        }
460      }
461    }
462  }
463
464  /**
465   * Check whether regions are assigned to servers consistent with the explicit
466   * hints that are persisted in the hbase:meta table.
467   * Also keep track of the number of the regions are assigned to the
468   * primary region server.
469   * @return the number of regions are assigned to the primary region server
470   * @throws IOException
471   */
472  private int getNumRegionisOnPrimaryRS() throws IOException {
473    final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
474    final AtomicInteger totalRegionNum = new AtomicInteger(0);
475    LOG.info("The start of region placement verification");
476    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
477      @Override
478      public boolean visit(Result result) throws IOException {
479        try {
480          @SuppressWarnings("deprecation")
481          RegionInfo info = MetaTableAccessor.getRegionInfo(result);
482          if(info.getTable().getNamespaceAsString()
483              .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
484            return true;
485          }
486          byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
487              HConstants.SERVER_QUALIFIER);
488          byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
489              FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
490          // Add the favored nodes into assignment plan
491          ServerName[] favoredServerList =
492              FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
493          favoredNodesAssignmentPlan.put(info, favoredServerList);
494
495          Position[] positions = Position.values();
496          if (info != null) {
497            totalRegionNum.incrementAndGet();
498            if (server != null) {
499              ServerName serverName =
500                  ServerName.valueOf(Bytes.toString(server), -1);
501              if (favoredNodes != null) {
502                String placement = "[NOT FAVORED NODE]";
503                for (int i = 0; i < favoredServerList.length; i++) {
504                  if (favoredServerList[i].equals(serverName)) {
505                    placement = positions[i].toString();
506                    if (i == Position.PRIMARY.ordinal()) {
507                      regionOnPrimaryNum.incrementAndGet();
508                    }
509                    break;
510                  }
511                }
512                LOG.info(info.getRegionNameAsString() + " on " +
513                    serverName + " " + placement);
514              } else {
515                LOG.info(info.getRegionNameAsString() + " running on " +
516                    serverName + " but there is no favored region server");
517              }
518            } else {
519              LOG.info(info.getRegionNameAsString() +
520                  " not assigned to any server");
521            }
522          }
523          return true;
524        } catch (RuntimeException e) {
525          LOG.error("Result=" + result);
526          throw e;
527        }
528      }
529    };
530    MetaTableAccessor.fullScanRegions(CONNECTION, visitor);
531    LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
532        totalRegionNum.intValue() + " regions running on the primary" +
533        " region servers" );
534    return regionOnPrimaryNum.intValue() ;
535  }
536
537  /**
538   * Create a table with specified table name and region number.
539   * @param tablename
540   * @param regionNum
541   * @return
542   * @throws IOException
543   */
544  private static void createTable(TableName tableName, int regionNum)
545      throws IOException {
546    int expectedRegions = regionNum;
547    byte[][] splitKeys = new byte[expectedRegions - 1][];
548    for (int i = 1; i < expectedRegions; i++) {
549      byte splitKey = (byte) i;
550      splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
551    }
552
553    HTableDescriptor desc = new HTableDescriptor(tableName);
554    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
555    admin.createTable(desc, splitKeys);
556
557    try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
558      List<HRegionLocation> regions = r.getAllRegionLocations();
559      assertEquals("Tried to create " + expectedRegions + " regions "
560          + "but only found " + regions.size(), expectedRegions, regions.size());
561    }
562  }
563}