001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.lengthOfPBMagic;
021import static org.apache.hadoop.hbase.zookeeper.ZKMetadata.removeMetaData;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.MetaTableAccessor;
033import org.apache.hadoop.hbase.RegionLocations;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.StartMiniClusterOption;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
041import org.apache.hadoop.hbase.master.region.MasterRegion;
042import org.apache.hadoop.hbase.procedure2.Procedure;
043import org.apache.hadoop.hbase.regionserver.RegionScanner;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.zookeeper.ZKUtil;
047import org.junit.AfterClass;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052
053import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
055
056/**
057 * Testcase for HBASE-26193.
058 */
059@Category({ MasterTests.class, LargeTests.class })
060public class TestMigrateAndMirrorMetaLocations {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestMigrateAndMirrorMetaLocations.class);
065
066  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
067
068  @BeforeClass
069  public static void setUp() throws Exception {
070    UTIL.startMiniCluster(3);
071    HBaseTestingUtility.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 2);
072  }
073
074  @AfterClass
075  public static void tearDown() throws IOException {
076    UTIL.shutdownMiniCluster();
077  }
078
079  private void assertLocationEquals(Result result, int replicaCount) throws Exception {
080    RegionLocations locs = MetaTableAccessor.getRegionLocations(result);
081    assertEquals(replicaCount, locs.size());
082    for (int i = 0; i < replicaCount; i++) {
083      String znode = UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(i);
084      byte[] data = ZKUtil.getData(UTIL.getZooKeeperWatcher(), znode);
085      data = removeMetaData(data);
086      int prefixLen = lengthOfPBMagic();
087      ZooKeeperProtos.MetaRegionServer zkProto = ZooKeeperProtos.MetaRegionServer.parser()
088        .parseFrom(data, prefixLen, data.length - prefixLen);
089      ServerName sn = ProtobufUtil.toServerName(zkProto.getServer());
090      assertEquals(locs.getRegionLocation(i).getServerName(), sn);
091    }
092    assertEquals(replicaCount, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
093  }
094
095  private void checkMirrorLocation(int replicaCount) throws Exception {
096    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
097    try (RegionScanner scanner =
098      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
099      List<Cell> cells = new ArrayList<>();
100      boolean moreRows = scanner.next(cells);
101      // should only have one row as we have only one meta region, different replicas will be in the
102      // same row
103      assertFalse(moreRows);
104      assertFalse(cells.isEmpty());
105      Result result = Result.create(cells);
106      // make sure we publish the correct location to zookeeper too
107      assertLocationEquals(result, replicaCount);
108    }
109  }
110
111  private void waitUntilNoSCP() throws IOException {
112    UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
113      .filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
114  }
115
116  @Test
117  public void test() throws Exception {
118    checkMirrorLocation(2);
119    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
120    try (RegionScanner scanner =
121      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
122      List<Cell> cells = new ArrayList<>();
123      scanner.next(cells);
124      Cell cell = cells.get(0);
125      // delete the only row
126      masterRegion.update(
127        r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
128          .addFamily(HConstants.CATALOG_FAMILY)));
129      masterRegion.flush(true);
130    }
131    // restart the whole cluster, to see if we can migrate the data on zookeeper to master local
132    // region
133    UTIL.shutdownMiniHBaseCluster();
134    UTIL.startMiniHBaseCluster(StartMiniClusterOption.builder().numRegionServers(3).build());
135    masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
136    try (RegionScanner scanner =
137      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
138      List<Cell> cells = new ArrayList<>();
139      boolean moreRows = scanner.next(cells);
140      assertFalse(moreRows);
141      // should have the migrated data
142      assertFalse(cells.isEmpty());
143    }
144    // wait until all meta regions have been assigned
145    UTIL.waitFor(30000,
146      () -> UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size() == 2);
147    // make sure all the SCPs are finished
148    waitUntilNoSCP();
149    checkMirrorLocation(2);
150
151    // increase replica count to 3
152    HBaseTestingUtility.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
153    checkMirrorLocation(3);
154
155    byte[] replica2Data = ZKUtil.getData(UTIL.getZooKeeperWatcher(),
156      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2));
157
158    // decrease replica count to 1
159    HBaseTestingUtility.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 1);
160    checkMirrorLocation(1);
161
162    // restart the whole cluster, put an extra replica znode on zookeeper, to see if we will remove
163    // it
164    UTIL.shutdownMiniHBaseCluster();
165    ZKUtil.createAndFailSilent(UTIL.getZooKeeperWatcher(),
166      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2), replica2Data);
167    UTIL.startMiniHBaseCluster(StartMiniClusterOption.builder().numRegionServers(3).build());
168    // should have removed the extra replica znode as it is part of the start up process, when
169    // initializing AM
170    assertEquals(1, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
171    // make sure all the SCPs are finished
172    waitUntilNoSCP();
173    checkMirrorLocation(1);
174  }
175}