001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.lengthOfPBMagic;
021import static org.apache.hadoop.hbase.zookeeper.ZKMetadata.removeMetaData;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import org.apache.hadoop.hbase.CatalogFamilyFormat;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.RegionLocations;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.StartTestingClusterOption;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Delete;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.Scan;
039import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
040import org.apache.hadoop.hbase.master.region.MasterRegion;
041import org.apache.hadoop.hbase.procedure2.Procedure;
042import org.apache.hadoop.hbase.regionserver.RegionScanner;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.testclassification.MasterTests;
045import org.apache.hadoop.hbase.zookeeper.ZKUtil;
046import org.junit.jupiter.api.AfterAll;
047import org.junit.jupiter.api.BeforeAll;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050
051import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
053
054/**
055 * Testcase for HBASE-26193.
056 */
057@Tag(MasterTests.TAG)
058@Tag(LargeTests.TAG)
059public class TestMigrateAndMirrorMetaLocations {
060
061  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
062
063  @BeforeAll
064  public static void setUp() throws Exception {
065    UTIL.startMiniCluster(3);
066    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 2);
067  }
068
069  @AfterAll
070  public static void tearDown() throws IOException {
071    UTIL.shutdownMiniCluster();
072  }
073
074  private void assertLocationEquals(Result result, int replicaCount) throws Exception {
075    RegionLocations locs = CatalogFamilyFormat.getRegionLocations(result);
076    assertEquals(replicaCount, locs.size());
077    for (int i = 0; i < replicaCount; i++) {
078      String znode = UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(i);
079      byte[] data = ZKUtil.getData(UTIL.getZooKeeperWatcher(), znode);
080      data = removeMetaData(data);
081      int prefixLen = lengthOfPBMagic();
082      ZooKeeperProtos.MetaRegionServer zkProto = ZooKeeperProtos.MetaRegionServer.parser()
083        .parseFrom(data, prefixLen, data.length - prefixLen);
084      ServerName sn = ProtobufUtil.toServerName(zkProto.getServer());
085      assertEquals(locs.getRegionLocation(i).getServerName(), sn);
086    }
087    assertEquals(replicaCount, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
088  }
089
090  private void checkMirrorLocation(int replicaCount) throws Exception {
091    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
092    try (RegionScanner scanner =
093      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
094      List<Cell> cells = new ArrayList<>();
095      boolean moreRows = scanner.next(cells);
096      // should only have one row as we have only one meta region, different replicas will be in the
097      // same row
098      assertFalse(moreRows);
099      assertFalse(cells.isEmpty());
100      Result result = Result.create(cells);
101      // make sure we publish the correct location to zookeeper too
102      assertLocationEquals(result, replicaCount);
103    }
104  }
105
106  private void waitUntilNoSCP() throws IOException {
107    UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
108      .filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
109  }
110
111  @Test
112  public void test() throws Exception {
113    checkMirrorLocation(2);
114    MasterRegion masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
115    try (RegionScanner scanner =
116      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
117      List<Cell> cells = new ArrayList<>();
118      scanner.next(cells);
119      Cell cell = cells.get(0);
120      // delete the only row
121      masterRegion.update(
122        r -> r.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
123          .addFamily(HConstants.CATALOG_FAMILY)));
124      masterRegion.flush(true);
125    }
126    // restart the whole cluster, to see if we can migrate the data on zookeeper to master local
127    // region
128    UTIL.shutdownMiniHBaseCluster();
129    UTIL.startMiniHBaseCluster(StartTestingClusterOption.builder().numRegionServers(3).build());
130    masterRegion = UTIL.getMiniHBaseCluster().getMaster().getMasterRegion();
131    try (RegionScanner scanner =
132      masterRegion.getRegionScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
133      List<Cell> cells = new ArrayList<>();
134      boolean moreRows = scanner.next(cells);
135      assertFalse(moreRows);
136      // should have the migrated data
137      assertFalse(cells.isEmpty());
138    }
139    // wait until all meta regions have been assigned
140    UTIL.waitFor(30000,
141      () -> UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size() == 2);
142    // make sure all the SCPs are finished
143    waitUntilNoSCP();
144    checkMirrorLocation(2);
145
146    // increase replica count to 3
147    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
148    checkMirrorLocation(3);
149
150    byte[] replica2Data = ZKUtil.getData(UTIL.getZooKeeperWatcher(),
151      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2));
152
153    // decrease replica count to 1
154    HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 1);
155    checkMirrorLocation(1);
156
157    // restart the whole cluster, put an extra replica znode on zookeeper, to see if we will remove
158    // it
159    UTIL.shutdownMiniHBaseCluster();
160    ZKUtil.createAndFailSilent(UTIL.getZooKeeperWatcher(),
161      UTIL.getZooKeeperWatcher().getZNodePaths().getZNodeForReplica(2), replica2Data);
162    UTIL.startMiniHBaseCluster(StartTestingClusterOption.builder().numRegionServers(3).build());
163    // should have removed the extra replica znode as it is part of the start up process, when
164    // initializing AM
165    assertEquals(1, UTIL.getZooKeeperWatcher().getMetaReplicaNodes().size());
166    // make sure all the SCPs are finished
167    waitUntilNoSCP();
168    checkMirrorLocation(1);
169  }
170}