001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertThrows;
023
024import java.io.IOException;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.NamespaceDescriptor;
031import org.apache.hadoop.hbase.StartTestingClusterOption;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineNamespaceProcedure;
038import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
039import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
040import org.apache.hadoop.hbase.procedure2.Procedure;
041import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
042import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
043import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.CommonFSUtils;
048import org.apache.hadoop.hbase.util.FSTableDescriptors;
049import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
050import org.junit.jupiter.api.AfterAll;
051import org.junit.jupiter.api.BeforeAll;
052import org.junit.jupiter.api.Tag;
053import org.junit.jupiter.api.Test;
054
055import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
057
058/**
059 * Testcase for HBASE-21154.
060 */
061@Tag(MasterTests.TAG)
062@Tag(LargeTests.TAG)
063public class TestMigrateNamespaceTable {
064
065  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
066
067  private static volatile boolean CONTINUE = false;
068
069  // used to halt the migration procedure
070  public static final class SuspendProcedure extends Procedure<MasterProcedureEnv>
071    implements TableProcedureInterface {
072
073    @Override
074    public TableName getTableName() {
075      return TableName.META_TABLE_NAME;
076    }
077
078    @Override
079    public TableOperationType getTableOperationType() {
080      return TableOperationType.CREATE;
081    }
082
083    @Override
084    protected LockState acquireLock(final MasterProcedureEnv env) {
085      if (env.getProcedureScheduler().waitTableExclusiveLock(this, getTableName())) {
086        return LockState.LOCK_EVENT_WAIT;
087      }
088      return LockState.LOCK_ACQUIRED;
089    }
090
091    @Override
092    protected void releaseLock(final MasterProcedureEnv env) {
093      env.getProcedureScheduler().wakeTableExclusiveLock(this, getTableName());
094    }
095
096    @Override
097    protected boolean holdLock(MasterProcedureEnv env) {
098      return true;
099    }
100
101    @Override
102    protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
103      throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
104      if (CONTINUE) {
105        return null;
106      }
107      throw suspend(1000, true);
108    }
109
110    @Override
111    protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
112      setState(ProcedureProtos.ProcedureState.RUNNABLE);
113      env.getProcedureScheduler().addFront(this);
114      return false;
115    }
116
117    @Override
118    protected void rollback(MasterProcedureEnv env) throws IOException, InterruptedException {
119      throw new UnsupportedOperationException();
120    }
121
122    @Override
123    protected boolean abort(MasterProcedureEnv env) {
124      return true;
125    }
126
127    @Override
128    protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
129    }
130
131    @Override
132    protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
133    }
134  }
135
136  @BeforeAll
137  public static void setUp() throws Exception {
138    StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(1)
139      .numAlwaysStandByMasters(1).numRegionServers(1).build();
140    UTIL.startMiniCluster(option);
141  }
142
143  @AfterAll
144  public static void tearDown() throws Exception {
145    UTIL.shutdownMiniCluster();
146  }
147
148  // simulate upgrading scenario, where we do not have the ns family
149  private void removeNamespaceFamily() throws IOException {
150    FileSystem fs = UTIL.getTestFileSystem();
151    Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
152    Path tableDir = CommonFSUtils.getTableDir(rootDir, TableName.META_TABLE_NAME);
153    TableDescriptor metaTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
154    TableDescriptor noNsMetaTableDesc = TableDescriptorBuilder.newBuilder(metaTableDesc)
155      .removeColumnFamily(HConstants.NAMESPACE_FAMILY).build();
156    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, noNsMetaTableDesc,
157      true);
158    for (FileStatus status : fs.listStatus(tableDir)) {
159      if (!status.isDirectory()) {
160        continue;
161      }
162      Path familyPath = new Path(status.getPath(), HConstants.NAMESPACE_FAMILY_STR);
163      fs.delete(familyPath, true);
164    }
165  }
166
167  private void addNamespace(Table table, NamespaceDescriptor nd) throws IOException {
168    table.put(new Put(Bytes.toBytes(nd.getName())).addColumn(
169      TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES,
170      TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES,
171      ProtobufUtil.toProtoNamespaceDescriptor(nd).toByteArray()));
172  }
173
174  @Test
175  public void testMigrate() throws IOException, InterruptedException {
176    UTIL.getAdmin().createTable(TableDescriptorBuilder.NAMESPACE_TABLEDESC);
177    try (Table table = UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) {
178      for (int i = 0; i < 5; i++) {
179        NamespaceDescriptor nd = NamespaceDescriptor.create("Test-NS-" + i)
180          .addConfiguration("key-" + i, "value-" + i).build();
181        addNamespace(table, nd);
182        AbstractStateMachineNamespaceProcedure
183          .createDirectory(UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(), nd);
184      }
185      // add default and system
186      addNamespace(table, NamespaceDescriptor.DEFAULT_NAMESPACE);
187      addNamespace(table, NamespaceDescriptor.SYSTEM_NAMESPACE);
188    }
189    MasterThread masterThread = UTIL.getMiniHBaseCluster().getMasterThread();
190    masterThread.getMaster().getMasterProcedureExecutor().submitProcedure(new SuspendProcedure());
191    masterThread.getMaster().stop("For testing");
192    masterThread.join();
193
194    removeNamespaceFamily();
195
196    UTIL.getMiniHBaseCluster().startMaster();
197
198    // 5 + default and system('hbase')
199    assertEquals(7, UTIL.getAdmin().listNamespaceDescriptors().length);
200    for (int i = 0; i < 5; i++) {
201      NamespaceDescriptor nd = UTIL.getAdmin().getNamespaceDescriptor("Test-NS-" + i);
202      assertEquals("Test-NS-" + i, nd.getName());
203      assertEquals(1, nd.getConfiguration().size());
204      assertEquals("value-" + i, nd.getConfigurationValue("key-" + i));
205    }
206    // before migration done, modification on the namespace is not supported
207    TableNamespaceManager tableNsMgr =
208      UTIL.getMiniHBaseCluster().getMaster().getClusterSchema().getTableNamespaceManager();
209    assertThrows(IOException.class, () -> tableNsMgr.deleteNamespace("Test-NS-0"));
210    assertThrows(IOException.class,
211      () -> tableNsMgr.addOrUpdateNamespace(NamespaceDescriptor.create("NNN").build()));
212    CONTINUE = true;
213    UTIL.waitFor(30000, () -> UTIL.getAdmin().isTableDisabled(TableName.NAMESPACE_TABLE_NAME));
214    // this time it is allowed to change the namespace
215    UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("NNN").build());
216
217    masterThread = UTIL.getMiniHBaseCluster().getMasterThread();
218    masterThread.getMaster().stop("For testing");
219    masterThread.join();
220    UTIL.getMiniHBaseCluster().startMaster();
221
222    // make sure that we could still restart the cluster after disabling the namespace table.
223    assertEquals(8, UTIL.getAdmin().listNamespaceDescriptors().length);
224
225    // let's delete the namespace table
226    UTIL.getAdmin().deleteTable(TableName.NAMESPACE_TABLE_NAME);
227    assertFalse(UTIL.getAdmin().tableExists(TableName.NAMESPACE_TABLE_NAME));
228  }
229}