001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertThrows;
023
024import java.io.IOException;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.NamespaceDescriptor;
032import org.apache.hadoop.hbase.StartTestingClusterOption;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineNamespaceProcedure;
039import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
040import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
041import org.apache.hadoop.hbase.procedure2.Procedure;
042import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
043import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
044import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.apache.hadoop.hbase.util.FSTableDescriptors;
050import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
051import org.junit.AfterClass;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056
057import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
059
060/**
061 * Testcase for HBASE-21154.
062 */
063@Category({ MasterTests.class, LargeTests.class })
064public class TestMigrateNamespaceTable {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068    HBaseClassTestRule.forClass(TestMigrateNamespaceTable.class);
069
070  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
071
072  private static volatile boolean CONTINUE = false;
073
074  // used to halt the migration procedure
075  public static final class SuspendProcedure extends Procedure<MasterProcedureEnv>
076    implements TableProcedureInterface {
077
078    @Override
079    public TableName getTableName() {
080      return TableName.META_TABLE_NAME;
081    }
082
083    @Override
084    public TableOperationType getTableOperationType() {
085      return TableOperationType.CREATE;
086    }
087
088    @Override
089    protected LockState acquireLock(final MasterProcedureEnv env) {
090      if (env.getProcedureScheduler().waitTableExclusiveLock(this, getTableName())) {
091        return LockState.LOCK_EVENT_WAIT;
092      }
093      return LockState.LOCK_ACQUIRED;
094    }
095
096    @Override
097    protected void releaseLock(final MasterProcedureEnv env) {
098      env.getProcedureScheduler().wakeTableExclusiveLock(this, getTableName());
099    }
100
101    @Override
102    protected boolean holdLock(MasterProcedureEnv env) {
103      return true;
104    }
105
106    @Override
107    protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
108      throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
109      if (CONTINUE) {
110        return null;
111      }
112      throw suspend(1000, true);
113    }
114
115    @Override
116    protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
117      setState(ProcedureProtos.ProcedureState.RUNNABLE);
118      env.getProcedureScheduler().addFront(this);
119      return false;
120    }
121
122    @Override
123    protected void rollback(MasterProcedureEnv env) throws IOException, InterruptedException {
124      throw new UnsupportedOperationException();
125    }
126
127    @Override
128    protected boolean abort(MasterProcedureEnv env) {
129      return true;
130    }
131
132    @Override
133    protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
134    }
135
136    @Override
137    protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
138    }
139  }
140
141  @BeforeClass
142  public static void setUp() throws Exception {
143    StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(1)
144      .numAlwaysStandByMasters(1).numRegionServers(1).build();
145    UTIL.startMiniCluster(option);
146  }
147
148  @AfterClass
149  public static void tearDown() throws Exception {
150    UTIL.shutdownMiniCluster();
151  }
152
153  // simulate upgrading scenario, where we do not have the ns family
154  private void removeNamespaceFamily() throws IOException {
155    FileSystem fs = UTIL.getTestFileSystem();
156    Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
157    Path tableDir = CommonFSUtils.getTableDir(rootDir, TableName.META_TABLE_NAME);
158    TableDescriptor metaTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
159    TableDescriptor noNsMetaTableDesc = TableDescriptorBuilder.newBuilder(metaTableDesc)
160      .removeColumnFamily(HConstants.NAMESPACE_FAMILY).build();
161    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, noNsMetaTableDesc,
162      true);
163    for (FileStatus status : fs.listStatus(tableDir)) {
164      if (!status.isDirectory()) {
165        continue;
166      }
167      Path familyPath = new Path(status.getPath(), HConstants.NAMESPACE_FAMILY_STR);
168      fs.delete(familyPath, true);
169    }
170  }
171
172  private void addNamespace(Table table, NamespaceDescriptor nd) throws IOException {
173    table.put(new Put(Bytes.toBytes(nd.getName())).addColumn(
174      TableDescriptorBuilder.NAMESPACE_FAMILY_INFO_BYTES,
175      TableDescriptorBuilder.NAMESPACE_COL_DESC_BYTES,
176      ProtobufUtil.toProtoNamespaceDescriptor(nd).toByteArray()));
177  }
178
179  @Test
180  public void testMigrate() throws IOException, InterruptedException {
181    UTIL.getAdmin().createTable(TableDescriptorBuilder.NAMESPACE_TABLEDESC);
182    try (Table table = UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) {
183      for (int i = 0; i < 5; i++) {
184        NamespaceDescriptor nd = NamespaceDescriptor.create("Test-NS-" + i)
185          .addConfiguration("key-" + i, "value-" + i).build();
186        addNamespace(table, nd);
187        AbstractStateMachineNamespaceProcedure
188          .createDirectory(UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(), nd);
189      }
190      // add default and system
191      addNamespace(table, NamespaceDescriptor.DEFAULT_NAMESPACE);
192      addNamespace(table, NamespaceDescriptor.SYSTEM_NAMESPACE);
193    }
194    MasterThread masterThread = UTIL.getMiniHBaseCluster().getMasterThread();
195    masterThread.getMaster().getMasterProcedureExecutor().submitProcedure(new SuspendProcedure());
196    masterThread.getMaster().stop("For testing");
197    masterThread.join();
198
199    removeNamespaceFamily();
200
201    UTIL.getMiniHBaseCluster().startMaster();
202
203    // 5 + default and system('hbase')
204    assertEquals(7, UTIL.getAdmin().listNamespaceDescriptors().length);
205    for (int i = 0; i < 5; i++) {
206      NamespaceDescriptor nd = UTIL.getAdmin().getNamespaceDescriptor("Test-NS-" + i);
207      assertEquals("Test-NS-" + i, nd.getName());
208      assertEquals(1, nd.getConfiguration().size());
209      assertEquals("value-" + i, nd.getConfigurationValue("key-" + i));
210    }
211    // before migration done, modification on the namespace is not supported
212    TableNamespaceManager tableNsMgr =
213      UTIL.getMiniHBaseCluster().getMaster().getClusterSchema().getTableNamespaceManager();
214    assertThrows(IOException.class, () -> tableNsMgr.deleteNamespace("Test-NS-0"));
215    assertThrows(IOException.class,
216      () -> tableNsMgr.addOrUpdateNamespace(NamespaceDescriptor.create("NNN").build()));
217    CONTINUE = true;
218    UTIL.waitFor(30000, () -> UTIL.getAdmin().isTableDisabled(TableName.NAMESPACE_TABLE_NAME));
219    // this time it is allowed to change the namespace
220    UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("NNN").build());
221
222    masterThread = UTIL.getMiniHBaseCluster().getMasterThread();
223    masterThread.getMaster().stop("For testing");
224    masterThread.join();
225    UTIL.getMiniHBaseCluster().startMaster();
226
227    // make sure that we could still restart the cluster after disabling the namespace table.
228    assertEquals(8, UTIL.getAdmin().listNamespaceDescriptors().length);
229
230    // let's delete the namespace table
231    UTIL.getAdmin().deleteTable(TableName.NAMESPACE_TABLE_NAME);
232    assertFalse(UTIL.getAdmin().tableExists(TableName.NAMESPACE_TABLE_NAME));
233  }
234}