001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertNull;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.io.IOException;
027import java.util.EnumSet;
028import java.util.concurrent.atomic.AtomicInteger;
029import org.apache.hadoop.fs.CreateFlag;
030import org.apache.hadoop.fs.FSDataOutputStream;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Options;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.fs.permission.FsPermission;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.MetaTableAccessor;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.master.HMaster;
041import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
042import org.apache.hadoop.hbase.procedure2.Procedure;
043import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
044import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.CommonFSUtils;
048import org.apache.hadoop.hbase.util.FSTableDescriptors;
049import org.apache.hadoop.hbase.util.ModifyRegionUtils;
050import org.apache.hadoop.hdfs.DistributedFileSystem;
051import org.apache.hadoop.util.Progressable;
052import org.junit.jupiter.api.AfterAll;
053import org.junit.jupiter.api.AfterEach;
054import org.junit.jupiter.api.BeforeAll;
055import org.junit.jupiter.api.Tag;
056import org.junit.jupiter.api.Test;
057
058/**
059 * Test for HBASE-30539: {@link CreateTableProcedure} must not move on to {@code hbase:meta} and
060 * region assignment when the table descriptor could not be written.
061 * <p>
062 * {@code FSTableDescriptors.writeTableDescriptor} retries 10 times. If every attempt fails it must
063 * throw, so that the create procedure stays in {@code CREATE_TABLE_WRITE_FS_LAYOUT} and retries,
064 * instead of publishing a table whose region servers can never load a descriptor ("Missing table
065 * descriptor for ...") and whose assign procedure retries forever.
066 */
067@Tag(MasterTests.TAG)
068@Tag(MediumTests.TAG)
069public class TestCreateTableProcedureDescriptorWriteFailure {
070
071  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
072
073  /** Same as {@code maxAttempts} in {@code FSTableDescriptors.writeTableDescriptor}. */
074  private static final int WRITE_ATTEMPTS = 10;
075
076  /** Substring identifying the file creations to fail; null means do not inject. */
077  private static volatile String failPathContains = null;
078  /** Stop injecting once this many failures were injected; default is to fail forever. */
079  private static volatile int maxInjectedFailures = Integer.MAX_VALUE;
080  private static final AtomicInteger INJECTED_FAILURES = new AtomicInteger();
081
082  /**
083   * A {@link DistributedFileSystem} that refuses to create any file whose path contains
084   * {@link #failPathContains}, simulating the NameNode answering "could only be written to 0 of the
085   * 1 minReplication nodes".
086   */
087  public static class FailingTableInfoFileSystem extends DistributedFileSystem {
088    @Override
089    public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> cflags,
090      int bufferSize, short replication, long blockSize, Progressable progress,
091      Options.ChecksumOpt checksumOpt) throws IOException {
092      String marker = failPathContains;
093      if (
094        marker != null && f.toString().contains(marker)
095          && INJECTED_FAILURES.get() < maxInjectedFailures
096      ) {
097        INJECTED_FAILURES.incrementAndGet();
098        throw new IOException("Injected HDFS failure creating " + f);
099      }
100      return super.create(f, permission, cflags, bufferSize, replication, blockSize, progress,
101        checksumOpt);
102    }
103  }
104
105  @BeforeAll
106  public static void setUp() throws Exception {
107    UTIL.getConfiguration().setClass("fs.hdfs.impl", FailingTableInfoFileSystem.class,
108      FileSystem.class);
109    UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
110    UTIL.startMiniCluster(1);
111  }
112
113  @AfterAll
114  public static void tearDownCluster() throws Exception {
115    failPathContains = null;
116    UTIL.shutdownMiniCluster();
117  }
118
119  @AfterEach
120  public void resetInjection() {
121    failPathContains = null;
122    maxInjectedFailures = Integer.MAX_VALUE;
123    INJECTED_FAILURES.set(0);
124  }
125
126  /**
127   * Every write of the table descriptor fails, so the {@value #WRITE_ATTEMPTS} in-method attempts
128   * of {@code writeTableDescriptor} are exhausted, over and over. The create procedure must keep
129   * retrying {@code CREATE_TABLE_WRITE_FS_LAYOUT}: it must not fail, finish, add the regions to
130   * {@code hbase:meta} or start any {@link TransitRegionStateProcedure} for region assignment. Once
131   * the injection is cleared, i.e. HDFS recovers, the same procedure completes, the descriptor is
132   * on the file system and the region server can load it.
133   */
134  @Test
135  public void testCreateTableDoesNotProceedWhenTableDescriptorWriteFails() throws Exception {
136    final TableName tableName = TableName.valueOf("testDescriptorWriteFailure");
137    final HMaster master = UTIL.getHBaseCluster().getMaster();
138    final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
139    final FileSystem fs = master.getMasterFileSystem().getFileSystem();
140    final Path tableDir =
141      CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
142
143    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1");
144    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
145
146    // Make every write of <table>/.tabledesc/.tableinfo.* fail.
147    failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/";
148    long procId =
149      procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
150
151    // 1. The procedure keeps retrying the file system layout step: at least two rounds of
152    // WRITE_ATTEMPTS refused writes.
153    UTIL.waitFor(60_000, 100, () -> INJECTED_FAILURES.get() >= 2 * WRITE_ATTEMPTS);
154
155    // 2. It must neither have failed, nor finished, nor gone on to hbase:meta / assignment.
156    Procedure<MasterProcedureEnv> createProc = procExec.getProcedure(procId);
157    assertNotNull(createProc, "create procedure should still be tracked");
158    assertFalse(createProc.isFailed(), "retryable failure must not fail the procedure");
159    assertFalse(createProc.isFinished(), "procedure cannot finish without a descriptor");
160    assertTrue(MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).isEmpty(),
161      "regions must not be added to hbase:meta before the descriptor is written");
162    assertNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir),
163      "no table descriptor should exist on the file system");
164    for (Procedure<MasterProcedureEnv> p : procExec.getProcedures()) {
165      assertFalse(p instanceof TransitRegionStateProcedure && p.getParentProcId() == procId,
166        "no region assignment may be started: " + p);
167    }
168
169    // 3. Once HDFS recovers, the same procedure completes and the table is usable.
170    failPathContains = null;
171    ProcedureTestingUtility.waitProcedure(procExec, procId);
172    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
173    assertNotNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir),
174      "table descriptor must be on the file system after recovery");
175    assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());
176    UTIL.waitTableAvailable(tableName);
177    assertNotNull(UTIL.getHBaseCluster().getRegionServer(0).getTableDescriptors().get(tableName),
178      "region server must be able to load the descriptor");
179  }
180
181  /**
182   * A few transient failures, fewer than the {@value #WRITE_ATTEMPTS} in-method attempts, are
183   * absorbed by the retry loop in {@code writeTableDescriptor}: the procedure never notices.
184   */
185  @Test
186  public void testTransientWriteFailuresAbsorbedByWriteRetries() throws Exception {
187    final TableName tableName = TableName.valueOf("testTransientDescriptorWriteFailure");
188    final int failures = 3;
189    assertTrue(failures < WRITE_ATTEMPTS);
190
191    failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/";
192    maxInjectedFailures = failures;
193    long procId = submitCreateTable(tableName);
194
195    ProcedureExecutor<MasterProcedureEnv> procExec =
196      UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
197    ProcedureTestingUtility.waitProcedure(procExec, procId);
198    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
199
200    assertEquals(failures, INJECTED_FAILURES.get(), "exactly the injected failures happened");
201    assertTableCreatedWithDescriptor(tableName);
202  }
203
204  /**
205   * More failures than the in-method attempts: the first round of {@value #WRITE_ATTEMPTS} writes
206   * is exhausted and {@code CREATE_TABLE_WRITE_FS_LAYOUT} fails, then the procedure retries that
207   * step, the write succeeds and the procedure completes without ever having added the table to
208   * {@code hbase:meta} in between.
209   */
210  @Test
211  public void testProcedureRetriesAfterWriteAttemptsExhaustedThenSucceeds() throws Exception {
212    final TableName tableName = TableName.valueOf("testDescriptorWriteFailureThenSuccess");
213    final int failures = WRITE_ATTEMPTS + 2;
214
215    failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/";
216    maxInjectedFailures = failures;
217    long procId = submitCreateTable(tableName);
218
219    ProcedureExecutor<MasterProcedureEnv> procExec =
220      UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
221    ProcedureTestingUtility.waitProcedure(procExec, procId);
222    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
223
224    assertEquals(failures, INJECTED_FAILURES.get(), "exactly the injected failures happened");
225    assertTableCreatedWithDescriptor(tableName);
226  }
227
228  private long submitCreateTable(TableName tableName) throws IOException {
229    ProcedureExecutor<MasterProcedureEnv> procExec =
230      UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
231    TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1");
232    RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
233    return procExec
234      .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
235  }
236
237  private void assertTableCreatedWithDescriptor(TableName tableName) throws Exception {
238    HMaster master = UTIL.getHBaseCluster().getMaster();
239    FileSystem fs = master.getMasterFileSystem().getFileSystem();
240    Path tableDir = CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
241    assertNotNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir),
242      "table descriptor must be on the file system");
243    assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());
244    UTIL.waitTableAvailable(tableName);
245    assertNotNull(UTIL.getHBaseCluster().getRegionServer(0).getTableDescriptors().get(tableName),
246      "region server must be able to load the descriptor");
247  }
248}