001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.procedure; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertNotNull; 023import static org.junit.jupiter.api.Assertions.assertNull; 024import static org.junit.jupiter.api.Assertions.assertTrue; 025 026import java.io.IOException; 027import java.util.EnumSet; 028import java.util.concurrent.atomic.AtomicInteger; 029import org.apache.hadoop.fs.CreateFlag; 030import org.apache.hadoop.fs.FSDataOutputStream; 031import org.apache.hadoop.fs.FileSystem; 032import org.apache.hadoop.fs.Options; 033import org.apache.hadoop.fs.Path; 034import org.apache.hadoop.fs.permission.FsPermission; 035import org.apache.hadoop.hbase.HBaseTestingUtil; 036import org.apache.hadoop.hbase.MetaTableAccessor; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.TableDescriptor; 040import org.apache.hadoop.hbase.master.HMaster; 041import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; 042import org.apache.hadoop.hbase.procedure2.Procedure; 043import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 044import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 045import org.apache.hadoop.hbase.testclassification.MasterTests; 046import org.apache.hadoop.hbase.testclassification.MediumTests; 047import org.apache.hadoop.hbase.util.CommonFSUtils; 048import org.apache.hadoop.hbase.util.FSTableDescriptors; 049import org.apache.hadoop.hbase.util.ModifyRegionUtils; 050import org.apache.hadoop.hdfs.DistributedFileSystem; 051import org.apache.hadoop.util.Progressable; 052import org.junit.jupiter.api.AfterAll; 053import org.junit.jupiter.api.AfterEach; 054import org.junit.jupiter.api.BeforeAll; 055import org.junit.jupiter.api.Tag; 056import org.junit.jupiter.api.Test; 057 058/** 059 * Test for HBASE-30539: {@link CreateTableProcedure} must not move on to {@code hbase:meta} and 060 * region assignment when the table descriptor could not be written. 061 * <p> 062 * {@code FSTableDescriptors.writeTableDescriptor} retries 10 times. If every attempt fails it must 063 * throw, so that the create procedure stays in {@code CREATE_TABLE_WRITE_FS_LAYOUT} and retries, 064 * instead of publishing a table whose region servers can never load a descriptor ("Missing table 065 * descriptor for ...") and whose assign procedure retries forever. 066 */ 067@Tag(MasterTests.TAG) 068@Tag(MediumTests.TAG) 069public class TestCreateTableProcedureDescriptorWriteFailure { 070 071 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 072 073 /** Same as {@code maxAttempts} in {@code FSTableDescriptors.writeTableDescriptor}. */ 074 private static final int WRITE_ATTEMPTS = 10; 075 076 /** Substring identifying the file creations to fail; null means do not inject. */ 077 private static volatile String failPathContains = null; 078 /** Stop injecting once this many failures were injected; default is to fail forever. */ 079 private static volatile int maxInjectedFailures = Integer.MAX_VALUE; 080 private static final AtomicInteger INJECTED_FAILURES = new AtomicInteger(); 081 082 /** 083 * A {@link DistributedFileSystem} that refuses to create any file whose path contains 084 * {@link #failPathContains}, simulating the NameNode answering "could only be written to 0 of the 085 * 1 minReplication nodes". 086 */ 087 public static class FailingTableInfoFileSystem extends DistributedFileSystem { 088 @Override 089 public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> cflags, 090 int bufferSize, short replication, long blockSize, Progressable progress, 091 Options.ChecksumOpt checksumOpt) throws IOException { 092 String marker = failPathContains; 093 if ( 094 marker != null && f.toString().contains(marker) 095 && INJECTED_FAILURES.get() < maxInjectedFailures 096 ) { 097 INJECTED_FAILURES.incrementAndGet(); 098 throw new IOException("Injected HDFS failure creating " + f); 099 } 100 return super.create(f, permission, cflags, bufferSize, replication, blockSize, progress, 101 checksumOpt); 102 } 103 } 104 105 @BeforeAll 106 public static void setUp() throws Exception { 107 UTIL.getConfiguration().setClass("fs.hdfs.impl", FailingTableInfoFileSystem.class, 108 FileSystem.class); 109 UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); 110 UTIL.startMiniCluster(1); 111 } 112 113 @AfterAll 114 public static void tearDownCluster() throws Exception { 115 failPathContains = null; 116 UTIL.shutdownMiniCluster(); 117 } 118 119 @AfterEach 120 public void resetInjection() { 121 failPathContains = null; 122 maxInjectedFailures = Integer.MAX_VALUE; 123 INJECTED_FAILURES.set(0); 124 } 125 126 /** 127 * Every write of the table descriptor fails, so the {@value #WRITE_ATTEMPTS} in-method attempts 128 * of {@code writeTableDescriptor} are exhausted, over and over. The create procedure must keep 129 * retrying {@code CREATE_TABLE_WRITE_FS_LAYOUT}: it must not fail, finish, add the regions to 130 * {@code hbase:meta} or start any {@link TransitRegionStateProcedure} for region assignment. Once 131 * the injection is cleared, i.e. HDFS recovers, the same procedure completes, the descriptor is 132 * on the file system and the region server can load it. 133 */ 134 @Test 135 public void testCreateTableDoesNotProceedWhenTableDescriptorWriteFails() throws Exception { 136 final TableName tableName = TableName.valueOf("testDescriptorWriteFailure"); 137 final HMaster master = UTIL.getHBaseCluster().getMaster(); 138 final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); 139 final FileSystem fs = master.getMasterFileSystem().getFileSystem(); 140 final Path tableDir = 141 CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); 142 143 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1"); 144 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 145 146 // Make every write of <table>/.tabledesc/.tableinfo.* fail. 147 failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/"; 148 long procId = 149 procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 150 151 // 1. The procedure keeps retrying the file system layout step: at least two rounds of 152 // WRITE_ATTEMPTS refused writes. 153 UTIL.waitFor(60_000, 100, () -> INJECTED_FAILURES.get() >= 2 * WRITE_ATTEMPTS); 154 155 // 2. It must neither have failed, nor finished, nor gone on to hbase:meta / assignment. 156 Procedure<MasterProcedureEnv> createProc = procExec.getProcedure(procId); 157 assertNotNull(createProc, "create procedure should still be tracked"); 158 assertFalse(createProc.isFailed(), "retryable failure must not fail the procedure"); 159 assertFalse(createProc.isFinished(), "procedure cannot finish without a descriptor"); 160 assertTrue(MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).isEmpty(), 161 "regions must not be added to hbase:meta before the descriptor is written"); 162 assertNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir), 163 "no table descriptor should exist on the file system"); 164 for (Procedure<MasterProcedureEnv> p : procExec.getProcedures()) { 165 assertFalse(p instanceof TransitRegionStateProcedure && p.getParentProcId() == procId, 166 "no region assignment may be started: " + p); 167 } 168 169 // 3. Once HDFS recovers, the same procedure completes and the table is usable. 170 failPathContains = null; 171 ProcedureTestingUtility.waitProcedure(procExec, procId); 172 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 173 assertNotNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir), 174 "table descriptor must be on the file system after recovery"); 175 assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size()); 176 UTIL.waitTableAvailable(tableName); 177 assertNotNull(UTIL.getHBaseCluster().getRegionServer(0).getTableDescriptors().get(tableName), 178 "region server must be able to load the descriptor"); 179 } 180 181 /** 182 * A few transient failures, fewer than the {@value #WRITE_ATTEMPTS} in-method attempts, are 183 * absorbed by the retry loop in {@code writeTableDescriptor}: the procedure never notices. 184 */ 185 @Test 186 public void testTransientWriteFailuresAbsorbedByWriteRetries() throws Exception { 187 final TableName tableName = TableName.valueOf("testTransientDescriptorWriteFailure"); 188 final int failures = 3; 189 assertTrue(failures < WRITE_ATTEMPTS); 190 191 failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/"; 192 maxInjectedFailures = failures; 193 long procId = submitCreateTable(tableName); 194 195 ProcedureExecutor<MasterProcedureEnv> procExec = 196 UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); 197 ProcedureTestingUtility.waitProcedure(procExec, procId); 198 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 199 200 assertEquals(failures, INJECTED_FAILURES.get(), "exactly the injected failures happened"); 201 assertTableCreatedWithDescriptor(tableName); 202 } 203 204 /** 205 * More failures than the in-method attempts: the first round of {@value #WRITE_ATTEMPTS} writes 206 * is exhausted and {@code CREATE_TABLE_WRITE_FS_LAYOUT} fails, then the procedure retries that 207 * step, the write succeeds and the procedure completes without ever having added the table to 208 * {@code hbase:meta} in between. 209 */ 210 @Test 211 public void testProcedureRetriesAfterWriteAttemptsExhaustedThenSucceeds() throws Exception { 212 final TableName tableName = TableName.valueOf("testDescriptorWriteFailureThenSuccess"); 213 final int failures = WRITE_ATTEMPTS + 2; 214 215 failPathContains = "/" + tableName.getNameAsString() + "/.tabledesc/"; 216 maxInjectedFailures = failures; 217 long procId = submitCreateTable(tableName); 218 219 ProcedureExecutor<MasterProcedureEnv> procExec = 220 UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); 221 ProcedureTestingUtility.waitProcedure(procExec, procId); 222 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); 223 224 assertEquals(failures, INJECTED_FAILURES.get(), "exactly the injected failures happened"); 225 assertTableCreatedWithDescriptor(tableName); 226 } 227 228 private long submitCreateTable(TableName tableName) throws IOException { 229 ProcedureExecutor<MasterProcedureEnv> procExec = 230 UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); 231 TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1"); 232 RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); 233 return procExec 234 .submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); 235 } 236 237 private void assertTableCreatedWithDescriptor(TableName tableName) throws Exception { 238 HMaster master = UTIL.getHBaseCluster().getMaster(); 239 FileSystem fs = master.getMasterFileSystem().getFileSystem(); 240 Path tableDir = CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); 241 assertNotNull(FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir), 242 "table descriptor must be on the file system"); 243 assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size()); 244 UTIL.waitTableAvailable(tableName); 245 assertNotNull(UTIL.getHBaseCluster().getRegionServer(0).getTableDescriptors().get(tableName), 246 "region server must be able to load the descriptor"); 247 } 248}