001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.master.procedure; 020 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.List; 025import java.util.stream.Collectors; 026import org.apache.hadoop.fs.FileStatus; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.fs.Path; 029import org.apache.hadoop.hbase.MetaTableAccessor; 030import org.apache.hadoop.hbase.TableName; 031import org.apache.hadoop.hbase.TableNotDisabledException; 032import org.apache.hadoop.hbase.TableNotFoundException; 033import org.apache.hadoop.hbase.backup.HFileArchiver; 034import org.apache.hadoop.hbase.client.Delete; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.client.RegionReplicaUtil; 037import org.apache.hadoop.hbase.client.Result; 038import org.apache.hadoop.hbase.client.ResultScanner; 039import org.apache.hadoop.hbase.client.Scan; 040import org.apache.hadoop.hbase.client.Table; 041import org.apache.hadoop.hbase.favored.FavoredNodesManager; 042import org.apache.hadoop.hbase.filter.KeyOnlyFilter; 043import org.apache.hadoop.hbase.master.MasterCoprocessorHost; 044import org.apache.hadoop.hbase.master.MasterFileSystem; 045import org.apache.hadoop.hbase.mob.MobConstants; 046import org.apache.hadoop.hbase.mob.MobUtils; 047import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; 048import org.apache.hadoop.hbase.util.CommonFSUtils; 049import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 050import org.apache.hadoop.hbase.util.FSUtils; 051import org.apache.yetus.audience.InterfaceAudience; 052import org.slf4j.Logger; 053import org.slf4j.LoggerFactory; 054 055import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 056import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 057import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; 058import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState; 059 060@InterfaceAudience.Private 061public class DeleteTableProcedure 062 extends AbstractStateMachineTableProcedure<DeleteTableState> { 063 private static final Logger LOG = LoggerFactory.getLogger(DeleteTableProcedure.class); 064 065 private List<RegionInfo> regions; 066 private TableName tableName; 067 068 public DeleteTableProcedure() { 069 // Required by the Procedure framework to create the procedure on replay 070 super(); 071 } 072 073 public DeleteTableProcedure(final MasterProcedureEnv env, final TableName tableName) { 074 this(env, tableName, null); 075 } 076 077 public DeleteTableProcedure(final MasterProcedureEnv env, final TableName tableName, 078 final ProcedurePrepareLatch syncLatch) { 079 super(env, syncLatch); 080 this.tableName = tableName; 081 } 082 083 @Override 084 protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState state) 085 throws InterruptedException { 086 if (LOG.isTraceEnabled()) { 087 LOG.trace(this + " execute state=" + state); 088 } 089 try { 090 switch (state) { 091 case DELETE_TABLE_PRE_OPERATION: 092 // Verify if we can delete the table 093 boolean deletable = prepareDelete(env); 094 releaseSyncLatch(); 095 if (!deletable) { 096 assert isFailed() : "the delete should have an exception here"; 097 return Flow.NO_MORE_STATE; 098 } 099 100 // TODO: Move out... in the acquireLock() 101 LOG.debug("Waiting for RIT for {}", this); 102 regions = env.getAssignmentManager().getRegionStates() 103 .getRegionsOfTableForDeleting(getTableName()); 104 assert regions != null && !regions.isEmpty() : "unexpected 0 regions"; 105 ProcedureSyncWait.waitRegionInTransition(env, regions); 106 107 // Call coprocessors 108 preDelete(env); 109 110 setNextState(DeleteTableState.DELETE_TABLE_CLEAR_FS_LAYOUT); 111 break; 112 case DELETE_TABLE_CLEAR_FS_LAYOUT: 113 LOG.debug("Deleting regions from filesystem for {}", this); 114 DeleteTableProcedure.deleteFromFs(env, getTableName(), regions, true); 115 setNextState(DeleteTableState.DELETE_TABLE_REMOVE_FROM_META); 116 break; 117 case DELETE_TABLE_REMOVE_FROM_META: 118 LOG.debug("Deleting regions from META for {}", this); 119 DeleteTableProcedure.deleteFromMeta(env, getTableName(), regions); 120 setNextState(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS); 121 regions = null; 122 break; 123 case DELETE_TABLE_UNASSIGN_REGIONS: 124 LOG.debug("Deleting assignment state for {}", this); 125 DeleteTableProcedure.deleteAssignmentState(env, getTableName()); 126 setNextState(DeleteTableState.DELETE_TABLE_POST_OPERATION); 127 break; 128 case DELETE_TABLE_POST_OPERATION: 129 postDelete(env); 130 LOG.debug("Finished {}", this); 131 return Flow.NO_MORE_STATE; 132 default: 133 throw new UnsupportedOperationException("unhandled state=" + state); 134 } 135 } catch (IOException e) { 136 if (isRollbackSupported(state)) { 137 setFailure("master-delete-table", e); 138 } else { 139 LOG.warn("Retriable error trying to delete table=" + getTableName() + " state=" + state, e); 140 } 141 } 142 return Flow.HAS_MORE_STATE; 143 } 144 145 @Override 146 protected boolean abort(MasterProcedureEnv env) { 147 // TODO: Current behavior is: with no rollback and no abort support, procedure may get stuck 148 // looping in retrying failing a step forever. Default behavior of abort is changed to support 149 // aborting all procedures. Override the default wisely. Following code retains the current 150 // behavior. Revisit it later. 151 return isRollbackSupported(getCurrentState()) ? super.abort(env) : false; 152 } 153 154 @Override 155 protected void rollbackState(final MasterProcedureEnv env, final DeleteTableState state) { 156 if (state == DeleteTableState.DELETE_TABLE_PRE_OPERATION) { 157 // nothing to rollback, pre-delete is just table-state checks. 158 // We can fail if the table does not exist or is not disabled. 159 // TODO: coprocessor rollback semantic is still undefined. 160 releaseSyncLatch(); 161 return; 162 } 163 164 // The delete doesn't have a rollback. The execution will succeed, at some point. 165 throw new UnsupportedOperationException("unhandled state=" + state); 166 } 167 168 @Override 169 protected boolean isRollbackSupported(final DeleteTableState state) { 170 switch (state) { 171 case DELETE_TABLE_PRE_OPERATION: 172 return true; 173 default: 174 return false; 175 } 176 } 177 178 @Override 179 protected DeleteTableState getState(final int stateId) { 180 return DeleteTableState.forNumber(stateId); 181 } 182 183 @Override 184 protected int getStateId(final DeleteTableState state) { 185 return state.getNumber(); 186 } 187 188 @Override 189 protected DeleteTableState getInitialState() { 190 return DeleteTableState.DELETE_TABLE_PRE_OPERATION; 191 } 192 193 @Override 194 protected boolean holdLock(MasterProcedureEnv env) { 195 return true; 196 } 197 198 @Override 199 public TableName getTableName() { 200 return tableName; 201 } 202 203 @Override 204 public TableOperationType getTableOperationType() { 205 return TableOperationType.DELETE; 206 } 207 208 @Override 209 protected void serializeStateData(ProcedureStateSerializer serializer) 210 throws IOException { 211 super.serializeStateData(serializer); 212 213 MasterProcedureProtos.DeleteTableStateData.Builder state = 214 MasterProcedureProtos.DeleteTableStateData.newBuilder() 215 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) 216 .setTableName(ProtobufUtil.toProtoTableName(tableName)); 217 if (regions != null) { 218 for (RegionInfo hri: regions) { 219 state.addRegionInfo(ProtobufUtil.toRegionInfo(hri)); 220 } 221 } 222 serializer.serialize(state.build()); 223 } 224 225 @Override 226 protected void deserializeStateData(ProcedureStateSerializer serializer) 227 throws IOException { 228 super.deserializeStateData(serializer); 229 230 MasterProcedureProtos.DeleteTableStateData state = 231 serializer.deserialize(MasterProcedureProtos.DeleteTableStateData.class); 232 setUser(MasterProcedureUtil.toUserInfo(state.getUserInfo())); 233 tableName = ProtobufUtil.toTableName(state.getTableName()); 234 if (state.getRegionInfoCount() == 0) { 235 regions = null; 236 } else { 237 regions = new ArrayList<>(state.getRegionInfoCount()); 238 for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) { 239 regions.add(ProtobufUtil.toRegionInfo(hri)); 240 } 241 } 242 } 243 244 private boolean prepareDelete(final MasterProcedureEnv env) throws IOException { 245 try { 246 env.getMasterServices().checkTableModifiable(tableName); 247 } catch (TableNotFoundException|TableNotDisabledException e) { 248 setFailure("master-delete-table", e); 249 return false; 250 } 251 return true; 252 } 253 254 private boolean preDelete(final MasterProcedureEnv env) 255 throws IOException, InterruptedException { 256 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 257 if (cpHost != null) { 258 final TableName tableName = this.tableName; 259 cpHost.preDeleteTableAction(tableName, getUser()); 260 } 261 return true; 262 } 263 264 private void postDelete(final MasterProcedureEnv env) 265 throws IOException, InterruptedException { 266 deleteTableStates(env, tableName); 267 268 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); 269 if (cpHost != null) { 270 final TableName tableName = this.tableName; 271 cpHost.postCompletedDeleteTableAction(tableName, getUser()); 272 } 273 } 274 275 protected static void deleteFromFs(final MasterProcedureEnv env, 276 final TableName tableName, final List<RegionInfo> regions, 277 final boolean archive) throws IOException { 278 final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); 279 final FileSystem fs = mfs.getFileSystem(); 280 final Path tempdir = mfs.getTempDir(); 281 282 final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tableName); 283 final Path tempTableDir = CommonFSUtils.getTableDir(tempdir, tableName); 284 285 if (fs.exists(tableDir)) { 286 // Ensure temp exists 287 if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) { 288 throw new IOException("HBase temp directory '" + tempdir + "' creation failure."); 289 } 290 291 // Ensure parent exists 292 if (!fs.exists(tempTableDir.getParent()) && !fs.mkdirs(tempTableDir.getParent())) { 293 throw new IOException("HBase temp directory '" + tempdir + "' creation failure."); 294 } 295 296 if (fs.exists(tempTableDir)) { 297 // TODO 298 // what's in this dir? something old? probably something manual from the user... 299 // let's get rid of this stuff... 300 FileStatus[] files = fs.listStatus(tempTableDir); 301 if (files != null && files.length > 0) { 302 List<Path> regionDirList = Arrays.stream(files) 303 .filter(FileStatus::isDirectory) 304 .map(FileStatus::getPath) 305 .collect(Collectors.toList()); 306 HFileArchiver.archiveRegions(env.getMasterConfiguration(), fs, mfs.getRootDir(), 307 tempTableDir, regionDirList); 308 } 309 fs.delete(tempTableDir, true); 310 } 311 312 // Move the table in /hbase/.tmp 313 if (!fs.rename(tableDir, tempTableDir)) { 314 throw new IOException("Unable to move '" + tableDir + "' to temp '" + tempTableDir + "'"); 315 } 316 } 317 318 // Archive regions from FS (temp directory) 319 if (archive) { 320 List<Path> regionDirList = regions.stream().filter(RegionReplicaUtil::isDefaultReplica) 321 .map(region -> FSUtils.getRegionDirFromTableDir(tempTableDir, region)) 322 .collect(Collectors.toList()); 323 HFileArchiver.archiveRegions(env.getMasterConfiguration(), fs, mfs.getRootDir(), tempTableDir, 324 regionDirList); 325 if (!regionDirList.isEmpty()) { 326 LOG.debug("Archived {} regions", tableName); 327 } 328 } 329 330 // Archive mob data 331 Path mobTableDir = 332 CommonFSUtils.getTableDir(new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME), tableName); 333 Path regionDir = 334 new Path(mobTableDir, MobUtils.getMobRegionInfo(tableName).getEncodedName()); 335 if (fs.exists(regionDir)) { 336 HFileArchiver.archiveRegion(fs, mfs.getRootDir(), mobTableDir, regionDir); 337 } 338 339 // Delete table directory from FS (temp directory) 340 if (!fs.delete(tempTableDir, true) && fs.exists(tempTableDir)) { 341 throw new IOException("Couldn't delete " + tempTableDir); 342 } 343 344 // Delete the table directory where the mob files are saved 345 if (mobTableDir != null && fs.exists(mobTableDir)) { 346 if (!fs.delete(mobTableDir, true)) { 347 throw new IOException("Couldn't delete mob dir " + mobTableDir); 348 } 349 } 350 351 // Delete the directory on wal filesystem 352 FileSystem walFs = mfs.getWALFileSystem(); 353 Path tableWALDir = CommonFSUtils.getWALTableDir(env.getMasterConfiguration(), tableName); 354 if (walFs.exists(tableWALDir) && !walFs.delete(tableWALDir, true)) { 355 throw new IOException("Couldn't delete table dir on wal filesystem" + tableWALDir); 356 } 357 } 358 359 /** 360 * There may be items for this table still up in hbase:meta in the case where the info:regioninfo 361 * column was empty because of some write error. Remove ALL rows from hbase:meta that have to do 362 * with this table. 363 * <p/> 364 * See HBASE-12980. 365 */ 366 private static void cleanRegionsInMeta(final MasterProcedureEnv env, final TableName tableName) 367 throws IOException { 368 Scan tableScan = MetaTableAccessor.getScanForTableName(env.getMasterConfiguration(), tableName) 369 .setFilter(new KeyOnlyFilter()); 370 long now = EnvironmentEdgeManager.currentTime(); 371 List<Delete> deletes = new ArrayList<>(); 372 try ( 373 Table metaTable = env.getMasterServices().getConnection().getTable(TableName.META_TABLE_NAME); 374 ResultScanner scanner = metaTable.getScanner(tableScan)) { 375 for (;;) { 376 Result result = scanner.next(); 377 if (result == null) { 378 break; 379 } 380 deletes.add(new Delete(result.getRow(), now)); 381 } 382 if (!deletes.isEmpty()) { 383 LOG.warn("Deleting some vestigial " + deletes.size() + " rows of " + tableName + " from " + 384 TableName.META_TABLE_NAME); 385 metaTable.delete(deletes); 386 } 387 } 388 } 389 390 protected static void deleteFromMeta(final MasterProcedureEnv env, final TableName tableName, 391 List<RegionInfo> regions) throws IOException { 392 // Clean any remaining rows for this table. 393 cleanRegionsInMeta(env, tableName); 394 395 // clean region references from the server manager 396 env.getMasterServices().getServerManager().removeRegions(regions); 397 398 // Clear Favored Nodes for this table 399 FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager(); 400 if (fnm != null) { 401 fnm.deleteFavoredNodesForRegions(regions); 402 } 403 404 deleteTableDescriptorCache(env, tableName); 405 } 406 407 protected static void deleteAssignmentState(final MasterProcedureEnv env, 408 final TableName tableName) throws IOException { 409 // Clean up regions of the table in RegionStates. 410 LOG.debug("Removing '" + tableName + "' from region states."); 411 env.getMasterServices().getAssignmentManager().deleteTable(tableName); 412 413 // If entry for this table states, remove it. 414 LOG.debug("Marking '" + tableName + "' as deleted."); 415 env.getMasterServices().getTableStateManager().setDeletedTable(tableName); 416 } 417 418 protected static void deleteTableDescriptorCache(final MasterProcedureEnv env, 419 final TableName tableName) throws IOException { 420 LOG.debug("Removing '" + tableName + "' descriptor."); 421 env.getMasterServices().getTableDescriptors().remove(tableName); 422 } 423 424 protected static void deleteTableStates(final MasterProcedureEnv env, final TableName tableName) 425 throws IOException { 426 if (!tableName.isSystemTable()) { 427 ProcedureSyncWait.getMasterQuotaManager(env).removeTableFromNamespaceQuota(tableName); 428 } 429 } 430}