001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.master.procedure;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.List;
025import java.util.stream.Collectors;
026
027import org.apache.hadoop.fs.FileStatus;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.TableNotDisabledException;
033import org.apache.hadoop.hbase.TableNotFoundException;
034import org.apache.hadoop.hbase.backup.HFileArchiver;
035import org.apache.hadoop.hbase.client.Connection;
036import org.apache.hadoop.hbase.client.Delete;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionReplicaUtil;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.ResultScanner;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.favored.FavoredNodesManager;
044import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
045import org.apache.hadoop.hbase.master.MasterFileSystem;
046import org.apache.hadoop.hbase.mob.MobConstants;
047import org.apache.hadoop.hbase.mob.MobUtils;
048import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
049import org.apache.hadoop.hbase.util.FSUtils;
050import org.apache.yetus.audience.InterfaceAudience;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState;
057
058@InterfaceAudience.Private
059public class DeleteTableProcedure
060    extends AbstractStateMachineTableProcedure<DeleteTableState> {
061  private static final Logger LOG = LoggerFactory.getLogger(DeleteTableProcedure.class);
062
063  private List<RegionInfo> regions;
064  private TableName tableName;
065
066  public DeleteTableProcedure() {
067    // Required by the Procedure framework to create the procedure on replay
068    super();
069  }
070
071  public DeleteTableProcedure(final MasterProcedureEnv env, final TableName tableName) {
072    this(env, tableName, null);
073  }
074
075  public DeleteTableProcedure(final MasterProcedureEnv env, final TableName tableName,
076      final ProcedurePrepareLatch syncLatch) {
077    super(env, syncLatch);
078    this.tableName = tableName;
079  }
080
081  @Override
082  protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState state)
083      throws InterruptedException {
084    if (LOG.isTraceEnabled()) {
085      LOG.trace(this + " execute state=" + state);
086    }
087    try {
088      switch (state) {
089        case DELETE_TABLE_PRE_OPERATION:
090          // Verify if we can delete the table
091          boolean deletable = prepareDelete(env);
092          releaseSyncLatch();
093          if (!deletable) {
094            assert isFailed() : "the delete should have an exception here";
095            return Flow.NO_MORE_STATE;
096          }
097
098          // TODO: Move out... in the acquireLock()
099          LOG.debug("Waiting for RIT for {}", this);
100          regions = env.getAssignmentManager().getRegionStates().getRegionsOfTable(getTableName());
101          assert regions != null && !regions.isEmpty() : "unexpected 0 regions";
102          ProcedureSyncWait.waitRegionInTransition(env, regions);
103
104          // Call coprocessors
105          preDelete(env);
106
107          setNextState(DeleteTableState.DELETE_TABLE_CLEAR_FS_LAYOUT);
108          break;
109        case DELETE_TABLE_CLEAR_FS_LAYOUT:
110          LOG.debug("Deleting regions from filesystem for {}", this);
111          DeleteTableProcedure.deleteFromFs(env, getTableName(), regions, true);
112          setNextState(DeleteTableState.DELETE_TABLE_REMOVE_FROM_META);
113          break;
114        case DELETE_TABLE_REMOVE_FROM_META:
115          LOG.debug("Deleting regions from META for {}", this);
116          DeleteTableProcedure.deleteFromMeta(env, getTableName(), regions);
117          setNextState(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS);
118          regions = null;
119          break;
120        case DELETE_TABLE_UNASSIGN_REGIONS:
121          LOG.debug("Deleting assignment state for {}", this);
122          DeleteTableProcedure.deleteAssignmentState(env, getTableName());
123          setNextState(DeleteTableState.DELETE_TABLE_POST_OPERATION);
124          break;
125        case DELETE_TABLE_POST_OPERATION:
126          postDelete(env);
127          LOG.debug("Finished {}", this);
128          return Flow.NO_MORE_STATE;
129        default:
130          throw new UnsupportedOperationException("unhandled state=" + state);
131      }
132    } catch (IOException e) {
133      if (isRollbackSupported(state)) {
134        setFailure("master-delete-table", e);
135      } else {
136        LOG.warn("Retriable error trying to delete table=" + getTableName() + " state=" + state, e);
137      }
138    }
139    return Flow.HAS_MORE_STATE;
140  }
141
142  @Override
143  protected boolean abort(MasterProcedureEnv env) {
144    // TODO: Current behavior is: with no rollback and no abort support, procedure may get stuck
145    // looping in retrying failing a step forever. Default behavior of abort is changed to support
146    // aborting all procedures. Override the default wisely. Following code retains the current
147    // behavior. Revisit it later.
148    return isRollbackSupported(getCurrentState()) ? super.abort(env) : false;
149  }
150
151  @Override
152  protected void rollbackState(final MasterProcedureEnv env, final DeleteTableState state) {
153    if (state == DeleteTableState.DELETE_TABLE_PRE_OPERATION) {
154      // nothing to rollback, pre-delete is just table-state checks.
155      // We can fail if the table does not exist or is not disabled.
156      // TODO: coprocessor rollback semantic is still undefined.
157      releaseSyncLatch();
158      return;
159    }
160
161    // The delete doesn't have a rollback. The execution will succeed, at some point.
162    throw new UnsupportedOperationException("unhandled state=" + state);
163  }
164
165  @Override
166  protected boolean isRollbackSupported(final DeleteTableState state) {
167    switch (state) {
168      case DELETE_TABLE_PRE_OPERATION:
169        return true;
170      default:
171        return false;
172    }
173  }
174
175  @Override
176  protected DeleteTableState getState(final int stateId) {
177    return DeleteTableState.forNumber(stateId);
178  }
179
180  @Override
181  protected int getStateId(final DeleteTableState state) {
182    return state.getNumber();
183  }
184
185  @Override
186  protected DeleteTableState getInitialState() {
187    return DeleteTableState.DELETE_TABLE_PRE_OPERATION;
188  }
189
190  @Override
191  protected boolean holdLock(MasterProcedureEnv env) {
192    return true;
193  }
194
195  @Override
196  public TableName getTableName() {
197    return tableName;
198  }
199
200  @Override
201  public TableOperationType getTableOperationType() {
202    return TableOperationType.DELETE;
203  }
204
205  @Override
206  protected void serializeStateData(ProcedureStateSerializer serializer)
207      throws IOException {
208    super.serializeStateData(serializer);
209
210    MasterProcedureProtos.DeleteTableStateData.Builder state =
211      MasterProcedureProtos.DeleteTableStateData.newBuilder()
212        .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser()))
213        .setTableName(ProtobufUtil.toProtoTableName(tableName));
214    if (regions != null) {
215      for (RegionInfo hri: regions) {
216        state.addRegionInfo(ProtobufUtil.toRegionInfo(hri));
217      }
218    }
219    serializer.serialize(state.build());
220  }
221
222  @Override
223  protected void deserializeStateData(ProcedureStateSerializer serializer)
224      throws IOException {
225    super.deserializeStateData(serializer);
226
227    MasterProcedureProtos.DeleteTableStateData state =
228        serializer.deserialize(MasterProcedureProtos.DeleteTableStateData.class);
229    setUser(MasterProcedureUtil.toUserInfo(state.getUserInfo()));
230    tableName = ProtobufUtil.toTableName(state.getTableName());
231    if (state.getRegionInfoCount() == 0) {
232      regions = null;
233    } else {
234      regions = new ArrayList<>(state.getRegionInfoCount());
235      for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) {
236        regions.add(ProtobufUtil.toRegionInfo(hri));
237      }
238    }
239  }
240
241  private boolean prepareDelete(final MasterProcedureEnv env) throws IOException {
242    try {
243      env.getMasterServices().checkTableModifiable(tableName);
244    } catch (TableNotFoundException|TableNotDisabledException e) {
245      setFailure("master-delete-table", e);
246      return false;
247    }
248    return true;
249  }
250
251  private boolean preDelete(final MasterProcedureEnv env)
252      throws IOException, InterruptedException {
253    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
254    if (cpHost != null) {
255      final TableName tableName = this.tableName;
256      cpHost.preDeleteTableAction(tableName, getUser());
257    }
258    return true;
259  }
260
261  private void postDelete(final MasterProcedureEnv env)
262      throws IOException, InterruptedException {
263    deleteTableStates(env, tableName);
264
265    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
266    if (cpHost != null) {
267      final TableName tableName = this.tableName;
268      cpHost.postCompletedDeleteTableAction(tableName, getUser());
269    }
270  }
271
272  protected static void deleteFromFs(final MasterProcedureEnv env,
273      final TableName tableName, final List<RegionInfo> regions,
274      final boolean archive) throws IOException {
275    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
276    final FileSystem fs = mfs.getFileSystem();
277    final Path tempdir = mfs.getTempDir();
278
279    final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
280    final Path tempTableDir = FSUtils.getTableDir(tempdir, tableName);
281
282    if (fs.exists(tableDir)) {
283      // Ensure temp exists
284      if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
285        throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
286      }
287
288      // Ensure parent exists
289      if (!fs.exists(tempTableDir.getParent()) && !fs.mkdirs(tempTableDir.getParent())) {
290        throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
291      }
292
293      if (fs.exists(tempTableDir)) {
294        // TODO
295        // what's in this dir? something old? probably something manual from the user...
296        // let's get rid of this stuff...
297        FileStatus[] files = fs.listStatus(tempTableDir);
298        if (files != null && files.length > 0) {
299          List<Path> regionDirList = Arrays.stream(files)
300            .filter(FileStatus::isDirectory)
301            .map(FileStatus::getPath)
302            .collect(Collectors.toList());
303          HFileArchiver.archiveRegions(env.getMasterConfiguration(), fs, mfs.getRootDir(),
304            tempTableDir, regionDirList);
305        }
306        fs.delete(tempTableDir, true);
307      }
308
309      // Move the table in /hbase/.tmp
310      if (!fs.rename(tableDir, tempTableDir)) {
311        throw new IOException("Unable to move '" + tableDir + "' to temp '" + tempTableDir + "'");
312      }
313    }
314
315    // Archive regions from FS (temp directory)
316    if (archive) {
317      List<Path> regionDirList = regions.stream().filter(RegionReplicaUtil::isDefaultReplica)
318        .map(region -> FSUtils.getRegionDirFromTableDir(tempTableDir, region))
319        .collect(Collectors.toList());
320      HFileArchiver.archiveRegions(env.getMasterConfiguration(), fs, mfs.getRootDir(), tempTableDir,
321        regionDirList);
322      LOG.debug("Table '{}' archived!", tableName);
323    }
324
325    // Archive mob data
326    Path mobTableDir = FSUtils.getTableDir(new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME),
327            tableName);
328    Path regionDir =
329            new Path(mobTableDir, MobUtils.getMobRegionInfo(tableName).getEncodedName());
330    if (fs.exists(regionDir)) {
331      HFileArchiver.archiveRegion(fs, mfs.getRootDir(), mobTableDir, regionDir);
332    }
333
334    // Delete table directory from FS (temp directory)
335    if (!fs.delete(tempTableDir, true) && fs.exists(tempTableDir)) {
336      throw new IOException("Couldn't delete " + tempTableDir);
337    }
338
339    // Delete the table directory where the mob files are saved
340    if (mobTableDir != null && fs.exists(mobTableDir)) {
341      if (!fs.delete(mobTableDir, true)) {
342        throw new IOException("Couldn't delete mob dir " + mobTableDir);
343      }
344    }
345
346    // Delete the directory on wal filesystem
347    FileSystem walFs = mfs.getWALFileSystem();
348    Path tableWALDir = FSUtils.getWALTableDir(env.getMasterConfiguration(), tableName);
349    if (walFs.exists(tableWALDir) && !walFs.delete(tableWALDir, true)) {
350      throw new IOException("Couldn't delete table dir on wal filesystem" + tableWALDir);
351    }
352  }
353
354  /**
355   * There may be items for this table still up in hbase:meta in the case where the info:regioninfo
356   * column was empty because of some write error. Remove ALL rows from hbase:meta that have to do
357   * with this table. See HBASE-12980.
358   */
359  private static void cleanRegionsInMeta(final MasterProcedureEnv env, final TableName tableName)
360      throws IOException {
361    Connection connection = env.getMasterServices().getConnection();
362    Scan tableScan = MetaTableAccessor.getScanForTableName(connection, tableName);
363    try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
364      List<Delete> deletes = new ArrayList<>();
365      try (ResultScanner resScanner = metaTable.getScanner(tableScan)) {
366        for (Result result : resScanner) {
367          deletes.add(new Delete(result.getRow()));
368        }
369      }
370      if (!deletes.isEmpty()) {
371        LOG.warn("Deleting some vestigial " + deletes.size() + " rows of " + tableName + " from "
372            + TableName.META_TABLE_NAME);
373        metaTable.delete(deletes);
374      }
375    }
376  }
377
378  protected static void deleteFromMeta(final MasterProcedureEnv env, final TableName tableName,
379      List<RegionInfo> regions) throws IOException {
380    // Clean any remaining rows for this table.
381    cleanRegionsInMeta(env, tableName);
382
383    // clean region references from the server manager
384    env.getMasterServices().getServerManager().removeRegions(regions);
385
386    // Clear Favored Nodes for this table
387    FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager();
388    if (fnm != null) {
389      fnm.deleteFavoredNodesForRegions(regions);
390    }
391
392    deleteTableDescriptorCache(env, tableName);
393  }
394
395  protected static void deleteAssignmentState(final MasterProcedureEnv env,
396      final TableName tableName) throws IOException {
397    // Clean up regions of the table in RegionStates.
398    LOG.debug("Removing '" + tableName + "' from region states.");
399    env.getMasterServices().getAssignmentManager().deleteTable(tableName);
400
401    // If entry for this table states, remove it.
402    LOG.debug("Marking '" + tableName + "' as deleted.");
403    env.getMasterServices().getTableStateManager().setDeletedTable(tableName);
404  }
405
406  protected static void deleteTableDescriptorCache(final MasterProcedureEnv env,
407      final TableName tableName) throws IOException {
408    LOG.debug("Removing '" + tableName + "' descriptor.");
409    env.getMasterServices().getTableDescriptors().remove(tableName);
410  }
411
412  protected static void deleteTableStates(final MasterProcedureEnv env, final TableName tableName)
413      throws IOException {
414    if (!tableName.isSystemTable()) {
415      ProcedureSyncWait.getMasterQuotaManager(env).removeTableFromNamespaceQuota(tableName);
416    }
417  }
418}