001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import com.google.errorprone.annotations.RestrictedApi;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.Iterator;
025import java.util.List;
026import java.util.Map;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.TableExistsException;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.errorhandling.ForeignException;
038import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
039import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
040import org.apache.hadoop.hbase.master.MasterFileSystem;
041import org.apache.hadoop.hbase.master.MetricsSnapshot;
042import org.apache.hadoop.hbase.master.RegionState;
043import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
044import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
045import org.apache.hadoop.hbase.monitoring.MonitoredTask;
046import org.apache.hadoop.hbase.monitoring.TaskMonitor;
047import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
048import org.apache.hadoop.hbase.procedure2.util.StringUtils;
049import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
050import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
051import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
052import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
053import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
054import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
055import org.apache.hadoop.hbase.util.CommonFSUtils;
056import org.apache.hadoop.hbase.util.FSTableDescriptors;
057import org.apache.hadoop.hbase.util.Pair;
058import org.apache.yetus.audience.InterfaceAudience;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
063
064import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
065import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
066import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState;
067import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotStateData;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreParentToChildRegionsPair;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
070
071@InterfaceAudience.Private
072public class CloneSnapshotProcedure extends AbstractStateMachineTableProcedure<CloneSnapshotState> {
073  private static final Logger LOG = LoggerFactory.getLogger(CloneSnapshotProcedure.class);
074
075  private TableDescriptor tableDescriptor;
076  private SnapshotDescription snapshot;
077  private boolean restoreAcl;
078  private String customSFT;
079  private List<RegionInfo> newRegions = null;
080  private Map<String, Pair<String, String>> parentsToChildrenPairMap = new HashMap<>();
081
082  // Monitor
083  private MonitoredTask monitorStatus = null;
084
085  /**
086   * Constructor (for failover)
087   */
088  public CloneSnapshotProcedure() {
089  }
090
091  public CloneSnapshotProcedure(final MasterProcedureEnv env, final TableDescriptor tableDescriptor,
092    final SnapshotDescription snapshot) {
093    this(env, tableDescriptor, snapshot, false);
094  }
095
096  /**
097   * Constructor
098   * @param env             MasterProcedureEnv
099   * @param tableDescriptor the table to operate on
100   * @param snapshot        snapshot to clone from
101   */
102  public CloneSnapshotProcedure(final MasterProcedureEnv env, final TableDescriptor tableDescriptor,
103    final SnapshotDescription snapshot, final boolean restoreAcl) {
104    this(env, tableDescriptor, snapshot, restoreAcl, null);
105  }
106
107  public CloneSnapshotProcedure(final MasterProcedureEnv env, final TableDescriptor tableDescriptor,
108    final SnapshotDescription snapshot, final boolean restoreAcl, final String customSFT) {
109    super(env);
110    this.tableDescriptor = tableDescriptor;
111    this.snapshot = snapshot;
112    this.restoreAcl = restoreAcl;
113    this.customSFT = customSFT;
114
115    getMonitorStatus();
116  }
117
118  /**
119   * Set up monitor status if it is not created.
120   */
121  private MonitoredTask getMonitorStatus() {
122    if (monitorStatus == null) {
123      monitorStatus = TaskMonitor.get()
124        .createStatus("Cloning  snapshot '" + snapshot.getName() + "' to table " + getTableName());
125    }
126    return monitorStatus;
127  }
128
129  private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException {
130    Configuration conf = env.getMasterServices().getConfiguration();
131    if (
132      restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
133        && SnapshotDescriptionUtils.isSecurityAvailable(conf)
134    ) {
135      RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf);
136    }
137  }
138
139  @Override
140  protected Flow executeFromState(final MasterProcedureEnv env, final CloneSnapshotState state)
141    throws InterruptedException {
142    LOG.trace("{} execute state={}", this, state);
143    try {
144      switch (state) {
145        case CLONE_SNAPSHOT_PRE_OPERATION:
146          // Verify if we can clone the table
147          prepareClone(env);
148
149          preCloneSnapshot(env);
150          setNextState(CloneSnapshotState.CLONE_SNAPSHOT_WRITE_FS_LAYOUT);
151          break;
152        case CLONE_SNAPSHOT_WRITE_FS_LAYOUT:
153          updateTableDescriptorWithSFT();
154          newRegions = createFilesystemLayout(env, tableDescriptor, newRegions);
155          env.getMasterServices().getTableDescriptors().update(tableDescriptor, true);
156          setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ADD_TO_META);
157          break;
158        case CLONE_SNAPSHOT_ADD_TO_META:
159          addRegionsToMeta(env);
160          setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ASSIGN_REGIONS);
161          break;
162        case CLONE_SNAPSHOT_ASSIGN_REGIONS:
163          CreateTableProcedure.setEnablingState(env, getTableName());
164
165          // Separate newRegions to split regions and regions to assign
166          List<RegionInfo> splitRegions = new ArrayList<>();
167          List<RegionInfo> regionsToAssign = new ArrayList<>();
168          newRegions.forEach(ri -> {
169            if (ri.isOffline() && (ri.isSplit() || ri.isSplitParent())) {
170              splitRegions.add(ri);
171            } else {
172              regionsToAssign.add(ri);
173            }
174          });
175
176          // For split regions, add them to RegionStates
177          AssignmentManager am = env.getAssignmentManager();
178          splitRegions
179            .forEach(ri -> am.getRegionStates().updateRegionState(ri, RegionState.State.SPLIT));
180
181          addChildProcedure(
182            env.getAssignmentManager().createRoundRobinAssignProcedures(regionsToAssign));
183          setNextState(CloneSnapshotState.CLONE_SNAPSHOT_UPDATE_DESC_CACHE);
184          break;
185        case CLONE_SNAPSHOT_UPDATE_DESC_CACHE:
186          // XXX: this stage should be named as set table enabled, as now we will cache the
187          // descriptor after writing fs layout.
188          CreateTableProcedure.setEnabledState(env, getTableName());
189          setNextState(CloneSnapshotState.CLONE_SNAPHOST_RESTORE_ACL);
190          break;
191        case CLONE_SNAPHOST_RESTORE_ACL:
192          restoreSnapshotAcl(env);
193          setNextState(CloneSnapshotState.CLONE_SNAPSHOT_POST_OPERATION);
194          break;
195        case CLONE_SNAPSHOT_POST_OPERATION:
196          postCloneSnapshot(env);
197
198          MetricsSnapshot metricsSnapshot = new MetricsSnapshot();
199          metricsSnapshot.addSnapshotClone(
200            getMonitorStatus().getCompletionTimestamp() - getMonitorStatus().getStartTime());
201          getMonitorStatus().markComplete("Clone snapshot '" + snapshot.getName() + "' completed!");
202          return Flow.NO_MORE_STATE;
203        default:
204          throw new UnsupportedOperationException("unhandled state=" + state);
205      }
206    } catch (IOException e) {
207      if (isRollbackSupported(state)) {
208        setFailure("master-clone-snapshot", e);
209      } else {
210        LOG.warn("Retriable error trying to clone snapshot=" + snapshot.getName() + " to table="
211          + getTableName() + " state=" + state, e);
212      }
213    }
214    return Flow.HAS_MORE_STATE;
215  }
216
217  /**
218   * If a StoreFileTracker is specified we strip the TableDescriptor from previous SFT config and
219   * set the specified SFT on the table level
220   */
221  private void updateTableDescriptorWithSFT() {
222    if (StringUtils.isEmpty(customSFT)) {
223      return;
224    }
225
226    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor);
227    builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, customSFT);
228    for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {
229      ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(family);
230      cfBuilder.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL, null);
231      cfBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, null);
232      builder.modifyColumnFamily(cfBuilder.build());
233    }
234    tableDescriptor = builder.build();
235  }
236
237  private void validateSFT() {
238    if (StringUtils.isEmpty(customSFT)) {
239      return;
240    }
241
242    // if customSFT is invalid getTrackerClass will throw a RuntimeException
243    Configuration sftConfig = new Configuration();
244    sftConfig.set(StoreFileTrackerFactory.TRACKER_IMPL, customSFT);
245    StoreFileTrackerFactory.getTrackerClass(sftConfig);
246  }
247
248  @Override
249  protected void rollbackState(final MasterProcedureEnv env, final CloneSnapshotState state)
250    throws IOException {
251    if (state == CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION) {
252      DeleteTableProcedure.deleteTableStates(env, getTableName());
253      // TODO-MAYBE: call the deleteTable coprocessor event?
254      return;
255    }
256
257    // The procedure doesn't have a rollback. The execution will succeed, at some point.
258    throw new UnsupportedOperationException("unhandled state=" + state);
259  }
260
261  @Override
262  protected boolean isRollbackSupported(final CloneSnapshotState state) {
263    switch (state) {
264      case CLONE_SNAPSHOT_PRE_OPERATION:
265        return true;
266      default:
267        return false;
268    }
269  }
270
271  @Override
272  protected CloneSnapshotState getState(final int stateId) {
273    return CloneSnapshotState.valueOf(stateId);
274  }
275
276  @Override
277  protected int getStateId(final CloneSnapshotState state) {
278    return state.getNumber();
279  }
280
281  @Override
282  protected CloneSnapshotState getInitialState() {
283    return CloneSnapshotState.CLONE_SNAPSHOT_PRE_OPERATION;
284  }
285
286  @Override
287  public TableName getTableName() {
288    return tableDescriptor.getTableName();
289  }
290
291  @Override
292  public TableOperationType getTableOperationType() {
293    return TableOperationType.CREATE; // Clone is creating a table
294  }
295
296  @Override
297  public void toStringClassDetails(StringBuilder sb) {
298    sb.append(getClass().getSimpleName());
299    sb.append(" (table=");
300    sb.append(getTableName());
301    sb.append(" snapshot=");
302    sb.append(snapshot);
303    sb.append(")");
304  }
305
306  @Override
307  protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
308    super.serializeStateData(serializer);
309
310    CloneSnapshotStateData.Builder cloneSnapshotMsg = CloneSnapshotStateData.newBuilder()
311      .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())).setSnapshot(this.snapshot)
312      .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor));
313
314    cloneSnapshotMsg.setRestoreAcl(restoreAcl);
315    if (newRegions != null) {
316      for (RegionInfo hri : newRegions) {
317        cloneSnapshotMsg.addRegionInfo(ProtobufUtil.toRegionInfo(hri));
318      }
319    }
320    if (!parentsToChildrenPairMap.isEmpty()) {
321      final Iterator<Map.Entry<String, Pair<String, String>>> it =
322        parentsToChildrenPairMap.entrySet().iterator();
323      while (it.hasNext()) {
324        final Map.Entry<String, Pair<String, String>> entry = it.next();
325
326        RestoreParentToChildRegionsPair.Builder parentToChildrenPair =
327          RestoreParentToChildRegionsPair.newBuilder().setParentRegionName(entry.getKey())
328            .setChild1RegionName(entry.getValue().getFirst())
329            .setChild2RegionName(entry.getValue().getSecond());
330        cloneSnapshotMsg.addParentToChildRegionsPairList(parentToChildrenPair);
331      }
332    }
333    if (!StringUtils.isEmpty(customSFT)) {
334      cloneSnapshotMsg.setCustomSFT(customSFT);
335    }
336    serializer.serialize(cloneSnapshotMsg.build());
337  }
338
339  @Override
340  protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
341    super.deserializeStateData(serializer);
342
343    CloneSnapshotStateData cloneSnapshotMsg = serializer.deserialize(CloneSnapshotStateData.class);
344    setUser(MasterProcedureUtil.toUserInfo(cloneSnapshotMsg.getUserInfo()));
345    snapshot = cloneSnapshotMsg.getSnapshot();
346    tableDescriptor = ProtobufUtil.toTableDescriptor(cloneSnapshotMsg.getTableSchema());
347    if (cloneSnapshotMsg.hasRestoreAcl()) {
348      restoreAcl = cloneSnapshotMsg.getRestoreAcl();
349    }
350    if (cloneSnapshotMsg.getRegionInfoCount() == 0) {
351      newRegions = null;
352    } else {
353      newRegions = new ArrayList<>(cloneSnapshotMsg.getRegionInfoCount());
354      for (HBaseProtos.RegionInfo hri : cloneSnapshotMsg.getRegionInfoList()) {
355        newRegions.add(ProtobufUtil.toRegionInfo(hri));
356      }
357    }
358    if (cloneSnapshotMsg.getParentToChildRegionsPairListCount() > 0) {
359      parentsToChildrenPairMap = new HashMap<>();
360      for (RestoreParentToChildRegionsPair parentToChildrenPair : cloneSnapshotMsg
361        .getParentToChildRegionsPairListList()) {
362        parentsToChildrenPairMap.put(parentToChildrenPair.getParentRegionName(), new Pair<>(
363          parentToChildrenPair.getChild1RegionName(), parentToChildrenPair.getChild2RegionName()));
364      }
365    }
366    if (!StringUtils.isEmpty(cloneSnapshotMsg.getCustomSFT())) {
367      customSFT = cloneSnapshotMsg.getCustomSFT();
368    }
369    // Make sure that the monitor status is set up
370    getMonitorStatus();
371  }
372
373  /**
374   * Action before any real action of cloning from snapshot.
375   * @param env MasterProcedureEnv
376   */
377  private void prepareClone(final MasterProcedureEnv env) throws IOException {
378    final TableName tableName = getTableName();
379    if (env.getMasterServices().getTableDescriptors().exists(tableName)) {
380      throw new TableExistsException(tableName);
381    }
382
383    validateSFT();
384  }
385
386  /**
387   * Action before cloning from snapshot.
388   * @param env MasterProcedureEnv nn
389   */
390  private void preCloneSnapshot(final MasterProcedureEnv env)
391    throws IOException, InterruptedException {
392    if (!getTableName().isSystemTable()) {
393      // Check and update namespace quota
394      final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
395
396      SnapshotManifest manifest =
397        SnapshotManifest.open(env.getMasterConfiguration(), mfs.getFileSystem(),
398          SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot);
399
400      ProcedureSyncWait.getMasterQuotaManager(env).checkNamespaceTableAndRegionQuota(getTableName(),
401        manifest.getRegionManifestsMap().size());
402    }
403
404    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
405    if (cpHost != null) {
406      cpHost.preCreateTableAction(tableDescriptor, null, getUser());
407    }
408  }
409
410  /**
411   * Action after cloning from snapshot.
412   * @param env MasterProcedureEnv nn
413   */
414  private void postCloneSnapshot(final MasterProcedureEnv env)
415    throws IOException, InterruptedException {
416    final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
417    if (cpHost != null) {
418      final RegionInfo[] regions =
419        (newRegions == null) ? null : newRegions.toArray(new RegionInfo[newRegions.size()]);
420      cpHost.postCompletedCreateTableAction(tableDescriptor, regions, getUser());
421    }
422  }
423
424  /**
425   * Create regions in file system.
426   * @param env MasterProcedureEnv n
427   */
428  private List<RegionInfo> createFilesystemLayout(final MasterProcedureEnv env,
429    final TableDescriptor tableDescriptor, final List<RegionInfo> newRegions) throws IOException {
430    return createFsLayout(env, tableDescriptor, newRegions, new CreateHdfsRegions() {
431      @Override
432      public List<RegionInfo> createHdfsRegions(final MasterProcedureEnv env,
433        final Path tableRootDir, final TableName tableName, final List<RegionInfo> newRegions)
434        throws IOException {
435
436        final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
437        final FileSystem fs = mfs.getFileSystem();
438        final Path rootDir = mfs.getRootDir();
439        final Configuration conf = env.getMasterConfiguration();
440        final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
441
442        getMonitorStatus().setStatus("Clone snapshot - creating regions for table: " + tableName);
443
444        try {
445          // 1. Execute the on-disk Clone
446          Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
447          SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
448          RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs, manifest,
449            tableDescriptor, tableRootDir, monitorException, monitorStatus);
450          RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
451
452          // Clone operation should not have stuff to restore or remove
453          Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
454            "A clone should not have regions to restore");
455          Preconditions.checkArgument(!metaChanges.hasRegionsToRemove(),
456            "A clone should not have regions to remove");
457
458          // At this point the clone is complete. Next step is enabling the table.
459          String msg =
460            "Clone snapshot=" + snapshot.getName() + " on table=" + tableName + " completed!";
461          LOG.info(msg);
462          monitorStatus.setStatus(msg + " Waiting for table to be enabled...");
463
464          // 2. Let the next step to add the regions to meta
465          return metaChanges.getRegionsToAdd();
466        } catch (Exception e) {
467          String msg = "clone snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
468            + " failed because " + e.getMessage();
469          LOG.error(msg, e);
470          IOException rse =
471            new RestoreSnapshotException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
472
473          // these handlers aren't futures so we need to register the error here.
474          monitorException.receive(new ForeignException("Master CloneSnapshotProcedure", rse));
475          throw rse;
476        }
477      }
478    });
479  }
480
481  /**
482   * Create region layout in file system.
483   * @param env MasterProcedureEnv n
484   */
485  private List<RegionInfo> createFsLayout(final MasterProcedureEnv env,
486    final TableDescriptor tableDescriptor, List<RegionInfo> newRegions,
487    final CreateHdfsRegions hdfsRegionHandler) throws IOException {
488    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
489
490    // 1. Create Table Descriptor
491    // using a copy of descriptor, table will be created enabling first
492    final Path tableDir =
493      CommonFSUtils.getTableDir(mfs.getRootDir(), tableDescriptor.getTableName());
494    if (CommonFSUtils.isExists(mfs.getFileSystem(), tableDir)) {
495      // if the region dirs exist, will cause exception and unlimited retry (see HBASE-24546)
496      LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tableDir);
497      CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tableDir);
498    }
499    ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors()))
500      .createTableDescriptorForTableDirectory(tableDir,
501        TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false);
502
503    // 2. Create Regions
504    newRegions = hdfsRegionHandler.createHdfsRegions(env, mfs.getRootDir(),
505      tableDescriptor.getTableName(), newRegions);
506
507    return newRegions;
508  }
509
510  /**
511   * Add regions to hbase:meta table.
512   * @param env MasterProcedureEnv n
513   */
514  private void addRegionsToMeta(final MasterProcedureEnv env) throws IOException {
515    newRegions = CreateTableProcedure.addTableToMeta(env, tableDescriptor, newRegions);
516
517    // TODO: parentsToChildrenPairMap is always empty, which makes updateMetaParentRegions()
518    // a no-op. This part seems unnecessary. Figure out. - Appy 12/21/17
519    RestoreSnapshotHelper.RestoreMetaChanges metaChanges =
520      new RestoreSnapshotHelper.RestoreMetaChanges(tableDescriptor, parentsToChildrenPairMap);
521    metaChanges.updateMetaParentRegions(env.getMasterServices().getConnection(), newRegions);
522  }
523
524  /**
525   * Exposed for Testing: HBASE-26462
526   */
527  @RestrictedApi(explanation = "Should only be called in tests", link = "",
528      allowedOnPath = ".*/src/test/.*")
529  public boolean getRestoreAcl() {
530    return restoreAcl;
531  }
532
533}