001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.HashMap;
023import java.util.List;
024
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileStatus;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.backup.BackupInfo;
031import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
032import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
033import org.apache.hadoop.hbase.backup.BackupRequest;
034import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
035import org.apache.hadoop.hbase.backup.BackupType;
036import org.apache.hadoop.hbase.backup.HBackupFileSystem;
037import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.apache.hadoop.hbase.util.FSUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
047
048/**
049 * Base class for backup operation. Concrete implementation for
050 * full and incremental backup are delegated to corresponding sub-classes:
051 * {@link FullTableBackupClient} and {@link IncrementalTableBackupClient}
052 *
053 */
054@InterfaceAudience.Private
055public abstract class TableBackupClient {
056
057  public static final String BACKUP_CLIENT_IMPL_CLASS = "backup.client.impl.class";
058
059  @VisibleForTesting
060  public static final String BACKUP_TEST_MODE_STAGE = "backup.test.mode.stage";
061
062  private static final Logger LOG = LoggerFactory.getLogger(TableBackupClient.class);
063
064  protected Configuration conf;
065  protected Connection conn;
066  protected String backupId;
067  protected List<TableName> tableList;
068  protected HashMap<String, Long> newTimestamps = null;
069
070  protected BackupManager backupManager;
071  protected BackupInfo backupInfo;
072  protected FileSystem fs;
073
074  public TableBackupClient() {
075  }
076
077  public TableBackupClient(final Connection conn, final String backupId, BackupRequest request)
078      throws IOException {
079    init(conn, backupId, request);
080  }
081
082  public void init(final Connection conn, final String backupId, BackupRequest request)
083      throws IOException {
084    if (request.getBackupType() == BackupType.FULL) {
085      backupManager = new BackupManager(conn, conn.getConfiguration());
086    } else {
087      backupManager = new IncrementalBackupManager(conn, conn.getConfiguration());
088    }
089    this.backupId = backupId;
090    this.tableList = request.getTableList();
091    this.conn = conn;
092    this.conf = conn.getConfiguration();
093    this.fs = FSUtils.getCurrentFileSystem(conf);
094    backupInfo =
095        backupManager.createBackupInfo(backupId, request.getBackupType(), tableList,
096          request.getTargetRootDir(), request.getTotalTasks(), request.getBandwidth());
097    if (tableList == null || tableList.isEmpty()) {
098      this.tableList = new ArrayList<>(backupInfo.getTables());
099    }
100    // Start new session
101    backupManager.startBackupSession();
102  }
103
104  /**
105   * Begin the overall backup.
106   * @param backupInfo backup info
107   * @throws IOException exception
108   */
109  protected void beginBackup(BackupManager backupManager, BackupInfo backupInfo)
110      throws IOException {
111
112    BackupSystemTable.snapshot(conn);
113    backupManager.setBackupInfo(backupInfo);
114    // set the start timestamp of the overall backup
115    long startTs = EnvironmentEdgeManager.currentTime();
116    backupInfo.setStartTs(startTs);
117    // set overall backup status: ongoing
118    backupInfo.setState(BackupState.RUNNING);
119    backupInfo.setPhase(BackupPhase.REQUEST);
120    LOG.info("Backup " + backupInfo.getBackupId() + " started at " + startTs + ".");
121
122    backupManager.updateBackupInfo(backupInfo);
123    if (LOG.isDebugEnabled()) {
124      LOG.debug("Backup session " + backupInfo.getBackupId() + " has been started.");
125    }
126  }
127
128  protected String getMessage(Exception e) {
129    String msg = e.getMessage();
130    if (msg == null || msg.equals("")) {
131      msg = e.getClass().getName();
132    }
133    return msg;
134  }
135
136  /**
137   * Delete HBase snapshot for backup.
138   * @param backupInfo backup info
139   * @throws IOException exception
140   */
141  protected static void deleteSnapshots(final Connection conn, BackupInfo backupInfo,
142      Configuration conf) throws IOException {
143    LOG.debug("Trying to delete snapshot for full backup.");
144    for (String snapshotName : backupInfo.getSnapshotNames()) {
145      if (snapshotName == null) {
146        continue;
147      }
148      LOG.debug("Trying to delete snapshot: " + snapshotName);
149
150      try (Admin admin = conn.getAdmin()) {
151        admin.deleteSnapshot(snapshotName);
152      }
153      LOG.debug("Deleting the snapshot " + snapshotName + " for backup " + backupInfo.getBackupId()
154          + " succeeded.");
155    }
156  }
157
158  /**
159   * Clean up directories with prefix "exportSnapshot-", which are generated when exporting
160   * snapshots.
161   * @throws IOException exception
162   */
163  protected static void cleanupExportSnapshotLog(Configuration conf) throws IOException {
164    FileSystem fs = FSUtils.getCurrentFileSystem(conf);
165    Path stagingDir =
166        new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory()
167            .toString()));
168    FileStatus[] files = FSUtils.listStatus(fs, stagingDir);
169    if (files == null) {
170      return;
171    }
172    for (FileStatus file : files) {
173      if (file.getPath().getName().startsWith("exportSnapshot-")) {
174        LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName());
175        if (FSUtils.delete(fs, file.getPath(), true) == false) {
176          LOG.warn("Can not delete " + file.getPath());
177        }
178      }
179    }
180  }
181
182  /**
183   * Clean up the uncompleted data at target directory if the ongoing backup has already entered
184   * the copy phase.
185   */
186  protected static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) {
187    try {
188      // clean up the uncompleted data at target directory if the ongoing backup has already entered
189      // the copy phase
190      LOG.debug("Trying to cleanup up target dir. Current backup phase: "
191          + backupInfo.getPhase());
192      if (backupInfo.getPhase().equals(BackupPhase.SNAPSHOTCOPY)
193          || backupInfo.getPhase().equals(BackupPhase.INCREMENTAL_COPY)
194          || backupInfo.getPhase().equals(BackupPhase.STORE_MANIFEST)) {
195        FileSystem outputFs =
196            FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf);
197
198        // now treat one backup as a transaction, clean up data that has been partially copied at
199        // table level
200        for (TableName table : backupInfo.getTables()) {
201          Path targetDirPath =
202              new Path(HBackupFileSystem.getTableBackupDir(backupInfo.getBackupRootDir(),
203                backupInfo.getBackupId(), table));
204          if (outputFs.delete(targetDirPath, true)) {
205            LOG.debug("Cleaning up uncompleted backup data at " + targetDirPath.toString()
206                + " done.");
207          } else {
208            LOG.debug("No data has been copied to " + targetDirPath.toString() + ".");
209          }
210
211          Path tableDir = targetDirPath.getParent();
212          FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir);
213          if (backups == null || backups.length == 0) {
214            outputFs.delete(tableDir, true);
215            LOG.debug(tableDir.toString() + " is empty, remove it.");
216          }
217        }
218      }
219
220    } catch (IOException e1) {
221      LOG.error("Cleaning up uncompleted backup data of " + backupInfo.getBackupId() + " at "
222          + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + ".");
223    }
224  }
225
226  /**
227   * Fail the overall backup.
228   * @param backupInfo backup info
229   * @param e exception
230   * @throws IOException exception
231   */
232  protected void failBackup(Connection conn, BackupInfo backupInfo, BackupManager backupManager,
233      Exception e, String msg, BackupType type, Configuration conf) throws IOException {
234    try {
235      LOG.error(msg + getMessage(e), e);
236      // If this is a cancel exception, then we've already cleaned.
237      // set the failure timestamp of the overall backup
238      backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
239      // set failure message
240      backupInfo.setFailedMsg(e.getMessage());
241      // set overall backup status: failed
242      backupInfo.setState(BackupState.FAILED);
243      // compose the backup failed data
244      String backupFailedData =
245          "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs()
246              + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" + backupInfo.getPhase()
247              + ",failedmessage=" + backupInfo.getFailedMsg();
248      LOG.error(backupFailedData);
249      cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
250      // If backup session is updated to FAILED state - means we
251      // processed recovery already.
252      backupManager.updateBackupInfo(backupInfo);
253      backupManager.finishBackupSession();
254      LOG.error("Backup " + backupInfo.getBackupId() + " failed.");
255    } catch (IOException ee) {
256      LOG.error("Please run backup repair tool manually to restore backup system integrity");
257      throw ee;
258    }
259  }
260
261  public static void cleanupAndRestoreBackupSystem(Connection conn, BackupInfo backupInfo,
262      Configuration conf) throws IOException {
263    BackupType type = backupInfo.getType();
264    // if full backup, then delete HBase snapshots if there already are snapshots taken
265    // and also clean up export snapshot log files if exist
266    if (type == BackupType.FULL) {
267      deleteSnapshots(conn, backupInfo, conf);
268      cleanupExportSnapshotLog(conf);
269    }
270    BackupSystemTable.restoreFromSnapshot(conn);
271    BackupSystemTable.deleteSnapshot(conn);
272    // clean up the uncompleted data at target directory if the ongoing backup has already entered
273    // the copy phase
274    // For incremental backup, DistCp logs will be cleaned with the targetDir.
275    cleanupTargetDir(backupInfo, conf);
276  }
277
278  /**
279   * Add manifest for the current backup. The manifest is stored within the table backup directory.
280   * @param backupInfo The current backup info
281   * @throws IOException exception
282   */
283  protected void addManifest(BackupInfo backupInfo, BackupManager backupManager, BackupType type,
284      Configuration conf) throws IOException {
285    // set the overall backup phase : store manifest
286    backupInfo.setPhase(BackupPhase.STORE_MANIFEST);
287
288    BackupManifest manifest;
289
290    // Since we have each table's backup in its own directory structure,
291    // we'll store its manifest with the table directory.
292    for (TableName table : backupInfo.getTables()) {
293      manifest = new BackupManifest(backupInfo, table);
294      ArrayList<BackupImage> ancestors = backupManager.getAncestors(backupInfo, table);
295      for (BackupImage image : ancestors) {
296        manifest.addDependentImage(image);
297      }
298
299      if (type == BackupType.INCREMENTAL) {
300        // We'll store the log timestamps for this table only in its manifest.
301        HashMap<TableName, HashMap<String, Long>> tableTimestampMap = new HashMap<>();
302        tableTimestampMap.put(table, backupInfo.getIncrTimestampMap().get(table));
303        manifest.setIncrTimestampMap(tableTimestampMap);
304        ArrayList<BackupImage> ancestorss = backupManager.getAncestors(backupInfo);
305        for (BackupImage image : ancestorss) {
306          manifest.addDependentImage(image);
307        }
308      }
309      manifest.store(conf);
310    }
311
312    // For incremental backup, we store a overall manifest in
313    // <backup-root-dir>/WALs/<backup-id>
314    // This is used when created the next incremental backup
315    if (type == BackupType.INCREMENTAL) {
316      manifest = new BackupManifest(backupInfo);
317      // set the table region server start and end timestamps for incremental backup
318      manifest.setIncrTimestampMap(backupInfo.getIncrTimestampMap());
319      ArrayList<BackupImage> ancestors = backupManager.getAncestors(backupInfo);
320      for (BackupImage image : ancestors) {
321        manifest.addDependentImage(image);
322      }
323      manifest.store(conf);
324    }
325  }
326
327  /**
328   * Get backup request meta data dir as string.
329   * @param backupInfo backup info
330   * @return meta data dir
331   */
332  protected String obtainBackupMetaDataStr(BackupInfo backupInfo) {
333    StringBuffer sb = new StringBuffer();
334    sb.append("type=" + backupInfo.getType() + ",tablelist=");
335    for (TableName table : backupInfo.getTables()) {
336      sb.append(table + ";");
337    }
338    if (sb.lastIndexOf(";") > 0) {
339      sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1);
340    }
341    sb.append(",targetRootDir=" + backupInfo.getBackupRootDir());
342
343    return sb.toString();
344  }
345
346  /**
347   * Clean up directories with prefix "_distcp_logs-", which are generated when DistCp copying
348   * hlogs.
349   * @throws IOException exception
350   */
351  protected void cleanupDistCpLog(BackupInfo backupInfo, Configuration conf) throws IOException {
352    Path rootPath = new Path(backupInfo.getHLogTargetDir()).getParent();
353    FileStatus[] files = FSUtils.listStatus(fs, rootPath);
354    if (files == null) {
355      return;
356    }
357    for (FileStatus file : files) {
358      if (file.getPath().getName().startsWith("_distcp_logs")) {
359        LOG.debug("Delete log files of DistCp: " + file.getPath().getName());
360        FSUtils.delete(fs, file.getPath(), true);
361      }
362    }
363  }
364
365  /**
366   * Complete the overall backup.
367   * @param backupInfo backup info
368   * @throws IOException exception
369   */
370  protected void completeBackup(final Connection conn, BackupInfo backupInfo,
371      BackupManager backupManager, BackupType type, Configuration conf) throws IOException {
372    // set the complete timestamp of the overall backup
373    backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
374    // set overall backup status: complete
375    backupInfo.setState(BackupState.COMPLETE);
376    backupInfo.setProgress(100);
377    // add and store the manifest for the backup
378    addManifest(backupInfo, backupManager, type, conf);
379
380    // compose the backup complete data
381    String backupCompleteData =
382        obtainBackupMetaDataStr(backupInfo) + ",startts=" + backupInfo.getStartTs()
383            + ",completets=" + backupInfo.getCompleteTs() + ",bytescopied="
384            + backupInfo.getTotalBytesCopied();
385    if (LOG.isDebugEnabled()) {
386      LOG.debug("Backup " + backupInfo.getBackupId() + " finished: " + backupCompleteData);
387    }
388
389    // when full backup is done:
390    // - delete HBase snapshot
391    // - clean up directories with prefix "exportSnapshot-", which are generated when exporting
392    // snapshots
393    if (type == BackupType.FULL) {
394      deleteSnapshots(conn, backupInfo, conf);
395      cleanupExportSnapshotLog(conf);
396    } else if (type == BackupType.INCREMENTAL) {
397      cleanupDistCpLog(backupInfo, conf);
398    }
399    BackupSystemTable.deleteSnapshot(conn);
400    backupManager.updateBackupInfo(backupInfo);
401
402    // Finish active session
403    backupManager.finishBackupSession();
404
405    LOG.info("Backup " + backupInfo.getBackupId() + " completed.");
406  }
407
408  /**
409   * Backup request execution.
410   *
411   * @throws IOException if the execution of the backup fails
412   */
413  public abstract void execute() throws IOException;
414
415  @VisibleForTesting
416  protected Stage getTestStage() {
417    return Stage.valueOf("stage_"+ conf.getInt(BACKUP_TEST_MODE_STAGE, 0));
418  }
419
420  @VisibleForTesting
421  protected void failStageIf(Stage stage) throws IOException {
422    Stage current = getTestStage();
423    if (current == stage) {
424      throw new IOException("Failed stage " + stage+" in testing");
425    }
426  }
427
428  public enum Stage {
429    stage_0, stage_1, stage_2, stage_3, stage_4
430  }
431}