001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.List;
025import java.util.TreeSet;
026import org.apache.commons.lang3.StringUtils;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.LocatedFileStatus;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.fs.RemoteIterator;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.backup.BackupType;
034import org.apache.hadoop.hbase.backup.HBackupFileSystem;
035import org.apache.hadoop.hbase.backup.RestoreRequest;
036import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
037import org.apache.hadoop.hbase.backup.util.BackupUtils;
038import org.apache.hadoop.hbase.backup.util.RestoreTool;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.io.hfile.HFile;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046/**
047 * Restore table implementation
048 */
049@InterfaceAudience.Private
050public class RestoreTablesClient {
051  private static final Logger LOG = LoggerFactory.getLogger(RestoreTablesClient.class);
052
053  private Configuration conf;
054  private Connection conn;
055  private String backupId;
056  private TableName[] sTableArray;
057  private TableName[] tTableArray;
058  private String backupRootDir;
059  private Path restoreRootDir;
060  private boolean isOverwrite;
061
062  public RestoreTablesClient(Connection conn, RestoreRequest request) throws IOException {
063    this.backupRootDir = request.getBackupRootDir();
064    this.backupId = request.getBackupId();
065    this.sTableArray = request.getFromTables();
066    this.tTableArray = request.getToTables();
067    if (tTableArray == null || tTableArray.length == 0) {
068      this.tTableArray = sTableArray;
069    }
070    this.isOverwrite = request.isOverwrite();
071    this.conn = conn;
072    this.conf = conn.getConfiguration();
073    if (request.getRestoreRootDir() != null) {
074      restoreRootDir = new Path(request.getRestoreRootDir());
075    } else {
076      FileSystem fs = FileSystem.get(conf);
077      this.restoreRootDir = BackupUtils.getTmpRestoreOutputDir(fs, conf);
078    }
079  }
080
081  /**
082   * Validate target tables.
083   * @param tTableArray target tables
084   * @param isOverwrite overwrite existing table
085   * @throws IOException exception
086   */
087  private void checkTargetTables(TableName[] tTableArray, boolean isOverwrite) throws IOException {
088    ArrayList<TableName> existTableList = new ArrayList<>();
089    ArrayList<TableName> disabledTableList = new ArrayList<>();
090
091    // check if the tables already exist
092    try (Admin admin = conn.getAdmin()) {
093      for (TableName tableName : tTableArray) {
094        if (admin.tableExists(tableName)) {
095          existTableList.add(tableName);
096          if (admin.isTableDisabled(tableName)) {
097            disabledTableList.add(tableName);
098          }
099        } else {
100          LOG.info("HBase table " + tableName
101            + " does not exist. It will be created during restore process");
102        }
103      }
104    }
105
106    if (existTableList.size() > 0) {
107      if (!isOverwrite) {
108        LOG.error("Existing table (" + existTableList + ") found in the restore target, please add "
109          + "\"-o\" as overwrite option in the command if you mean"
110          + " to restore to these existing tables");
111        throw new IOException(
112          "Existing table found in target while no \"-o\" " + "as overwrite option found");
113      } else {
114        if (disabledTableList.size() > 0) {
115          LOG.error("Found offline table in the restore target, "
116            + "please enable them before restore with \"-overwrite\" option");
117          LOG.info("Offline table list in restore target: " + disabledTableList);
118          throw new IOException(
119            "Found offline table in the target when restore with \"-overwrite\" option");
120        }
121      }
122    }
123  }
124
125  /**
126   * Restore operation handle each backupImage in array.
127   * @param images           array BackupImage
128   * @param sTable           table to be restored
129   * @param tTable           table to be restored to
130   * @param truncateIfExists truncate table
131   * @throws IOException exception
132   */
133
134  private void restoreImages(BackupImage[] images, TableName sTable, TableName tTable,
135    boolean truncateIfExists) throws IOException {
136    // First image MUST be image of a FULL backup
137    BackupImage image = images[0];
138    String rootDir = image.getRootDir();
139    String backupId = image.getBackupId();
140    Path backupRoot = new Path(rootDir);
141    RestoreTool restoreTool = new RestoreTool(conf, backupRoot, restoreRootDir, backupId);
142    Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId);
143    String lastIncrBackupId = images.length == 1 ? null : images[images.length - 1].getBackupId();
144    // We need hFS only for full restore (see the code)
145    BackupManifest manifest = HBackupFileSystem.getManifest(conf, backupRoot, backupId);
146    if (manifest.getType() == BackupType.FULL) {
147      LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full" + " backup image "
148        + tableBackupPath.toString());
149      conf.set(JOB_NAME_CONF_KEY, "Full_Restore-" + backupId + "-" + tTable);
150      restoreTool.fullRestoreTable(conn, tableBackupPath, sTable, tTable, truncateIfExists,
151        lastIncrBackupId);
152      conf.unset(JOB_NAME_CONF_KEY);
153    } else { // incremental Backup
154      throw new IOException("Unexpected backup type " + image.getType());
155    }
156
157    if (images.length == 1) {
158      // full backup restore done
159      return;
160    }
161
162    List<Path> dirList = new ArrayList<>();
163    // add full backup path
164    // full backup path comes first
165    for (int i = 1; i < images.length; i++) {
166      BackupImage im = images[i];
167      String fileBackupDir =
168        HBackupFileSystem.getTableBackupDir(im.getRootDir(), im.getBackupId(), sTable);
169      List<Path> list = getFilesRecursively(fileBackupDir);
170      dirList.addAll(list);
171
172    }
173
174    if (dirList.isEmpty()) {
175      LOG.info("No incremental changes since full backup for '" + sTable
176        + "', skipping incremental restore step.");
177      return;
178    }
179
180    String dirs = StringUtils.join(dirList, ",");
181    LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from log dirs: " + dirs);
182    Path[] paths = new Path[dirList.size()];
183    dirList.toArray(paths);
184    conf.set(JOB_NAME_CONF_KEY, "Incremental_Restore-" + backupId + "-" + tTable);
185    restoreTool.incrementalRestoreTable(conn, tableBackupPath, paths, new TableName[] { sTable },
186      new TableName[] { tTable }, lastIncrBackupId);
187    LOG.info(sTable + " has been successfully restored to " + tTable);
188  }
189
190  private List<Path> getFilesRecursively(String fileBackupDir)
191    throws IllegalArgumentException, IOException {
192    FileSystem fs = FileSystem.get(new Path(fileBackupDir).toUri(), new Configuration());
193    List<Path> list = new ArrayList<>();
194    RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true);
195    while (it.hasNext()) {
196      Path p = it.next().getPath();
197      if (HFile.isHFileFormat(fs, p)) {
198        list.add(p);
199      }
200    }
201    return list;
202  }
203
204  /**
205   * Restore operation. Stage 2: resolved Backup Image dependency
206   * @param sTableArray The array of tables to be restored
207   * @param tTableArray The array of mapping tables to restore to
208   * @throws IOException exception
209   */
210  private void restore(BackupManifest manifest, TableName[] sTableArray, TableName[] tTableArray,
211    boolean isOverwrite) throws IOException {
212    TreeSet<BackupImage> restoreImageSet = new TreeSet<>();
213
214    for (int i = 0; i < sTableArray.length; i++) {
215      TableName table = sTableArray[i];
216
217      // Get the image list of this backup for restore in time order from old
218      // to new.
219      List<BackupImage> list = new ArrayList<>();
220      list.add(manifest.getBackupImage());
221      TreeSet<BackupImage> set = new TreeSet<>(list);
222      List<BackupImage> depList = manifest.getDependentListByTable(table);
223      set.addAll(depList);
224      BackupImage[] arr = new BackupImage[set.size()];
225      set.toArray(arr);
226      restoreImages(arr, table, tTableArray[i], isOverwrite);
227      restoreImageSet.addAll(list);
228      if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
229        LOG.info("Restore includes the following image(s):");
230        for (BackupImage image : restoreImageSet) {
231          LOG.info("Backup: " + image.getBackupId() + " "
232            + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table));
233        }
234      }
235    }
236    LOG.debug("restoreStage finished");
237  }
238
239  static long getTsFromBackupId(String backupId) {
240    if (backupId == null) {
241      return 0;
242    }
243    return Long.parseLong(backupId.substring(backupId.lastIndexOf("_") + 1));
244  }
245
246  static boolean withinRange(long a, long lower, long upper) {
247    return a >= lower && a <= upper;
248  }
249
250  public void execute() throws IOException {
251    // case VALIDATION:
252    // check the target tables
253    checkTargetTables(tTableArray, isOverwrite);
254
255    // case RESTORE_IMAGES:
256    // check and load backup image manifest for the tables
257    Path rootPath = new Path(backupRootDir);
258    BackupManifest manifest = HBackupFileSystem.getManifest(conf, rootPath, backupId);
259
260    restore(manifest, sTableArray, tTableArray, isOverwrite);
261  }
262}