001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.List;
026import java.util.TreeSet;
027import org.apache.commons.lang3.StringUtils;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.LocatedFileStatus;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.fs.RemoteIterator;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.backup.BackupType;
035import org.apache.hadoop.hbase.backup.HBackupFileSystem;
036import org.apache.hadoop.hbase.backup.RestoreRequest;
037import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
038import org.apache.hadoop.hbase.backup.util.RestoreTool;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.io.hfile.HFile;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046/**
047 * Restore table implementation
048 */
049@InterfaceAudience.Private
050public class RestoreTablesClient {
051  private static final Logger LOG = LoggerFactory.getLogger(RestoreTablesClient.class);
052
053  private Configuration conf;
054  private Connection conn;
055  private String backupId;
056  private TableName[] sTableArray;
057  private TableName[] tTableArray;
058  private String targetRootDir;
059  private boolean isOverwrite;
060
061  public RestoreTablesClient(Connection conn, RestoreRequest request) {
062    this.targetRootDir = request.getBackupRootDir();
063    this.backupId = request.getBackupId();
064    this.sTableArray = request.getFromTables();
065    this.tTableArray = request.getToTables();
066    if (tTableArray == null || tTableArray.length == 0) {
067      this.tTableArray = sTableArray;
068    }
069    this.isOverwrite = request.isOverwrite();
070    this.conn = conn;
071    this.conf = conn.getConfiguration();
072  }
073
074  /**
075   * Validate target tables.
076   * @param tTableArray target tables
077   * @param isOverwrite overwrite existing table
078   * @throws IOException exception
079   */
080  private void checkTargetTables(TableName[] tTableArray, boolean isOverwrite) throws IOException {
081    ArrayList<TableName> existTableList = new ArrayList<>();
082    ArrayList<TableName> disabledTableList = new ArrayList<>();
083
084    // check if the tables already exist
085    try (Admin admin = conn.getAdmin()) {
086      for (TableName tableName : tTableArray) {
087        if (admin.tableExists(tableName)) {
088          existTableList.add(tableName);
089          if (admin.isTableDisabled(tableName)) {
090            disabledTableList.add(tableName);
091          }
092        } else {
093          LOG.info("HBase table " + tableName
094            + " does not exist. It will be created during restore process");
095        }
096      }
097    }
098
099    if (existTableList.size() > 0) {
100      if (!isOverwrite) {
101        LOG.error("Existing table (" + existTableList + ") found in the restore target, please add "
102          + "\"-o\" as overwrite option in the command if you mean"
103          + " to restore to these existing tables");
104        throw new IOException(
105          "Existing table found in target while no \"-o\" " + "as overwrite option found");
106      } else {
107        if (disabledTableList.size() > 0) {
108          LOG.error("Found offline table in the restore target, "
109            + "please enable them before restore with \"-overwrite\" option");
110          LOG.info("Offline table list in restore target: " + disabledTableList);
111          throw new IOException(
112            "Found offline table in the target when restore with \"-overwrite\" option");
113        }
114      }
115    }
116  }
117
118  /**
119   * Restore operation handle each backupImage in array.
120   * @param images           array BackupImage
121   * @param sTable           table to be restored
122   * @param tTable           table to be restored to
123   * @param truncateIfExists truncate table
124   * @throws IOException exception
125   */
126
127  private void restoreImages(BackupImage[] images, TableName sTable, TableName tTable,
128    boolean truncateIfExists) throws IOException {
129    // First image MUST be image of a FULL backup
130    BackupImage image = images[0];
131    String rootDir = image.getRootDir();
132    String backupId = image.getBackupId();
133    Path backupRoot = new Path(rootDir);
134    RestoreTool restoreTool = new RestoreTool(conf, backupRoot, backupId);
135    Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId);
136    String lastIncrBackupId = images.length == 1 ? null : images[images.length - 1].getBackupId();
137    // We need hFS only for full restore (see the code)
138    BackupManifest manifest = HBackupFileSystem.getManifest(conf, backupRoot, backupId);
139    if (manifest.getType() == BackupType.FULL) {
140      LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full" + " backup image "
141        + tableBackupPath.toString());
142      conf.set(JOB_NAME_CONF_KEY, "Full_Restore-" + backupId + "-" + tTable);
143      restoreTool.fullRestoreTable(conn, tableBackupPath, sTable, tTable, truncateIfExists,
144        lastIncrBackupId);
145      conf.unset(JOB_NAME_CONF_KEY);
146    } else { // incremental Backup
147      throw new IOException("Unexpected backup type " + image.getType());
148    }
149
150    if (images.length == 1) {
151      // full backup restore done
152      return;
153    }
154
155    List<Path> dirList = new ArrayList<>();
156    // add full backup path
157    // full backup path comes first
158    for (int i = 1; i < images.length; i++) {
159      BackupImage im = images[i];
160      String fileBackupDir =
161        HBackupFileSystem.getTableBackupDir(im.getRootDir(), im.getBackupId(), sTable);
162      List<Path> list = getFilesRecursively(fileBackupDir);
163      dirList.addAll(list);
164
165    }
166
167    if (dirList.isEmpty()) {
168      LOG.warn("Nothing has changed, so there is no need to restore '" + sTable + "'");
169      return;
170    }
171
172    String dirs = StringUtils.join(dirList, ",");
173    LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from log dirs: " + dirs);
174    Path[] paths = new Path[dirList.size()];
175    dirList.toArray(paths);
176    conf.set(JOB_NAME_CONF_KEY, "Incremental_Restore-" + backupId + "-" + tTable);
177    restoreTool.incrementalRestoreTable(conn, tableBackupPath, paths, new TableName[] { sTable },
178      new TableName[] { tTable }, lastIncrBackupId);
179    LOG.info(sTable + " has been successfully restored to " + tTable);
180  }
181
182  private List<Path> getFilesRecursively(String fileBackupDir)
183    throws IllegalArgumentException, IOException {
184    FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration());
185    List<Path> list = new ArrayList<>();
186    RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true);
187    while (it.hasNext()) {
188      Path p = it.next().getPath();
189      if (HFile.isHFileFormat(fs, p)) {
190        list.add(p);
191      }
192    }
193    return list;
194  }
195
196  /**
197   * Restore operation. Stage 2: resolved Backup Image dependency
198   * @param backupManifestMap : tableName, Manifest
199   * @param sTableArray       The array of tables to be restored
200   * @param tTableArray       The array of mapping tables to restore to
201   * @throws IOException exception
202   */
203  private void restore(HashMap<TableName, BackupManifest> backupManifestMap,
204    TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException {
205    TreeSet<BackupImage> restoreImageSet = new TreeSet<>();
206
207    for (int i = 0; i < sTableArray.length; i++) {
208      TableName table = sTableArray[i];
209
210      BackupManifest manifest = backupManifestMap.get(table);
211      // Get the image list of this backup for restore in time order from old
212      // to new.
213      List<BackupImage> list = new ArrayList<>();
214      list.add(manifest.getBackupImage());
215      TreeSet<BackupImage> set = new TreeSet<>(list);
216      List<BackupImage> depList = manifest.getDependentListByTable(table);
217      set.addAll(depList);
218      BackupImage[] arr = new BackupImage[set.size()];
219      set.toArray(arr);
220      restoreImages(arr, table, tTableArray[i], isOverwrite);
221      restoreImageSet.addAll(list);
222      if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
223        LOG.info("Restore includes the following image(s):");
224        for (BackupImage image : restoreImageSet) {
225          LOG.info("Backup: " + image.getBackupId() + " "
226            + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table));
227        }
228      }
229    }
230    LOG.debug("restoreStage finished");
231  }
232
233  static long getTsFromBackupId(String backupId) {
234    if (backupId == null) {
235      return 0;
236    }
237    return Long.parseLong(backupId.substring(backupId.lastIndexOf("_") + 1));
238  }
239
240  static boolean withinRange(long a, long lower, long upper) {
241    return a >= lower && a <= upper;
242  }
243
244  public void execute() throws IOException {
245    // case VALIDATION:
246    // check the target tables
247    checkTargetTables(tTableArray, isOverwrite);
248
249    // case RESTORE_IMAGES:
250    HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>();
251    // check and load backup image manifest for the tables
252    Path rootPath = new Path(targetRootDir);
253    HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath,
254      backupId);
255
256    restore(backupManifestMap, sTableArray, tTableArray, isOverwrite);
257  }
258}