001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUPID_PREFIX;
021
022import com.google.errorprone.annotations.RestrictedApi;
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Comparator;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.LocatedFileStatus;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.fs.RemoteIterator;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.backup.impl.BackupManifest;
034import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
040
041/**
042 * View to an on-disk Backup Image FileSytem Provides the set of methods necessary to interact with
043 * the on-disk Backup Image data.
044 */
045@InterfaceAudience.Private
046public final class HBackupFileSystem {
047  public static final Logger LOG = LoggerFactory.getLogger(HBackupFileSystem.class);
048
049  /**
050   * This is utility class.
051   */
052  private HBackupFileSystem() {
053  }
054
055  /**
056   * Given the backup root dir, backup id and the table name, return the backup image location.
057   * Return value look like:
058   * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where
059   * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory
060   * @param backupRootDir backup root directory
061   * @param backupId      backup id
062   * @param tableName     table name
063   * @return backupPath String for the particular table
064   */
065  public static String getTableBackupDir(String backupRootDir, String backupId,
066    TableName tableName) {
067    return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR
068      + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString()
069      + Path.SEPARATOR;
070  }
071
072  /**
073   * Get backup temporary directory
074   * @param backupRootDir backup root
075   * @return backup tmp directory path
076   */
077  public static Path getBackupTmpDirPath(String backupRootDir) {
078    return new Path(backupRootDir, ".tmp");
079  }
080
081  /**
082   * Get backup tmp directory for backupId
083   * @param backupRoot backup root
084   * @param backupId   backup id
085   * @return backup tmp directory path
086   */
087  public static Path getBackupTmpDirPathForBackupId(String backupRoot, String backupId) {
088    return new Path(getBackupTmpDirPath(backupRoot), backupId);
089  }
090
091  public static Path getBackupPath(String backupRootDir, String backupId) {
092    return new Path(backupRootDir + Path.SEPARATOR + backupId);
093  }
094
095  /**
096   * Given the backup root dir, backup id and the table name, return the backup image location,
097   * which is also where the backup manifest file is. return value look like:
098   * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where
099   * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory
100   * @param backupRootPath backup root path
101   * @param tableName      table name
102   * @param backupId       backup Id
103   * @return backupPath for the particular table
104   */
105  public static Path getTableBackupPath(TableName tableName, Path backupRootPath, String backupId) {
106    return new Path(getTableBackupDir(backupRootPath.toString(), backupId, tableName));
107  }
108
109  private static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId)
110    throws IOException {
111    return getManifestPath(conf, backupRootPath, backupId, true);
112  }
113
114  /* Visible for testing only */
115  @RestrictedApi(explanation = "Should only be called internally or in tests", link = "",
116      allowedOnPath = "(.*/src/test/.*|.*/org/apache/hadoop/hbase/backup/HBackupFileSystem.java)")
117  static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId,
118    boolean throwIfNotFound) throws IOException {
119    FileSystem fs = backupRootPath.getFileSystem(conf);
120    Path manifestPath = new Path(getBackupPath(backupRootPath.toString(), backupId) + Path.SEPARATOR
121      + BackupManifest.MANIFEST_FILE_NAME);
122    if (throwIfNotFound && !fs.exists(manifestPath)) {
123      String errorMsg = "Could not find backup manifest " + BackupManifest.MANIFEST_FILE_NAME
124        + " for " + backupId + ". File " + manifestPath + " does not exists. Did " + backupId
125        + " correspond to previously taken backup ?";
126      throw new IOException(errorMsg);
127    }
128    return manifestPath;
129  }
130
131  public static Path getRootDirFromBackupPath(Path backupPath, String backupId) {
132    if (backupPath.getName().equals(BackupManifest.MANIFEST_FILE_NAME)) {
133      backupPath = backupPath.getParent();
134    }
135    Preconditions.checkArgument(backupPath.getName().equals(backupId),
136      String.format("Backup path %s must end in backupId %s", backupPath, backupId));
137    return backupPath.getParent();
138  }
139
140  public static BackupManifest getManifest(Configuration conf, Path backupRootPath, String backupId)
141    throws IOException {
142    BackupManifest manifest =
143      new BackupManifest(conf, getManifestPath(conf, backupRootPath, backupId));
144    return manifest;
145  }
146
147  public static List<BackupImage> getAllBackupImages(Configuration conf, Path backupRootPath)
148    throws IOException {
149    FileSystem fs = FileSystem.get(backupRootPath.toUri(), conf);
150    RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(backupRootPath);
151
152    List<BackupImage> images = new ArrayList<>();
153
154    while (it.hasNext()) {
155      LocatedFileStatus lfs = it.next();
156      if (!lfs.isDirectory()) {
157        continue;
158      }
159
160      String backupId = lfs.getPath().getName();
161      try {
162        BackupManifest manifest = getManifest(conf, backupRootPath, backupId);
163        images.add(manifest.getBackupImage());
164      } catch (IOException e) {
165        LOG.error("Cannot load backup manifest from: " + lfs.getPath(), e);
166      }
167    }
168
169    // Sort images by timestamp in descending order
170    images.sort(Comparator.comparingLong(m -> -getTimestamp(m.getBackupId())));
171
172    return images;
173  }
174
175  private static long getTimestamp(String backupId) {
176    return Long.parseLong(backupId.substring(BACKUPID_PREFIX.length()));
177  }
178}