001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.apache.hadoop.hbase.HConstants.HBASE_TEMP_DIRECTORY;
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUPID_PREFIX;
022
023import com.google.errorprone.annotations.RestrictedApi;
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Comparator;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.LocatedFileStatus;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.fs.RemoteIterator;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.backup.impl.BackupManifest;
035import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
041
042/**
043 * View to an on-disk Backup Image FileSytem Provides the set of methods necessary to interact with
044 * the on-disk Backup Image data.
045 */
046@InterfaceAudience.Private
047public final class HBackupFileSystem {
048  public static final Logger LOG = LoggerFactory.getLogger(HBackupFileSystem.class);
049
050  /**
051   * This is utility class.
052   */
053  private HBackupFileSystem() {
054  }
055
056  /**
057   * Given the backup root dir, backup id and the table name, return the backup image location.
058   * Return value look like:
059   * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where
060   * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory
061   * @param backupRootDir backup root directory
062   * @param backupId      backup id
063   * @param tableName     table name
064   * @return backupPath String for the particular table
065   */
066  public static String getTableBackupDir(String backupRootDir, String backupId,
067    TableName tableName) {
068    return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR
069      + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString()
070      + Path.SEPARATOR;
071  }
072
073  /**
074   * Get backup temporary directory
075   * @param backupRootDir backup root
076   * @return backup tmp directory path
077   */
078  public static Path getBackupTmpDirPath(String backupRootDir) {
079    return new Path(backupRootDir, ".tmp");
080  }
081
082  /**
083   * Get backup tmp directory for backupId
084   * @param backupRoot backup root
085   * @param backupId   backup id
086   * @return backup tmp directory path
087   */
088  public static Path getBackupTmpDirPathForBackupId(String backupRoot, String backupId) {
089    return new Path(getBackupTmpDirPath(backupRoot), backupId);
090  }
091
092  public static Path getBackupPath(String backupRootDir, String backupId) {
093    return new Path(backupRootDir + Path.SEPARATOR + backupId);
094  }
095
096  /**
097   * Given the backup root dir, backup id and the table name, return the backup image location,
098   * which is also where the backup manifest file is. return value look like:
099   * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where
100   * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory
101   * @param backupRootPath backup root path
102   * @param tableName      table name
103   * @param backupId       backup Id
104   * @return backupPath for the particular table
105   */
106  public static Path getTableBackupPath(TableName tableName, Path backupRootPath, String backupId) {
107    return new Path(getTableBackupDir(backupRootPath.toString(), backupId, tableName));
108  }
109
110  private static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId)
111    throws IOException {
112    return getManifestPath(conf, backupRootPath, backupId, true);
113  }
114
115  /* Visible for testing only */
116  @RestrictedApi(explanation = "Should only be called internally or in tests", link = "",
117      allowedOnPath = "(.*/src/test/.*|.*/org/apache/hadoop/hbase/backup/HBackupFileSystem.java)")
118  static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId,
119    boolean throwIfNotFound) throws IOException {
120    FileSystem fs = backupRootPath.getFileSystem(conf);
121    Path manifestPath = new Path(getBackupPath(backupRootPath.toString(), backupId) + Path.SEPARATOR
122      + BackupManifest.MANIFEST_FILE_NAME);
123    if (throwIfNotFound && !fs.exists(manifestPath)) {
124      String errorMsg = "Could not find backup manifest " + BackupManifest.MANIFEST_FILE_NAME
125        + " for " + backupId + ". File " + manifestPath + " does not exists. Did " + backupId
126        + " correspond to previously taken backup ?";
127      throw new IOException(errorMsg);
128    }
129    return manifestPath;
130  }
131
132  public static Path getRootDirFromBackupPath(Path backupPath, String backupId) {
133    if (backupPath.getName().equals(BackupManifest.MANIFEST_FILE_NAME)) {
134      backupPath = backupPath.getParent();
135    }
136    Preconditions.checkArgument(backupPath.getName().equals(backupId),
137      String.format("Backup path %s must end in backupId %s", backupPath, backupId));
138    return backupPath.getParent();
139  }
140
141  public static BackupManifest getManifest(Configuration conf, Path backupRootPath, String backupId)
142    throws IOException {
143    BackupManifest manifest =
144      new BackupManifest(conf, getManifestPath(conf, backupRootPath, backupId));
145    return manifest;
146  }
147
148  public static List<BackupImage> getAllBackupImages(Configuration conf, Path backupRootPath)
149    throws IOException {
150    FileSystem fs = FileSystem.get(backupRootPath.toUri(), conf);
151    RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(backupRootPath);
152
153    List<BackupImage> images = new ArrayList<>();
154
155    while (it.hasNext()) {
156      LocatedFileStatus lfs = it.next();
157      if (!lfs.isDirectory()) {
158        continue;
159      }
160
161      String backupId = lfs.getPath().getName();
162      if (HBASE_TEMP_DIRECTORY.equals(backupId)) {
163        continue;
164      }
165
166      try {
167        BackupManifest manifest = getManifest(conf, backupRootPath, backupId);
168        images.add(manifest.getBackupImage());
169      } catch (IOException e) {
170        LOG.error("Cannot load backup manifest from: {}", lfs.getPath(), e);
171      }
172    }
173
174    // Sort images by timestamp in descending order
175    images.sort(Comparator.comparingLong(m -> -getTimestamp(m.getBackupId())));
176
177    return images;
178  }
179
180  private static long getTimestamp(String backupId) {
181    return Long.parseLong(backupId.substring(BACKUPID_PREFIX.length()));
182  }
183}