001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.backup.util;
020
021import java.io.FileNotFoundException;
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.HashMap;
026import java.util.List;
027import java.util.TreeMap;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.FileUtil;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.backup.BackupRestoreFactory;
037import org.apache.hadoop.hbase.backup.HBackupFileSystem;
038import org.apache.hadoop.hbase.backup.RestoreJob;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
044import org.apache.hadoop.hbase.io.HFileLink;
045import org.apache.hadoop.hbase.io.hfile.HFile;
046import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
047import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
048import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
049import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
052import org.apache.hadoop.hbase.util.FSTableDescriptors;
053import org.apache.yetus.audience.InterfaceAudience;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
058
059/**
060 * A collection for methods used by multiple classes to restore HBase tables.
061 */
062@InterfaceAudience.Private
063public class RestoreTool {
064  public static final Logger LOG = LoggerFactory.getLogger(BackupUtils.class);
065  private final static long TABLE_AVAILABILITY_WAIT_TIME = 180000;
066
067  private final String[] ignoreDirs = { HConstants.RECOVERED_EDITS_DIR };
068  protected Configuration conf;
069  protected Path backupRootPath;
070  protected String backupId;
071  protected FileSystem fs;
072
073  // store table name and snapshot dir mapping
074  private final HashMap<TableName, Path> snapshotMap = new HashMap<>();
075
076  public RestoreTool(Configuration conf, final Path backupRootPath, final String backupId)
077      throws IOException {
078    this.conf = conf;
079    this.backupRootPath = backupRootPath;
080    this.backupId = backupId;
081    this.fs = backupRootPath.getFileSystem(conf);
082  }
083
084  /**
085   * return value represent path for:
086   * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn"
087   * @param tableName table name
088   * @return path to table archive
089   * @throws IOException exception
090   */
091  Path getTableArchivePath(TableName tableName) throws IOException {
092    Path baseDir =
093        new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId),
094            HConstants.HFILE_ARCHIVE_DIRECTORY);
095    Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR);
096    Path archivePath = new Path(dataDir, tableName.getNamespaceAsString());
097    Path tableArchivePath = new Path(archivePath, tableName.getQualifierAsString());
098    if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) {
099      LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists");
100      tableArchivePath = null; // empty table has no archive
101    }
102    return tableArchivePath;
103  }
104
105  /**
106   * Gets region list
107   * @param tableName table name
108   * @return RegionList region list
109   * @throws IOException exception
110   */
111  ArrayList<Path> getRegionList(TableName tableName) throws IOException {
112    Path tableArchivePath = getTableArchivePath(tableName);
113    ArrayList<Path> regionDirList = new ArrayList<>();
114    FileStatus[] children = fs.listStatus(tableArchivePath);
115    for (FileStatus childStatus : children) {
116      // here child refer to each region(Name)
117      Path child = childStatus.getPath();
118      regionDirList.add(child);
119    }
120    return regionDirList;
121  }
122
123  void modifyTableSync(Connection conn, TableDescriptor desc) throws IOException {
124    try (Admin admin = conn.getAdmin()) {
125      admin.modifyTable(desc);
126      int attempt = 0;
127      int maxAttempts = 600;
128      while (!admin.isTableAvailable(desc.getTableName())) {
129        Thread.sleep(100);
130        attempt++;
131        if (attempt++ > maxAttempts) {
132          throw new IOException("Timeout expired " + (maxAttempts * 100) + "ms");
133        }
134      }
135    } catch (Exception e) {
136      throw new IOException(e);
137    }
138  }
139
140  /**
141   * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently
142   * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in
143   * the future
144   * @param conn HBase connection
145   * @param tableBackupPath backup path
146   * @param logDirs : incremental backup folders, which contains WAL
147   * @param tableNames : source tableNames(table names were backuped)
148   * @param newTableNames : target tableNames(table names to be restored to)
149   * @param incrBackupId incremental backup Id
150   * @throws IOException exception
151   */
152  public void incrementalRestoreTable(Connection conn, Path tableBackupPath, Path[] logDirs,
153      TableName[] tableNames, TableName[] newTableNames, String incrBackupId) throws IOException {
154    try (Admin admin = conn.getAdmin()) {
155      if (tableNames.length != newTableNames.length) {
156        throw new IOException("Number of source tables and target tables does not match!");
157      }
158      FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
159
160      // for incremental backup image, expect the table already created either by user or previous
161      // full backup. Here, check that all new tables exists
162      for (TableName tableName : newTableNames) {
163        if (!admin.tableExists(tableName)) {
164          throw new IOException("HBase table " + tableName
165              + " does not exist. Create the table first, e.g. by restoring a full backup.");
166        }
167      }
168      // adjust table schema
169      for (int i = 0; i < tableNames.length; i++) {
170        TableName tableName = tableNames[i];
171        TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, incrBackupId);
172        LOG.debug("Found descriptor " + tableDescriptor + " through " + incrBackupId);
173
174        TableName newTableName = newTableNames[i];
175        TableDescriptor newTableDescriptor = admin.getDescriptor(newTableName);
176        List<ColumnFamilyDescriptor> families = Arrays.asList(tableDescriptor.getColumnFamilies());
177        List<ColumnFamilyDescriptor> existingFamilies =
178            Arrays.asList(newTableDescriptor.getColumnFamilies());
179        TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newTableDescriptor);
180        boolean schemaChangeNeeded = false;
181        for (ColumnFamilyDescriptor family : families) {
182          if (!existingFamilies.contains(family)) {
183            builder.setColumnFamily(family);
184            schemaChangeNeeded = true;
185          }
186        }
187        for (ColumnFamilyDescriptor family : existingFamilies) {
188          if (!families.contains(family)) {
189            builder.removeColumnFamily(family.getName());
190            schemaChangeNeeded = true;
191          }
192        }
193        if (schemaChangeNeeded) {
194          modifyTableSync(conn, builder.build());
195          LOG.info("Changed " + newTableDescriptor.getTableName() + " to: " + newTableDescriptor);
196        }
197      }
198      RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf);
199
200      restoreService.run(logDirs, tableNames, newTableNames, false);
201    }
202  }
203
204  public void fullRestoreTable(Connection conn, Path tableBackupPath, TableName tableName,
205      TableName newTableName, boolean truncateIfExists, String lastIncrBackupId)
206          throws IOException {
207    createAndRestoreTable(conn, tableName, newTableName, tableBackupPath, truncateIfExists,
208      lastIncrBackupId);
209  }
210
211  /**
212   * Returns value represent path for path to backup table snapshot directory:
213   * "/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot"
214   * @param backupRootPath backup root path
215   * @param tableName table name
216   * @param backupId backup Id
217   * @return path for snapshot
218   */
219  Path getTableSnapshotPath(Path backupRootPath, TableName tableName, String backupId) {
220    return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId),
221        HConstants.SNAPSHOT_DIR_NAME);
222  }
223
224  /**
225   * Returns value represent path for:
226   * ""/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot/
227   *    snapshot_1396650097621_namespace_table"
228   * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
229   * .data.manifest (trunk)
230   * @param tableName table name
231   * @return path to table info
232   * @throws IOException exception
233   */
234  Path getTableInfoPath(TableName tableName) throws IOException {
235    Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
236    Path tableInfoPath = null;
237
238    // can't build the path directly as the timestamp values are different
239    FileStatus[] snapshots = fs.listStatus(tableSnapShotPath,
240        new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
241    for (FileStatus snapshot : snapshots) {
242      tableInfoPath = snapshot.getPath();
243      // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
244      if (tableInfoPath.getName().endsWith("data.manifest")) {
245        break;
246      }
247    }
248    return tableInfoPath;
249  }
250
251  /**
252   * Get table descriptor
253   * @param tableName is the table backed up
254   * @return {@link TableDescriptor} saved in backup image of the table
255   */
256  TableDescriptor getTableDesc(TableName tableName) throws IOException {
257    Path tableInfoPath = this.getTableInfoPath(tableName);
258    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
259    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
260    TableDescriptor tableDescriptor = manifest.getTableDescriptor();
261    if (!tableDescriptor.getTableName().equals(tableName)) {
262      LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
263              + tableInfoPath.toString());
264      LOG.error("tableDescriptor.getNameAsString() = "
265              + tableDescriptor.getTableName().getNameAsString());
266      throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
267          + " under tableInfoPath: " + tableInfoPath.toString());
268    }
269    return tableDescriptor;
270  }
271
272  private TableDescriptor getTableDescriptor(FileSystem fileSys, TableName tableName,
273      String lastIncrBackupId) throws IOException {
274    if (lastIncrBackupId != null) {
275      String target =
276          BackupUtils.getTableBackupDir(backupRootPath.toString(),
277            lastIncrBackupId, tableName);
278      return FSTableDescriptors.getTableDescriptorFromFs(fileSys, new Path(target));
279    }
280    return null;
281  }
282
283  private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName,
284      Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException {
285    if (newTableName == null) {
286      newTableName = tableName;
287    }
288    FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
289
290    // get table descriptor first
291    TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
292    if (tableDescriptor != null) {
293      LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId);
294    }
295
296    if (tableDescriptor == null) {
297      Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
298      if (fileSys.exists(tableSnapshotPath)) {
299        // snapshot path exist means the backup path is in HDFS
300        // check whether snapshot dir already recorded for target table
301        if (snapshotMap.get(tableName) != null) {
302          SnapshotDescription desc =
303              SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
304          SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
305          tableDescriptor = manifest.getTableDescriptor();
306        } else {
307          tableDescriptor = getTableDesc(tableName);
308          snapshotMap.put(tableName, getTableInfoPath(tableName));
309        }
310        if (tableDescriptor == null) {
311          LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
312        }
313      } else {
314        throw new IOException("Table snapshot directory: " +
315            tableSnapshotPath + " does not exist.");
316      }
317    }
318
319    Path tableArchivePath = getTableArchivePath(tableName);
320    if (tableArchivePath == null) {
321      if (tableDescriptor != null) {
322        // find table descriptor but no archive dir means the table is empty, create table and exit
323        if (LOG.isDebugEnabled()) {
324          LOG.debug("find table descriptor but no archive dir for table " + tableName
325              + ", will only create table");
326        }
327        tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
328        checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor,
329          truncateIfExists);
330        return;
331      } else {
332        throw new IllegalStateException("Cannot restore hbase table because directory '"
333            + " tableArchivePath is null.");
334      }
335    }
336
337    if (tableDescriptor == null) {
338      tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build();
339    } else {
340      tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
341    }
342
343    // record all region dirs:
344    // load all files in dir
345    try {
346      ArrayList<Path> regionPathList = getRegionList(tableName);
347
348      // should only try to create the table with all region informations, so we could pre-split
349      // the regions in fine grain
350      checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList,
351        tableDescriptor, truncateIfExists);
352      RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf);
353      Path[] paths = new Path[regionPathList.size()];
354      regionPathList.toArray(paths);
355      restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true);
356
357    } catch (Exception e) {
358      LOG.error(e.toString(), e);
359      throw new IllegalStateException("Cannot restore hbase table", e);
360    }
361  }
362
363  /**
364   * Gets region list
365   * @param tableArchivePath table archive path
366   * @return RegionList region list
367   * @throws IOException exception
368   */
369  ArrayList<Path> getRegionList(Path tableArchivePath) throws IOException {
370    ArrayList<Path> regionDirList = new ArrayList<>();
371    FileStatus[] children = fs.listStatus(tableArchivePath);
372    for (FileStatus childStatus : children) {
373      // here child refer to each region(Name)
374      Path child = childStatus.getPath();
375      regionDirList.add(child);
376    }
377    return regionDirList;
378  }
379
380  /**
381   * Calculate region boundaries and add all the column families to the table descriptor
382   * @param regionDirList region dir list
383   * @return a set of keys to store the boundaries
384   */
385  byte[][] generateBoundaryKeys(ArrayList<Path> regionDirList) throws IOException {
386    TreeMap<byte[], Integer> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
387    // Build a set of keys to store the boundaries
388    // calculate region boundaries and add all the column families to the table descriptor
389    for (Path regionDir : regionDirList) {
390      LOG.debug("Parsing region dir: " + regionDir);
391      Path hfofDir = regionDir;
392
393      if (!fs.exists(hfofDir)) {
394        LOG.warn("HFileOutputFormat dir " + hfofDir + " not found");
395      }
396
397      FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
398      if (familyDirStatuses == null) {
399        throw new IOException("No families found in " + hfofDir);
400      }
401
402      for (FileStatus stat : familyDirStatuses) {
403        if (!stat.isDirectory()) {
404          LOG.warn("Skipping non-directory " + stat.getPath());
405          continue;
406        }
407        boolean isIgnore = false;
408        String pathName = stat.getPath().getName();
409        for (String ignore : ignoreDirs) {
410          if (pathName.contains(ignore)) {
411            LOG.warn("Skipping non-family directory" + pathName);
412            isIgnore = true;
413            break;
414          }
415        }
416        if (isIgnore) {
417          continue;
418        }
419        Path familyDir = stat.getPath();
420        LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]");
421        // Skip _logs, etc
422        if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) {
423          continue;
424        }
425
426        // start to parse hfile inside one family dir
427        Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
428        for (Path hfile : hfiles) {
429          if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".")
430              || StoreFileInfo.isReference(hfile.getName())
431              || HFileLink.isHFileLink(hfile.getName())) {
432            continue;
433          }
434          HFile.Reader reader = HFile.createReader(fs, hfile, conf);
435          final byte[] first, last;
436          try {
437            reader.loadFileInfo();
438            first = reader.getFirstRowKey().get();
439            last = reader.getLastRowKey().get();
440            LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first="
441                + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
442
443            // To eventually infer start key-end key boundaries
444            Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0;
445            map.put(first, value + 1);
446            value = map.containsKey(last) ? (Integer) map.get(last) : 0;
447            map.put(last, value - 1);
448          } finally {
449            reader.close();
450          }
451        }
452      }
453    }
454    return LoadIncrementalHFiles.inferBoundaries(map);
455  }
456
457  /**
458   * Prepare the table for bulkload, most codes copied from
459   * {@link LoadIncrementalHFiles#createTable(TableName, String, Admin)}
460   * @param conn connection
461   * @param tableBackupPath path
462   * @param tableName table name
463   * @param targetTableName target table name
464   * @param regionDirList region directory list
465   * @param htd table descriptor
466   * @param truncateIfExists truncates table if exists
467   * @throws IOException exception
468   */
469  private void checkAndCreateTable(Connection conn, Path tableBackupPath, TableName tableName,
470      TableName targetTableName, ArrayList<Path> regionDirList, TableDescriptor htd,
471      boolean truncateIfExists) throws IOException {
472    try (Admin admin = conn.getAdmin()) {
473      boolean createNew = false;
474      if (admin.tableExists(targetTableName)) {
475        if (truncateIfExists) {
476          LOG.info("Truncating exising target table '" + targetTableName
477              + "', preserving region splits");
478          admin.disableTable(targetTableName);
479          admin.truncateTable(targetTableName, true);
480        } else {
481          LOG.info("Using exising target table '" + targetTableName + "'");
482        }
483      } else {
484        createNew = true;
485      }
486      if (createNew) {
487        LOG.info("Creating target table '" + targetTableName + "'");
488        byte[][] keys;
489        if (regionDirList == null || regionDirList.size() == 0) {
490          admin.createTable(htd, null);
491        } else {
492          keys = generateBoundaryKeys(regionDirList);
493          // create table using table descriptor and region boundaries
494          admin.createTable(htd, keys);
495        }
496
497      }
498      long startTime = EnvironmentEdgeManager.currentTime();
499      while (!admin.isTableAvailable(targetTableName)) {
500        try {
501          Thread.sleep(100);
502        } catch (InterruptedException ie) {
503          Thread.currentThread().interrupt();
504        }
505        if (EnvironmentEdgeManager.currentTime() - startTime > TABLE_AVAILABILITY_WAIT_TIME) {
506          throw new IOException("Time out " + TABLE_AVAILABILITY_WAIT_TIME + "ms expired, table "
507              + targetTableName + " is still not available");
508        }
509      }
510    }
511  }
512}