001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupInfo.withState;
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS;
022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER;
024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS;
025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_ENABLE_CONTINUOUS_BACKUP;
031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_ENABLE_CONTINUOUS_BACKUP_DESC;
032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_FORCE_DELETE;
033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_FORCE_DELETE_DESC;
034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM;
035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC;
036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
042import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
043import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
044import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
045import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
046import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
047import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
048import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
049import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
050import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
051import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
052import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
053import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD;
054import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
055import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
056
057import java.io.IOException;
058import java.net.URI;
059import java.text.ParseException;
060import java.text.SimpleDateFormat;
061import java.util.ArrayList;
062import java.util.Collections;
063import java.util.HashSet;
064import java.util.List;
065import java.util.Map;
066import java.util.Optional;
067import java.util.Set;
068import java.util.TimeZone;
069import java.util.concurrent.TimeUnit;
070import org.apache.commons.lang3.StringUtils;
071import org.apache.hadoop.conf.Configuration;
072import org.apache.hadoop.conf.Configured;
073import org.apache.hadoop.fs.FileStatus;
074import org.apache.hadoop.fs.FileSystem;
075import org.apache.hadoop.fs.Path;
076import org.apache.hadoop.hbase.HBaseConfiguration;
077import org.apache.hadoop.hbase.TableName;
078import org.apache.hadoop.hbase.backup.BackupAdmin;
079import org.apache.hadoop.hbase.backup.BackupInfo;
080import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
081import org.apache.hadoop.hbase.backup.BackupRequest;
082import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
083import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
084import org.apache.hadoop.hbase.backup.BackupType;
085import org.apache.hadoop.hbase.backup.HBackupFileSystem;
086import org.apache.hadoop.hbase.backup.util.BackupFileSystemManager;
087import org.apache.hadoop.hbase.backup.util.BackupSet;
088import org.apache.hadoop.hbase.backup.util.BackupUtils;
089import org.apache.hadoop.hbase.client.Admin;
090import org.apache.hadoop.hbase.client.Connection;
091import org.apache.hadoop.hbase.client.ConnectionFactory;
092import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
093import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
094import org.apache.hadoop.hbase.util.Pair;
095import org.apache.yetus.audience.InterfaceAudience;
096
097import org.apache.hbase.thirdparty.com.google.common.base.Splitter;
098import org.apache.hbase.thirdparty.com.google.common.base.Strings;
099import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
100import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
101import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
102import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
103
104/**
105 * General backup commands, options and usage messages
106 */
107@InterfaceAudience.Private
108public final class BackupCommands {
109  public final static String INCORRECT_USAGE = "Incorrect usage";
110
111  public final static String TOP_LEVEL_NOT_ALLOWED =
112    "Top level (root) folder is not allowed to be a backup destination";
113
114  public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
115    + "where COMMAND is one of:\n" + "  create     create a new backup image\n"
116    + "  delete     delete an existing backup image\n"
117    + "  describe   show the detailed information of a backup image\n"
118    + "  history    show history of all successful backups\n"
119    + "  progress   show the progress of the latest backup request\n"
120    + "  set        backup set management\n" + "  repair     repair backup system table\n"
121    + "  merge      merge backup images\n"
122    + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
123
124  public static final String CREATE_CMD_USAGE =
125    "Usage: hbase backup create <type> <backup_path> [options]\n"
126      + "  type           \"full\" to create a full backup image\n"
127      + "                 \"incremental\" to create an incremental backup image\n"
128      + "  backup_path     Full path to store the backup image\n";
129
130  public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
131    + "  backup_id       Backup image id (optional). If no id specified, the command will show\n"
132    + "                  progress for currently running backup session.";
133  public static final String NO_INFO_FOUND = "No info was found for backup id: ";
134  public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
135
136  public static final String DESCRIBE_CMD_USAGE =
137    "Usage: hbase backup describe <backup_id>\n" + "  backup_id       Backup image id\n";
138
139  public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
140
141  public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]";
142
143  public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
144
145  public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
146    + "  name            Backup set name\n" + "  tables          Comma separated list of tables.\n"
147    + "COMMAND is one of:\n" + "  add             add tables to a set, create a set if needed\n"
148    + "  remove          remove tables from a set\n"
149    + "  list            list all backup sets in the system\n" + "  describe        describe set\n"
150    + "  delete          delete backup set\n";
151  public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
152    + "  backup_ids      Comma separated list of backup image ids.\n";
153
154  public static final String USAGE_FOOTER = "";
155
156  public static abstract class Command extends Configured {
157    CommandLine cmdline;
158    Connection conn;
159
160    Command(Configuration conf) {
161      if (conf == null) {
162        conf = HBaseConfiguration.create();
163      }
164      setConf(conf);
165    }
166
167    public void execute() throws IOException {
168      if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
169        printUsage();
170        throw new IOException(INCORRECT_USAGE);
171      }
172
173      if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
174        String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
175        // Set MR job queuename to configuration
176        getConf().set("mapreduce.job.queuename", queueName);
177      }
178
179      // Create connection
180      conn = ConnectionFactory.createConnection(getConf());
181      if (requiresNoActiveSession()) {
182        // Check active session
183        try (BackupSystemTable table = new BackupSystemTable(conn)) {
184          List<BackupInfo> sessions =
185            table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING));
186
187          if (sessions.size() > 0) {
188            System.err.println("Found backup session in a RUNNING state: ");
189            System.err.println(sessions.get(0));
190            System.err.println("This may indicate that a previous session has failed abnormally.");
191            System.err.println("In this case, backup recovery is recommended.");
192            throw new IOException("Active session found, aborted command execution");
193          }
194        }
195      }
196      if (requiresConsistentState()) {
197        // Check failed delete
198        try (BackupSystemTable table = new BackupSystemTable(conn)) {
199          String[] ids = table.getListOfBackupIdsFromDeleteOperation();
200
201          if (ids != null && ids.length > 0) {
202            System.err.println("Found failed backup DELETE coommand. ");
203            System.err.println("Backup system recovery is required.");
204            throw new IOException("Failed backup DELETE found, aborted command execution");
205          }
206
207          ids = table.getListOfBackupIdsFromMergeOperation();
208          if (ids != null && ids.length > 0) {
209            System.err.println("Found failed backup MERGE coommand. ");
210            System.err.println("Backup system recovery is required.");
211            throw new IOException("Failed backup MERGE found, aborted command execution");
212          }
213        }
214      }
215    }
216
217    public void finish() throws IOException {
218      if (conn != null) {
219        conn.close();
220      }
221    }
222
223    protected abstract void printUsage();
224
225    /**
226     * The command can't be run if active backup session is in progress
227     * @return true if no active sessions are in progress
228     */
229    protected boolean requiresNoActiveSession() {
230      return false;
231    }
232
233    /**
234     * Command requires consistent state of a backup system Backup system may become inconsistent
235     * because of an abnormal termination of a backup session or delete command
236     * @return true, if yes
237     */
238    protected boolean requiresConsistentState() {
239      return false;
240    }
241  }
242
243  private BackupCommands() {
244    throw new AssertionError("Instantiating utility class...");
245  }
246
247  public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
248    Command cmd;
249    switch (type) {
250      case CREATE:
251        cmd = new CreateCommand(conf, cmdline);
252        break;
253      case DESCRIBE:
254        cmd = new DescribeCommand(conf, cmdline);
255        break;
256      case PROGRESS:
257        cmd = new ProgressCommand(conf, cmdline);
258        break;
259      case DELETE:
260        cmd = new DeleteCommand(conf, cmdline);
261        break;
262      case HISTORY:
263        cmd = new HistoryCommand(conf, cmdline);
264        break;
265      case SET:
266        cmd = new BackupSetCommand(conf, cmdline);
267        break;
268      case REPAIR:
269        cmd = new RepairCommand(conf, cmdline);
270        break;
271      case MERGE:
272        cmd = new MergeCommand(conf, cmdline);
273        break;
274      case HELP:
275      default:
276        cmd = new HelpCommand(conf, cmdline);
277        break;
278    }
279    return cmd;
280  }
281
282  static int numOfArgs(String[] args) {
283    if (args == null) {
284      return 0;
285    }
286
287    return args.length;
288  }
289
290  public static class CreateCommand extends Command {
291    CreateCommand(Configuration conf, CommandLine cmdline) {
292      super(conf);
293      this.cmdline = cmdline;
294    }
295
296    @Override
297    protected boolean requiresNoActiveSession() {
298      return true;
299    }
300
301    @Override
302    protected boolean requiresConsistentState() {
303      return true;
304    }
305
306    @Override
307    public void execute() throws IOException {
308      if (cmdline == null || cmdline.getArgs() == null) {
309        printUsage();
310        throw new IOException(INCORRECT_USAGE);
311      }
312      String[] args = cmdline.getArgs();
313      if (args.length != 3) {
314        printUsage();
315        throw new IOException(INCORRECT_USAGE);
316      }
317
318      if (
319        !BackupType.FULL.toString().equalsIgnoreCase(args[1])
320          && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])
321      ) {
322        System.out.println("ERROR: invalid backup type: " + args[1]);
323        printUsage();
324        throw new IOException(INCORRECT_USAGE);
325      }
326      if (!verifyPath(args[2])) {
327        System.out.println("ERROR: invalid backup destination: " + args[2]);
328        printUsage();
329        throw new IOException(INCORRECT_USAGE);
330      }
331      String targetBackupDir = args[2];
332      // Check if backup destination is top level (root) folder - not allowed
333      if (isRootFolder(targetBackupDir)) {
334        throw new IOException(TOP_LEVEL_NOT_ALLOWED);
335      }
336      String tables;
337
338      // Check if we have both: backup set and list of tables
339      if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
340        System.out
341          .println("ERROR: You can specify either backup set or list" + " of tables, but not both");
342        printUsage();
343        throw new IOException(INCORRECT_USAGE);
344      }
345      // Creates connection
346      super.execute();
347      // Check backup set
348      String setName = null;
349      if (cmdline.hasOption(OPTION_SET)) {
350        setName = cmdline.getOptionValue(OPTION_SET);
351        tables = getTablesForSet(setName);
352
353        if (tables == null) {
354          System.out
355            .println("ERROR: Backup set '" + setName + "' is either empty or does not exist");
356          printUsage();
357          throw new IOException(INCORRECT_USAGE);
358        }
359      } else {
360        tables = cmdline.getOptionValue(OPTION_TABLE);
361      }
362      int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH)
363        ? Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH))
364        : -1;
365      int workers = cmdline.hasOption(OPTION_WORKERS)
366        ? Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS))
367        : -1;
368
369      boolean ignoreChecksum = cmdline.hasOption(OPTION_IGNORECHECKSUM);
370
371      BackupType backupType = BackupType.valueOf(args[1].toUpperCase());
372      List<TableName> tableNameList = null;
373      if (tables != null) {
374        tableNameList = Lists.newArrayList(BackupUtils.parseTableNames(tables));
375      }
376      boolean continuousBackup = cmdline.hasOption(OPTION_ENABLE_CONTINUOUS_BACKUP);
377      if (continuousBackup && !BackupType.FULL.equals(backupType)) {
378        System.out.println("ERROR: Continuous backup can Only be specified for Full Backup");
379        printUsage();
380        throw new IOException(INCORRECT_USAGE);
381      }
382
383      /*
384       * The `continuousBackup` flag is specified only during the first full backup to initiate
385       * continuous WAL replication. After that, it is redundant because the tables are already set
386       * up for continuous backup. If the `continuousBackup` flag is not explicitly enabled, we need
387       * to determine the backup mode based on the current state of the specified tables: - If all
388       * the specified tables are already part of continuous backup, we treat the request as a
389       * continuous backup request and proceed accordingly (since these tables are already
390       * continuously backed up, no additional setup is needed). - If none of the specified tables
391       * are part of continuous backup, we treat the request as a normal full backup without
392       * continuous backup. - If the request includes a mix of tables—some with continuous backup
393       * enabled and others without—we cannot determine a clear backup strategy. In this case, we
394       * throw an error. If all tables are already in continuous backup mode, we explicitly set the
395       * `continuousBackup` flag to `true` so that the request is processed using the continuous
396       * backup approach rather than the normal full backup flow.
397       */
398      if (!continuousBackup && tableNameList != null && !tableNameList.isEmpty()) {
399        try (BackupSystemTable backupSystemTable = new BackupSystemTable(conn)) {
400          Set<TableName> continuousBackupTableSet =
401            backupSystemTable.getContinuousBackupTableSet().keySet();
402
403          boolean allTablesInContinuousBackup = continuousBackupTableSet.containsAll(tableNameList);
404          boolean noTablesInContinuousBackup =
405            tableNameList.stream().noneMatch(continuousBackupTableSet::contains);
406
407          // Ensure that all tables are either fully in continuous backup or not at all
408          if (!allTablesInContinuousBackup && !noTablesInContinuousBackup) {
409            System.err
410              .println("ERROR: Some tables are already in continuous backup, while others are not. "
411                + "Cannot mix both in a single request.");
412            printUsage();
413            throw new IOException(INCORRECT_USAGE);
414          }
415
416          // If all tables are already in continuous backup, enable the flag
417          if (allTablesInContinuousBackup) {
418            continuousBackup = true;
419          }
420        }
421      }
422
423      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
424        BackupRequest.Builder builder = new BackupRequest.Builder();
425        BackupRequest request = builder.withBackupType(backupType).withTableList(tableNameList)
426          .withTargetRootDir(targetBackupDir).withTotalTasks(workers)
427          .withBandwidthPerTasks(bandwidth).withNoChecksumVerify(ignoreChecksum)
428          .withBackupSetName(setName).withContinuousBackupEnabled(continuousBackup).build();
429        String backupId = admin.backupTables(request);
430        System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
431      } catch (IOException e) {
432        System.out.println("Backup session finished. Status: FAILURE");
433        throw e;
434      }
435    }
436
437    private boolean isRootFolder(String targetBackupDir) {
438      Path p = new Path(targetBackupDir);
439      return p.isRoot();
440    }
441
442    private boolean verifyPath(String path) {
443      try {
444        Path p = new Path(path);
445        Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
446        URI uri = p.toUri();
447
448        if (uri.getScheme() == null) {
449          return false;
450        }
451
452        FileSystem.get(uri, conf);
453        return true;
454      } catch (Exception e) {
455        return false;
456      }
457    }
458
459    private String getTablesForSet(String name) throws IOException {
460      try (final BackupSystemTable table = new BackupSystemTable(conn)) {
461        List<TableName> tables = table.describeBackupSet(name);
462
463        if (tables == null) {
464          return null;
465        }
466
467        return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
468      }
469    }
470
471    @Override
472    protected void printUsage() {
473      System.out.println(CREATE_CMD_USAGE);
474      Options options = new Options();
475      options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
476      options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
477      options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
478      options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
479      options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
480      options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC);
481      options.addOption(OPTION_IGNORECHECKSUM, false, OPTION_IGNORECHECKSUM_DESC);
482      options.addOption(OPTION_ENABLE_CONTINUOUS_BACKUP, false,
483        OPTION_ENABLE_CONTINUOUS_BACKUP_DESC);
484
485      HelpFormatter helpFormatter = new HelpFormatter();
486      helpFormatter.setLeftPadding(2);
487      helpFormatter.setDescPadding(8);
488      helpFormatter.setWidth(100);
489      helpFormatter.setSyntaxPrefix("Options:");
490      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
491    }
492  }
493
494  public static class HelpCommand extends Command {
495    HelpCommand(Configuration conf, CommandLine cmdline) {
496      super(conf);
497      this.cmdline = cmdline;
498    }
499
500    @Override
501    public void execute() throws IOException {
502      if (cmdline == null) {
503        printUsage();
504        throw new IOException(INCORRECT_USAGE);
505      }
506
507      String[] args = cmdline.getArgs();
508      if (args == null || args.length == 0) {
509        printUsage();
510        throw new IOException(INCORRECT_USAGE);
511      }
512
513      if (args.length != 2) {
514        System.out.println("ERROR: Only supports help message of a single command type");
515        printUsage();
516        throw new IOException(INCORRECT_USAGE);
517      }
518
519      String type = args[1];
520
521      if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
522        System.out.println(CREATE_CMD_USAGE);
523      } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
524        System.out.println(DESCRIBE_CMD_USAGE);
525      } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
526        System.out.println(HISTORY_CMD_USAGE);
527      } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
528        System.out.println(PROGRESS_CMD_USAGE);
529      } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
530        System.out.println(DELETE_CMD_USAGE);
531      } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
532        System.out.println(SET_CMD_USAGE);
533      } else {
534        System.out.println("Unknown command : " + type);
535        printUsage();
536      }
537    }
538
539    @Override
540    protected void printUsage() {
541      System.out.println(USAGE);
542    }
543  }
544
545  public static class DescribeCommand extends Command {
546    DescribeCommand(Configuration conf, CommandLine cmdline) {
547      super(conf);
548      this.cmdline = cmdline;
549    }
550
551    @Override
552    public void execute() throws IOException {
553      if (cmdline == null || cmdline.getArgs() == null) {
554        printUsage();
555        throw new IOException(INCORRECT_USAGE);
556      }
557      String[] args = cmdline.getArgs();
558      if (args.length != 2) {
559        printUsage();
560        throw new IOException(INCORRECT_USAGE);
561      }
562
563      super.execute();
564
565      String backupId = args[1];
566      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
567        BackupInfo info = sysTable.readBackupInfo(backupId);
568        if (info == null) {
569          System.out.println("ERROR: " + backupId + " does not exist");
570          printUsage();
571          throw new IOException(INCORRECT_USAGE);
572        }
573        System.out.println(info.getShortDescription());
574      }
575    }
576
577    @Override
578    protected void printUsage() {
579      System.out.println(DESCRIBE_CMD_USAGE);
580    }
581  }
582
583  public static class ProgressCommand extends Command {
584    ProgressCommand(Configuration conf, CommandLine cmdline) {
585      super(conf);
586      this.cmdline = cmdline;
587    }
588
589    @Override
590    public void execute() throws IOException {
591
592      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
593        System.out.println(
594          "No backup id was specified, " + "will retrieve the most recent (ongoing) session");
595      }
596      String[] args = cmdline == null ? null : cmdline.getArgs();
597      if (args != null && args.length > 2) {
598        System.err.println("ERROR: wrong number of arguments: " + args.length);
599        printUsage();
600        throw new IOException(INCORRECT_USAGE);
601      }
602
603      super.execute();
604
605      String backupId = (args == null || args.length <= 1) ? null : args[1];
606      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
607        BackupInfo info = null;
608
609        if (backupId != null) {
610          info = sysTable.readBackupInfo(backupId);
611        } else {
612          List<BackupInfo> infos =
613            sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING));
614          if (infos != null && infos.size() > 0) {
615            info = infos.get(0);
616            backupId = info.getBackupId();
617            System.out.println("Found ongoing session with backupId=" + backupId);
618          }
619        }
620        int progress = info == null ? -1 : info.getProgress();
621        if (progress < 0) {
622          if (backupId != null) {
623            System.out.println(NO_INFO_FOUND + backupId);
624          } else {
625            System.err.println(NO_ACTIVE_SESSION_FOUND);
626          }
627        } else {
628          System.out.println(backupId + " progress=" + progress + "%");
629        }
630      }
631    }
632
633    @Override
634    protected void printUsage() {
635      System.out.println(PROGRESS_CMD_USAGE);
636    }
637  }
638
639  public static class DeleteCommand extends Command {
640    DeleteCommand(Configuration conf, CommandLine cmdline) {
641      super(conf);
642      this.cmdline = cmdline;
643    }
644
645    @Override
646    protected boolean requiresNoActiveSession() {
647      return true;
648    }
649
650    @Override
651    public void execute() throws IOException {
652
653      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) {
654        printUsage();
655        throw new IOException(INCORRECT_USAGE);
656      }
657
658      if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) {
659        printUsage();
660        throw new IOException(INCORRECT_USAGE);
661      }
662
663      boolean isForceDelete = cmdline.hasOption(OPTION_FORCE_DELETE);
664      super.execute();
665      if (cmdline.hasOption(OPTION_KEEP)) {
666        executeDeleteOlderThan(cmdline, isForceDelete);
667      } else if (cmdline.hasOption(OPTION_LIST)) {
668        executeDeleteListOfBackups(cmdline, isForceDelete);
669      }
670
671      cleanUpUnusedBackupWALs();
672    }
673
674    private void executeDeleteOlderThan(CommandLine cmdline, boolean isForceDelete)
675      throws IOException {
676      String value = cmdline.getOptionValue(OPTION_KEEP);
677      int days = 0;
678      try {
679        days = Integer.parseInt(value);
680      } catch (NumberFormatException e) {
681        throw new IOException(value + " is not an integer number");
682      }
683      final long fdays = days;
684      BackupInfo.Filter dateFilter = info -> {
685        long currentTime = EnvironmentEdgeManager.currentTime();
686        long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000;
687        return info.getCompleteTs() <= maxTsToDelete;
688      };
689      List<BackupInfo> history = null;
690      try (final BackupSystemTable sysTable = new BackupSystemTable(conn);
691        BackupAdminImpl admin = new BackupAdminImpl(conn)) {
692        history = sysTable.getBackupHistory(dateFilter);
693        String[] backupIds = convertToBackupIds(history);
694        validatePITRBackupDeletion(backupIds, isForceDelete);
695        int deleted = admin.deleteBackups(backupIds);
696        System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: "
697          + backupIds.length);
698      } catch (IOException e) {
699        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
700          + "system integrity");
701        throw e;
702      }
703    }
704
705    private String[] convertToBackupIds(List<BackupInfo> history) {
706      String[] ids = new String[history.size()];
707      for (int i = 0; i < ids.length; i++) {
708        ids[i] = history.get(i).getBackupId();
709      }
710      return ids;
711    }
712
713    private void executeDeleteListOfBackups(CommandLine cmdline, boolean isForceDelete)
714      throws IOException {
715      String value = cmdline.getOptionValue(OPTION_LIST);
716      String[] backupIds = value.split(",");
717      validatePITRBackupDeletion(backupIds, isForceDelete);
718      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
719        int deleted = admin.deleteBackups(backupIds);
720        System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length);
721      } catch (IOException e) {
722        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
723          + "system integrity");
724        throw e;
725      }
726
727    }
728
729    /**
730     * Validates whether the specified backups can be deleted while preserving Point-In-Time
731     * Recovery (PITR) capabilities. If a backup is the only remaining full backup enabling PITR for
732     * certain tables, deletion is prevented unless forced.
733     * @param backupIds     Array of backup IDs to validate.
734     * @param isForceDelete Flag indicating whether deletion should proceed regardless of PITR
735     *                      constraints.
736     * @throws IOException If a backup is essential for PITR and force deletion is not enabled.
737     */
738    private void validatePITRBackupDeletion(String[] backupIds, boolean isForceDelete)
739      throws IOException {
740      if (!isForceDelete) {
741        for (String backupId : backupIds) {
742          List<TableName> affectedTables = getTablesDependentOnBackupForPITR(backupId);
743          if (!affectedTables.isEmpty()) {
744            String errMsg = String.format(
745              "Backup %s is the only FULL backup remaining that enables PITR for tables: %s. "
746                + "Use the force option to delete it anyway.",
747              backupId, affectedTables);
748            System.err.println(errMsg);
749            throw new IOException(errMsg);
750          }
751        }
752      }
753    }
754
755    /**
756     * Identifies tables that rely on the specified backup for PITR (Point-In-Time Recovery). A
757     * table is considered dependent on the backup if it does not have any other valid full backups
758     * that can cover the PITR window enabled by the specified backup.
759     * @param backupId The ID of the backup being evaluated for PITR coverage.
760     * @return A list of tables that are dependent on the specified backup for PITR recovery.
761     * @throws IOException If there is an error retrieving the backup metadata or backup system
762     *                     table.
763     */
764    private List<TableName> getTablesDependentOnBackupForPITR(String backupId) throws IOException {
765      List<TableName> dependentTables = new ArrayList<>();
766
767      try (final BackupSystemTable backupSystemTable = new BackupSystemTable(conn)) {
768        // Fetch the target backup's info using the backup ID
769        BackupInfo targetBackup = backupSystemTable.readBackupInfo(backupId);
770        if (targetBackup == null) {
771          throw new IOException("Backup info not found for backupId: " + backupId);
772        }
773
774        // Only full backups are mandatory for PITR
775        if (!BackupType.FULL.equals(targetBackup.getType())) {
776          return List.of();
777        }
778
779        // Retrieve the tables with continuous backup enabled along with their start times
780        Map<TableName, Long> continuousBackupStartTimes =
781          backupSystemTable.getContinuousBackupTableSet();
782
783        // Calculate the PITR window by fetching configuration and current time
784        long pitrWindowDays = getConf().getLong(CONF_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS,
785          DEFAULT_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS);
786        long currentTime = EnvironmentEdgeManager.getDelegate().currentTime();
787        final long maxAllowedPITRTime = currentTime - TimeUnit.DAYS.toMillis(pitrWindowDays);
788
789        // Check each table associated with the target backup
790        for (TableName table : targetBackup.getTableNames()) {
791          // Skip tables without continuous backup enabled
792          if (!continuousBackupStartTimes.containsKey(table)) {
793            continue;
794          }
795
796          // Calculate the PITR window this backup covers for the table
797          Optional<Pair<Long, Long>> coveredPitrWindow = getCoveredPitrWindowForTable(targetBackup,
798            continuousBackupStartTimes.get(table), maxAllowedPITRTime, currentTime);
799
800          // If this backup does not cover a valid PITR window for the table, skip
801          if (coveredPitrWindow.isEmpty()) {
802            continue;
803          }
804
805          // Check if there is any other valid backup that can cover the PITR window
806          List<BackupInfo> allBackups =
807            backupSystemTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE));
808          boolean hasAnotherValidBackup =
809            canAnyOtherBackupCover(allBackups, targetBackup, table, coveredPitrWindow.get(),
810              continuousBackupStartTimes.get(table), maxAllowedPITRTime, currentTime);
811
812          // If no other valid backup exists, add the table to the dependent list
813          if (!hasAnotherValidBackup) {
814            dependentTables.add(table);
815          }
816        }
817      }
818
819      return dependentTables;
820    }
821
822    /**
823     * Calculates the PITR (Point-In-Time Recovery) window that the given backup enables for a
824     * table.
825     * @param backupInfo                Metadata of the backup being evaluated.
826     * @param continuousBackupStartTime When continuous backups started for the table.
827     * @param maxAllowedPITRTime        The earliest timestamp from which PITR is supported in the
828     *                                  cluster.
829     * @param currentTime               Current time.
830     * @return Optional PITR window as a pair (start, end), or empty if backup is not useful for
831     *         PITR.
832     */
833    private Optional<Pair<Long, Long>> getCoveredPitrWindowForTable(BackupInfo backupInfo,
834      long continuousBackupStartTime, long maxAllowedPITRTime, long currentTime) {
835
836      long backupStartTs = backupInfo.getStartTs();
837      long backupEndTs = backupInfo.getCompleteTs();
838      long effectiveStart = Math.max(continuousBackupStartTime, maxAllowedPITRTime);
839
840      if (backupStartTs < continuousBackupStartTime) {
841        return Optional.empty();
842      }
843
844      return Optional.of(Pair.newPair(Math.max(backupEndTs, effectiveStart), currentTime));
845    }
846
847    /**
848     * Checks if any backup (excluding the current backup) can cover the specified PITR window for
849     * the given table. A backup can cover the PITR window if it fully encompasses the target time
850     * range specified.
851     * @param allBackups                List of all backups available.
852     * @param currentBackup             The current backup that should not be considered for
853     *                                  coverage.
854     * @param table                     The table for which we need to check backup coverage.
855     * @param targetWindow              A pair representing the target PITR window (start and end
856     *                                  times).
857     * @param continuousBackupStartTime When continuous backups started for the table.
858     * @param maxAllowedPITRTime        The earliest timestamp from which PITR is supported in the
859     *                                  cluster.
860     * @param currentTime               Current time.
861     * @return {@code true} if any backup (excluding the current one) fully covers the target PITR
862     *         window; {@code false} otherwise.
863     */
864    private boolean canAnyOtherBackupCover(List<BackupInfo> allBackups, BackupInfo currentBackup,
865      TableName table, Pair<Long, Long> targetWindow, long continuousBackupStartTime,
866      long maxAllowedPITRTime, long currentTime) {
867
868      long targetStart = targetWindow.getFirst();
869      long targetEnd = targetWindow.getSecond();
870
871      // Iterate through all backups (including the current one)
872      for (BackupInfo backup : allBackups) {
873        // Skip if the backup is not full or doesn't contain the table
874        if (!BackupType.FULL.equals(backup.getType())) continue;
875        if (!backup.getTableNames().contains(table)) continue;
876
877        // Skip the current backup itself
878        if (backup.equals(currentBackup)) continue;
879
880        // Get the covered PITR window for this backup
881        Optional<Pair<Long, Long>> coveredWindow = getCoveredPitrWindowForTable(backup,
882          continuousBackupStartTime, maxAllowedPITRTime, currentTime);
883
884        if (coveredWindow.isPresent()) {
885          Pair<Long, Long> covered = coveredWindow.get();
886
887          // The backup must fully cover the target window
888          if (covered.getFirst() <= targetStart && covered.getSecond() >= targetEnd) {
889            return true;
890          }
891        }
892      }
893
894      return false;
895    }
896
897    /**
898     * Cleans up Write-Ahead Logs (WALs) that are no longer required for PITR after a successful
899     * backup deletion. If no full backups are present, all WALs are deleted, tables are removed
900     * from continuous backup metadata, and the associated replication peer is disabled.
901     */
902    private void cleanUpUnusedBackupWALs() throws IOException {
903      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
904      String backupWalDir = conf.get(CONF_CONTINUOUS_BACKUP_WAL_DIR);
905
906      if (Strings.isNullOrEmpty(backupWalDir)) {
907        System.out.println("No WAL directory specified for continuous backup. Skipping cleanup.");
908        return;
909      }
910
911      try (Admin admin = conn.getAdmin();
912        BackupSystemTable sysTable = new BackupSystemTable(conn)) {
913        // Get list of tables under continuous backup
914        Map<TableName, Long> continuousBackupTables = sysTable.getContinuousBackupTableSet();
915        if (continuousBackupTables.isEmpty()) {
916          System.out.println("No continuous backups configured. Skipping WAL cleanup.");
917          return;
918        }
919
920        // Find the earliest timestamp after which WALs are still needed
921        long cutoffTimestamp = determineWALCleanupCutoffTime(sysTable);
922        if (cutoffTimestamp == 0) {
923          // No full backup exists. PITR cannot function without a base full backup.
924          // Clean up all WALs, remove tables from backup metadata, and disable the replication
925          // peer.
926          System.out
927            .println("No full backups found. Cleaning up all WALs and disabling replication peer.");
928
929          disableContinuousBackupReplicationPeer(admin);
930          removeAllTablesFromContinuousBackup(sysTable);
931          deleteAllBackupWALFiles(conf, backupWalDir);
932          return;
933        }
934
935        // Update metadata before actual cleanup to avoid inconsistencies
936        updateBackupTableStartTimes(sysTable, cutoffTimestamp);
937
938        // Delete WAL files older than cutoff timestamp
939        deleteOldWALFiles(conf, backupWalDir, cutoffTimestamp);
940
941      }
942    }
943
944    /**
945     * Determines the cutoff time for cleaning WAL files.
946     * @param sysTable Backup system table
947     * @return cutoff timestamp or 0 if not found
948     */
949    long determineWALCleanupCutoffTime(BackupSystemTable sysTable) throws IOException {
950      List<BackupInfo> backupInfos =
951        sysTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE));
952      Collections.reverse(backupInfos); // Start from oldest
953
954      for (BackupInfo backupInfo : backupInfos) {
955        if (BackupType.FULL.equals(backupInfo.getType())) {
956          return backupInfo.getStartTs();
957        }
958      }
959      return 0;
960    }
961
962    private void disableContinuousBackupReplicationPeer(Admin admin) throws IOException {
963      for (ReplicationPeerDescription peer : admin.listReplicationPeers()) {
964        if (peer.getPeerId().equals(CONTINUOUS_BACKUP_REPLICATION_PEER) && peer.isEnabled()) {
965          admin.disableReplicationPeer(CONTINUOUS_BACKUP_REPLICATION_PEER);
966          System.out.println("Disabled replication peer: " + CONTINUOUS_BACKUP_REPLICATION_PEER);
967          break;
968        }
969      }
970    }
971
972    /**
973     * Updates the start time for continuous backups if older than cutoff timestamp.
974     * @param sysTable        Backup system table
975     * @param cutoffTimestamp Timestamp before which WALs are no longer needed
976     */
977    void updateBackupTableStartTimes(BackupSystemTable sysTable, long cutoffTimestamp)
978      throws IOException {
979
980      Map<TableName, Long> backupTables = sysTable.getContinuousBackupTableSet();
981      Set<TableName> tablesToUpdate = new HashSet<>();
982
983      for (Map.Entry<TableName, Long> entry : backupTables.entrySet()) {
984        if (entry.getValue() < cutoffTimestamp) {
985          tablesToUpdate.add(entry.getKey());
986        }
987      }
988
989      if (!tablesToUpdate.isEmpty()) {
990        sysTable.updateContinuousBackupTableSet(tablesToUpdate, cutoffTimestamp);
991      }
992    }
993
994    private void removeAllTablesFromContinuousBackup(BackupSystemTable sysTable)
995      throws IOException {
996      Map<TableName, Long> allTables = sysTable.getContinuousBackupTableSet();
997      if (!allTables.isEmpty()) {
998        sysTable.removeContinuousBackupTableSet(allTables.keySet());
999        System.out.println("Removed all tables from continuous backup metadata.");
1000      }
1001    }
1002
1003    private void deleteAllBackupWALFiles(Configuration conf, String backupWalDir)
1004      throws IOException {
1005      try {
1006        BackupFileSystemManager manager =
1007          new BackupFileSystemManager(CONTINUOUS_BACKUP_REPLICATION_PEER, conf, backupWalDir);
1008        FileSystem fs = manager.getBackupFs();
1009        Path walDir = manager.getWalsDir();
1010        Path bulkloadDir = manager.getBulkLoadFilesDir();
1011
1012        // Delete contents under WAL directory
1013        if (fs.exists(walDir)) {
1014          FileStatus[] walContents = fs.listStatus(walDir);
1015          for (FileStatus item : walContents) {
1016            fs.delete(item.getPath(), true); // recursive delete of each child
1017          }
1018          System.out.println("Deleted all contents under WAL directory: " + walDir);
1019        }
1020
1021        // Delete contents under bulk load directory
1022        if (fs.exists(bulkloadDir)) {
1023          FileStatus[] bulkContents = fs.listStatus(bulkloadDir);
1024          for (FileStatus item : bulkContents) {
1025            fs.delete(item.getPath(), true); // recursive delete of each child
1026          }
1027          System.out.println("Deleted all contents under Bulk Load directory: " + bulkloadDir);
1028        }
1029
1030      } catch (IOException e) {
1031        System.out.println("WARNING: Failed to delete contents under backup directories: "
1032          + backupWalDir + ". Error: " + e.getMessage());
1033        throw e;
1034      }
1035    }
1036
1037    /**
1038     * Cleans up old WAL and bulk-loaded files based on the determined cutoff timestamp.
1039     */
1040    void deleteOldWALFiles(Configuration conf, String backupWalDir, long cutoffTime)
1041      throws IOException {
1042      System.out.println("Starting WAL cleanup in backup directory: " + backupWalDir
1043        + " with cutoff time: " + cutoffTime);
1044
1045      BackupFileSystemManager manager =
1046        new BackupFileSystemManager(CONTINUOUS_BACKUP_REPLICATION_PEER, conf, backupWalDir);
1047      FileSystem fs = manager.getBackupFs();
1048      Path walDir = manager.getWalsDir();
1049      Path bulkloadDir = manager.getBulkLoadFilesDir();
1050
1051      SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
1052      dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
1053
1054      System.out.println("Listing directories under: " + walDir);
1055
1056      FileStatus[] directories = fs.listStatus(walDir);
1057
1058      for (FileStatus dirStatus : directories) {
1059        if (!dirStatus.isDirectory()) {
1060          continue; // Skip files, we only want directories
1061        }
1062
1063        Path dirPath = dirStatus.getPath();
1064        String dirName = dirPath.getName();
1065
1066        try {
1067          long dayStart = parseDayDirectory(dirName, dateFormat);
1068          System.out
1069            .println("Checking WAL directory: " + dirName + " (Start Time: " + dayStart + ")");
1070
1071          // If WAL files of that day are older than cutoff time, delete them
1072          if (dayStart + ONE_DAY_IN_MILLISECONDS - 1 < cutoffTime) {
1073            System.out.println("Deleting outdated WAL directory: " + dirPath);
1074            fs.delete(dirPath, true);
1075            Path bulkloadPath = new Path(bulkloadDir, dirName);
1076            System.out.println("Deleting corresponding bulk-load directory: " + bulkloadPath);
1077            fs.delete(bulkloadPath, true);
1078          }
1079        } catch (ParseException e) {
1080          System.out.println("WARNING: Failed to parse directory name '" + dirName
1081            + "'. Skipping. Error: " + e.getMessage());
1082        } catch (IOException e) {
1083          System.err.println("WARNING: Failed to delete directory '" + dirPath
1084            + "'. Skipping. Error: " + e.getMessage());
1085        }
1086      }
1087
1088      System.out.println("Completed WAL cleanup for backup directory: " + backupWalDir);
1089    }
1090
1091    private long parseDayDirectory(String dayDir, SimpleDateFormat dateFormat)
1092      throws ParseException {
1093      return dateFormat.parse(dayDir).getTime();
1094    }
1095
1096    @Override
1097    protected void printUsage() {
1098      System.out.println(DELETE_CMD_USAGE);
1099      Options options = new Options();
1100      options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC);
1101      options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC);
1102      options.addOption(OPTION_FORCE_DELETE, false, OPTION_FORCE_DELETE_DESC);
1103
1104      HelpFormatter helpFormatter = new HelpFormatter();
1105      helpFormatter.setLeftPadding(2);
1106      helpFormatter.setDescPadding(8);
1107      helpFormatter.setWidth(100);
1108      helpFormatter.setSyntaxPrefix("Options:");
1109      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
1110
1111    }
1112  }
1113
1114  public static class RepairCommand extends Command {
1115    RepairCommand(Configuration conf, CommandLine cmdline) {
1116      super(conf);
1117      this.cmdline = cmdline;
1118    }
1119
1120    @Override
1121    public void execute() throws IOException {
1122      super.execute();
1123
1124      String[] args = cmdline == null ? null : cmdline.getArgs();
1125      if (args != null && args.length > 1) {
1126        System.err.println("ERROR: wrong number of arguments: " + args.length);
1127        printUsage();
1128        throw new IOException(INCORRECT_USAGE);
1129      }
1130
1131      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
1132      try (final Connection conn = ConnectionFactory.createConnection(conf);
1133        final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1134        // Failed backup
1135        BackupInfo backupInfo;
1136        List<BackupInfo> list =
1137          sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING));
1138        if (list.size() == 0) {
1139          // No failed sessions found
1140          System.out.println("REPAIR status: no failed sessions found."
1141            + " Checking failed delete backup operation ...");
1142          repairFailedBackupDeletionIfAny(conn, sysTable);
1143          repairFailedBackupMergeIfAny(conn, sysTable);
1144          return;
1145        }
1146        backupInfo = list.get(0);
1147        // If this is a cancel exception, then we've already cleaned.
1148        // set the failure timestamp of the overall backup
1149        backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
1150        // set failure message
1151        backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
1152        // set overall backup status: failed
1153        backupInfo.setState(BackupState.FAILED);
1154        // compose the backup failed data
1155        String backupFailedData = "BackupId=" + backupInfo.getBackupId() + ",startts="
1156          + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
1157          + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
1158        System.out.println(backupFailedData);
1159        TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
1160        // If backup session is updated to FAILED state - means we
1161        // processed recovery already.
1162        sysTable.updateBackupInfo(backupInfo);
1163        sysTable.finishBackupExclusiveOperation();
1164        System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
1165      }
1166    }
1167
1168    private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
1169      throws IOException {
1170      String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
1171      if (backupIds == null || backupIds.length == 0) {
1172        System.out.println("No failed backup DELETE operation found");
1173        // Delete backup table snapshot if exists
1174        BackupSystemTable.deleteSnapshot(conn);
1175        return;
1176      }
1177      System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
1178      System.out.println("Running DELETE again ...");
1179      // Restore table from snapshot
1180      BackupSystemTable.restoreFromSnapshot(conn);
1181      // Finish previous failed session
1182      sysTable.finishBackupExclusiveOperation();
1183      try (BackupAdmin admin = new BackupAdminImpl(conn)) {
1184        admin.deleteBackups(backupIds);
1185      }
1186      System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
1187    }
1188
1189    public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
1190      throws IOException {
1191
1192      String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
1193      if (backupIds == null || backupIds.length == 0) {
1194        System.out.println("No failed backup MERGE operation found");
1195        // Delete backup table snapshot if exists
1196        BackupSystemTable.deleteSnapshot(conn);
1197        return;
1198      }
1199      System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
1200      // Check if backup .tmp exists
1201      BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]);
1202      String backupRoot = bInfo.getBackupRootDir();
1203      FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration());
1204      String backupId = BackupUtils.findMostRecentBackupId(backupIds);
1205      Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId);
1206      if (fs.exists(tmpPath)) {
1207        // Move data back
1208        Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId);
1209        if (!fs.delete(destPath, true)) {
1210          System.out.println("Failed to delete " + destPath);
1211        }
1212        boolean res = fs.rename(tmpPath, destPath);
1213        if (!res) {
1214          throw new IOException(
1215            "MERGE repair: failed  to rename from " + tmpPath + " to " + destPath);
1216        }
1217        System.out
1218          .println("MERGE repair: renamed from " + tmpPath + " to " + destPath + " res=" + res);
1219      } else {
1220        checkRemoveBackupImages(fs, backupRoot, backupIds);
1221      }
1222      // Restore table from snapshot
1223      BackupSystemTable.restoreFromSnapshot(conn);
1224      // Unlock backup system
1225      sysTable.finishBackupExclusiveOperation();
1226      // Finish previous failed session
1227      sysTable.finishMergeOperation();
1228
1229      System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds));
1230    }
1231
1232    private static void checkRemoveBackupImages(FileSystem fs, String backupRoot,
1233      String[] backupIds) throws IOException {
1234      String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);
1235      for (String backupId : backupIds) {
1236        if (backupId.equals(mergedBackupId)) {
1237          continue;
1238        }
1239        Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId);
1240        if (fs.exists(path)) {
1241          if (!fs.delete(path, true)) {
1242            System.out.println("MERGE repair removing: " + path + " - FAILED");
1243          } else {
1244            System.out.println("MERGE repair removing: " + path + " - OK");
1245          }
1246        }
1247      }
1248    }
1249
1250    @Override
1251    protected void printUsage() {
1252      System.out.println(REPAIR_CMD_USAGE);
1253    }
1254  }
1255
1256  public static class MergeCommand extends Command {
1257    MergeCommand(Configuration conf, CommandLine cmdline) {
1258      super(conf);
1259      this.cmdline = cmdline;
1260    }
1261
1262    @Override
1263    protected boolean requiresNoActiveSession() {
1264      return true;
1265    }
1266
1267    @Override
1268    protected boolean requiresConsistentState() {
1269      return true;
1270    }
1271
1272    @Override
1273    public void execute() throws IOException {
1274      super.execute();
1275
1276      String[] args = cmdline == null ? null : cmdline.getArgs();
1277      if (args == null || (args.length != 2)) {
1278        System.err
1279          .println("ERROR: wrong number of arguments: " + (args == null ? null : args.length));
1280        printUsage();
1281        throw new IOException(INCORRECT_USAGE);
1282      }
1283
1284      String[] backupIds = args[1].split(",");
1285      if (backupIds.length < 2) {
1286        String msg = "ERROR: can not merge a single backup image. "
1287          + "Number of images must be greater than 1.";
1288        System.err.println(msg);
1289        throw new IOException(msg);
1290
1291      }
1292      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
1293      try (final Connection conn = ConnectionFactory.createConnection(conf);
1294        final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1295        admin.mergeBackups(backupIds);
1296      }
1297    }
1298
1299    @Override
1300    protected void printUsage() {
1301      System.out.println(MERGE_CMD_USAGE);
1302    }
1303  }
1304
1305  public static class HistoryCommand extends Command {
1306    private final static int DEFAULT_HISTORY_LENGTH = 10;
1307
1308    HistoryCommand(Configuration conf, CommandLine cmdline) {
1309      super(conf);
1310      this.cmdline = cmdline;
1311    }
1312
1313    @Override
1314    public void execute() throws IOException {
1315      int n = parseHistoryLength();
1316      final TableName tableName = getTableName();
1317      final String setName = getTableSetName();
1318      BackupInfo.Filter tableNameFilter = info -> {
1319        if (tableName == null) {
1320          return true;
1321        }
1322
1323        List<TableName> names = info.getTableNames();
1324        return names.contains(tableName);
1325      };
1326      BackupInfo.Filter tableSetFilter = info -> {
1327        if (setName == null) {
1328          return true;
1329        }
1330
1331        String backupId = info.getBackupId();
1332        return backupId.startsWith(setName);
1333      };
1334      Path backupRootPath = getBackupRootPath();
1335      List<BackupInfo> history;
1336      if (backupRootPath == null) {
1337        // Load from backup system table
1338        super.execute();
1339        try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1340          history = sysTable.getBackupHistory(tableNameFilter, tableSetFilter);
1341          history = history.subList(0, Math.min(n, history.size()));
1342        }
1343      } else {
1344        // load from backup FS
1345        history =
1346          BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
1347      }
1348      for (BackupInfo info : history) {
1349        System.out.println(info.getShortDescription());
1350      }
1351    }
1352
1353    private Path getBackupRootPath() throws IOException {
1354      String value = null;
1355      try {
1356        value = cmdline.getOptionValue(OPTION_PATH);
1357
1358        if (value == null) {
1359          return null;
1360        }
1361
1362        return new Path(value);
1363      } catch (IllegalArgumentException e) {
1364        System.out.println("ERROR: Illegal argument for backup root path: " + value);
1365        printUsage();
1366        throw new IOException(INCORRECT_USAGE);
1367      }
1368    }
1369
1370    private TableName getTableName() throws IOException {
1371      String value = cmdline.getOptionValue(OPTION_TABLE);
1372
1373      if (value == null) {
1374        return null;
1375      }
1376
1377      try {
1378        return TableName.valueOf(value);
1379      } catch (IllegalArgumentException e) {
1380        System.out.println("Illegal argument for table name: " + value);
1381        printUsage();
1382        throw new IOException(INCORRECT_USAGE);
1383      }
1384    }
1385
1386    private String getTableSetName() {
1387      return cmdline.getOptionValue(OPTION_SET);
1388    }
1389
1390    private int parseHistoryLength() throws IOException {
1391      String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
1392      try {
1393        if (value == null) {
1394          return DEFAULT_HISTORY_LENGTH;
1395        }
1396
1397        return Integer.parseInt(value);
1398      } catch (NumberFormatException e) {
1399        System.out.println("Illegal argument for history length: " + value);
1400        printUsage();
1401        throw new IOException(INCORRECT_USAGE);
1402      }
1403    }
1404
1405    @Override
1406    protected void printUsage() {
1407      System.out.println(HISTORY_CMD_USAGE);
1408      Options options = new Options();
1409      options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
1410      options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
1411      options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
1412      options.addOption(OPTION_SET, true, OPTION_SET_DESC);
1413
1414      HelpFormatter helpFormatter = new HelpFormatter();
1415      helpFormatter.setLeftPadding(2);
1416      helpFormatter.setDescPadding(8);
1417      helpFormatter.setWidth(100);
1418      helpFormatter.setSyntaxPrefix("Options:");
1419      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
1420    }
1421  }
1422
1423  public static class BackupSetCommand extends Command {
1424    private final static String SET_ADD_CMD = "add";
1425    private final static String SET_REMOVE_CMD = "remove";
1426    private final static String SET_DELETE_CMD = "delete";
1427    private final static String SET_DESCRIBE_CMD = "describe";
1428    private final static String SET_LIST_CMD = "list";
1429
1430    BackupSetCommand(Configuration conf, CommandLine cmdline) {
1431      super(conf);
1432      this.cmdline = cmdline;
1433    }
1434
1435    @Override
1436    public void execute() throws IOException {
1437      // Command-line must have at least one element
1438      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
1439        printUsage();
1440        throw new IOException(INCORRECT_USAGE);
1441      }
1442
1443      String[] args = cmdline.getArgs();
1444      String cmdStr = args[1];
1445      BackupCommand cmd = getCommand(cmdStr);
1446
1447      switch (cmd) {
1448        case SET_ADD:
1449          processSetAdd(args);
1450          break;
1451        case SET_REMOVE:
1452          processSetRemove(args);
1453          break;
1454        case SET_DELETE:
1455          processSetDelete(args);
1456          break;
1457        case SET_DESCRIBE:
1458          processSetDescribe(args);
1459          break;
1460        case SET_LIST:
1461          processSetList();
1462          break;
1463        default:
1464          break;
1465      }
1466    }
1467
1468    private void processSetList() throws IOException {
1469      super.execute();
1470
1471      // List all backup set names
1472      // does not expect any args
1473      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1474        List<BackupSet> list = admin.listBackupSets();
1475        for (BackupSet bs : list) {
1476          System.out.println(bs);
1477        }
1478      }
1479    }
1480
1481    private void processSetDescribe(String[] args) throws IOException {
1482      if (args == null || args.length != 3) {
1483        printUsage();
1484        throw new IOException(INCORRECT_USAGE);
1485      }
1486      super.execute();
1487
1488      String setName = args[2];
1489      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1490        List<TableName> tables = sysTable.describeBackupSet(setName);
1491        BackupSet set = tables == null ? null : new BackupSet(setName, tables);
1492        if (set == null) {
1493          System.out.println("Set '" + setName + "' does not exist.");
1494        } else {
1495          System.out.println(set);
1496        }
1497      }
1498    }
1499
1500    private void processSetDelete(String[] args) throws IOException {
1501      if (args == null || args.length != 3) {
1502        printUsage();
1503        throw new IOException(INCORRECT_USAGE);
1504      }
1505      super.execute();
1506
1507      String setName = args[2];
1508      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1509        boolean result = admin.deleteBackupSet(setName);
1510        if (result) {
1511          System.out.println("Delete set " + setName + " OK.");
1512        } else {
1513          System.out.println("Set " + setName + " does not exist");
1514        }
1515      }
1516    }
1517
1518    private void processSetRemove(String[] args) throws IOException {
1519      if (args == null || args.length != 4) {
1520        printUsage();
1521        throw new IOException(INCORRECT_USAGE);
1522      }
1523      super.execute();
1524
1525      String setName = args[2];
1526      String[] tables = args[3].split(",");
1527      TableName[] tableNames = toTableNames(tables);
1528      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1529        admin.removeFromBackupSet(setName, tableNames);
1530      }
1531    }
1532
1533    private TableName[] toTableNames(String[] tables) {
1534      TableName[] arr = new TableName[tables.length];
1535      for (int i = 0; i < tables.length; i++) {
1536        arr[i] = TableName.valueOf(tables[i]);
1537      }
1538      return arr;
1539    }
1540
1541    private void processSetAdd(String[] args) throws IOException {
1542      if (args == null || args.length != 4) {
1543        printUsage();
1544        throw new IOException(INCORRECT_USAGE);
1545      }
1546      super.execute();
1547      String setName = args[2];
1548      TableName[] tableNames =
1549        Splitter.on(',').splitToStream(args[3]).map(TableName::valueOf).toArray(TableName[]::new);
1550      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1551        admin.addToBackupSet(setName, tableNames);
1552      }
1553    }
1554
1555    private BackupCommand getCommand(String cmdStr) throws IOException {
1556      switch (cmdStr) {
1557        case SET_ADD_CMD:
1558          return BackupCommand.SET_ADD;
1559        case SET_REMOVE_CMD:
1560          return BackupCommand.SET_REMOVE;
1561        case SET_DELETE_CMD:
1562          return BackupCommand.SET_DELETE;
1563        case SET_DESCRIBE_CMD:
1564          return BackupCommand.SET_DESCRIBE;
1565        case SET_LIST_CMD:
1566          return BackupCommand.SET_LIST;
1567        default:
1568          System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
1569          printUsage();
1570          throw new IOException(INCORRECT_USAGE);
1571      }
1572    }
1573
1574    @Override
1575    protected void printUsage() {
1576      System.out.println(SET_CMD_USAGE);
1577    }
1578  }
1579}