001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.backup.impl;
020
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
042import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
043
044import java.io.IOException;
045import java.net.URI;
046import java.util.List;
047
048import org.apache.commons.lang3.StringUtils;
049import org.apache.hadoop.conf.Configuration;
050import org.apache.hadoop.conf.Configured;
051import org.apache.hadoop.fs.FileSystem;
052import org.apache.hadoop.fs.Path;
053import org.apache.hadoop.hbase.HBaseConfiguration;
054import org.apache.hadoop.hbase.TableName;
055import org.apache.hadoop.hbase.backup.BackupAdmin;
056import org.apache.hadoop.hbase.backup.BackupInfo;
057import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
058import org.apache.hadoop.hbase.backup.BackupRequest;
059import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
060import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
061import org.apache.hadoop.hbase.backup.BackupType;
062import org.apache.hadoop.hbase.backup.HBackupFileSystem;
063import org.apache.hadoop.hbase.backup.util.BackupSet;
064import org.apache.hadoop.hbase.backup.util.BackupUtils;
065import org.apache.hadoop.hbase.client.Connection;
066import org.apache.hadoop.hbase.client.ConnectionFactory;
067import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
068import org.apache.yetus.audience.InterfaceAudience;
069
070import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
071import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
072import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
073import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
074
075/**
076 * General backup commands, options and usage messages
077 */
078@InterfaceAudience.Private
079public final class BackupCommands {
080  public final static String INCORRECT_USAGE = "Incorrect usage";
081
082  public final static String TOP_LEVEL_NOT_ALLOWED =
083      "Top level (root) folder is not allowed to be a backup destination";
084
085  public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
086      + "where COMMAND is one of:\n" + "  create     create a new backup image\n"
087      + "  delete     delete an existing backup image\n"
088      + "  describe   show the detailed information of a backup image\n"
089      + "  history    show history of all successful backups\n"
090      + "  progress   show the progress of the latest backup request\n"
091      + "  set        backup set management\n"
092      + "  repair     repair backup system table\n"
093      + "  merge      merge backup images\n"
094      + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
095
096  public static final String CREATE_CMD_USAGE =
097      "Usage: hbase backup create <type> <backup_path> [options]\n"
098          + "  type           \"full\" to create a full backup image\n"
099          + "                 \"incremental\" to create an incremental backup image\n"
100          + "  backup_path     Full path to store the backup image\n";
101
102  public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
103      + "  backup_id       Backup image id (optional). If no id specified, the command will show\n"
104      + "                  progress for currently running backup session.";
105  public static final String NO_INFO_FOUND = "No info was found for backup id: ";
106  public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
107
108  public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup describe <backup_id>\n"
109      + "  backup_id       Backup image id\n";
110
111  public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
112
113  public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]";
114
115  public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
116
117  public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
118      + "  name            Backup set name\n"
119      + "  tables          Comma separated list of tables.\n" + "COMMAND is one of:\n"
120      + "  add             add tables to a set, create a set if needed\n"
121      + "  remove          remove tables from a set\n"
122      + "  list            list all backup sets in the system\n"
123      + "  describe        describe set\n" + "  delete          delete backup set\n";
124  public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
125      + "  backup_ids      Comma separated list of backup image ids.\n";
126
127  public static final String USAGE_FOOTER = "";
128
129  public static abstract class Command extends Configured {
130    CommandLine cmdline;
131    Connection conn;
132
133    Command(Configuration conf) {
134      if (conf == null) {
135        conf = HBaseConfiguration.create();
136      }
137      setConf(conf);
138    }
139
140    public void execute() throws IOException {
141      if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
142        printUsage();
143        throw new IOException(INCORRECT_USAGE);
144      }
145
146      // Create connection
147      conn = ConnectionFactory.createConnection(getConf());
148      if (requiresNoActiveSession()) {
149        // Check active session
150        try (BackupSystemTable table = new BackupSystemTable(conn)) {
151          List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING);
152
153          if (sessions.size() > 0) {
154            System.err.println("Found backup session in a RUNNING state: ");
155            System.err.println(sessions.get(0));
156            System.err.println("This may indicate that a previous session has failed abnormally.");
157            System.err.println("In this case, backup recovery is recommended.");
158            throw new IOException("Active session found, aborted command execution");
159          }
160        }
161      }
162      if (requiresConsistentState()) {
163        // Check failed delete
164        try (BackupSystemTable table = new BackupSystemTable(conn)) {
165          String[] ids = table.getListOfBackupIdsFromDeleteOperation();
166
167          if (ids != null && ids.length > 0) {
168            System.err.println("Found failed backup DELETE coommand. ");
169            System.err.println("Backup system recovery is required.");
170            throw new IOException("Failed backup DELETE found, aborted command execution");
171          }
172
173          ids = table.getListOfBackupIdsFromMergeOperation();
174          if (ids != null && ids.length > 0) {
175            System.err.println("Found failed backup MERGE coommand. ");
176            System.err.println("Backup system recovery is required.");
177            throw new IOException("Failed backup MERGE found, aborted command execution");
178          }
179        }
180      }
181    }
182
183    public void finish() throws IOException {
184      if (conn != null) {
185        conn.close();
186      }
187    }
188
189    protected abstract void printUsage();
190
191    /**
192     * The command can't be run if active backup session is in progress
193     * @return true if no active sessions are in progress
194     */
195    protected boolean requiresNoActiveSession() {
196      return false;
197    }
198
199    /**
200     * Command requires consistent state of a backup system Backup system may become inconsistent
201     * because of an abnormal termination of a backup session or delete command
202     * @return true, if yes
203     */
204    protected boolean requiresConsistentState() {
205      return false;
206    }
207  }
208
209  private BackupCommands() {
210    throw new AssertionError("Instantiating utility class...");
211  }
212
213  public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
214    Command cmd;
215    switch (type) {
216      case CREATE:
217        cmd = new CreateCommand(conf, cmdline);
218        break;
219      case DESCRIBE:
220        cmd = new DescribeCommand(conf, cmdline);
221        break;
222      case PROGRESS:
223        cmd = new ProgressCommand(conf, cmdline);
224        break;
225      case DELETE:
226        cmd = new DeleteCommand(conf, cmdline);
227        break;
228      case HISTORY:
229        cmd = new HistoryCommand(conf, cmdline);
230        break;
231      case SET:
232        cmd = new BackupSetCommand(conf, cmdline);
233        break;
234      case REPAIR:
235        cmd = new RepairCommand(conf, cmdline);
236        break;
237      case MERGE:
238        cmd = new MergeCommand(conf, cmdline);
239        break;
240      case HELP:
241      default:
242        cmd = new HelpCommand(conf, cmdline);
243        break;
244    }
245    return cmd;
246  }
247
248  static int numOfArgs(String[] args) {
249    if (args == null) {
250      return 0;
251    }
252
253    return args.length;
254  }
255
256  public static class CreateCommand extends Command {
257    CreateCommand(Configuration conf, CommandLine cmdline) {
258      super(conf);
259      this.cmdline = cmdline;
260    }
261
262    @Override
263    protected boolean requiresNoActiveSession() {
264      return true;
265    }
266
267    @Override
268    protected boolean requiresConsistentState() {
269      return true;
270    }
271
272    @Override
273    public void execute() throws IOException {
274      if (cmdline == null || cmdline.getArgs() == null) {
275        printUsage();
276        throw new IOException(INCORRECT_USAGE);
277      }
278      String[] args = cmdline.getArgs();
279      if (args.length != 3) {
280        printUsage();
281        throw new IOException(INCORRECT_USAGE);
282      }
283
284      if (!BackupType.FULL.toString().equalsIgnoreCase(args[1])
285          && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) {
286        System.out.println("ERROR: invalid backup type: " + args[1]);
287        printUsage();
288        throw new IOException(INCORRECT_USAGE);
289      }
290      if (!verifyPath(args[2])) {
291        System.out.println("ERROR: invalid backup destination: " + args[2]);
292        printUsage();
293        throw new IOException(INCORRECT_USAGE);
294      }
295      String targetBackupDir = args[2];
296      // Check if backup destination is top level (root) folder - not allowed
297      if (isRootFolder(targetBackupDir)) {
298        throw new IOException(TOP_LEVEL_NOT_ALLOWED);
299      }
300      String tables;
301
302      // Check if we have both: backup set and list of tables
303      if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
304        System.out.println("ERROR: You can specify either backup set or list"
305            + " of tables, but not both");
306        printUsage();
307        throw new IOException(INCORRECT_USAGE);
308      }
309      // Creates connection
310      super.execute();
311      // Check backup set
312      String setName = null;
313      if (cmdline.hasOption(OPTION_SET)) {
314        setName = cmdline.getOptionValue(OPTION_SET);
315        tables = getTablesForSet(setName, getConf());
316
317        if (tables == null) {
318          System.out.println("ERROR: Backup set '" + setName
319              + "' is either empty or does not exist");
320          printUsage();
321          throw new IOException(INCORRECT_USAGE);
322        }
323      } else {
324        tables = cmdline.getOptionValue(OPTION_TABLE);
325      }
326      int bandwidth =
327          cmdline.hasOption(OPTION_BANDWIDTH) ? Integer.parseInt(cmdline
328              .getOptionValue(OPTION_BANDWIDTH)) : -1;
329      int workers =
330          cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
331              .getOptionValue(OPTION_WORKERS)) : -1;
332
333      if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
334        String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
335        // Set system property value for MR job
336        System.setProperty("mapreduce.job.queuename", queueName);
337      }
338
339      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
340        BackupRequest.Builder builder = new BackupRequest.Builder();
341        BackupRequest request =
342            builder
343                .withBackupType(BackupType.valueOf(args[1].toUpperCase()))
344                .withTableList(
345                  tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null)
346                .withTargetRootDir(targetBackupDir).withTotalTasks(workers)
347                .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build();
348        String backupId = admin.backupTables(request);
349        System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
350      } catch (IOException e) {
351        System.out.println("Backup session finished. Status: FAILURE");
352        throw e;
353      }
354    }
355
356    private boolean isRootFolder(String targetBackupDir) {
357      Path p = new Path(targetBackupDir);
358      return p.isRoot();
359    }
360
361    private boolean verifyPath(String path) {
362      try {
363        Path p = new Path(path);
364        Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
365        URI uri = p.toUri();
366
367        if (uri.getScheme() == null) {
368          return false;
369        }
370
371        FileSystem.get(uri, conf);
372        return true;
373      } catch (Exception e) {
374        return false;
375      }
376    }
377
378    private String getTablesForSet(String name, Configuration conf) throws IOException {
379      try (final BackupSystemTable table = new BackupSystemTable(conn)) {
380        List<TableName> tables = table.describeBackupSet(name);
381
382        if (tables == null) {
383          return null;
384        }
385
386        return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
387      }
388    }
389
390    @Override
391    protected void printUsage() {
392      System.out.println(CREATE_CMD_USAGE);
393      Options options = new Options();
394      options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
395      options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
396      options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
397      options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
398      options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
399      options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC);
400
401      HelpFormatter helpFormatter = new HelpFormatter();
402      helpFormatter.setLeftPadding(2);
403      helpFormatter.setDescPadding(8);
404      helpFormatter.setWidth(100);
405      helpFormatter.setSyntaxPrefix("Options:");
406      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
407    }
408  }
409
410  public static class HelpCommand extends Command {
411    HelpCommand(Configuration conf, CommandLine cmdline) {
412      super(conf);
413      this.cmdline = cmdline;
414    }
415
416    @Override
417    public void execute() throws IOException {
418      if (cmdline == null) {
419        printUsage();
420        throw new IOException(INCORRECT_USAGE);
421      }
422
423      String[] args = cmdline.getArgs();
424      if (args == null || args.length == 0) {
425        printUsage();
426        throw new IOException(INCORRECT_USAGE);
427      }
428
429      if (args.length != 2) {
430        System.out.println("ERROR: Only supports help message of a single command type");
431        printUsage();
432        throw new IOException(INCORRECT_USAGE);
433      }
434
435      String type = args[1];
436
437      if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
438        System.out.println(CREATE_CMD_USAGE);
439      } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
440        System.out.println(DESCRIBE_CMD_USAGE);
441      } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
442        System.out.println(HISTORY_CMD_USAGE);
443      } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
444        System.out.println(PROGRESS_CMD_USAGE);
445      } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
446        System.out.println(DELETE_CMD_USAGE);
447      } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
448        System.out.println(SET_CMD_USAGE);
449      } else {
450        System.out.println("Unknown command : " + type);
451        printUsage();
452      }
453    }
454
455    @Override
456    protected void printUsage() {
457      System.out.println(USAGE);
458    }
459  }
460
461  public static class DescribeCommand extends Command {
462    DescribeCommand(Configuration conf, CommandLine cmdline) {
463      super(conf);
464      this.cmdline = cmdline;
465    }
466
467    @Override
468    public void execute() throws IOException {
469      if (cmdline == null || cmdline.getArgs() == null) {
470        printUsage();
471        throw new IOException(INCORRECT_USAGE);
472      }
473      String[] args = cmdline.getArgs();
474      if (args.length != 2) {
475        printUsage();
476        throw new IOException(INCORRECT_USAGE);
477      }
478
479      super.execute();
480
481      String backupId = args[1];
482      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
483        BackupInfo info = sysTable.readBackupInfo(backupId);
484        if (info == null) {
485          System.out.println("ERROR: " + backupId + " does not exist");
486          printUsage();
487          throw new IOException(INCORRECT_USAGE);
488        }
489        System.out.println(info.getShortDescription());
490      }
491    }
492
493    @Override
494    protected void printUsage() {
495      System.out.println(DESCRIBE_CMD_USAGE);
496    }
497  }
498
499  public static class ProgressCommand extends Command {
500    ProgressCommand(Configuration conf, CommandLine cmdline) {
501      super(conf);
502      this.cmdline = cmdline;
503    }
504
505    @Override
506    public void execute() throws IOException {
507
508      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
509        System.out.println("No backup id was specified, "
510            + "will retrieve the most recent (ongoing) session");
511      }
512      String[] args = cmdline == null ? null : cmdline.getArgs();
513      if (args != null && args.length > 2) {
514        System.err.println("ERROR: wrong number of arguments: " + args.length);
515        printUsage();
516        throw new IOException(INCORRECT_USAGE);
517      }
518
519      super.execute();
520
521      String backupId = (args == null || args.length <= 1) ? null : args[1];
522      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
523        BackupInfo info = null;
524
525        if (backupId != null) {
526          info = sysTable.readBackupInfo(backupId);
527        } else {
528          List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING);
529          if (infos != null && infos.size() > 0) {
530            info = infos.get(0);
531            backupId = info.getBackupId();
532            System.out.println("Found ongoing session with backupId=" + backupId);
533          }
534        }
535        int progress = info == null ? -1 : info.getProgress();
536        if (progress < 0) {
537          if (backupId != null) {
538            System.out.println(NO_INFO_FOUND + backupId);
539          } else {
540            System.err.println(NO_ACTIVE_SESSION_FOUND);
541          }
542        } else {
543          System.out.println(backupId + " progress=" + progress + "%");
544        }
545      }
546    }
547
548    @Override
549    protected void printUsage() {
550      System.out.println(PROGRESS_CMD_USAGE);
551    }
552  }
553
554  public static class DeleteCommand extends Command {
555    DeleteCommand(Configuration conf, CommandLine cmdline) {
556      super(conf);
557      this.cmdline = cmdline;
558    }
559
560    @Override
561    protected boolean requiresNoActiveSession() {
562      return true;
563    }
564
565    @Override
566    public void execute() throws IOException {
567
568      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) {
569        printUsage();
570        throw new IOException(INCORRECT_USAGE);
571      }
572
573      if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) {
574        printUsage();
575        throw new IOException(INCORRECT_USAGE);
576      }
577      super.execute();
578      if (cmdline.hasOption(OPTION_KEEP)) {
579        executeDeleteOlderThan(cmdline);
580      } else if (cmdline.hasOption(OPTION_LIST)) {
581        executeDeleteListOfBackups(cmdline);
582      }
583    }
584
585    private void executeDeleteOlderThan(CommandLine cmdline) throws IOException {
586      String value = cmdline.getOptionValue(OPTION_KEEP);
587      int days = 0;
588      try {
589        days = Integer.parseInt(value);
590      } catch (NumberFormatException e) {
591        throw new IOException(value + " is not an integer number");
592      }
593      final long fdays = days;
594      BackupInfo.Filter dateFilter = new BackupInfo.Filter() {
595        @Override
596        public boolean apply(BackupInfo info) {
597          long currentTime = EnvironmentEdgeManager.currentTime();
598          long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000;
599          return info.getCompleteTs() <= maxTsToDelete;
600        }
601      };
602      List<BackupInfo> history = null;
603      try (final BackupSystemTable sysTable = new BackupSystemTable(conn);
604          BackupAdminImpl admin = new BackupAdminImpl(conn)) {
605        history = sysTable.getBackupHistory(-1, dateFilter);
606        String[] backupIds = convertToBackupIds(history);
607        int deleted = admin.deleteBackups(backupIds);
608        System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: "
609            + backupIds.length);
610      } catch (IOException e) {
611        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
612            + "system integrity");
613        throw e;
614      }
615    }
616
617    private String[] convertToBackupIds(List<BackupInfo> history) {
618      String[] ids = new String[history.size()];
619      for (int i = 0; i < ids.length; i++) {
620        ids[i] = history.get(i).getBackupId();
621      }
622      return ids;
623    }
624
625    private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException {
626      String value = cmdline.getOptionValue(OPTION_LIST);
627      String[] backupIds = value.split(",");
628
629      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
630        int deleted = admin.deleteBackups(backupIds);
631        System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length);
632      } catch (IOException e) {
633        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
634            + "system integrity");
635        throw e;
636      }
637
638    }
639
640    @Override
641    protected void printUsage() {
642      System.out.println(DELETE_CMD_USAGE);
643      Options options = new Options();
644      options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC);
645      options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC);
646
647      HelpFormatter helpFormatter = new HelpFormatter();
648      helpFormatter.setLeftPadding(2);
649      helpFormatter.setDescPadding(8);
650      helpFormatter.setWidth(100);
651      helpFormatter.setSyntaxPrefix("Options:");
652      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
653
654    }
655  }
656
657  public static class RepairCommand extends Command {
658    RepairCommand(Configuration conf, CommandLine cmdline) {
659      super(conf);
660      this.cmdline = cmdline;
661    }
662
663    @Override
664    public void execute() throws IOException {
665      super.execute();
666
667      String[] args = cmdline == null ? null : cmdline.getArgs();
668      if (args != null && args.length > 1) {
669        System.err.println("ERROR: wrong number of arguments: " + args.length);
670        printUsage();
671        throw new IOException(INCORRECT_USAGE);
672      }
673
674      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
675      try (final Connection conn = ConnectionFactory.createConnection(conf);
676          final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
677        // Failed backup
678        BackupInfo backupInfo;
679        List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
680        if (list.size() == 0) {
681          // No failed sessions found
682          System.out.println("REPAIR status: no failed sessions found."
683              + " Checking failed delete backup operation ...");
684          repairFailedBackupDeletionIfAny(conn, sysTable);
685          repairFailedBackupMergeIfAny(conn, sysTable);
686          return;
687        }
688        backupInfo = list.get(0);
689        // If this is a cancel exception, then we've already cleaned.
690        // set the failure timestamp of the overall backup
691        backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
692        // set failure message
693        backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
694        // set overall backup status: failed
695        backupInfo.setState(BackupState.FAILED);
696        // compose the backup failed data
697        String backupFailedData =
698            "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs()
699                + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
700                + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
701        System.out.println(backupFailedData);
702        TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
703        // If backup session is updated to FAILED state - means we
704        // processed recovery already.
705        sysTable.updateBackupInfo(backupInfo);
706        sysTable.finishBackupExclusiveOperation();
707        System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
708      }
709    }
710
711    private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
712        throws IOException {
713      String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
714      if (backupIds == null || backupIds.length == 0) {
715        System.out.println("No failed backup DELETE operation found");
716        // Delete backup table snapshot if exists
717        BackupSystemTable.deleteSnapshot(conn);
718        return;
719      }
720      System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
721      System.out.println("Running DELETE again ...");
722      // Restore table from snapshot
723      BackupSystemTable.restoreFromSnapshot(conn);
724      // Finish previous failed session
725      sysTable.finishBackupExclusiveOperation();
726      try (BackupAdmin admin = new BackupAdminImpl(conn)) {
727        admin.deleteBackups(backupIds);
728      }
729      System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
730    }
731
732    public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
733        throws IOException {
734
735      String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
736      if (backupIds == null || backupIds.length == 0) {
737        System.out.println("No failed backup MERGE operation found");
738        // Delete backup table snapshot if exists
739        BackupSystemTable.deleteSnapshot(conn);
740        return;
741      }
742      System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
743      // Check if backup .tmp exists
744      BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]);
745      String backupRoot = bInfo.getBackupRootDir();
746      FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration());
747      String backupId = BackupUtils.findMostRecentBackupId(backupIds);
748      Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId);
749      if (fs.exists(tmpPath)) {
750        // Move data back
751        Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId);
752        if (!fs.delete(destPath, true)) {
753          System.out.println("Failed to delete " + destPath);
754        }
755        boolean res = fs.rename(tmpPath, destPath);
756        if (!res) {
757          throw new IOException("MERGE repair: failed  to rename from "+ tmpPath+" to "+ destPath);
758        }
759        System.out.println("MERGE repair: renamed from "+ tmpPath+" to "+ destPath+" res="+ res);
760      } else {
761        checkRemoveBackupImages(fs, backupRoot, backupIds);
762      }
763      // Restore table from snapshot
764      BackupSystemTable.restoreFromSnapshot(conn);
765      // Unlock backup system
766      sysTable.finishBackupExclusiveOperation();
767      // Finish previous failed session
768      sysTable.finishMergeOperation();
769
770      System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds));
771    }
772
773    private static void checkRemoveBackupImages(FileSystem fs, String backupRoot,
774      String[] backupIds) throws IOException {
775      String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);
776      for (String backupId: backupIds) {
777        if (backupId.equals(mergedBackupId)) {
778          continue;
779        }
780        Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId);
781        if (fs.exists(path)) {
782          if (!fs.delete(path, true)) {
783            System.out.println("MERGE repair removing: "+ path +" - FAILED");
784          } else {
785            System.out.println("MERGE repair removing: "+ path +" - OK");
786          }
787        }
788      }
789    }
790
791    @Override
792    protected void printUsage() {
793      System.out.println(REPAIR_CMD_USAGE);
794    }
795  }
796
797  public static class MergeCommand extends Command {
798    MergeCommand(Configuration conf, CommandLine cmdline) {
799      super(conf);
800      this.cmdline = cmdline;
801    }
802
803    @Override
804    protected boolean requiresNoActiveSession() {
805      return true;
806    }
807
808    @Override
809    protected boolean requiresConsistentState() {
810      return true;
811    }
812
813    @Override
814    public void execute() throws IOException {
815      super.execute();
816
817      String[] args = cmdline == null ? null : cmdline.getArgs();
818      if (args == null || (args.length != 2)) {
819        System.err.println("ERROR: wrong number of arguments: "
820            + (args == null ? null : args.length));
821        printUsage();
822        throw new IOException(INCORRECT_USAGE);
823      }
824
825      String[] backupIds = args[1].split(",");
826      if (backupIds.length < 2) {
827        String msg = "ERROR: can not merge a single backup image. "+
828            "Number of images must be greater than 1.";
829        System.err.println(msg);
830        throw new IOException(msg);
831
832      }
833      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
834      try (final Connection conn = ConnectionFactory.createConnection(conf);
835          final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
836        admin.mergeBackups(backupIds);
837      }
838    }
839
840    @Override
841    protected void printUsage() {
842      System.out.println(MERGE_CMD_USAGE);
843    }
844  }
845
846  public static class HistoryCommand extends Command {
847    private final static int DEFAULT_HISTORY_LENGTH = 10;
848
849    HistoryCommand(Configuration conf, CommandLine cmdline) {
850      super(conf);
851      this.cmdline = cmdline;
852    }
853
854    @Override
855    public void execute() throws IOException {
856      int n = parseHistoryLength();
857      final TableName tableName = getTableName();
858      final String setName = getTableSetName();
859      BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
860        @Override
861        public boolean apply(BackupInfo info) {
862          if (tableName == null) {
863            return true;
864          }
865
866          List<TableName> names = info.getTableNames();
867          return names.contains(tableName);
868        }
869      };
870      BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
871        @Override
872        public boolean apply(BackupInfo info) {
873          if (setName == null) {
874            return true;
875          }
876
877          String backupId = info.getBackupId();
878          return backupId.startsWith(setName);
879        }
880      };
881      Path backupRootPath = getBackupRootPath();
882      List<BackupInfo> history;
883      if (backupRootPath == null) {
884        // Load from backup system table
885        super.execute();
886        try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
887          history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
888        }
889      } else {
890        // load from backup FS
891        history =
892            BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
893      }
894      for (BackupInfo info : history) {
895        System.out.println(info.getShortDescription());
896      }
897    }
898
899    private Path getBackupRootPath() throws IOException {
900      String value = null;
901      try {
902        value = cmdline.getOptionValue(OPTION_PATH);
903
904        if (value == null) {
905          return null;
906        }
907
908        return new Path(value);
909      } catch (IllegalArgumentException e) {
910        System.out.println("ERROR: Illegal argument for backup root path: " + value);
911        printUsage();
912        throw new IOException(INCORRECT_USAGE);
913      }
914    }
915
916    private TableName getTableName() throws IOException {
917      String value = cmdline.getOptionValue(OPTION_TABLE);
918
919      if (value == null) {
920        return null;
921      }
922
923      try {
924        return TableName.valueOf(value);
925      } catch (IllegalArgumentException e) {
926        System.out.println("Illegal argument for table name: " + value);
927        printUsage();
928        throw new IOException(INCORRECT_USAGE);
929      }
930    }
931
932    private String getTableSetName() {
933      return cmdline.getOptionValue(OPTION_SET);
934    }
935
936    private int parseHistoryLength() throws IOException {
937      String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
938      try {
939        if (value == null) {
940          return DEFAULT_HISTORY_LENGTH;
941        }
942
943        return Integer.parseInt(value);
944      } catch (NumberFormatException e) {
945        System.out.println("Illegal argument for history length: " + value);
946        printUsage();
947        throw new IOException(INCORRECT_USAGE);
948      }
949    }
950
951    @Override
952    protected void printUsage() {
953      System.out.println(HISTORY_CMD_USAGE);
954      Options options = new Options();
955      options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
956      options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
957      options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
958      options.addOption(OPTION_SET, true, OPTION_SET_DESC);
959
960      HelpFormatter helpFormatter = new HelpFormatter();
961      helpFormatter.setLeftPadding(2);
962      helpFormatter.setDescPadding(8);
963      helpFormatter.setWidth(100);
964      helpFormatter.setSyntaxPrefix("Options:");
965      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
966    }
967  }
968
969  public static class BackupSetCommand extends Command {
970    private final static String SET_ADD_CMD = "add";
971    private final static String SET_REMOVE_CMD = "remove";
972    private final static String SET_DELETE_CMD = "delete";
973    private final static String SET_DESCRIBE_CMD = "describe";
974    private final static String SET_LIST_CMD = "list";
975
976    BackupSetCommand(Configuration conf, CommandLine cmdline) {
977      super(conf);
978      this.cmdline = cmdline;
979    }
980
981    @Override
982    public void execute() throws IOException {
983      // Command-line must have at least one element
984      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
985        printUsage();
986        throw new IOException(INCORRECT_USAGE);
987      }
988
989      String[] args = cmdline.getArgs();
990      String cmdStr = args[1];
991      BackupCommand cmd = getCommand(cmdStr);
992
993      switch (cmd) {
994        case SET_ADD:
995          processSetAdd(args);
996          break;
997        case SET_REMOVE:
998          processSetRemove(args);
999          break;
1000        case SET_DELETE:
1001          processSetDelete(args);
1002          break;
1003        case SET_DESCRIBE:
1004          processSetDescribe(args);
1005          break;
1006        case SET_LIST:
1007          processSetList(args);
1008          break;
1009        default:
1010          break;
1011      }
1012    }
1013
1014    private void processSetList(String[] args) throws IOException {
1015      super.execute();
1016
1017      // List all backup set names
1018      // does not expect any args
1019      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1020        List<BackupSet> list = admin.listBackupSets();
1021        for (BackupSet bs : list) {
1022          System.out.println(bs);
1023        }
1024      }
1025    }
1026
1027    private void processSetDescribe(String[] args) throws IOException {
1028      if (args == null || args.length != 3) {
1029        printUsage();
1030        throw new IOException(INCORRECT_USAGE);
1031      }
1032      super.execute();
1033
1034      String setName = args[2];
1035      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1036        List<TableName> tables = sysTable.describeBackupSet(setName);
1037        BackupSet set = tables == null ? null : new BackupSet(setName, tables);
1038        if (set == null) {
1039          System.out.println("Set '" + setName + "' does not exist.");
1040        } else {
1041          System.out.println(set);
1042        }
1043      }
1044    }
1045
1046    private void processSetDelete(String[] args) throws IOException {
1047      if (args == null || args.length != 3) {
1048        printUsage();
1049        throw new IOException(INCORRECT_USAGE);
1050      }
1051      super.execute();
1052
1053      String setName = args[2];
1054      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1055        boolean result = admin.deleteBackupSet(setName);
1056        if (result) {
1057          System.out.println("Delete set " + setName + " OK.");
1058        } else {
1059          System.out.println("Set " + setName + " does not exist");
1060        }
1061      }
1062    }
1063
1064    private void processSetRemove(String[] args) throws IOException {
1065      if (args == null || args.length != 4) {
1066        printUsage();
1067        throw new IOException(INCORRECT_USAGE);
1068      }
1069      super.execute();
1070
1071      String setName = args[2];
1072      String[] tables = args[3].split(",");
1073      TableName[] tableNames = toTableNames(tables);
1074      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1075        admin.removeFromBackupSet(setName, tableNames);
1076      }
1077    }
1078
1079    private TableName[] toTableNames(String[] tables) {
1080      TableName[] arr = new TableName[tables.length];
1081      for (int i = 0; i < tables.length; i++) {
1082        arr[i] = TableName.valueOf(tables[i]);
1083      }
1084      return arr;
1085    }
1086
1087    private void processSetAdd(String[] args) throws IOException {
1088      if (args == null || args.length != 4) {
1089        printUsage();
1090        throw new IOException(INCORRECT_USAGE);
1091      }
1092      super.execute();
1093
1094      String setName = args[2];
1095      String[] tables = args[3].split(",");
1096      TableName[] tableNames = new TableName[tables.length];
1097      for (int i = 0; i < tables.length; i++) {
1098        tableNames[i] = TableName.valueOf(tables[i]);
1099      }
1100      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1101        admin.addToBackupSet(setName, tableNames);
1102      }
1103
1104    }
1105
1106    private BackupCommand getCommand(String cmdStr) throws IOException {
1107      switch (cmdStr) {
1108        case SET_ADD_CMD:
1109          return BackupCommand.SET_ADD;
1110        case SET_REMOVE_CMD:
1111          return BackupCommand.SET_REMOVE;
1112        case SET_DELETE_CMD:
1113          return BackupCommand.SET_DELETE;
1114        case SET_DESCRIBE_CMD:
1115          return BackupCommand.SET_DESCRIBE;
1116        case SET_LIST_CMD:
1117          return BackupCommand.SET_LIST;
1118        default:
1119          System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
1120          printUsage();
1121          throw new IOException(INCORRECT_USAGE);
1122      }
1123    }
1124
1125    @Override
1126    protected void printUsage() {
1127      System.out.println(SET_CMD_USAGE);
1128    }
1129  }
1130}