001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
042
043import java.io.IOException;
044import java.net.URI;
045import java.util.List;
046import org.apache.commons.lang3.StringUtils;
047import org.apache.hadoop.conf.Configuration;
048import org.apache.hadoop.conf.Configured;
049import org.apache.hadoop.fs.FileSystem;
050import org.apache.hadoop.fs.Path;
051import org.apache.hadoop.hbase.HBaseConfiguration;
052import org.apache.hadoop.hbase.TableName;
053import org.apache.hadoop.hbase.backup.BackupAdmin;
054import org.apache.hadoop.hbase.backup.BackupInfo;
055import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
056import org.apache.hadoop.hbase.backup.BackupRequest;
057import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
058import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
059import org.apache.hadoop.hbase.backup.BackupType;
060import org.apache.hadoop.hbase.backup.HBackupFileSystem;
061import org.apache.hadoop.hbase.backup.util.BackupSet;
062import org.apache.hadoop.hbase.backup.util.BackupUtils;
063import org.apache.hadoop.hbase.client.Connection;
064import org.apache.hadoop.hbase.client.ConnectionFactory;
065import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
066import org.apache.yetus.audience.InterfaceAudience;
067
068import org.apache.hbase.thirdparty.com.google.common.base.Splitter;
069import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
070import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
071import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
072import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
073
074/**
075 * General backup commands, options and usage messages
076 */
077@InterfaceAudience.Private
078public final class BackupCommands {
079  public final static String INCORRECT_USAGE = "Incorrect usage";
080
081  public final static String TOP_LEVEL_NOT_ALLOWED =
082    "Top level (root) folder is not allowed to be a backup destination";
083
084  public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
085    + "where COMMAND is one of:\n" + "  create     create a new backup image\n"
086    + "  delete     delete an existing backup image\n"
087    + "  describe   show the detailed information of a backup image\n"
088    + "  history    show history of all successful backups\n"
089    + "  progress   show the progress of the latest backup request\n"
090    + "  set        backup set management\n" + "  repair     repair backup system table\n"
091    + "  merge      merge backup images\n"
092    + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
093
094  public static final String CREATE_CMD_USAGE =
095    "Usage: hbase backup create <type> <backup_path> [options]\n"
096      + "  type           \"full\" to create a full backup image\n"
097      + "                 \"incremental\" to create an incremental backup image\n"
098      + "  backup_path     Full path to store the backup image\n";
099
100  public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
101    + "  backup_id       Backup image id (optional). If no id specified, the command will show\n"
102    + "                  progress for currently running backup session.";
103  public static final String NO_INFO_FOUND = "No info was found for backup id: ";
104  public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
105
106  public static final String DESCRIBE_CMD_USAGE =
107    "Usage: hbase backup describe <backup_id>\n" + "  backup_id       Backup image id\n";
108
109  public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
110
111  public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]";
112
113  public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
114
115  public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
116    + "  name            Backup set name\n" + "  tables          Comma separated list of tables.\n"
117    + "COMMAND is one of:\n" + "  add             add tables to a set, create a set if needed\n"
118    + "  remove          remove tables from a set\n"
119    + "  list            list all backup sets in the system\n" + "  describe        describe set\n"
120    + "  delete          delete backup set\n";
121  public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
122    + "  backup_ids      Comma separated list of backup image ids.\n";
123
124  public static final String USAGE_FOOTER = "";
125
126  public static abstract class Command extends Configured {
127    CommandLine cmdline;
128    Connection conn;
129
130    Command(Configuration conf) {
131      if (conf == null) {
132        conf = HBaseConfiguration.create();
133      }
134      setConf(conf);
135    }
136
137    public void execute() throws IOException {
138      if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
139        printUsage();
140        throw new IOException(INCORRECT_USAGE);
141      }
142
143      // Create connection
144      conn = ConnectionFactory.createConnection(getConf());
145      if (requiresNoActiveSession()) {
146        // Check active session
147        try (BackupSystemTable table = new BackupSystemTable(conn)) {
148          List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING);
149
150          if (sessions.size() > 0) {
151            System.err.println("Found backup session in a RUNNING state: ");
152            System.err.println(sessions.get(0));
153            System.err.println("This may indicate that a previous session has failed abnormally.");
154            System.err.println("In this case, backup recovery is recommended.");
155            throw new IOException("Active session found, aborted command execution");
156          }
157        }
158      }
159      if (requiresConsistentState()) {
160        // Check failed delete
161        try (BackupSystemTable table = new BackupSystemTable(conn)) {
162          String[] ids = table.getListOfBackupIdsFromDeleteOperation();
163
164          if (ids != null && ids.length > 0) {
165            System.err.println("Found failed backup DELETE coommand. ");
166            System.err.println("Backup system recovery is required.");
167            throw new IOException("Failed backup DELETE found, aborted command execution");
168          }
169
170          ids = table.getListOfBackupIdsFromMergeOperation();
171          if (ids != null && ids.length > 0) {
172            System.err.println("Found failed backup MERGE coommand. ");
173            System.err.println("Backup system recovery is required.");
174            throw new IOException("Failed backup MERGE found, aborted command execution");
175          }
176        }
177      }
178    }
179
180    public void finish() throws IOException {
181      if (conn != null) {
182        conn.close();
183      }
184    }
185
186    protected abstract void printUsage();
187
188    /**
189     * The command can't be run if active backup session is in progress
190     * @return true if no active sessions are in progress
191     */
192    protected boolean requiresNoActiveSession() {
193      return false;
194    }
195
196    /**
197     * Command requires consistent state of a backup system Backup system may become inconsistent
198     * because of an abnormal termination of a backup session or delete command
199     * @return true, if yes
200     */
201    protected boolean requiresConsistentState() {
202      return false;
203    }
204  }
205
206  private BackupCommands() {
207    throw new AssertionError("Instantiating utility class...");
208  }
209
210  public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
211    Command cmd;
212    switch (type) {
213      case CREATE:
214        cmd = new CreateCommand(conf, cmdline);
215        break;
216      case DESCRIBE:
217        cmd = new DescribeCommand(conf, cmdline);
218        break;
219      case PROGRESS:
220        cmd = new ProgressCommand(conf, cmdline);
221        break;
222      case DELETE:
223        cmd = new DeleteCommand(conf, cmdline);
224        break;
225      case HISTORY:
226        cmd = new HistoryCommand(conf, cmdline);
227        break;
228      case SET:
229        cmd = new BackupSetCommand(conf, cmdline);
230        break;
231      case REPAIR:
232        cmd = new RepairCommand(conf, cmdline);
233        break;
234      case MERGE:
235        cmd = new MergeCommand(conf, cmdline);
236        break;
237      case HELP:
238      default:
239        cmd = new HelpCommand(conf, cmdline);
240        break;
241    }
242    return cmd;
243  }
244
245  static int numOfArgs(String[] args) {
246    if (args == null) {
247      return 0;
248    }
249
250    return args.length;
251  }
252
253  public static class CreateCommand extends Command {
254    CreateCommand(Configuration conf, CommandLine cmdline) {
255      super(conf);
256      this.cmdline = cmdline;
257    }
258
259    @Override
260    protected boolean requiresNoActiveSession() {
261      return true;
262    }
263
264    @Override
265    protected boolean requiresConsistentState() {
266      return true;
267    }
268
269    @Override
270    public void execute() throws IOException {
271      if (cmdline == null || cmdline.getArgs() == null) {
272        printUsage();
273        throw new IOException(INCORRECT_USAGE);
274      }
275      String[] args = cmdline.getArgs();
276      if (args.length != 3) {
277        printUsage();
278        throw new IOException(INCORRECT_USAGE);
279      }
280
281      if (
282        !BackupType.FULL.toString().equalsIgnoreCase(args[1])
283          && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])
284      ) {
285        System.out.println("ERROR: invalid backup type: " + args[1]);
286        printUsage();
287        throw new IOException(INCORRECT_USAGE);
288      }
289      if (!verifyPath(args[2])) {
290        System.out.println("ERROR: invalid backup destination: " + args[2]);
291        printUsage();
292        throw new IOException(INCORRECT_USAGE);
293      }
294      String targetBackupDir = args[2];
295      // Check if backup destination is top level (root) folder - not allowed
296      if (isRootFolder(targetBackupDir)) {
297        throw new IOException(TOP_LEVEL_NOT_ALLOWED);
298      }
299      String tables;
300
301      // Check if we have both: backup set and list of tables
302      if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
303        System.out
304          .println("ERROR: You can specify either backup set or list" + " of tables, but not both");
305        printUsage();
306        throw new IOException(INCORRECT_USAGE);
307      }
308      // Creates connection
309      super.execute();
310      // Check backup set
311      String setName = null;
312      if (cmdline.hasOption(OPTION_SET)) {
313        setName = cmdline.getOptionValue(OPTION_SET);
314        tables = getTablesForSet(setName);
315
316        if (tables == null) {
317          System.out
318            .println("ERROR: Backup set '" + setName + "' is either empty or does not exist");
319          printUsage();
320          throw new IOException(INCORRECT_USAGE);
321        }
322      } else {
323        tables = cmdline.getOptionValue(OPTION_TABLE);
324      }
325      int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH)
326        ? Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH))
327        : -1;
328      int workers = cmdline.hasOption(OPTION_WORKERS)
329        ? Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS))
330        : -1;
331
332      if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
333        String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
334        // Set system property value for MR job
335        System.setProperty("mapreduce.job.queuename", queueName);
336      }
337
338      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
339        BackupRequest.Builder builder = new BackupRequest.Builder();
340        BackupRequest request = builder.withBackupType(BackupType.valueOf(args[1].toUpperCase()))
341          .withTableList(
342            tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null)
343          .withTargetRootDir(targetBackupDir).withTotalTasks(workers)
344          .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build();
345        String backupId = admin.backupTables(request);
346        System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
347      } catch (IOException e) {
348        System.out.println("Backup session finished. Status: FAILURE");
349        throw e;
350      }
351    }
352
353    private boolean isRootFolder(String targetBackupDir) {
354      Path p = new Path(targetBackupDir);
355      return p.isRoot();
356    }
357
358    private boolean verifyPath(String path) {
359      try {
360        Path p = new Path(path);
361        Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
362        URI uri = p.toUri();
363
364        if (uri.getScheme() == null) {
365          return false;
366        }
367
368        FileSystem.get(uri, conf);
369        return true;
370      } catch (Exception e) {
371        return false;
372      }
373    }
374
375    private String getTablesForSet(String name) throws IOException {
376      try (final BackupSystemTable table = new BackupSystemTable(conn)) {
377        List<TableName> tables = table.describeBackupSet(name);
378
379        if (tables == null) {
380          return null;
381        }
382
383        return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
384      }
385    }
386
387    @Override
388    protected void printUsage() {
389      System.out.println(CREATE_CMD_USAGE);
390      Options options = new Options();
391      options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
392      options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
393      options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
394      options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
395      options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
396      options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC);
397
398      HelpFormatter helpFormatter = new HelpFormatter();
399      helpFormatter.setLeftPadding(2);
400      helpFormatter.setDescPadding(8);
401      helpFormatter.setWidth(100);
402      helpFormatter.setSyntaxPrefix("Options:");
403      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
404    }
405  }
406
407  public static class HelpCommand extends Command {
408    HelpCommand(Configuration conf, CommandLine cmdline) {
409      super(conf);
410      this.cmdline = cmdline;
411    }
412
413    @Override
414    public void execute() throws IOException {
415      if (cmdline == null) {
416        printUsage();
417        throw new IOException(INCORRECT_USAGE);
418      }
419
420      String[] args = cmdline.getArgs();
421      if (args == null || args.length == 0) {
422        printUsage();
423        throw new IOException(INCORRECT_USAGE);
424      }
425
426      if (args.length != 2) {
427        System.out.println("ERROR: Only supports help message of a single command type");
428        printUsage();
429        throw new IOException(INCORRECT_USAGE);
430      }
431
432      String type = args[1];
433
434      if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
435        System.out.println(CREATE_CMD_USAGE);
436      } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
437        System.out.println(DESCRIBE_CMD_USAGE);
438      } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
439        System.out.println(HISTORY_CMD_USAGE);
440      } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
441        System.out.println(PROGRESS_CMD_USAGE);
442      } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
443        System.out.println(DELETE_CMD_USAGE);
444      } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
445        System.out.println(SET_CMD_USAGE);
446      } else {
447        System.out.println("Unknown command : " + type);
448        printUsage();
449      }
450    }
451
452    @Override
453    protected void printUsage() {
454      System.out.println(USAGE);
455    }
456  }
457
458  public static class DescribeCommand extends Command {
459    DescribeCommand(Configuration conf, CommandLine cmdline) {
460      super(conf);
461      this.cmdline = cmdline;
462    }
463
464    @Override
465    public void execute() throws IOException {
466      if (cmdline == null || cmdline.getArgs() == null) {
467        printUsage();
468        throw new IOException(INCORRECT_USAGE);
469      }
470      String[] args = cmdline.getArgs();
471      if (args.length != 2) {
472        printUsage();
473        throw new IOException(INCORRECT_USAGE);
474      }
475
476      super.execute();
477
478      String backupId = args[1];
479      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
480        BackupInfo info = sysTable.readBackupInfo(backupId);
481        if (info == null) {
482          System.out.println("ERROR: " + backupId + " does not exist");
483          printUsage();
484          throw new IOException(INCORRECT_USAGE);
485        }
486        System.out.println(info.getShortDescription());
487      }
488    }
489
490    @Override
491    protected void printUsage() {
492      System.out.println(DESCRIBE_CMD_USAGE);
493    }
494  }
495
496  public static class ProgressCommand extends Command {
497    ProgressCommand(Configuration conf, CommandLine cmdline) {
498      super(conf);
499      this.cmdline = cmdline;
500    }
501
502    @Override
503    public void execute() throws IOException {
504
505      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
506        System.out.println(
507          "No backup id was specified, " + "will retrieve the most recent (ongoing) session");
508      }
509      String[] args = cmdline == null ? null : cmdline.getArgs();
510      if (args != null && args.length > 2) {
511        System.err.println("ERROR: wrong number of arguments: " + args.length);
512        printUsage();
513        throw new IOException(INCORRECT_USAGE);
514      }
515
516      super.execute();
517
518      String backupId = (args == null || args.length <= 1) ? null : args[1];
519      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
520        BackupInfo info = null;
521
522        if (backupId != null) {
523          info = sysTable.readBackupInfo(backupId);
524        } else {
525          List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING);
526          if (infos != null && infos.size() > 0) {
527            info = infos.get(0);
528            backupId = info.getBackupId();
529            System.out.println("Found ongoing session with backupId=" + backupId);
530          }
531        }
532        int progress = info == null ? -1 : info.getProgress();
533        if (progress < 0) {
534          if (backupId != null) {
535            System.out.println(NO_INFO_FOUND + backupId);
536          } else {
537            System.err.println(NO_ACTIVE_SESSION_FOUND);
538          }
539        } else {
540          System.out.println(backupId + " progress=" + progress + "%");
541        }
542      }
543    }
544
545    @Override
546    protected void printUsage() {
547      System.out.println(PROGRESS_CMD_USAGE);
548    }
549  }
550
551  public static class DeleteCommand extends Command {
552    DeleteCommand(Configuration conf, CommandLine cmdline) {
553      super(conf);
554      this.cmdline = cmdline;
555    }
556
557    @Override
558    protected boolean requiresNoActiveSession() {
559      return true;
560    }
561
562    @Override
563    public void execute() throws IOException {
564
565      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) {
566        printUsage();
567        throw new IOException(INCORRECT_USAGE);
568      }
569
570      if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) {
571        printUsage();
572        throw new IOException(INCORRECT_USAGE);
573      }
574      super.execute();
575      if (cmdline.hasOption(OPTION_KEEP)) {
576        executeDeleteOlderThan(cmdline);
577      } else if (cmdline.hasOption(OPTION_LIST)) {
578        executeDeleteListOfBackups(cmdline);
579      }
580    }
581
582    private void executeDeleteOlderThan(CommandLine cmdline) throws IOException {
583      String value = cmdline.getOptionValue(OPTION_KEEP);
584      int days = 0;
585      try {
586        days = Integer.parseInt(value);
587      } catch (NumberFormatException e) {
588        throw new IOException(value + " is not an integer number");
589      }
590      final long fdays = days;
591      BackupInfo.Filter dateFilter = new BackupInfo.Filter() {
592        @Override
593        public boolean apply(BackupInfo info) {
594          long currentTime = EnvironmentEdgeManager.currentTime();
595          long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000;
596          return info.getCompleteTs() <= maxTsToDelete;
597        }
598      };
599      List<BackupInfo> history = null;
600      try (final BackupSystemTable sysTable = new BackupSystemTable(conn);
601        BackupAdminImpl admin = new BackupAdminImpl(conn)) {
602        history = sysTable.getBackupHistory(-1, dateFilter);
603        String[] backupIds = convertToBackupIds(history);
604        int deleted = admin.deleteBackups(backupIds);
605        System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: "
606          + backupIds.length);
607      } catch (IOException e) {
608        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
609          + "system integrity");
610        throw e;
611      }
612    }
613
614    private String[] convertToBackupIds(List<BackupInfo> history) {
615      String[] ids = new String[history.size()];
616      for (int i = 0; i < ids.length; i++) {
617        ids[i] = history.get(i).getBackupId();
618      }
619      return ids;
620    }
621
622    private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException {
623      String value = cmdline.getOptionValue(OPTION_LIST);
624      String[] backupIds = value.split(",");
625
626      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
627        int deleted = admin.deleteBackups(backupIds);
628        System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length);
629      } catch (IOException e) {
630        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
631          + "system integrity");
632        throw e;
633      }
634
635    }
636
637    @Override
638    protected void printUsage() {
639      System.out.println(DELETE_CMD_USAGE);
640      Options options = new Options();
641      options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC);
642      options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC);
643
644      HelpFormatter helpFormatter = new HelpFormatter();
645      helpFormatter.setLeftPadding(2);
646      helpFormatter.setDescPadding(8);
647      helpFormatter.setWidth(100);
648      helpFormatter.setSyntaxPrefix("Options:");
649      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
650
651    }
652  }
653
654  public static class RepairCommand extends Command {
655    RepairCommand(Configuration conf, CommandLine cmdline) {
656      super(conf);
657      this.cmdline = cmdline;
658    }
659
660    @Override
661    public void execute() throws IOException {
662      super.execute();
663
664      String[] args = cmdline == null ? null : cmdline.getArgs();
665      if (args != null && args.length > 1) {
666        System.err.println("ERROR: wrong number of arguments: " + args.length);
667        printUsage();
668        throw new IOException(INCORRECT_USAGE);
669      }
670
671      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
672      try (final Connection conn = ConnectionFactory.createConnection(conf);
673        final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
674        // Failed backup
675        BackupInfo backupInfo;
676        List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
677        if (list.size() == 0) {
678          // No failed sessions found
679          System.out.println("REPAIR status: no failed sessions found."
680            + " Checking failed delete backup operation ...");
681          repairFailedBackupDeletionIfAny(conn, sysTable);
682          repairFailedBackupMergeIfAny(conn, sysTable);
683          return;
684        }
685        backupInfo = list.get(0);
686        // If this is a cancel exception, then we've already cleaned.
687        // set the failure timestamp of the overall backup
688        backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
689        // set failure message
690        backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
691        // set overall backup status: failed
692        backupInfo.setState(BackupState.FAILED);
693        // compose the backup failed data
694        String backupFailedData = "BackupId=" + backupInfo.getBackupId() + ",startts="
695          + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
696          + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
697        System.out.println(backupFailedData);
698        TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
699        // If backup session is updated to FAILED state - means we
700        // processed recovery already.
701        sysTable.updateBackupInfo(backupInfo);
702        sysTable.finishBackupExclusiveOperation();
703        System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
704      }
705    }
706
707    private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
708      throws IOException {
709      String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
710      if (backupIds == null || backupIds.length == 0) {
711        System.out.println("No failed backup DELETE operation found");
712        // Delete backup table snapshot if exists
713        BackupSystemTable.deleteSnapshot(conn);
714        return;
715      }
716      System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
717      System.out.println("Running DELETE again ...");
718      // Restore table from snapshot
719      BackupSystemTable.restoreFromSnapshot(conn);
720      // Finish previous failed session
721      sysTable.finishBackupExclusiveOperation();
722      try (BackupAdmin admin = new BackupAdminImpl(conn)) {
723        admin.deleteBackups(backupIds);
724      }
725      System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
726    }
727
728    public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
729      throws IOException {
730
731      String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
732      if (backupIds == null || backupIds.length == 0) {
733        System.out.println("No failed backup MERGE operation found");
734        // Delete backup table snapshot if exists
735        BackupSystemTable.deleteSnapshot(conn);
736        return;
737      }
738      System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
739      // Check if backup .tmp exists
740      BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]);
741      String backupRoot = bInfo.getBackupRootDir();
742      FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration());
743      String backupId = BackupUtils.findMostRecentBackupId(backupIds);
744      Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId);
745      if (fs.exists(tmpPath)) {
746        // Move data back
747        Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId);
748        if (!fs.delete(destPath, true)) {
749          System.out.println("Failed to delete " + destPath);
750        }
751        boolean res = fs.rename(tmpPath, destPath);
752        if (!res) {
753          throw new IOException(
754            "MERGE repair: failed  to rename from " + tmpPath + " to " + destPath);
755        }
756        System.out
757          .println("MERGE repair: renamed from " + tmpPath + " to " + destPath + " res=" + res);
758      } else {
759        checkRemoveBackupImages(fs, backupRoot, backupIds);
760      }
761      // Restore table from snapshot
762      BackupSystemTable.restoreFromSnapshot(conn);
763      // Unlock backup system
764      sysTable.finishBackupExclusiveOperation();
765      // Finish previous failed session
766      sysTable.finishMergeOperation();
767
768      System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds));
769    }
770
771    private static void checkRemoveBackupImages(FileSystem fs, String backupRoot,
772      String[] backupIds) throws IOException {
773      String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);
774      for (String backupId : backupIds) {
775        if (backupId.equals(mergedBackupId)) {
776          continue;
777        }
778        Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId);
779        if (fs.exists(path)) {
780          if (!fs.delete(path, true)) {
781            System.out.println("MERGE repair removing: " + path + " - FAILED");
782          } else {
783            System.out.println("MERGE repair removing: " + path + " - OK");
784          }
785        }
786      }
787    }
788
789    @Override
790    protected void printUsage() {
791      System.out.println(REPAIR_CMD_USAGE);
792    }
793  }
794
795  public static class MergeCommand extends Command {
796    MergeCommand(Configuration conf, CommandLine cmdline) {
797      super(conf);
798      this.cmdline = cmdline;
799    }
800
801    @Override
802    protected boolean requiresNoActiveSession() {
803      return true;
804    }
805
806    @Override
807    protected boolean requiresConsistentState() {
808      return true;
809    }
810
811    @Override
812    public void execute() throws IOException {
813      super.execute();
814
815      String[] args = cmdline == null ? null : cmdline.getArgs();
816      if (args == null || (args.length != 2)) {
817        System.err
818          .println("ERROR: wrong number of arguments: " + (args == null ? null : args.length));
819        printUsage();
820        throw new IOException(INCORRECT_USAGE);
821      }
822
823      String[] backupIds = args[1].split(",");
824      if (backupIds.length < 2) {
825        String msg = "ERROR: can not merge a single backup image. "
826          + "Number of images must be greater than 1.";
827        System.err.println(msg);
828        throw new IOException(msg);
829
830      }
831      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
832      try (final Connection conn = ConnectionFactory.createConnection(conf);
833        final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
834        admin.mergeBackups(backupIds);
835      }
836    }
837
838    @Override
839    protected void printUsage() {
840      System.out.println(MERGE_CMD_USAGE);
841    }
842  }
843
844  public static class HistoryCommand extends Command {
845    private final static int DEFAULT_HISTORY_LENGTH = 10;
846
847    HistoryCommand(Configuration conf, CommandLine cmdline) {
848      super(conf);
849      this.cmdline = cmdline;
850    }
851
852    @Override
853    public void execute() throws IOException {
854      int n = parseHistoryLength();
855      final TableName tableName = getTableName();
856      final String setName = getTableSetName();
857      BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
858        @Override
859        public boolean apply(BackupInfo info) {
860          if (tableName == null) {
861            return true;
862          }
863
864          List<TableName> names = info.getTableNames();
865          return names.contains(tableName);
866        }
867      };
868      BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
869        @Override
870        public boolean apply(BackupInfo info) {
871          if (setName == null) {
872            return true;
873          }
874
875          String backupId = info.getBackupId();
876          return backupId.startsWith(setName);
877        }
878      };
879      Path backupRootPath = getBackupRootPath();
880      List<BackupInfo> history;
881      if (backupRootPath == null) {
882        // Load from backup system table
883        super.execute();
884        try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
885          history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
886        }
887      } else {
888        // load from backup FS
889        history =
890          BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
891      }
892      for (BackupInfo info : history) {
893        System.out.println(info.getShortDescription());
894      }
895    }
896
897    private Path getBackupRootPath() throws IOException {
898      String value = null;
899      try {
900        value = cmdline.getOptionValue(OPTION_PATH);
901
902        if (value == null) {
903          return null;
904        }
905
906        return new Path(value);
907      } catch (IllegalArgumentException e) {
908        System.out.println("ERROR: Illegal argument for backup root path: " + value);
909        printUsage();
910        throw new IOException(INCORRECT_USAGE);
911      }
912    }
913
914    private TableName getTableName() throws IOException {
915      String value = cmdline.getOptionValue(OPTION_TABLE);
916
917      if (value == null) {
918        return null;
919      }
920
921      try {
922        return TableName.valueOf(value);
923      } catch (IllegalArgumentException e) {
924        System.out.println("Illegal argument for table name: " + value);
925        printUsage();
926        throw new IOException(INCORRECT_USAGE);
927      }
928    }
929
930    private String getTableSetName() {
931      return cmdline.getOptionValue(OPTION_SET);
932    }
933
934    private int parseHistoryLength() throws IOException {
935      String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
936      try {
937        if (value == null) {
938          return DEFAULT_HISTORY_LENGTH;
939        }
940
941        return Integer.parseInt(value);
942      } catch (NumberFormatException e) {
943        System.out.println("Illegal argument for history length: " + value);
944        printUsage();
945        throw new IOException(INCORRECT_USAGE);
946      }
947    }
948
949    @Override
950    protected void printUsage() {
951      System.out.println(HISTORY_CMD_USAGE);
952      Options options = new Options();
953      options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
954      options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
955      options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
956      options.addOption(OPTION_SET, true, OPTION_SET_DESC);
957
958      HelpFormatter helpFormatter = new HelpFormatter();
959      helpFormatter.setLeftPadding(2);
960      helpFormatter.setDescPadding(8);
961      helpFormatter.setWidth(100);
962      helpFormatter.setSyntaxPrefix("Options:");
963      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
964    }
965  }
966
967  public static class BackupSetCommand extends Command {
968    private final static String SET_ADD_CMD = "add";
969    private final static String SET_REMOVE_CMD = "remove";
970    private final static String SET_DELETE_CMD = "delete";
971    private final static String SET_DESCRIBE_CMD = "describe";
972    private final static String SET_LIST_CMD = "list";
973
974    BackupSetCommand(Configuration conf, CommandLine cmdline) {
975      super(conf);
976      this.cmdline = cmdline;
977    }
978
979    @Override
980    public void execute() throws IOException {
981      // Command-line must have at least one element
982      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
983        printUsage();
984        throw new IOException(INCORRECT_USAGE);
985      }
986
987      String[] args = cmdline.getArgs();
988      String cmdStr = args[1];
989      BackupCommand cmd = getCommand(cmdStr);
990
991      switch (cmd) {
992        case SET_ADD:
993          processSetAdd(args);
994          break;
995        case SET_REMOVE:
996          processSetRemove(args);
997          break;
998        case SET_DELETE:
999          processSetDelete(args);
1000          break;
1001        case SET_DESCRIBE:
1002          processSetDescribe(args);
1003          break;
1004        case SET_LIST:
1005          processSetList();
1006          break;
1007        default:
1008          break;
1009      }
1010    }
1011
1012    private void processSetList() throws IOException {
1013      super.execute();
1014
1015      // List all backup set names
1016      // does not expect any args
1017      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1018        List<BackupSet> list = admin.listBackupSets();
1019        for (BackupSet bs : list) {
1020          System.out.println(bs);
1021        }
1022      }
1023    }
1024
1025    private void processSetDescribe(String[] args) throws IOException {
1026      if (args == null || args.length != 3) {
1027        printUsage();
1028        throw new IOException(INCORRECT_USAGE);
1029      }
1030      super.execute();
1031
1032      String setName = args[2];
1033      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1034        List<TableName> tables = sysTable.describeBackupSet(setName);
1035        BackupSet set = tables == null ? null : new BackupSet(setName, tables);
1036        if (set == null) {
1037          System.out.println("Set '" + setName + "' does not exist.");
1038        } else {
1039          System.out.println(set);
1040        }
1041      }
1042    }
1043
1044    private void processSetDelete(String[] args) throws IOException {
1045      if (args == null || args.length != 3) {
1046        printUsage();
1047        throw new IOException(INCORRECT_USAGE);
1048      }
1049      super.execute();
1050
1051      String setName = args[2];
1052      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1053        boolean result = admin.deleteBackupSet(setName);
1054        if (result) {
1055          System.out.println("Delete set " + setName + " OK.");
1056        } else {
1057          System.out.println("Set " + setName + " does not exist");
1058        }
1059      }
1060    }
1061
1062    private void processSetRemove(String[] args) throws IOException {
1063      if (args == null || args.length != 4) {
1064        printUsage();
1065        throw new IOException(INCORRECT_USAGE);
1066      }
1067      super.execute();
1068
1069      String setName = args[2];
1070      String[] tables = args[3].split(",");
1071      TableName[] tableNames = toTableNames(tables);
1072      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1073        admin.removeFromBackupSet(setName, tableNames);
1074      }
1075    }
1076
1077    private TableName[] toTableNames(String[] tables) {
1078      TableName[] arr = new TableName[tables.length];
1079      for (int i = 0; i < tables.length; i++) {
1080        arr[i] = TableName.valueOf(tables[i]);
1081      }
1082      return arr;
1083    }
1084
1085    private void processSetAdd(String[] args) throws IOException {
1086      if (args == null || args.length != 4) {
1087        printUsage();
1088        throw new IOException(INCORRECT_USAGE);
1089      }
1090      super.execute();
1091      String setName = args[2];
1092      TableName[] tableNames =
1093        Splitter.on(',').splitToStream(args[3]).map(TableName::valueOf).toArray(TableName[]::new);
1094      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1095        admin.addToBackupSet(setName, tableNames);
1096      }
1097    }
1098
1099    private BackupCommand getCommand(String cmdStr) throws IOException {
1100      switch (cmdStr) {
1101        case SET_ADD_CMD:
1102          return BackupCommand.SET_ADD;
1103        case SET_REMOVE_CMD:
1104          return BackupCommand.SET_REMOVE;
1105        case SET_DELETE_CMD:
1106          return BackupCommand.SET_DELETE;
1107        case SET_DESCRIBE_CMD:
1108          return BackupCommand.SET_DESCRIBE;
1109        case SET_LIST_CMD:
1110          return BackupCommand.SET_LIST;
1111        default:
1112          System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
1113          printUsage();
1114          throw new IOException(INCORRECT_USAGE);
1115      }
1116    }
1117
1118    @Override
1119    protected void printUsage() {
1120      System.out.println(SET_CMD_USAGE);
1121    }
1122  }
1123}