001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
042
043import java.io.IOException;
044import java.net.URI;
045import java.util.List;
046import org.apache.commons.lang3.StringUtils;
047import org.apache.hadoop.conf.Configuration;
048import org.apache.hadoop.conf.Configured;
049import org.apache.hadoop.fs.FileSystem;
050import org.apache.hadoop.fs.Path;
051import org.apache.hadoop.hbase.HBaseConfiguration;
052import org.apache.hadoop.hbase.TableName;
053import org.apache.hadoop.hbase.backup.BackupAdmin;
054import org.apache.hadoop.hbase.backup.BackupInfo;
055import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
056import org.apache.hadoop.hbase.backup.BackupRequest;
057import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
058import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
059import org.apache.hadoop.hbase.backup.BackupType;
060import org.apache.hadoop.hbase.backup.HBackupFileSystem;
061import org.apache.hadoop.hbase.backup.util.BackupSet;
062import org.apache.hadoop.hbase.backup.util.BackupUtils;
063import org.apache.hadoop.hbase.client.Connection;
064import org.apache.hadoop.hbase.client.ConnectionFactory;
065import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
066import org.apache.yetus.audience.InterfaceAudience;
067
068import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
069import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
070import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
071import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
072
073/**
074 * General backup commands, options and usage messages
075 */
076@InterfaceAudience.Private
077public final class BackupCommands {
078  public final static String INCORRECT_USAGE = "Incorrect usage";
079
080  public final static String TOP_LEVEL_NOT_ALLOWED =
081    "Top level (root) folder is not allowed to be a backup destination";
082
083  public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
084    + "where COMMAND is one of:\n" + "  create     create a new backup image\n"
085    + "  delete     delete an existing backup image\n"
086    + "  describe   show the detailed information of a backup image\n"
087    + "  history    show history of all successful backups\n"
088    + "  progress   show the progress of the latest backup request\n"
089    + "  set        backup set management\n" + "  repair     repair backup system table\n"
090    + "  merge      merge backup images\n"
091    + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
092
093  public static final String CREATE_CMD_USAGE =
094    "Usage: hbase backup create <type> <backup_path> [options]\n"
095      + "  type           \"full\" to create a full backup image\n"
096      + "                 \"incremental\" to create an incremental backup image\n"
097      + "  backup_path     Full path to store the backup image\n";
098
099  public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
100    + "  backup_id       Backup image id (optional). If no id specified, the command will show\n"
101    + "                  progress for currently running backup session.";
102  public static final String NO_INFO_FOUND = "No info was found for backup id: ";
103  public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
104
105  public static final String DESCRIBE_CMD_USAGE =
106    "Usage: hbase backup describe <backup_id>\n" + "  backup_id       Backup image id\n";
107
108  public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
109
110  public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]";
111
112  public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
113
114  public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
115    + "  name            Backup set name\n" + "  tables          Comma separated list of tables.\n"
116    + "COMMAND is one of:\n" + "  add             add tables to a set, create a set if needed\n"
117    + "  remove          remove tables from a set\n"
118    + "  list            list all backup sets in the system\n" + "  describe        describe set\n"
119    + "  delete          delete backup set\n";
120  public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
121    + "  backup_ids      Comma separated list of backup image ids.\n";
122
123  public static final String USAGE_FOOTER = "";
124
125  public static abstract class Command extends Configured {
126    CommandLine cmdline;
127    Connection conn;
128
129    Command(Configuration conf) {
130      if (conf == null) {
131        conf = HBaseConfiguration.create();
132      }
133      setConf(conf);
134    }
135
136    public void execute() throws IOException {
137      if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
138        printUsage();
139        throw new IOException(INCORRECT_USAGE);
140      }
141
142      // Create connection
143      conn = ConnectionFactory.createConnection(getConf());
144      if (requiresNoActiveSession()) {
145        // Check active session
146        try (BackupSystemTable table = new BackupSystemTable(conn)) {
147          List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING);
148
149          if (sessions.size() > 0) {
150            System.err.println("Found backup session in a RUNNING state: ");
151            System.err.println(sessions.get(0));
152            System.err.println("This may indicate that a previous session has failed abnormally.");
153            System.err.println("In this case, backup recovery is recommended.");
154            throw new IOException("Active session found, aborted command execution");
155          }
156        }
157      }
158      if (requiresConsistentState()) {
159        // Check failed delete
160        try (BackupSystemTable table = new BackupSystemTable(conn)) {
161          String[] ids = table.getListOfBackupIdsFromDeleteOperation();
162
163          if (ids != null && ids.length > 0) {
164            System.err.println("Found failed backup DELETE coommand. ");
165            System.err.println("Backup system recovery is required.");
166            throw new IOException("Failed backup DELETE found, aborted command execution");
167          }
168
169          ids = table.getListOfBackupIdsFromMergeOperation();
170          if (ids != null && ids.length > 0) {
171            System.err.println("Found failed backup MERGE coommand. ");
172            System.err.println("Backup system recovery is required.");
173            throw new IOException("Failed backup MERGE found, aborted command execution");
174          }
175        }
176      }
177    }
178
179    public void finish() throws IOException {
180      if (conn != null) {
181        conn.close();
182      }
183    }
184
185    protected abstract void printUsage();
186
187    /**
188     * The command can't be run if active backup session is in progress
189     * @return true if no active sessions are in progress
190     */
191    protected boolean requiresNoActiveSession() {
192      return false;
193    }
194
195    /**
196     * Command requires consistent state of a backup system Backup system may become inconsistent
197     * because of an abnormal termination of a backup session or delete command
198     * @return true, if yes
199     */
200    protected boolean requiresConsistentState() {
201      return false;
202    }
203  }
204
205  private BackupCommands() {
206    throw new AssertionError("Instantiating utility class...");
207  }
208
209  public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
210    Command cmd;
211    switch (type) {
212      case CREATE:
213        cmd = new CreateCommand(conf, cmdline);
214        break;
215      case DESCRIBE:
216        cmd = new DescribeCommand(conf, cmdline);
217        break;
218      case PROGRESS:
219        cmd = new ProgressCommand(conf, cmdline);
220        break;
221      case DELETE:
222        cmd = new DeleteCommand(conf, cmdline);
223        break;
224      case HISTORY:
225        cmd = new HistoryCommand(conf, cmdline);
226        break;
227      case SET:
228        cmd = new BackupSetCommand(conf, cmdline);
229        break;
230      case REPAIR:
231        cmd = new RepairCommand(conf, cmdline);
232        break;
233      case MERGE:
234        cmd = new MergeCommand(conf, cmdline);
235        break;
236      case HELP:
237      default:
238        cmd = new HelpCommand(conf, cmdline);
239        break;
240    }
241    return cmd;
242  }
243
244  static int numOfArgs(String[] args) {
245    if (args == null) {
246      return 0;
247    }
248
249    return args.length;
250  }
251
252  public static class CreateCommand extends Command {
253    CreateCommand(Configuration conf, CommandLine cmdline) {
254      super(conf);
255      this.cmdline = cmdline;
256    }
257
258    @Override
259    protected boolean requiresNoActiveSession() {
260      return true;
261    }
262
263    @Override
264    protected boolean requiresConsistentState() {
265      return true;
266    }
267
268    @Override
269    public void execute() throws IOException {
270      if (cmdline == null || cmdline.getArgs() == null) {
271        printUsage();
272        throw new IOException(INCORRECT_USAGE);
273      }
274      String[] args = cmdline.getArgs();
275      if (args.length != 3) {
276        printUsage();
277        throw new IOException(INCORRECT_USAGE);
278      }
279
280      if (
281        !BackupType.FULL.toString().equalsIgnoreCase(args[1])
282          && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])
283      ) {
284        System.out.println("ERROR: invalid backup type: " + args[1]);
285        printUsage();
286        throw new IOException(INCORRECT_USAGE);
287      }
288      if (!verifyPath(args[2])) {
289        System.out.println("ERROR: invalid backup destination: " + args[2]);
290        printUsage();
291        throw new IOException(INCORRECT_USAGE);
292      }
293      String targetBackupDir = args[2];
294      // Check if backup destination is top level (root) folder - not allowed
295      if (isRootFolder(targetBackupDir)) {
296        throw new IOException(TOP_LEVEL_NOT_ALLOWED);
297      }
298      String tables;
299
300      // Check if we have both: backup set and list of tables
301      if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
302        System.out
303          .println("ERROR: You can specify either backup set or list" + " of tables, but not both");
304        printUsage();
305        throw new IOException(INCORRECT_USAGE);
306      }
307      // Creates connection
308      super.execute();
309      // Check backup set
310      String setName = null;
311      if (cmdline.hasOption(OPTION_SET)) {
312        setName = cmdline.getOptionValue(OPTION_SET);
313        tables = getTablesForSet(setName, getConf());
314
315        if (tables == null) {
316          System.out
317            .println("ERROR: Backup set '" + setName + "' is either empty or does not exist");
318          printUsage();
319          throw new IOException(INCORRECT_USAGE);
320        }
321      } else {
322        tables = cmdline.getOptionValue(OPTION_TABLE);
323      }
324      int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH)
325        ? Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH))
326        : -1;
327      int workers = cmdline.hasOption(OPTION_WORKERS)
328        ? Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS))
329        : -1;
330
331      if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
332        String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
333        // Set system property value for MR job
334        System.setProperty("mapreduce.job.queuename", queueName);
335      }
336
337      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
338        BackupRequest.Builder builder = new BackupRequest.Builder();
339        BackupRequest request = builder.withBackupType(BackupType.valueOf(args[1].toUpperCase()))
340          .withTableList(
341            tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null)
342          .withTargetRootDir(targetBackupDir).withTotalTasks(workers)
343          .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build();
344        String backupId = admin.backupTables(request);
345        System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
346      } catch (IOException e) {
347        System.out.println("Backup session finished. Status: FAILURE");
348        throw e;
349      }
350    }
351
352    private boolean isRootFolder(String targetBackupDir) {
353      Path p = new Path(targetBackupDir);
354      return p.isRoot();
355    }
356
357    private boolean verifyPath(String path) {
358      try {
359        Path p = new Path(path);
360        Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
361        URI uri = p.toUri();
362
363        if (uri.getScheme() == null) {
364          return false;
365        }
366
367        FileSystem.get(uri, conf);
368        return true;
369      } catch (Exception e) {
370        return false;
371      }
372    }
373
374    private String getTablesForSet(String name, Configuration conf) throws IOException {
375      try (final BackupSystemTable table = new BackupSystemTable(conn)) {
376        List<TableName> tables = table.describeBackupSet(name);
377
378        if (tables == null) {
379          return null;
380        }
381
382        return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
383      }
384    }
385
386    @Override
387    protected void printUsage() {
388      System.out.println(CREATE_CMD_USAGE);
389      Options options = new Options();
390      options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
391      options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
392      options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
393      options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
394      options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
395      options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC);
396
397      HelpFormatter helpFormatter = new HelpFormatter();
398      helpFormatter.setLeftPadding(2);
399      helpFormatter.setDescPadding(8);
400      helpFormatter.setWidth(100);
401      helpFormatter.setSyntaxPrefix("Options:");
402      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
403    }
404  }
405
406  public static class HelpCommand extends Command {
407    HelpCommand(Configuration conf, CommandLine cmdline) {
408      super(conf);
409      this.cmdline = cmdline;
410    }
411
412    @Override
413    public void execute() throws IOException {
414      if (cmdline == null) {
415        printUsage();
416        throw new IOException(INCORRECT_USAGE);
417      }
418
419      String[] args = cmdline.getArgs();
420      if (args == null || args.length == 0) {
421        printUsage();
422        throw new IOException(INCORRECT_USAGE);
423      }
424
425      if (args.length != 2) {
426        System.out.println("ERROR: Only supports help message of a single command type");
427        printUsage();
428        throw new IOException(INCORRECT_USAGE);
429      }
430
431      String type = args[1];
432
433      if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
434        System.out.println(CREATE_CMD_USAGE);
435      } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
436        System.out.println(DESCRIBE_CMD_USAGE);
437      } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
438        System.out.println(HISTORY_CMD_USAGE);
439      } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
440        System.out.println(PROGRESS_CMD_USAGE);
441      } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
442        System.out.println(DELETE_CMD_USAGE);
443      } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
444        System.out.println(SET_CMD_USAGE);
445      } else {
446        System.out.println("Unknown command : " + type);
447        printUsage();
448      }
449    }
450
451    @Override
452    protected void printUsage() {
453      System.out.println(USAGE);
454    }
455  }
456
457  public static class DescribeCommand extends Command {
458    DescribeCommand(Configuration conf, CommandLine cmdline) {
459      super(conf);
460      this.cmdline = cmdline;
461    }
462
463    @Override
464    public void execute() throws IOException {
465      if (cmdline == null || cmdline.getArgs() == null) {
466        printUsage();
467        throw new IOException(INCORRECT_USAGE);
468      }
469      String[] args = cmdline.getArgs();
470      if (args.length != 2) {
471        printUsage();
472        throw new IOException(INCORRECT_USAGE);
473      }
474
475      super.execute();
476
477      String backupId = args[1];
478      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
479        BackupInfo info = sysTable.readBackupInfo(backupId);
480        if (info == null) {
481          System.out.println("ERROR: " + backupId + " does not exist");
482          printUsage();
483          throw new IOException(INCORRECT_USAGE);
484        }
485        System.out.println(info.getShortDescription());
486      }
487    }
488
489    @Override
490    protected void printUsage() {
491      System.out.println(DESCRIBE_CMD_USAGE);
492    }
493  }
494
495  public static class ProgressCommand extends Command {
496    ProgressCommand(Configuration conf, CommandLine cmdline) {
497      super(conf);
498      this.cmdline = cmdline;
499    }
500
501    @Override
502    public void execute() throws IOException {
503
504      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
505        System.out.println(
506          "No backup id was specified, " + "will retrieve the most recent (ongoing) session");
507      }
508      String[] args = cmdline == null ? null : cmdline.getArgs();
509      if (args != null && args.length > 2) {
510        System.err.println("ERROR: wrong number of arguments: " + args.length);
511        printUsage();
512        throw new IOException(INCORRECT_USAGE);
513      }
514
515      super.execute();
516
517      String backupId = (args == null || args.length <= 1) ? null : args[1];
518      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
519        BackupInfo info = null;
520
521        if (backupId != null) {
522          info = sysTable.readBackupInfo(backupId);
523        } else {
524          List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING);
525          if (infos != null && infos.size() > 0) {
526            info = infos.get(0);
527            backupId = info.getBackupId();
528            System.out.println("Found ongoing session with backupId=" + backupId);
529          }
530        }
531        int progress = info == null ? -1 : info.getProgress();
532        if (progress < 0) {
533          if (backupId != null) {
534            System.out.println(NO_INFO_FOUND + backupId);
535          } else {
536            System.err.println(NO_ACTIVE_SESSION_FOUND);
537          }
538        } else {
539          System.out.println(backupId + " progress=" + progress + "%");
540        }
541      }
542    }
543
544    @Override
545    protected void printUsage() {
546      System.out.println(PROGRESS_CMD_USAGE);
547    }
548  }
549
550  public static class DeleteCommand extends Command {
551    DeleteCommand(Configuration conf, CommandLine cmdline) {
552      super(conf);
553      this.cmdline = cmdline;
554    }
555
556    @Override
557    protected boolean requiresNoActiveSession() {
558      return true;
559    }
560
561    @Override
562    public void execute() throws IOException {
563
564      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) {
565        printUsage();
566        throw new IOException(INCORRECT_USAGE);
567      }
568
569      if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) {
570        printUsage();
571        throw new IOException(INCORRECT_USAGE);
572      }
573      super.execute();
574      if (cmdline.hasOption(OPTION_KEEP)) {
575        executeDeleteOlderThan(cmdline);
576      } else if (cmdline.hasOption(OPTION_LIST)) {
577        executeDeleteListOfBackups(cmdline);
578      }
579    }
580
581    private void executeDeleteOlderThan(CommandLine cmdline) throws IOException {
582      String value = cmdline.getOptionValue(OPTION_KEEP);
583      int days = 0;
584      try {
585        days = Integer.parseInt(value);
586      } catch (NumberFormatException e) {
587        throw new IOException(value + " is not an integer number");
588      }
589      final long fdays = days;
590      BackupInfo.Filter dateFilter = new BackupInfo.Filter() {
591        @Override
592        public boolean apply(BackupInfo info) {
593          long currentTime = EnvironmentEdgeManager.currentTime();
594          long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000;
595          return info.getCompleteTs() <= maxTsToDelete;
596        }
597      };
598      List<BackupInfo> history = null;
599      try (final BackupSystemTable sysTable = new BackupSystemTable(conn);
600        BackupAdminImpl admin = new BackupAdminImpl(conn)) {
601        history = sysTable.getBackupHistory(-1, dateFilter);
602        String[] backupIds = convertToBackupIds(history);
603        int deleted = admin.deleteBackups(backupIds);
604        System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: "
605          + backupIds.length);
606      } catch (IOException e) {
607        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
608          + "system integrity");
609        throw e;
610      }
611    }
612
613    private String[] convertToBackupIds(List<BackupInfo> history) {
614      String[] ids = new String[history.size()];
615      for (int i = 0; i < ids.length; i++) {
616        ids[i] = history.get(i).getBackupId();
617      }
618      return ids;
619    }
620
621    private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException {
622      String value = cmdline.getOptionValue(OPTION_LIST);
623      String[] backupIds = value.split(",");
624
625      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
626        int deleted = admin.deleteBackups(backupIds);
627        System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length);
628      } catch (IOException e) {
629        System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
630          + "system integrity");
631        throw e;
632      }
633
634    }
635
636    @Override
637    protected void printUsage() {
638      System.out.println(DELETE_CMD_USAGE);
639      Options options = new Options();
640      options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC);
641      options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC);
642
643      HelpFormatter helpFormatter = new HelpFormatter();
644      helpFormatter.setLeftPadding(2);
645      helpFormatter.setDescPadding(8);
646      helpFormatter.setWidth(100);
647      helpFormatter.setSyntaxPrefix("Options:");
648      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
649
650    }
651  }
652
653  public static class RepairCommand extends Command {
654    RepairCommand(Configuration conf, CommandLine cmdline) {
655      super(conf);
656      this.cmdline = cmdline;
657    }
658
659    @Override
660    public void execute() throws IOException {
661      super.execute();
662
663      String[] args = cmdline == null ? null : cmdline.getArgs();
664      if (args != null && args.length > 1) {
665        System.err.println("ERROR: wrong number of arguments: " + args.length);
666        printUsage();
667        throw new IOException(INCORRECT_USAGE);
668      }
669
670      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
671      try (final Connection conn = ConnectionFactory.createConnection(conf);
672        final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
673        // Failed backup
674        BackupInfo backupInfo;
675        List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
676        if (list.size() == 0) {
677          // No failed sessions found
678          System.out.println("REPAIR status: no failed sessions found."
679            + " Checking failed delete backup operation ...");
680          repairFailedBackupDeletionIfAny(conn, sysTable);
681          repairFailedBackupMergeIfAny(conn, sysTable);
682          return;
683        }
684        backupInfo = list.get(0);
685        // If this is a cancel exception, then we've already cleaned.
686        // set the failure timestamp of the overall backup
687        backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
688        // set failure message
689        backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
690        // set overall backup status: failed
691        backupInfo.setState(BackupState.FAILED);
692        // compose the backup failed data
693        String backupFailedData = "BackupId=" + backupInfo.getBackupId() + ",startts="
694          + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
695          + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
696        System.out.println(backupFailedData);
697        TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
698        // If backup session is updated to FAILED state - means we
699        // processed recovery already.
700        sysTable.updateBackupInfo(backupInfo);
701        sysTable.finishBackupExclusiveOperation();
702        System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
703      }
704    }
705
706    private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
707      throws IOException {
708      String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
709      if (backupIds == null || backupIds.length == 0) {
710        System.out.println("No failed backup DELETE operation found");
711        // Delete backup table snapshot if exists
712        BackupSystemTable.deleteSnapshot(conn);
713        return;
714      }
715      System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
716      System.out.println("Running DELETE again ...");
717      // Restore table from snapshot
718      BackupSystemTable.restoreFromSnapshot(conn);
719      // Finish previous failed session
720      sysTable.finishBackupExclusiveOperation();
721      try (BackupAdmin admin = new BackupAdminImpl(conn)) {
722        admin.deleteBackups(backupIds);
723      }
724      System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
725    }
726
727    public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
728      throws IOException {
729
730      String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
731      if (backupIds == null || backupIds.length == 0) {
732        System.out.println("No failed backup MERGE operation found");
733        // Delete backup table snapshot if exists
734        BackupSystemTable.deleteSnapshot(conn);
735        return;
736      }
737      System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
738      // Check if backup .tmp exists
739      BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]);
740      String backupRoot = bInfo.getBackupRootDir();
741      FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration());
742      String backupId = BackupUtils.findMostRecentBackupId(backupIds);
743      Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId);
744      if (fs.exists(tmpPath)) {
745        // Move data back
746        Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId);
747        if (!fs.delete(destPath, true)) {
748          System.out.println("Failed to delete " + destPath);
749        }
750        boolean res = fs.rename(tmpPath, destPath);
751        if (!res) {
752          throw new IOException(
753            "MERGE repair: failed  to rename from " + tmpPath + " to " + destPath);
754        }
755        System.out
756          .println("MERGE repair: renamed from " + tmpPath + " to " + destPath + " res=" + res);
757      } else {
758        checkRemoveBackupImages(fs, backupRoot, backupIds);
759      }
760      // Restore table from snapshot
761      BackupSystemTable.restoreFromSnapshot(conn);
762      // Unlock backup system
763      sysTable.finishBackupExclusiveOperation();
764      // Finish previous failed session
765      sysTable.finishMergeOperation();
766
767      System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds));
768    }
769
770    private static void checkRemoveBackupImages(FileSystem fs, String backupRoot,
771      String[] backupIds) throws IOException {
772      String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);
773      for (String backupId : backupIds) {
774        if (backupId.equals(mergedBackupId)) {
775          continue;
776        }
777        Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId);
778        if (fs.exists(path)) {
779          if (!fs.delete(path, true)) {
780            System.out.println("MERGE repair removing: " + path + " - FAILED");
781          } else {
782            System.out.println("MERGE repair removing: " + path + " - OK");
783          }
784        }
785      }
786    }
787
788    @Override
789    protected void printUsage() {
790      System.out.println(REPAIR_CMD_USAGE);
791    }
792  }
793
794  public static class MergeCommand extends Command {
795    MergeCommand(Configuration conf, CommandLine cmdline) {
796      super(conf);
797      this.cmdline = cmdline;
798    }
799
800    @Override
801    protected boolean requiresNoActiveSession() {
802      return true;
803    }
804
805    @Override
806    protected boolean requiresConsistentState() {
807      return true;
808    }
809
810    @Override
811    public void execute() throws IOException {
812      super.execute();
813
814      String[] args = cmdline == null ? null : cmdline.getArgs();
815      if (args == null || (args.length != 2)) {
816        System.err
817          .println("ERROR: wrong number of arguments: " + (args == null ? null : args.length));
818        printUsage();
819        throw new IOException(INCORRECT_USAGE);
820      }
821
822      String[] backupIds = args[1].split(",");
823      if (backupIds.length < 2) {
824        String msg = "ERROR: can not merge a single backup image. "
825          + "Number of images must be greater than 1.";
826        System.err.println(msg);
827        throw new IOException(msg);
828
829      }
830      Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
831      try (final Connection conn = ConnectionFactory.createConnection(conf);
832        final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
833        admin.mergeBackups(backupIds);
834      }
835    }
836
837    @Override
838    protected void printUsage() {
839      System.out.println(MERGE_CMD_USAGE);
840    }
841  }
842
843  public static class HistoryCommand extends Command {
844    private final static int DEFAULT_HISTORY_LENGTH = 10;
845
846    HistoryCommand(Configuration conf, CommandLine cmdline) {
847      super(conf);
848      this.cmdline = cmdline;
849    }
850
851    @Override
852    public void execute() throws IOException {
853      int n = parseHistoryLength();
854      final TableName tableName = getTableName();
855      final String setName = getTableSetName();
856      BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
857        @Override
858        public boolean apply(BackupInfo info) {
859          if (tableName == null) {
860            return true;
861          }
862
863          List<TableName> names = info.getTableNames();
864          return names.contains(tableName);
865        }
866      };
867      BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
868        @Override
869        public boolean apply(BackupInfo info) {
870          if (setName == null) {
871            return true;
872          }
873
874          String backupId = info.getBackupId();
875          return backupId.startsWith(setName);
876        }
877      };
878      Path backupRootPath = getBackupRootPath();
879      List<BackupInfo> history;
880      if (backupRootPath == null) {
881        // Load from backup system table
882        super.execute();
883        try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
884          history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
885        }
886      } else {
887        // load from backup FS
888        history =
889          BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
890      }
891      for (BackupInfo info : history) {
892        System.out.println(info.getShortDescription());
893      }
894    }
895
896    private Path getBackupRootPath() throws IOException {
897      String value = null;
898      try {
899        value = cmdline.getOptionValue(OPTION_PATH);
900
901        if (value == null) {
902          return null;
903        }
904
905        return new Path(value);
906      } catch (IllegalArgumentException e) {
907        System.out.println("ERROR: Illegal argument for backup root path: " + value);
908        printUsage();
909        throw new IOException(INCORRECT_USAGE);
910      }
911    }
912
913    private TableName getTableName() throws IOException {
914      String value = cmdline.getOptionValue(OPTION_TABLE);
915
916      if (value == null) {
917        return null;
918      }
919
920      try {
921        return TableName.valueOf(value);
922      } catch (IllegalArgumentException e) {
923        System.out.println("Illegal argument for table name: " + value);
924        printUsage();
925        throw new IOException(INCORRECT_USAGE);
926      }
927    }
928
929    private String getTableSetName() {
930      return cmdline.getOptionValue(OPTION_SET);
931    }
932
933    private int parseHistoryLength() throws IOException {
934      String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
935      try {
936        if (value == null) {
937          return DEFAULT_HISTORY_LENGTH;
938        }
939
940        return Integer.parseInt(value);
941      } catch (NumberFormatException e) {
942        System.out.println("Illegal argument for history length: " + value);
943        printUsage();
944        throw new IOException(INCORRECT_USAGE);
945      }
946    }
947
948    @Override
949    protected void printUsage() {
950      System.out.println(HISTORY_CMD_USAGE);
951      Options options = new Options();
952      options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
953      options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
954      options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
955      options.addOption(OPTION_SET, true, OPTION_SET_DESC);
956
957      HelpFormatter helpFormatter = new HelpFormatter();
958      helpFormatter.setLeftPadding(2);
959      helpFormatter.setDescPadding(8);
960      helpFormatter.setWidth(100);
961      helpFormatter.setSyntaxPrefix("Options:");
962      helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
963    }
964  }
965
966  public static class BackupSetCommand extends Command {
967    private final static String SET_ADD_CMD = "add";
968    private final static String SET_REMOVE_CMD = "remove";
969    private final static String SET_DELETE_CMD = "delete";
970    private final static String SET_DESCRIBE_CMD = "describe";
971    private final static String SET_LIST_CMD = "list";
972
973    BackupSetCommand(Configuration conf, CommandLine cmdline) {
974      super(conf);
975      this.cmdline = cmdline;
976    }
977
978    @Override
979    public void execute() throws IOException {
980      // Command-line must have at least one element
981      if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
982        printUsage();
983        throw new IOException(INCORRECT_USAGE);
984      }
985
986      String[] args = cmdline.getArgs();
987      String cmdStr = args[1];
988      BackupCommand cmd = getCommand(cmdStr);
989
990      switch (cmd) {
991        case SET_ADD:
992          processSetAdd(args);
993          break;
994        case SET_REMOVE:
995          processSetRemove(args);
996          break;
997        case SET_DELETE:
998          processSetDelete(args);
999          break;
1000        case SET_DESCRIBE:
1001          processSetDescribe(args);
1002          break;
1003        case SET_LIST:
1004          processSetList(args);
1005          break;
1006        default:
1007          break;
1008      }
1009    }
1010
1011    private void processSetList(String[] args) throws IOException {
1012      super.execute();
1013
1014      // List all backup set names
1015      // does not expect any args
1016      try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1017        List<BackupSet> list = admin.listBackupSets();
1018        for (BackupSet bs : list) {
1019          System.out.println(bs);
1020        }
1021      }
1022    }
1023
1024    private void processSetDescribe(String[] args) throws IOException {
1025      if (args == null || args.length != 3) {
1026        printUsage();
1027        throw new IOException(INCORRECT_USAGE);
1028      }
1029      super.execute();
1030
1031      String setName = args[2];
1032      try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
1033        List<TableName> tables = sysTable.describeBackupSet(setName);
1034        BackupSet set = tables == null ? null : new BackupSet(setName, tables);
1035        if (set == null) {
1036          System.out.println("Set '" + setName + "' does not exist.");
1037        } else {
1038          System.out.println(set);
1039        }
1040      }
1041    }
1042
1043    private void processSetDelete(String[] args) throws IOException {
1044      if (args == null || args.length != 3) {
1045        printUsage();
1046        throw new IOException(INCORRECT_USAGE);
1047      }
1048      super.execute();
1049
1050      String setName = args[2];
1051      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1052        boolean result = admin.deleteBackupSet(setName);
1053        if (result) {
1054          System.out.println("Delete set " + setName + " OK.");
1055        } else {
1056          System.out.println("Set " + setName + " does not exist");
1057        }
1058      }
1059    }
1060
1061    private void processSetRemove(String[] args) throws IOException {
1062      if (args == null || args.length != 4) {
1063        printUsage();
1064        throw new IOException(INCORRECT_USAGE);
1065      }
1066      super.execute();
1067
1068      String setName = args[2];
1069      String[] tables = args[3].split(",");
1070      TableName[] tableNames = toTableNames(tables);
1071      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1072        admin.removeFromBackupSet(setName, tableNames);
1073      }
1074    }
1075
1076    private TableName[] toTableNames(String[] tables) {
1077      TableName[] arr = new TableName[tables.length];
1078      for (int i = 0; i < tables.length; i++) {
1079        arr[i] = TableName.valueOf(tables[i]);
1080      }
1081      return arr;
1082    }
1083
1084    private void processSetAdd(String[] args) throws IOException {
1085      if (args == null || args.length != 4) {
1086        printUsage();
1087        throw new IOException(INCORRECT_USAGE);
1088      }
1089      super.execute();
1090
1091      String setName = args[2];
1092      String[] tables = args[3].split(",");
1093      TableName[] tableNames = new TableName[tables.length];
1094      for (int i = 0; i < tables.length; i++) {
1095        tableNames[i] = TableName.valueOf(tables[i]);
1096      }
1097      try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) {
1098        admin.addToBackupSet(setName, tableNames);
1099      }
1100
1101    }
1102
1103    private BackupCommand getCommand(String cmdStr) throws IOException {
1104      switch (cmdStr) {
1105        case SET_ADD_CMD:
1106          return BackupCommand.SET_ADD;
1107        case SET_REMOVE_CMD:
1108          return BackupCommand.SET_REMOVE;
1109        case SET_DELETE_CMD:
1110          return BackupCommand.SET_DELETE;
1111        case SET_DESCRIBE_CMD:
1112          return BackupCommand.SET_DESCRIBE;
1113        case SET_LIST_CMD:
1114          return BackupCommand.SET_LIST;
1115        default:
1116          System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
1117          printUsage();
1118          throw new IOException(INCORRECT_USAGE);
1119      }
1120    }
1121
1122    @Override
1123    protected void printUsage() {
1124      System.out.println(SET_CMD_USAGE);
1125    }
1126  }
1127}