001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup.impl; 019 020import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; 021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; 022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; 023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; 024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; 025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; 026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; 027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; 028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; 029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; 030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; 031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; 032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; 033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC; 034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; 035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; 036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; 037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; 038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; 039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; 040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; 041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; 042 043import java.io.IOException; 044import java.net.URI; 045import java.util.List; 046import org.apache.commons.lang3.StringUtils; 047import org.apache.hadoop.conf.Configuration; 048import org.apache.hadoop.conf.Configured; 049import org.apache.hadoop.fs.FileSystem; 050import org.apache.hadoop.fs.Path; 051import org.apache.hadoop.hbase.HBaseConfiguration; 052import org.apache.hadoop.hbase.TableName; 053import org.apache.hadoop.hbase.backup.BackupAdmin; 054import org.apache.hadoop.hbase.backup.BackupInfo; 055import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; 056import org.apache.hadoop.hbase.backup.BackupRequest; 057import org.apache.hadoop.hbase.backup.BackupRestoreConstants; 058import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; 059import org.apache.hadoop.hbase.backup.BackupType; 060import org.apache.hadoop.hbase.backup.HBackupFileSystem; 061import org.apache.hadoop.hbase.backup.util.BackupSet; 062import org.apache.hadoop.hbase.backup.util.BackupUtils; 063import org.apache.hadoop.hbase.client.Connection; 064import org.apache.hadoop.hbase.client.ConnectionFactory; 065import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 066import org.apache.yetus.audience.InterfaceAudience; 067 068import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 069import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 070import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; 071import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; 072 073/** 074 * General backup commands, options and usage messages 075 */ 076@InterfaceAudience.Private 077public final class BackupCommands { 078 public final static String INCORRECT_USAGE = "Incorrect usage"; 079 080 public final static String TOP_LEVEL_NOT_ALLOWED = 081 "Top level (root) folder is not allowed to be a backup destination"; 082 083 public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" 084 + "where COMMAND is one of:\n" + " create create a new backup image\n" 085 + " delete delete an existing backup image\n" 086 + " describe show the detailed information of a backup image\n" 087 + " history show history of all successful backups\n" 088 + " progress show the progress of the latest backup request\n" 089 + " set backup set management\n" + " repair repair backup system table\n" 090 + " merge merge backup images\n" 091 + "Run \'hbase backup COMMAND -h\' to see help message for each command\n"; 092 093 public static final String CREATE_CMD_USAGE = 094 "Usage: hbase backup create <type> <backup_path> [options]\n" 095 + " type \"full\" to create a full backup image\n" 096 + " \"incremental\" to create an incremental backup image\n" 097 + " backup_path Full path to store the backup image\n"; 098 099 public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n" 100 + " backup_id Backup image id (optional). If no id specified, the command will show\n" 101 + " progress for currently running backup session."; 102 public static final String NO_INFO_FOUND = "No info was found for backup id: "; 103 public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found."; 104 105 public static final String DESCRIBE_CMD_USAGE = 106 "Usage: hbase backup describe <backup_id>\n" + " backup_id Backup image id\n"; 107 108 public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; 109 110 public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]"; 111 112 public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; 113 114 public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" 115 + " name Backup set name\n" + " tables Comma separated list of tables.\n" 116 + "COMMAND is one of:\n" + " add add tables to a set, create a set if needed\n" 117 + " remove remove tables from a set\n" 118 + " list list all backup sets in the system\n" + " describe describe set\n" 119 + " delete delete backup set\n"; 120 public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n" 121 + " backup_ids Comma separated list of backup image ids.\n"; 122 123 public static final String USAGE_FOOTER = ""; 124 125 public static abstract class Command extends Configured { 126 CommandLine cmdline; 127 Connection conn; 128 129 Command(Configuration conf) { 130 if (conf == null) { 131 conf = HBaseConfiguration.create(); 132 } 133 setConf(conf); 134 } 135 136 public void execute() throws IOException { 137 if (cmdline.hasOption("h") || cmdline.hasOption("help")) { 138 printUsage(); 139 throw new IOException(INCORRECT_USAGE); 140 } 141 142 // Create connection 143 conn = ConnectionFactory.createConnection(getConf()); 144 if (requiresNoActiveSession()) { 145 // Check active session 146 try (BackupSystemTable table = new BackupSystemTable(conn)) { 147 List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING); 148 149 if (sessions.size() > 0) { 150 System.err.println("Found backup session in a RUNNING state: "); 151 System.err.println(sessions.get(0)); 152 System.err.println("This may indicate that a previous session has failed abnormally."); 153 System.err.println("In this case, backup recovery is recommended."); 154 throw new IOException("Active session found, aborted command execution"); 155 } 156 } 157 } 158 if (requiresConsistentState()) { 159 // Check failed delete 160 try (BackupSystemTable table = new BackupSystemTable(conn)) { 161 String[] ids = table.getListOfBackupIdsFromDeleteOperation(); 162 163 if (ids != null && ids.length > 0) { 164 System.err.println("Found failed backup DELETE coommand. "); 165 System.err.println("Backup system recovery is required."); 166 throw new IOException("Failed backup DELETE found, aborted command execution"); 167 } 168 169 ids = table.getListOfBackupIdsFromMergeOperation(); 170 if (ids != null && ids.length > 0) { 171 System.err.println("Found failed backup MERGE coommand. "); 172 System.err.println("Backup system recovery is required."); 173 throw new IOException("Failed backup MERGE found, aborted command execution"); 174 } 175 } 176 } 177 } 178 179 public void finish() throws IOException { 180 if (conn != null) { 181 conn.close(); 182 } 183 } 184 185 protected abstract void printUsage(); 186 187 /** 188 * The command can't be run if active backup session is in progress 189 * @return true if no active sessions are in progress 190 */ 191 protected boolean requiresNoActiveSession() { 192 return false; 193 } 194 195 /** 196 * Command requires consistent state of a backup system Backup system may become inconsistent 197 * because of an abnormal termination of a backup session or delete command 198 * @return true, if yes 199 */ 200 protected boolean requiresConsistentState() { 201 return false; 202 } 203 } 204 205 private BackupCommands() { 206 throw new AssertionError("Instantiating utility class..."); 207 } 208 209 public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { 210 Command cmd; 211 switch (type) { 212 case CREATE: 213 cmd = new CreateCommand(conf, cmdline); 214 break; 215 case DESCRIBE: 216 cmd = new DescribeCommand(conf, cmdline); 217 break; 218 case PROGRESS: 219 cmd = new ProgressCommand(conf, cmdline); 220 break; 221 case DELETE: 222 cmd = new DeleteCommand(conf, cmdline); 223 break; 224 case HISTORY: 225 cmd = new HistoryCommand(conf, cmdline); 226 break; 227 case SET: 228 cmd = new BackupSetCommand(conf, cmdline); 229 break; 230 case REPAIR: 231 cmd = new RepairCommand(conf, cmdline); 232 break; 233 case MERGE: 234 cmd = new MergeCommand(conf, cmdline); 235 break; 236 case HELP: 237 default: 238 cmd = new HelpCommand(conf, cmdline); 239 break; 240 } 241 return cmd; 242 } 243 244 static int numOfArgs(String[] args) { 245 if (args == null) { 246 return 0; 247 } 248 249 return args.length; 250 } 251 252 public static class CreateCommand extends Command { 253 CreateCommand(Configuration conf, CommandLine cmdline) { 254 super(conf); 255 this.cmdline = cmdline; 256 } 257 258 @Override 259 protected boolean requiresNoActiveSession() { 260 return true; 261 } 262 263 @Override 264 protected boolean requiresConsistentState() { 265 return true; 266 } 267 268 @Override 269 public void execute() throws IOException { 270 if (cmdline == null || cmdline.getArgs() == null) { 271 printUsage(); 272 throw new IOException(INCORRECT_USAGE); 273 } 274 String[] args = cmdline.getArgs(); 275 if (args.length != 3) { 276 printUsage(); 277 throw new IOException(INCORRECT_USAGE); 278 } 279 280 if ( 281 !BackupType.FULL.toString().equalsIgnoreCase(args[1]) 282 && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1]) 283 ) { 284 System.out.println("ERROR: invalid backup type: " + args[1]); 285 printUsage(); 286 throw new IOException(INCORRECT_USAGE); 287 } 288 if (!verifyPath(args[2])) { 289 System.out.println("ERROR: invalid backup destination: " + args[2]); 290 printUsage(); 291 throw new IOException(INCORRECT_USAGE); 292 } 293 String targetBackupDir = args[2]; 294 // Check if backup destination is top level (root) folder - not allowed 295 if (isRootFolder(targetBackupDir)) { 296 throw new IOException(TOP_LEVEL_NOT_ALLOWED); 297 } 298 String tables; 299 300 // Check if we have both: backup set and list of tables 301 if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) { 302 System.out 303 .println("ERROR: You can specify either backup set or list" + " of tables, but not both"); 304 printUsage(); 305 throw new IOException(INCORRECT_USAGE); 306 } 307 // Creates connection 308 super.execute(); 309 // Check backup set 310 String setName = null; 311 if (cmdline.hasOption(OPTION_SET)) { 312 setName = cmdline.getOptionValue(OPTION_SET); 313 tables = getTablesForSet(setName, getConf()); 314 315 if (tables == null) { 316 System.out 317 .println("ERROR: Backup set '" + setName + "' is either empty or does not exist"); 318 printUsage(); 319 throw new IOException(INCORRECT_USAGE); 320 } 321 } else { 322 tables = cmdline.getOptionValue(OPTION_TABLE); 323 } 324 int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH) 325 ? Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH)) 326 : -1; 327 int workers = cmdline.hasOption(OPTION_WORKERS) 328 ? Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS)) 329 : -1; 330 331 if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) { 332 String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME); 333 // Set system property value for MR job 334 System.setProperty("mapreduce.job.queuename", queueName); 335 } 336 337 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 338 BackupRequest.Builder builder = new BackupRequest.Builder(); 339 BackupRequest request = builder.withBackupType(BackupType.valueOf(args[1].toUpperCase())) 340 .withTableList( 341 tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null) 342 .withTargetRootDir(targetBackupDir).withTotalTasks(workers) 343 .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build(); 344 String backupId = admin.backupTables(request); 345 System.out.println("Backup session " + backupId + " finished. Status: SUCCESS"); 346 } catch (IOException e) { 347 System.out.println("Backup session finished. Status: FAILURE"); 348 throw e; 349 } 350 } 351 352 private boolean isRootFolder(String targetBackupDir) { 353 Path p = new Path(targetBackupDir); 354 return p.isRoot(); 355 } 356 357 private boolean verifyPath(String path) { 358 try { 359 Path p = new Path(path); 360 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 361 URI uri = p.toUri(); 362 363 if (uri.getScheme() == null) { 364 return false; 365 } 366 367 FileSystem.get(uri, conf); 368 return true; 369 } catch (Exception e) { 370 return false; 371 } 372 } 373 374 private String getTablesForSet(String name, Configuration conf) throws IOException { 375 try (final BackupSystemTable table = new BackupSystemTable(conn)) { 376 List<TableName> tables = table.describeBackupSet(name); 377 378 if (tables == null) { 379 return null; 380 } 381 382 return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); 383 } 384 } 385 386 @Override 387 protected void printUsage() { 388 System.out.println(CREATE_CMD_USAGE); 389 Options options = new Options(); 390 options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC); 391 options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); 392 options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC); 393 options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC); 394 options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC); 395 options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC); 396 397 HelpFormatter helpFormatter = new HelpFormatter(); 398 helpFormatter.setLeftPadding(2); 399 helpFormatter.setDescPadding(8); 400 helpFormatter.setWidth(100); 401 helpFormatter.setSyntaxPrefix("Options:"); 402 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 403 } 404 } 405 406 public static class HelpCommand extends Command { 407 HelpCommand(Configuration conf, CommandLine cmdline) { 408 super(conf); 409 this.cmdline = cmdline; 410 } 411 412 @Override 413 public void execute() throws IOException { 414 if (cmdline == null) { 415 printUsage(); 416 throw new IOException(INCORRECT_USAGE); 417 } 418 419 String[] args = cmdline.getArgs(); 420 if (args == null || args.length == 0) { 421 printUsage(); 422 throw new IOException(INCORRECT_USAGE); 423 } 424 425 if (args.length != 2) { 426 System.out.println("ERROR: Only supports help message of a single command type"); 427 printUsage(); 428 throw new IOException(INCORRECT_USAGE); 429 } 430 431 String type = args[1]; 432 433 if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { 434 System.out.println(CREATE_CMD_USAGE); 435 } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { 436 System.out.println(DESCRIBE_CMD_USAGE); 437 } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { 438 System.out.println(HISTORY_CMD_USAGE); 439 } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { 440 System.out.println(PROGRESS_CMD_USAGE); 441 } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { 442 System.out.println(DELETE_CMD_USAGE); 443 } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { 444 System.out.println(SET_CMD_USAGE); 445 } else { 446 System.out.println("Unknown command : " + type); 447 printUsage(); 448 } 449 } 450 451 @Override 452 protected void printUsage() { 453 System.out.println(USAGE); 454 } 455 } 456 457 public static class DescribeCommand extends Command { 458 DescribeCommand(Configuration conf, CommandLine cmdline) { 459 super(conf); 460 this.cmdline = cmdline; 461 } 462 463 @Override 464 public void execute() throws IOException { 465 if (cmdline == null || cmdline.getArgs() == null) { 466 printUsage(); 467 throw new IOException(INCORRECT_USAGE); 468 } 469 String[] args = cmdline.getArgs(); 470 if (args.length != 2) { 471 printUsage(); 472 throw new IOException(INCORRECT_USAGE); 473 } 474 475 super.execute(); 476 477 String backupId = args[1]; 478 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 479 BackupInfo info = sysTable.readBackupInfo(backupId); 480 if (info == null) { 481 System.out.println("ERROR: " + backupId + " does not exist"); 482 printUsage(); 483 throw new IOException(INCORRECT_USAGE); 484 } 485 System.out.println(info.getShortDescription()); 486 } 487 } 488 489 @Override 490 protected void printUsage() { 491 System.out.println(DESCRIBE_CMD_USAGE); 492 } 493 } 494 495 public static class ProgressCommand extends Command { 496 ProgressCommand(Configuration conf, CommandLine cmdline) { 497 super(conf); 498 this.cmdline = cmdline; 499 } 500 501 @Override 502 public void execute() throws IOException { 503 504 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) { 505 System.out.println( 506 "No backup id was specified, " + "will retrieve the most recent (ongoing) session"); 507 } 508 String[] args = cmdline == null ? null : cmdline.getArgs(); 509 if (args != null && args.length > 2) { 510 System.err.println("ERROR: wrong number of arguments: " + args.length); 511 printUsage(); 512 throw new IOException(INCORRECT_USAGE); 513 } 514 515 super.execute(); 516 517 String backupId = (args == null || args.length <= 1) ? null : args[1]; 518 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 519 BackupInfo info = null; 520 521 if (backupId != null) { 522 info = sysTable.readBackupInfo(backupId); 523 } else { 524 List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING); 525 if (infos != null && infos.size() > 0) { 526 info = infos.get(0); 527 backupId = info.getBackupId(); 528 System.out.println("Found ongoing session with backupId=" + backupId); 529 } 530 } 531 int progress = info == null ? -1 : info.getProgress(); 532 if (progress < 0) { 533 if (backupId != null) { 534 System.out.println(NO_INFO_FOUND + backupId); 535 } else { 536 System.err.println(NO_ACTIVE_SESSION_FOUND); 537 } 538 } else { 539 System.out.println(backupId + " progress=" + progress + "%"); 540 } 541 } 542 } 543 544 @Override 545 protected void printUsage() { 546 System.out.println(PROGRESS_CMD_USAGE); 547 } 548 } 549 550 public static class DeleteCommand extends Command { 551 DeleteCommand(Configuration conf, CommandLine cmdline) { 552 super(conf); 553 this.cmdline = cmdline; 554 } 555 556 @Override 557 protected boolean requiresNoActiveSession() { 558 return true; 559 } 560 561 @Override 562 public void execute() throws IOException { 563 564 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) { 565 printUsage(); 566 throw new IOException(INCORRECT_USAGE); 567 } 568 569 if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) { 570 printUsage(); 571 throw new IOException(INCORRECT_USAGE); 572 } 573 super.execute(); 574 if (cmdline.hasOption(OPTION_KEEP)) { 575 executeDeleteOlderThan(cmdline); 576 } else if (cmdline.hasOption(OPTION_LIST)) { 577 executeDeleteListOfBackups(cmdline); 578 } 579 } 580 581 private void executeDeleteOlderThan(CommandLine cmdline) throws IOException { 582 String value = cmdline.getOptionValue(OPTION_KEEP); 583 int days = 0; 584 try { 585 days = Integer.parseInt(value); 586 } catch (NumberFormatException e) { 587 throw new IOException(value + " is not an integer number"); 588 } 589 final long fdays = days; 590 BackupInfo.Filter dateFilter = new BackupInfo.Filter() { 591 @Override 592 public boolean apply(BackupInfo info) { 593 long currentTime = EnvironmentEdgeManager.currentTime(); 594 long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; 595 return info.getCompleteTs() <= maxTsToDelete; 596 } 597 }; 598 List<BackupInfo> history = null; 599 try (final BackupSystemTable sysTable = new BackupSystemTable(conn); 600 BackupAdminImpl admin = new BackupAdminImpl(conn)) { 601 history = sysTable.getBackupHistory(-1, dateFilter); 602 String[] backupIds = convertToBackupIds(history); 603 int deleted = admin.deleteBackups(backupIds); 604 System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " 605 + backupIds.length); 606 } catch (IOException e) { 607 System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " 608 + "system integrity"); 609 throw e; 610 } 611 } 612 613 private String[] convertToBackupIds(List<BackupInfo> history) { 614 String[] ids = new String[history.size()]; 615 for (int i = 0; i < ids.length; i++) { 616 ids[i] = history.get(i).getBackupId(); 617 } 618 return ids; 619 } 620 621 private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException { 622 String value = cmdline.getOptionValue(OPTION_LIST); 623 String[] backupIds = value.split(","); 624 625 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 626 int deleted = admin.deleteBackups(backupIds); 627 System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length); 628 } catch (IOException e) { 629 System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " 630 + "system integrity"); 631 throw e; 632 } 633 634 } 635 636 @Override 637 protected void printUsage() { 638 System.out.println(DELETE_CMD_USAGE); 639 Options options = new Options(); 640 options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC); 641 options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC); 642 643 HelpFormatter helpFormatter = new HelpFormatter(); 644 helpFormatter.setLeftPadding(2); 645 helpFormatter.setDescPadding(8); 646 helpFormatter.setWidth(100); 647 helpFormatter.setSyntaxPrefix("Options:"); 648 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 649 650 } 651 } 652 653 public static class RepairCommand extends Command { 654 RepairCommand(Configuration conf, CommandLine cmdline) { 655 super(conf); 656 this.cmdline = cmdline; 657 } 658 659 @Override 660 public void execute() throws IOException { 661 super.execute(); 662 663 String[] args = cmdline == null ? null : cmdline.getArgs(); 664 if (args != null && args.length > 1) { 665 System.err.println("ERROR: wrong number of arguments: " + args.length); 666 printUsage(); 667 throw new IOException(INCORRECT_USAGE); 668 } 669 670 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 671 try (final Connection conn = ConnectionFactory.createConnection(conf); 672 final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 673 // Failed backup 674 BackupInfo backupInfo; 675 List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING); 676 if (list.size() == 0) { 677 // No failed sessions found 678 System.out.println("REPAIR status: no failed sessions found." 679 + " Checking failed delete backup operation ..."); 680 repairFailedBackupDeletionIfAny(conn, sysTable); 681 repairFailedBackupMergeIfAny(conn, sysTable); 682 return; 683 } 684 backupInfo = list.get(0); 685 // If this is a cancel exception, then we've already cleaned. 686 // set the failure timestamp of the overall backup 687 backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); 688 // set failure message 689 backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo); 690 // set overall backup status: failed 691 backupInfo.setState(BackupState.FAILED); 692 // compose the backup failed data 693 String backupFailedData = "BackupId=" + backupInfo.getBackupId() + ",startts=" 694 + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" 695 + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); 696 System.out.println(backupFailedData); 697 TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf); 698 // If backup session is updated to FAILED state - means we 699 // processed recovery already. 700 sysTable.updateBackupInfo(backupInfo); 701 sysTable.finishBackupExclusiveOperation(); 702 System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo); 703 } 704 } 705 706 private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable) 707 throws IOException { 708 String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation(); 709 if (backupIds == null || backupIds.length == 0) { 710 System.out.println("No failed backup DELETE operation found"); 711 // Delete backup table snapshot if exists 712 BackupSystemTable.deleteSnapshot(conn); 713 return; 714 } 715 System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds)); 716 System.out.println("Running DELETE again ..."); 717 // Restore table from snapshot 718 BackupSystemTable.restoreFromSnapshot(conn); 719 // Finish previous failed session 720 sysTable.finishBackupExclusiveOperation(); 721 try (BackupAdmin admin = new BackupAdminImpl(conn)) { 722 admin.deleteBackups(backupIds); 723 } 724 System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds)); 725 } 726 727 public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable) 728 throws IOException { 729 730 String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation(); 731 if (backupIds == null || backupIds.length == 0) { 732 System.out.println("No failed backup MERGE operation found"); 733 // Delete backup table snapshot if exists 734 BackupSystemTable.deleteSnapshot(conn); 735 return; 736 } 737 System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds)); 738 // Check if backup .tmp exists 739 BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]); 740 String backupRoot = bInfo.getBackupRootDir(); 741 FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration()); 742 String backupId = BackupUtils.findMostRecentBackupId(backupIds); 743 Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId); 744 if (fs.exists(tmpPath)) { 745 // Move data back 746 Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId); 747 if (!fs.delete(destPath, true)) { 748 System.out.println("Failed to delete " + destPath); 749 } 750 boolean res = fs.rename(tmpPath, destPath); 751 if (!res) { 752 throw new IOException( 753 "MERGE repair: failed to rename from " + tmpPath + " to " + destPath); 754 } 755 System.out 756 .println("MERGE repair: renamed from " + tmpPath + " to " + destPath + " res=" + res); 757 } else { 758 checkRemoveBackupImages(fs, backupRoot, backupIds); 759 } 760 // Restore table from snapshot 761 BackupSystemTable.restoreFromSnapshot(conn); 762 // Unlock backup system 763 sysTable.finishBackupExclusiveOperation(); 764 // Finish previous failed session 765 sysTable.finishMergeOperation(); 766 767 System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds)); 768 } 769 770 private static void checkRemoveBackupImages(FileSystem fs, String backupRoot, 771 String[] backupIds) throws IOException { 772 String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds); 773 for (String backupId : backupIds) { 774 if (backupId.equals(mergedBackupId)) { 775 continue; 776 } 777 Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId); 778 if (fs.exists(path)) { 779 if (!fs.delete(path, true)) { 780 System.out.println("MERGE repair removing: " + path + " - FAILED"); 781 } else { 782 System.out.println("MERGE repair removing: " + path + " - OK"); 783 } 784 } 785 } 786 } 787 788 @Override 789 protected void printUsage() { 790 System.out.println(REPAIR_CMD_USAGE); 791 } 792 } 793 794 public static class MergeCommand extends Command { 795 MergeCommand(Configuration conf, CommandLine cmdline) { 796 super(conf); 797 this.cmdline = cmdline; 798 } 799 800 @Override 801 protected boolean requiresNoActiveSession() { 802 return true; 803 } 804 805 @Override 806 protected boolean requiresConsistentState() { 807 return true; 808 } 809 810 @Override 811 public void execute() throws IOException { 812 super.execute(); 813 814 String[] args = cmdline == null ? null : cmdline.getArgs(); 815 if (args == null || (args.length != 2)) { 816 System.err 817 .println("ERROR: wrong number of arguments: " + (args == null ? null : args.length)); 818 printUsage(); 819 throw new IOException(INCORRECT_USAGE); 820 } 821 822 String[] backupIds = args[1].split(","); 823 if (backupIds.length < 2) { 824 String msg = "ERROR: can not merge a single backup image. " 825 + "Number of images must be greater than 1."; 826 System.err.println(msg); 827 throw new IOException(msg); 828 829 } 830 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 831 try (final Connection conn = ConnectionFactory.createConnection(conf); 832 final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 833 admin.mergeBackups(backupIds); 834 } 835 } 836 837 @Override 838 protected void printUsage() { 839 System.out.println(MERGE_CMD_USAGE); 840 } 841 } 842 843 public static class HistoryCommand extends Command { 844 private final static int DEFAULT_HISTORY_LENGTH = 10; 845 846 HistoryCommand(Configuration conf, CommandLine cmdline) { 847 super(conf); 848 this.cmdline = cmdline; 849 } 850 851 @Override 852 public void execute() throws IOException { 853 int n = parseHistoryLength(); 854 final TableName tableName = getTableName(); 855 final String setName = getTableSetName(); 856 BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { 857 @Override 858 public boolean apply(BackupInfo info) { 859 if (tableName == null) { 860 return true; 861 } 862 863 List<TableName> names = info.getTableNames(); 864 return names.contains(tableName); 865 } 866 }; 867 BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { 868 @Override 869 public boolean apply(BackupInfo info) { 870 if (setName == null) { 871 return true; 872 } 873 874 String backupId = info.getBackupId(); 875 return backupId.startsWith(setName); 876 } 877 }; 878 Path backupRootPath = getBackupRootPath(); 879 List<BackupInfo> history; 880 if (backupRootPath == null) { 881 // Load from backup system table 882 super.execute(); 883 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 884 history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); 885 } 886 } else { 887 // load from backup FS 888 history = 889 BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter); 890 } 891 for (BackupInfo info : history) { 892 System.out.println(info.getShortDescription()); 893 } 894 } 895 896 private Path getBackupRootPath() throws IOException { 897 String value = null; 898 try { 899 value = cmdline.getOptionValue(OPTION_PATH); 900 901 if (value == null) { 902 return null; 903 } 904 905 return new Path(value); 906 } catch (IllegalArgumentException e) { 907 System.out.println("ERROR: Illegal argument for backup root path: " + value); 908 printUsage(); 909 throw new IOException(INCORRECT_USAGE); 910 } 911 } 912 913 private TableName getTableName() throws IOException { 914 String value = cmdline.getOptionValue(OPTION_TABLE); 915 916 if (value == null) { 917 return null; 918 } 919 920 try { 921 return TableName.valueOf(value); 922 } catch (IllegalArgumentException e) { 923 System.out.println("Illegal argument for table name: " + value); 924 printUsage(); 925 throw new IOException(INCORRECT_USAGE); 926 } 927 } 928 929 private String getTableSetName() { 930 return cmdline.getOptionValue(OPTION_SET); 931 } 932 933 private int parseHistoryLength() throws IOException { 934 String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER); 935 try { 936 if (value == null) { 937 return DEFAULT_HISTORY_LENGTH; 938 } 939 940 return Integer.parseInt(value); 941 } catch (NumberFormatException e) { 942 System.out.println("Illegal argument for history length: " + value); 943 printUsage(); 944 throw new IOException(INCORRECT_USAGE); 945 } 946 } 947 948 @Override 949 protected void printUsage() { 950 System.out.println(HISTORY_CMD_USAGE); 951 Options options = new Options(); 952 options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC); 953 options.addOption(OPTION_PATH, true, OPTION_PATH_DESC); 954 options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC); 955 options.addOption(OPTION_SET, true, OPTION_SET_DESC); 956 957 HelpFormatter helpFormatter = new HelpFormatter(); 958 helpFormatter.setLeftPadding(2); 959 helpFormatter.setDescPadding(8); 960 helpFormatter.setWidth(100); 961 helpFormatter.setSyntaxPrefix("Options:"); 962 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 963 } 964 } 965 966 public static class BackupSetCommand extends Command { 967 private final static String SET_ADD_CMD = "add"; 968 private final static String SET_REMOVE_CMD = "remove"; 969 private final static String SET_DELETE_CMD = "delete"; 970 private final static String SET_DESCRIBE_CMD = "describe"; 971 private final static String SET_LIST_CMD = "list"; 972 973 BackupSetCommand(Configuration conf, CommandLine cmdline) { 974 super(conf); 975 this.cmdline = cmdline; 976 } 977 978 @Override 979 public void execute() throws IOException { 980 // Command-line must have at least one element 981 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { 982 printUsage(); 983 throw new IOException(INCORRECT_USAGE); 984 } 985 986 String[] args = cmdline.getArgs(); 987 String cmdStr = args[1]; 988 BackupCommand cmd = getCommand(cmdStr); 989 990 switch (cmd) { 991 case SET_ADD: 992 processSetAdd(args); 993 break; 994 case SET_REMOVE: 995 processSetRemove(args); 996 break; 997 case SET_DELETE: 998 processSetDelete(args); 999 break; 1000 case SET_DESCRIBE: 1001 processSetDescribe(args); 1002 break; 1003 case SET_LIST: 1004 processSetList(args); 1005 break; 1006 default: 1007 break; 1008 } 1009 } 1010 1011 private void processSetList(String[] args) throws IOException { 1012 super.execute(); 1013 1014 // List all backup set names 1015 // does not expect any args 1016 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1017 List<BackupSet> list = admin.listBackupSets(); 1018 for (BackupSet bs : list) { 1019 System.out.println(bs); 1020 } 1021 } 1022 } 1023 1024 private void processSetDescribe(String[] args) throws IOException { 1025 if (args == null || args.length != 3) { 1026 printUsage(); 1027 throw new IOException(INCORRECT_USAGE); 1028 } 1029 super.execute(); 1030 1031 String setName = args[2]; 1032 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 1033 List<TableName> tables = sysTable.describeBackupSet(setName); 1034 BackupSet set = tables == null ? null : new BackupSet(setName, tables); 1035 if (set == null) { 1036 System.out.println("Set '" + setName + "' does not exist."); 1037 } else { 1038 System.out.println(set); 1039 } 1040 } 1041 } 1042 1043 private void processSetDelete(String[] args) throws IOException { 1044 if (args == null || args.length != 3) { 1045 printUsage(); 1046 throw new IOException(INCORRECT_USAGE); 1047 } 1048 super.execute(); 1049 1050 String setName = args[2]; 1051 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1052 boolean result = admin.deleteBackupSet(setName); 1053 if (result) { 1054 System.out.println("Delete set " + setName + " OK."); 1055 } else { 1056 System.out.println("Set " + setName + " does not exist"); 1057 } 1058 } 1059 } 1060 1061 private void processSetRemove(String[] args) throws IOException { 1062 if (args == null || args.length != 4) { 1063 printUsage(); 1064 throw new IOException(INCORRECT_USAGE); 1065 } 1066 super.execute(); 1067 1068 String setName = args[2]; 1069 String[] tables = args[3].split(","); 1070 TableName[] tableNames = toTableNames(tables); 1071 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1072 admin.removeFromBackupSet(setName, tableNames); 1073 } 1074 } 1075 1076 private TableName[] toTableNames(String[] tables) { 1077 TableName[] arr = new TableName[tables.length]; 1078 for (int i = 0; i < tables.length; i++) { 1079 arr[i] = TableName.valueOf(tables[i]); 1080 } 1081 return arr; 1082 } 1083 1084 private void processSetAdd(String[] args) throws IOException { 1085 if (args == null || args.length != 4) { 1086 printUsage(); 1087 throw new IOException(INCORRECT_USAGE); 1088 } 1089 super.execute(); 1090 1091 String setName = args[2]; 1092 String[] tables = args[3].split(","); 1093 TableName[] tableNames = new TableName[tables.length]; 1094 for (int i = 0; i < tables.length; i++) { 1095 tableNames[i] = TableName.valueOf(tables[i]); 1096 } 1097 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1098 admin.addToBackupSet(setName, tableNames); 1099 } 1100 1101 } 1102 1103 private BackupCommand getCommand(String cmdStr) throws IOException { 1104 switch (cmdStr) { 1105 case SET_ADD_CMD: 1106 return BackupCommand.SET_ADD; 1107 case SET_REMOVE_CMD: 1108 return BackupCommand.SET_REMOVE; 1109 case SET_DELETE_CMD: 1110 return BackupCommand.SET_DELETE; 1111 case SET_DESCRIBE_CMD: 1112 return BackupCommand.SET_DESCRIBE; 1113 case SET_LIST_CMD: 1114 return BackupCommand.SET_LIST; 1115 default: 1116 System.out.println("ERROR: Unknown command for 'set' :" + cmdStr); 1117 printUsage(); 1118 throw new IOException(INCORRECT_USAGE); 1119 } 1120 } 1121 1122 @Override 1123 protected void printUsage() { 1124 System.out.println(SET_CMD_USAGE); 1125 } 1126 } 1127}