001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup.impl; 019 020import static org.apache.hadoop.hbase.backup.BackupInfo.withState; 021import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS; 022import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR; 023import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER; 024import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS; 025import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; 026import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; 027import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; 028import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; 029import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; 030import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_ENABLE_CONTINUOUS_BACKUP; 031import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_ENABLE_CONTINUOUS_BACKUP_DESC; 032import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_FORCE_DELETE; 033import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_FORCE_DELETE_DESC; 034import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM; 035import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC; 036import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; 037import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; 038import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; 039import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; 040import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; 041import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; 042import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; 043import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; 044import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC; 045import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; 046import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; 047import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; 048import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; 049import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; 050import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; 051import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; 052import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; 053import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; 054import static org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS; 055import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT; 056 057import java.io.IOException; 058import java.net.URI; 059import java.text.ParseException; 060import java.text.SimpleDateFormat; 061import java.util.ArrayList; 062import java.util.Collections; 063import java.util.HashSet; 064import java.util.List; 065import java.util.Map; 066import java.util.Optional; 067import java.util.Set; 068import java.util.TimeZone; 069import java.util.concurrent.TimeUnit; 070import org.apache.commons.lang3.StringUtils; 071import org.apache.hadoop.conf.Configuration; 072import org.apache.hadoop.conf.Configured; 073import org.apache.hadoop.fs.FileStatus; 074import org.apache.hadoop.fs.FileSystem; 075import org.apache.hadoop.fs.Path; 076import org.apache.hadoop.hbase.HBaseConfiguration; 077import org.apache.hadoop.hbase.TableName; 078import org.apache.hadoop.hbase.backup.BackupAdmin; 079import org.apache.hadoop.hbase.backup.BackupInfo; 080import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; 081import org.apache.hadoop.hbase.backup.BackupRequest; 082import org.apache.hadoop.hbase.backup.BackupRestoreConstants; 083import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; 084import org.apache.hadoop.hbase.backup.BackupType; 085import org.apache.hadoop.hbase.backup.HBackupFileSystem; 086import org.apache.hadoop.hbase.backup.util.BackupFileSystemManager; 087import org.apache.hadoop.hbase.backup.util.BackupSet; 088import org.apache.hadoop.hbase.backup.util.BackupUtils; 089import org.apache.hadoop.hbase.client.Admin; 090import org.apache.hadoop.hbase.client.Connection; 091import org.apache.hadoop.hbase.client.ConnectionFactory; 092import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; 093import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 094import org.apache.hadoop.hbase.util.Pair; 095import org.apache.yetus.audience.InterfaceAudience; 096 097import org.apache.hbase.thirdparty.com.google.common.base.Splitter; 098import org.apache.hbase.thirdparty.com.google.common.base.Strings; 099import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 100import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 101import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; 102import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; 103 104/** 105 * General backup commands, options and usage messages 106 */ 107@InterfaceAudience.Private 108public final class BackupCommands { 109 public final static String INCORRECT_USAGE = "Incorrect usage"; 110 111 public final static String TOP_LEVEL_NOT_ALLOWED = 112 "Top level (root) folder is not allowed to be a backup destination"; 113 114 public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" 115 + "where COMMAND is one of:\n" + " create create a new backup image\n" 116 + " delete delete an existing backup image\n" 117 + " describe show the detailed information of a backup image\n" 118 + " history show history of all successful backups\n" 119 + " progress show the progress of the latest backup request\n" 120 + " set backup set management\n" + " repair repair backup system table\n" 121 + " merge merge backup images\n" 122 + "Run \'hbase backup COMMAND -h\' to see help message for each command\n"; 123 124 public static final String CREATE_CMD_USAGE = 125 "Usage: hbase backup create <type> <backup_path> [options]\n" 126 + " type \"full\" to create a full backup image\n" 127 + " \"incremental\" to create an incremental backup image\n" 128 + " backup_path Full path to store the backup image\n"; 129 130 public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n" 131 + " backup_id Backup image id (optional). If no id specified, the command will show\n" 132 + " progress for currently running backup session."; 133 public static final String NO_INFO_FOUND = "No info was found for backup id: "; 134 public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found."; 135 136 public static final String DESCRIBE_CMD_USAGE = 137 "Usage: hbase backup describe <backup_id>\n" + " backup_id Backup image id\n"; 138 139 public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; 140 141 public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]"; 142 143 public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; 144 145 public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" 146 + " name Backup set name\n" + " tables Comma separated list of tables.\n" 147 + "COMMAND is one of:\n" + " add add tables to a set, create a set if needed\n" 148 + " remove remove tables from a set\n" 149 + " list list all backup sets in the system\n" + " describe describe set\n" 150 + " delete delete backup set\n"; 151 public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n" 152 + " backup_ids Comma separated list of backup image ids.\n"; 153 154 public static final String USAGE_FOOTER = ""; 155 156 public static abstract class Command extends Configured { 157 CommandLine cmdline; 158 Connection conn; 159 160 Command(Configuration conf) { 161 if (conf == null) { 162 conf = HBaseConfiguration.create(); 163 } 164 setConf(conf); 165 } 166 167 public void execute() throws IOException { 168 if (cmdline.hasOption("h") || cmdline.hasOption("help")) { 169 printUsage(); 170 throw new IOException(INCORRECT_USAGE); 171 } 172 173 if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) { 174 String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME); 175 // Set MR job queuename to configuration 176 getConf().set("mapreduce.job.queuename", queueName); 177 } 178 179 // Create connection 180 conn = ConnectionFactory.createConnection(getConf()); 181 if (requiresNoActiveSession()) { 182 // Check active session 183 try (BackupSystemTable table = new BackupSystemTable(conn)) { 184 List<BackupInfo> sessions = 185 table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); 186 187 if (sessions.size() > 0) { 188 System.err.println("Found backup session in a RUNNING state: "); 189 System.err.println(sessions.get(0)); 190 System.err.println("This may indicate that a previous session has failed abnormally."); 191 System.err.println("In this case, backup recovery is recommended."); 192 throw new IOException("Active session found, aborted command execution"); 193 } 194 } 195 } 196 if (requiresConsistentState()) { 197 // Check failed delete 198 try (BackupSystemTable table = new BackupSystemTable(conn)) { 199 String[] ids = table.getListOfBackupIdsFromDeleteOperation(); 200 201 if (ids != null && ids.length > 0) { 202 System.err.println("Found failed backup DELETE coommand. "); 203 System.err.println("Backup system recovery is required."); 204 throw new IOException("Failed backup DELETE found, aborted command execution"); 205 } 206 207 ids = table.getListOfBackupIdsFromMergeOperation(); 208 if (ids != null && ids.length > 0) { 209 System.err.println("Found failed backup MERGE coommand. "); 210 System.err.println("Backup system recovery is required."); 211 throw new IOException("Failed backup MERGE found, aborted command execution"); 212 } 213 } 214 } 215 } 216 217 public void finish() throws IOException { 218 if (conn != null) { 219 conn.close(); 220 } 221 } 222 223 protected abstract void printUsage(); 224 225 /** 226 * The command can't be run if active backup session is in progress 227 * @return true if no active sessions are in progress 228 */ 229 protected boolean requiresNoActiveSession() { 230 return false; 231 } 232 233 /** 234 * Command requires consistent state of a backup system Backup system may become inconsistent 235 * because of an abnormal termination of a backup session or delete command 236 * @return true, if yes 237 */ 238 protected boolean requiresConsistentState() { 239 return false; 240 } 241 } 242 243 private BackupCommands() { 244 throw new AssertionError("Instantiating utility class..."); 245 } 246 247 public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { 248 Command cmd; 249 switch (type) { 250 case CREATE: 251 cmd = new CreateCommand(conf, cmdline); 252 break; 253 case DESCRIBE: 254 cmd = new DescribeCommand(conf, cmdline); 255 break; 256 case PROGRESS: 257 cmd = new ProgressCommand(conf, cmdline); 258 break; 259 case DELETE: 260 cmd = new DeleteCommand(conf, cmdline); 261 break; 262 case HISTORY: 263 cmd = new HistoryCommand(conf, cmdline); 264 break; 265 case SET: 266 cmd = new BackupSetCommand(conf, cmdline); 267 break; 268 case REPAIR: 269 cmd = new RepairCommand(conf, cmdline); 270 break; 271 case MERGE: 272 cmd = new MergeCommand(conf, cmdline); 273 break; 274 case HELP: 275 default: 276 cmd = new HelpCommand(conf, cmdline); 277 break; 278 } 279 return cmd; 280 } 281 282 static int numOfArgs(String[] args) { 283 if (args == null) { 284 return 0; 285 } 286 287 return args.length; 288 } 289 290 public static class CreateCommand extends Command { 291 CreateCommand(Configuration conf, CommandLine cmdline) { 292 super(conf); 293 this.cmdline = cmdline; 294 } 295 296 @Override 297 protected boolean requiresNoActiveSession() { 298 return true; 299 } 300 301 @Override 302 protected boolean requiresConsistentState() { 303 return true; 304 } 305 306 @Override 307 public void execute() throws IOException { 308 if (cmdline == null || cmdline.getArgs() == null) { 309 printUsage(); 310 throw new IOException(INCORRECT_USAGE); 311 } 312 String[] args = cmdline.getArgs(); 313 if (args.length != 3) { 314 printUsage(); 315 throw new IOException(INCORRECT_USAGE); 316 } 317 318 if ( 319 !BackupType.FULL.toString().equalsIgnoreCase(args[1]) 320 && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1]) 321 ) { 322 System.out.println("ERROR: invalid backup type: " + args[1]); 323 printUsage(); 324 throw new IOException(INCORRECT_USAGE); 325 } 326 if (!verifyPath(args[2])) { 327 System.out.println("ERROR: invalid backup destination: " + args[2]); 328 printUsage(); 329 throw new IOException(INCORRECT_USAGE); 330 } 331 String targetBackupDir = args[2]; 332 // Check if backup destination is top level (root) folder - not allowed 333 if (isRootFolder(targetBackupDir)) { 334 throw new IOException(TOP_LEVEL_NOT_ALLOWED); 335 } 336 String tables; 337 338 // Check if we have both: backup set and list of tables 339 if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) { 340 System.out 341 .println("ERROR: You can specify either backup set or list" + " of tables, but not both"); 342 printUsage(); 343 throw new IOException(INCORRECT_USAGE); 344 } 345 // Creates connection 346 super.execute(); 347 // Check backup set 348 String setName = null; 349 if (cmdline.hasOption(OPTION_SET)) { 350 setName = cmdline.getOptionValue(OPTION_SET); 351 tables = getTablesForSet(setName); 352 353 if (tables == null) { 354 System.out 355 .println("ERROR: Backup set '" + setName + "' is either empty or does not exist"); 356 printUsage(); 357 throw new IOException(INCORRECT_USAGE); 358 } 359 } else { 360 tables = cmdline.getOptionValue(OPTION_TABLE); 361 } 362 int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH) 363 ? Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH)) 364 : -1; 365 int workers = cmdline.hasOption(OPTION_WORKERS) 366 ? Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS)) 367 : -1; 368 369 boolean ignoreChecksum = cmdline.hasOption(OPTION_IGNORECHECKSUM); 370 371 BackupType backupType = BackupType.valueOf(args[1].toUpperCase()); 372 List<TableName> tableNameList = null; 373 if (tables != null) { 374 tableNameList = Lists.newArrayList(BackupUtils.parseTableNames(tables)); 375 } 376 boolean continuousBackup = cmdline.hasOption(OPTION_ENABLE_CONTINUOUS_BACKUP); 377 if (continuousBackup && !BackupType.FULL.equals(backupType)) { 378 System.out.println("ERROR: Continuous backup can Only be specified for Full Backup"); 379 printUsage(); 380 throw new IOException(INCORRECT_USAGE); 381 } 382 383 /* 384 * The `continuousBackup` flag is specified only during the first full backup to initiate 385 * continuous WAL replication. After that, it is redundant because the tables are already set 386 * up for continuous backup. If the `continuousBackup` flag is not explicitly enabled, we need 387 * to determine the backup mode based on the current state of the specified tables: - If all 388 * the specified tables are already part of continuous backup, we treat the request as a 389 * continuous backup request and proceed accordingly (since these tables are already 390 * continuously backed up, no additional setup is needed). - If none of the specified tables 391 * are part of continuous backup, we treat the request as a normal full backup without 392 * continuous backup. - If the request includes a mix of tables—some with continuous backup 393 * enabled and others without—we cannot determine a clear backup strategy. In this case, we 394 * throw an error. If all tables are already in continuous backup mode, we explicitly set the 395 * `continuousBackup` flag to `true` so that the request is processed using the continuous 396 * backup approach rather than the normal full backup flow. 397 */ 398 if (!continuousBackup && tableNameList != null && !tableNameList.isEmpty()) { 399 try (BackupSystemTable backupSystemTable = new BackupSystemTable(conn)) { 400 Set<TableName> continuousBackupTableSet = 401 backupSystemTable.getContinuousBackupTableSet().keySet(); 402 403 boolean allTablesInContinuousBackup = continuousBackupTableSet.containsAll(tableNameList); 404 boolean noTablesInContinuousBackup = 405 tableNameList.stream().noneMatch(continuousBackupTableSet::contains); 406 407 // Ensure that all tables are either fully in continuous backup or not at all 408 if (!allTablesInContinuousBackup && !noTablesInContinuousBackup) { 409 System.err 410 .println("ERROR: Some tables are already in continuous backup, while others are not. " 411 + "Cannot mix both in a single request."); 412 printUsage(); 413 throw new IOException(INCORRECT_USAGE); 414 } 415 416 // If all tables are already in continuous backup, enable the flag 417 if (allTablesInContinuousBackup) { 418 continuousBackup = true; 419 } 420 } 421 } 422 423 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 424 BackupRequest.Builder builder = new BackupRequest.Builder(); 425 BackupRequest request = builder.withBackupType(backupType).withTableList(tableNameList) 426 .withTargetRootDir(targetBackupDir).withTotalTasks(workers) 427 .withBandwidthPerTasks(bandwidth).withNoChecksumVerify(ignoreChecksum) 428 .withBackupSetName(setName).withContinuousBackupEnabled(continuousBackup).build(); 429 String backupId = admin.backupTables(request); 430 System.out.println("Backup session " + backupId + " finished. Status: SUCCESS"); 431 } catch (IOException e) { 432 System.out.println("Backup session finished. Status: FAILURE"); 433 throw e; 434 } 435 } 436 437 private boolean isRootFolder(String targetBackupDir) { 438 Path p = new Path(targetBackupDir); 439 return p.isRoot(); 440 } 441 442 private boolean verifyPath(String path) { 443 try { 444 Path p = new Path(path); 445 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 446 URI uri = p.toUri(); 447 448 if (uri.getScheme() == null) { 449 return false; 450 } 451 452 FileSystem.get(uri, conf); 453 return true; 454 } catch (Exception e) { 455 return false; 456 } 457 } 458 459 private String getTablesForSet(String name) throws IOException { 460 try (final BackupSystemTable table = new BackupSystemTable(conn)) { 461 List<TableName> tables = table.describeBackupSet(name); 462 463 if (tables == null) { 464 return null; 465 } 466 467 return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); 468 } 469 } 470 471 @Override 472 protected void printUsage() { 473 System.out.println(CREATE_CMD_USAGE); 474 Options options = new Options(); 475 options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC); 476 options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); 477 options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC); 478 options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC); 479 options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC); 480 options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC); 481 options.addOption(OPTION_IGNORECHECKSUM, false, OPTION_IGNORECHECKSUM_DESC); 482 options.addOption(OPTION_ENABLE_CONTINUOUS_BACKUP, false, 483 OPTION_ENABLE_CONTINUOUS_BACKUP_DESC); 484 485 HelpFormatter helpFormatter = new HelpFormatter(); 486 helpFormatter.setLeftPadding(2); 487 helpFormatter.setDescPadding(8); 488 helpFormatter.setWidth(100); 489 helpFormatter.setSyntaxPrefix("Options:"); 490 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 491 } 492 } 493 494 public static class HelpCommand extends Command { 495 HelpCommand(Configuration conf, CommandLine cmdline) { 496 super(conf); 497 this.cmdline = cmdline; 498 } 499 500 @Override 501 public void execute() throws IOException { 502 if (cmdline == null) { 503 printUsage(); 504 throw new IOException(INCORRECT_USAGE); 505 } 506 507 String[] args = cmdline.getArgs(); 508 if (args == null || args.length == 0) { 509 printUsage(); 510 throw new IOException(INCORRECT_USAGE); 511 } 512 513 if (args.length != 2) { 514 System.out.println("ERROR: Only supports help message of a single command type"); 515 printUsage(); 516 throw new IOException(INCORRECT_USAGE); 517 } 518 519 String type = args[1]; 520 521 if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { 522 System.out.println(CREATE_CMD_USAGE); 523 } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { 524 System.out.println(DESCRIBE_CMD_USAGE); 525 } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { 526 System.out.println(HISTORY_CMD_USAGE); 527 } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { 528 System.out.println(PROGRESS_CMD_USAGE); 529 } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { 530 System.out.println(DELETE_CMD_USAGE); 531 } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { 532 System.out.println(SET_CMD_USAGE); 533 } else { 534 System.out.println("Unknown command : " + type); 535 printUsage(); 536 } 537 } 538 539 @Override 540 protected void printUsage() { 541 System.out.println(USAGE); 542 } 543 } 544 545 public static class DescribeCommand extends Command { 546 DescribeCommand(Configuration conf, CommandLine cmdline) { 547 super(conf); 548 this.cmdline = cmdline; 549 } 550 551 @Override 552 public void execute() throws IOException { 553 if (cmdline == null || cmdline.getArgs() == null) { 554 printUsage(); 555 throw new IOException(INCORRECT_USAGE); 556 } 557 String[] args = cmdline.getArgs(); 558 if (args.length != 2) { 559 printUsage(); 560 throw new IOException(INCORRECT_USAGE); 561 } 562 563 super.execute(); 564 565 String backupId = args[1]; 566 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 567 BackupInfo info = sysTable.readBackupInfo(backupId); 568 if (info == null) { 569 System.out.println("ERROR: " + backupId + " does not exist"); 570 printUsage(); 571 throw new IOException(INCORRECT_USAGE); 572 } 573 System.out.println(info.getShortDescription()); 574 } 575 } 576 577 @Override 578 protected void printUsage() { 579 System.out.println(DESCRIBE_CMD_USAGE); 580 } 581 } 582 583 public static class ProgressCommand extends Command { 584 ProgressCommand(Configuration conf, CommandLine cmdline) { 585 super(conf); 586 this.cmdline = cmdline; 587 } 588 589 @Override 590 public void execute() throws IOException { 591 592 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) { 593 System.out.println( 594 "No backup id was specified, " + "will retrieve the most recent (ongoing) session"); 595 } 596 String[] args = cmdline == null ? null : cmdline.getArgs(); 597 if (args != null && args.length > 2) { 598 System.err.println("ERROR: wrong number of arguments: " + args.length); 599 printUsage(); 600 throw new IOException(INCORRECT_USAGE); 601 } 602 603 super.execute(); 604 605 String backupId = (args == null || args.length <= 1) ? null : args[1]; 606 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 607 BackupInfo info = null; 608 609 if (backupId != null) { 610 info = sysTable.readBackupInfo(backupId); 611 } else { 612 List<BackupInfo> infos = 613 sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); 614 if (infos != null && infos.size() > 0) { 615 info = infos.get(0); 616 backupId = info.getBackupId(); 617 System.out.println("Found ongoing session with backupId=" + backupId); 618 } 619 } 620 int progress = info == null ? -1 : info.getProgress(); 621 if (progress < 0) { 622 if (backupId != null) { 623 System.out.println(NO_INFO_FOUND + backupId); 624 } else { 625 System.err.println(NO_ACTIVE_SESSION_FOUND); 626 } 627 } else { 628 System.out.println(backupId + " progress=" + progress + "%"); 629 } 630 } 631 } 632 633 @Override 634 protected void printUsage() { 635 System.out.println(PROGRESS_CMD_USAGE); 636 } 637 } 638 639 public static class DeleteCommand extends Command { 640 DeleteCommand(Configuration conf, CommandLine cmdline) { 641 super(conf); 642 this.cmdline = cmdline; 643 } 644 645 @Override 646 protected boolean requiresNoActiveSession() { 647 return true; 648 } 649 650 @Override 651 public void execute() throws IOException { 652 653 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) { 654 printUsage(); 655 throw new IOException(INCORRECT_USAGE); 656 } 657 658 if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) { 659 printUsage(); 660 throw new IOException(INCORRECT_USAGE); 661 } 662 663 boolean isForceDelete = cmdline.hasOption(OPTION_FORCE_DELETE); 664 super.execute(); 665 if (cmdline.hasOption(OPTION_KEEP)) { 666 executeDeleteOlderThan(cmdline, isForceDelete); 667 } else if (cmdline.hasOption(OPTION_LIST)) { 668 executeDeleteListOfBackups(cmdline, isForceDelete); 669 } 670 671 cleanUpUnusedBackupWALs(); 672 } 673 674 private void executeDeleteOlderThan(CommandLine cmdline, boolean isForceDelete) 675 throws IOException { 676 String value = cmdline.getOptionValue(OPTION_KEEP); 677 int days = 0; 678 try { 679 days = Integer.parseInt(value); 680 } catch (NumberFormatException e) { 681 throw new IOException(value + " is not an integer number"); 682 } 683 final long fdays = days; 684 BackupInfo.Filter dateFilter = info -> { 685 long currentTime = EnvironmentEdgeManager.currentTime(); 686 long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; 687 return info.getCompleteTs() <= maxTsToDelete; 688 }; 689 List<BackupInfo> history = null; 690 try (final BackupSystemTable sysTable = new BackupSystemTable(conn); 691 BackupAdminImpl admin = new BackupAdminImpl(conn)) { 692 history = sysTable.getBackupHistory(dateFilter); 693 String[] backupIds = convertToBackupIds(history); 694 validatePITRBackupDeletion(backupIds, isForceDelete); 695 int deleted = admin.deleteBackups(backupIds); 696 System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " 697 + backupIds.length); 698 } catch (IOException e) { 699 System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " 700 + "system integrity"); 701 throw e; 702 } 703 } 704 705 private String[] convertToBackupIds(List<BackupInfo> history) { 706 String[] ids = new String[history.size()]; 707 for (int i = 0; i < ids.length; i++) { 708 ids[i] = history.get(i).getBackupId(); 709 } 710 return ids; 711 } 712 713 private void executeDeleteListOfBackups(CommandLine cmdline, boolean isForceDelete) 714 throws IOException { 715 String value = cmdline.getOptionValue(OPTION_LIST); 716 String[] backupIds = value.split(","); 717 validatePITRBackupDeletion(backupIds, isForceDelete); 718 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 719 int deleted = admin.deleteBackups(backupIds); 720 System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length); 721 } catch (IOException e) { 722 System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " 723 + "system integrity"); 724 throw e; 725 } 726 727 } 728 729 /** 730 * Validates whether the specified backups can be deleted while preserving Point-In-Time 731 * Recovery (PITR) capabilities. If a backup is the only remaining full backup enabling PITR for 732 * certain tables, deletion is prevented unless forced. 733 * @param backupIds Array of backup IDs to validate. 734 * @param isForceDelete Flag indicating whether deletion should proceed regardless of PITR 735 * constraints. 736 * @throws IOException If a backup is essential for PITR and force deletion is not enabled. 737 */ 738 private void validatePITRBackupDeletion(String[] backupIds, boolean isForceDelete) 739 throws IOException { 740 if (!isForceDelete) { 741 for (String backupId : backupIds) { 742 List<TableName> affectedTables = getTablesDependentOnBackupForPITR(backupId); 743 if (!affectedTables.isEmpty()) { 744 String errMsg = String.format( 745 "Backup %s is the only FULL backup remaining that enables PITR for tables: %s. " 746 + "Use the force option to delete it anyway.", 747 backupId, affectedTables); 748 System.err.println(errMsg); 749 throw new IOException(errMsg); 750 } 751 } 752 } 753 } 754 755 /** 756 * Identifies tables that rely on the specified backup for PITR (Point-In-Time Recovery). A 757 * table is considered dependent on the backup if it does not have any other valid full backups 758 * that can cover the PITR window enabled by the specified backup. 759 * @param backupId The ID of the backup being evaluated for PITR coverage. 760 * @return A list of tables that are dependent on the specified backup for PITR recovery. 761 * @throws IOException If there is an error retrieving the backup metadata or backup system 762 * table. 763 */ 764 private List<TableName> getTablesDependentOnBackupForPITR(String backupId) throws IOException { 765 List<TableName> dependentTables = new ArrayList<>(); 766 767 try (final BackupSystemTable backupSystemTable = new BackupSystemTable(conn)) { 768 // Fetch the target backup's info using the backup ID 769 BackupInfo targetBackup = backupSystemTable.readBackupInfo(backupId); 770 if (targetBackup == null) { 771 throw new IOException("Backup info not found for backupId: " + backupId); 772 } 773 774 // Only full backups are mandatory for PITR 775 if (!BackupType.FULL.equals(targetBackup.getType())) { 776 return List.of(); 777 } 778 779 // Retrieve the tables with continuous backup enabled along with their start times 780 Map<TableName, Long> continuousBackupStartTimes = 781 backupSystemTable.getContinuousBackupTableSet(); 782 783 // Calculate the PITR window by fetching configuration and current time 784 long pitrWindowDays = getConf().getLong(CONF_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS, 785 DEFAULT_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS); 786 long currentTime = EnvironmentEdgeManager.getDelegate().currentTime(); 787 final long maxAllowedPITRTime = currentTime - TimeUnit.DAYS.toMillis(pitrWindowDays); 788 789 // Check each table associated with the target backup 790 for (TableName table : targetBackup.getTableNames()) { 791 // Skip tables without continuous backup enabled 792 if (!continuousBackupStartTimes.containsKey(table)) { 793 continue; 794 } 795 796 // Calculate the PITR window this backup covers for the table 797 Optional<Pair<Long, Long>> coveredPitrWindow = getCoveredPitrWindowForTable(targetBackup, 798 continuousBackupStartTimes.get(table), maxAllowedPITRTime, currentTime); 799 800 // If this backup does not cover a valid PITR window for the table, skip 801 if (coveredPitrWindow.isEmpty()) { 802 continue; 803 } 804 805 // Check if there is any other valid backup that can cover the PITR window 806 List<BackupInfo> allBackups = 807 backupSystemTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE)); 808 boolean hasAnotherValidBackup = 809 canAnyOtherBackupCover(allBackups, targetBackup, table, coveredPitrWindow.get(), 810 continuousBackupStartTimes.get(table), maxAllowedPITRTime, currentTime); 811 812 // If no other valid backup exists, add the table to the dependent list 813 if (!hasAnotherValidBackup) { 814 dependentTables.add(table); 815 } 816 } 817 } 818 819 return dependentTables; 820 } 821 822 /** 823 * Calculates the PITR (Point-In-Time Recovery) window that the given backup enables for a 824 * table. 825 * @param backupInfo Metadata of the backup being evaluated. 826 * @param continuousBackupStartTime When continuous backups started for the table. 827 * @param maxAllowedPITRTime The earliest timestamp from which PITR is supported in the 828 * cluster. 829 * @param currentTime Current time. 830 * @return Optional PITR window as a pair (start, end), or empty if backup is not useful for 831 * PITR. 832 */ 833 private Optional<Pair<Long, Long>> getCoveredPitrWindowForTable(BackupInfo backupInfo, 834 long continuousBackupStartTime, long maxAllowedPITRTime, long currentTime) { 835 836 long backupStartTs = backupInfo.getStartTs(); 837 long backupEndTs = backupInfo.getCompleteTs(); 838 long effectiveStart = Math.max(continuousBackupStartTime, maxAllowedPITRTime); 839 840 if (backupStartTs < continuousBackupStartTime) { 841 return Optional.empty(); 842 } 843 844 return Optional.of(Pair.newPair(Math.max(backupEndTs, effectiveStart), currentTime)); 845 } 846 847 /** 848 * Checks if any backup (excluding the current backup) can cover the specified PITR window for 849 * the given table. A backup can cover the PITR window if it fully encompasses the target time 850 * range specified. 851 * @param allBackups List of all backups available. 852 * @param currentBackup The current backup that should not be considered for 853 * coverage. 854 * @param table The table for which we need to check backup coverage. 855 * @param targetWindow A pair representing the target PITR window (start and end 856 * times). 857 * @param continuousBackupStartTime When continuous backups started for the table. 858 * @param maxAllowedPITRTime The earliest timestamp from which PITR is supported in the 859 * cluster. 860 * @param currentTime Current time. 861 * @return {@code true} if any backup (excluding the current one) fully covers the target PITR 862 * window; {@code false} otherwise. 863 */ 864 private boolean canAnyOtherBackupCover(List<BackupInfo> allBackups, BackupInfo currentBackup, 865 TableName table, Pair<Long, Long> targetWindow, long continuousBackupStartTime, 866 long maxAllowedPITRTime, long currentTime) { 867 868 long targetStart = targetWindow.getFirst(); 869 long targetEnd = targetWindow.getSecond(); 870 871 // Iterate through all backups (including the current one) 872 for (BackupInfo backup : allBackups) { 873 // Skip if the backup is not full or doesn't contain the table 874 if (!BackupType.FULL.equals(backup.getType())) continue; 875 if (!backup.getTableNames().contains(table)) continue; 876 877 // Skip the current backup itself 878 if (backup.equals(currentBackup)) continue; 879 880 // Get the covered PITR window for this backup 881 Optional<Pair<Long, Long>> coveredWindow = getCoveredPitrWindowForTable(backup, 882 continuousBackupStartTime, maxAllowedPITRTime, currentTime); 883 884 if (coveredWindow.isPresent()) { 885 Pair<Long, Long> covered = coveredWindow.get(); 886 887 // The backup must fully cover the target window 888 if (covered.getFirst() <= targetStart && covered.getSecond() >= targetEnd) { 889 return true; 890 } 891 } 892 } 893 894 return false; 895 } 896 897 /** 898 * Cleans up Write-Ahead Logs (WALs) that are no longer required for PITR after a successful 899 * backup deletion. If no full backups are present, all WALs are deleted, tables are removed 900 * from continuous backup metadata, and the associated replication peer is disabled. 901 */ 902 private void cleanUpUnusedBackupWALs() throws IOException { 903 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 904 String backupWalDir = conf.get(CONF_CONTINUOUS_BACKUP_WAL_DIR); 905 906 if (Strings.isNullOrEmpty(backupWalDir)) { 907 System.out.println("No WAL directory specified for continuous backup. Skipping cleanup."); 908 return; 909 } 910 911 try (Admin admin = conn.getAdmin(); 912 BackupSystemTable sysTable = new BackupSystemTable(conn)) { 913 // Get list of tables under continuous backup 914 Map<TableName, Long> continuousBackupTables = sysTable.getContinuousBackupTableSet(); 915 if (continuousBackupTables.isEmpty()) { 916 System.out.println("No continuous backups configured. Skipping WAL cleanup."); 917 return; 918 } 919 920 // Find the earliest timestamp after which WALs are still needed 921 long cutoffTimestamp = determineWALCleanupCutoffTime(sysTable); 922 if (cutoffTimestamp == 0) { 923 // No full backup exists. PITR cannot function without a base full backup. 924 // Clean up all WALs, remove tables from backup metadata, and disable the replication 925 // peer. 926 System.out 927 .println("No full backups found. Cleaning up all WALs and disabling replication peer."); 928 929 disableContinuousBackupReplicationPeer(admin); 930 removeAllTablesFromContinuousBackup(sysTable); 931 deleteAllBackupWALFiles(conf, backupWalDir); 932 return; 933 } 934 935 // Update metadata before actual cleanup to avoid inconsistencies 936 updateBackupTableStartTimes(sysTable, cutoffTimestamp); 937 938 // Delete WAL files older than cutoff timestamp 939 deleteOldWALFiles(conf, backupWalDir, cutoffTimestamp); 940 941 } 942 } 943 944 /** 945 * Determines the cutoff time for cleaning WAL files. 946 * @param sysTable Backup system table 947 * @return cutoff timestamp or 0 if not found 948 */ 949 long determineWALCleanupCutoffTime(BackupSystemTable sysTable) throws IOException { 950 List<BackupInfo> backupInfos = 951 sysTable.getBackupHistory(withState(BackupInfo.BackupState.COMPLETE)); 952 Collections.reverse(backupInfos); // Start from oldest 953 954 for (BackupInfo backupInfo : backupInfos) { 955 if (BackupType.FULL.equals(backupInfo.getType())) { 956 return backupInfo.getStartTs(); 957 } 958 } 959 return 0; 960 } 961 962 private void disableContinuousBackupReplicationPeer(Admin admin) throws IOException { 963 for (ReplicationPeerDescription peer : admin.listReplicationPeers()) { 964 if (peer.getPeerId().equals(CONTINUOUS_BACKUP_REPLICATION_PEER) && peer.isEnabled()) { 965 admin.disableReplicationPeer(CONTINUOUS_BACKUP_REPLICATION_PEER); 966 System.out.println("Disabled replication peer: " + CONTINUOUS_BACKUP_REPLICATION_PEER); 967 break; 968 } 969 } 970 } 971 972 /** 973 * Updates the start time for continuous backups if older than cutoff timestamp. 974 * @param sysTable Backup system table 975 * @param cutoffTimestamp Timestamp before which WALs are no longer needed 976 */ 977 void updateBackupTableStartTimes(BackupSystemTable sysTable, long cutoffTimestamp) 978 throws IOException { 979 980 Map<TableName, Long> backupTables = sysTable.getContinuousBackupTableSet(); 981 Set<TableName> tablesToUpdate = new HashSet<>(); 982 983 for (Map.Entry<TableName, Long> entry : backupTables.entrySet()) { 984 if (entry.getValue() < cutoffTimestamp) { 985 tablesToUpdate.add(entry.getKey()); 986 } 987 } 988 989 if (!tablesToUpdate.isEmpty()) { 990 sysTable.updateContinuousBackupTableSet(tablesToUpdate, cutoffTimestamp); 991 } 992 } 993 994 private void removeAllTablesFromContinuousBackup(BackupSystemTable sysTable) 995 throws IOException { 996 Map<TableName, Long> allTables = sysTable.getContinuousBackupTableSet(); 997 if (!allTables.isEmpty()) { 998 sysTable.removeContinuousBackupTableSet(allTables.keySet()); 999 System.out.println("Removed all tables from continuous backup metadata."); 1000 } 1001 } 1002 1003 private void deleteAllBackupWALFiles(Configuration conf, String backupWalDir) 1004 throws IOException { 1005 try { 1006 BackupFileSystemManager manager = 1007 new BackupFileSystemManager(CONTINUOUS_BACKUP_REPLICATION_PEER, conf, backupWalDir); 1008 FileSystem fs = manager.getBackupFs(); 1009 Path walDir = manager.getWalsDir(); 1010 Path bulkloadDir = manager.getBulkLoadFilesDir(); 1011 1012 // Delete contents under WAL directory 1013 if (fs.exists(walDir)) { 1014 FileStatus[] walContents = fs.listStatus(walDir); 1015 for (FileStatus item : walContents) { 1016 fs.delete(item.getPath(), true); // recursive delete of each child 1017 } 1018 System.out.println("Deleted all contents under WAL directory: " + walDir); 1019 } 1020 1021 // Delete contents under bulk load directory 1022 if (fs.exists(bulkloadDir)) { 1023 FileStatus[] bulkContents = fs.listStatus(bulkloadDir); 1024 for (FileStatus item : bulkContents) { 1025 fs.delete(item.getPath(), true); // recursive delete of each child 1026 } 1027 System.out.println("Deleted all contents under Bulk Load directory: " + bulkloadDir); 1028 } 1029 1030 } catch (IOException e) { 1031 System.out.println("WARNING: Failed to delete contents under backup directories: " 1032 + backupWalDir + ". Error: " + e.getMessage()); 1033 throw e; 1034 } 1035 } 1036 1037 /** 1038 * Cleans up old WAL and bulk-loaded files based on the determined cutoff timestamp. 1039 */ 1040 void deleteOldWALFiles(Configuration conf, String backupWalDir, long cutoffTime) 1041 throws IOException { 1042 System.out.println("Starting WAL cleanup in backup directory: " + backupWalDir 1043 + " with cutoff time: " + cutoffTime); 1044 1045 BackupFileSystemManager manager = 1046 new BackupFileSystemManager(CONTINUOUS_BACKUP_REPLICATION_PEER, conf, backupWalDir); 1047 FileSystem fs = manager.getBackupFs(); 1048 Path walDir = manager.getWalsDir(); 1049 Path bulkloadDir = manager.getBulkLoadFilesDir(); 1050 1051 SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); 1052 dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); 1053 1054 System.out.println("Listing directories under: " + walDir); 1055 1056 FileStatus[] directories = fs.listStatus(walDir); 1057 1058 for (FileStatus dirStatus : directories) { 1059 if (!dirStatus.isDirectory()) { 1060 continue; // Skip files, we only want directories 1061 } 1062 1063 Path dirPath = dirStatus.getPath(); 1064 String dirName = dirPath.getName(); 1065 1066 try { 1067 long dayStart = parseDayDirectory(dirName, dateFormat); 1068 System.out 1069 .println("Checking WAL directory: " + dirName + " (Start Time: " + dayStart + ")"); 1070 1071 // If WAL files of that day are older than cutoff time, delete them 1072 if (dayStart + ONE_DAY_IN_MILLISECONDS - 1 < cutoffTime) { 1073 System.out.println("Deleting outdated WAL directory: " + dirPath); 1074 fs.delete(dirPath, true); 1075 Path bulkloadPath = new Path(bulkloadDir, dirName); 1076 System.out.println("Deleting corresponding bulk-load directory: " + bulkloadPath); 1077 fs.delete(bulkloadPath, true); 1078 } 1079 } catch (ParseException e) { 1080 System.out.println("WARNING: Failed to parse directory name '" + dirName 1081 + "'. Skipping. Error: " + e.getMessage()); 1082 } catch (IOException e) { 1083 System.err.println("WARNING: Failed to delete directory '" + dirPath 1084 + "'. Skipping. Error: " + e.getMessage()); 1085 } 1086 } 1087 1088 System.out.println("Completed WAL cleanup for backup directory: " + backupWalDir); 1089 } 1090 1091 private long parseDayDirectory(String dayDir, SimpleDateFormat dateFormat) 1092 throws ParseException { 1093 return dateFormat.parse(dayDir).getTime(); 1094 } 1095 1096 @Override 1097 protected void printUsage() { 1098 System.out.println(DELETE_CMD_USAGE); 1099 Options options = new Options(); 1100 options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC); 1101 options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC); 1102 options.addOption(OPTION_FORCE_DELETE, false, OPTION_FORCE_DELETE_DESC); 1103 1104 HelpFormatter helpFormatter = new HelpFormatter(); 1105 helpFormatter.setLeftPadding(2); 1106 helpFormatter.setDescPadding(8); 1107 helpFormatter.setWidth(100); 1108 helpFormatter.setSyntaxPrefix("Options:"); 1109 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 1110 1111 } 1112 } 1113 1114 public static class RepairCommand extends Command { 1115 RepairCommand(Configuration conf, CommandLine cmdline) { 1116 super(conf); 1117 this.cmdline = cmdline; 1118 } 1119 1120 @Override 1121 public void execute() throws IOException { 1122 super.execute(); 1123 1124 String[] args = cmdline == null ? null : cmdline.getArgs(); 1125 if (args != null && args.length > 1) { 1126 System.err.println("ERROR: wrong number of arguments: " + args.length); 1127 printUsage(); 1128 throw new IOException(INCORRECT_USAGE); 1129 } 1130 1131 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 1132 try (final Connection conn = ConnectionFactory.createConnection(conf); 1133 final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 1134 // Failed backup 1135 BackupInfo backupInfo; 1136 List<BackupInfo> list = 1137 sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); 1138 if (list.size() == 0) { 1139 // No failed sessions found 1140 System.out.println("REPAIR status: no failed sessions found." 1141 + " Checking failed delete backup operation ..."); 1142 repairFailedBackupDeletionIfAny(conn, sysTable); 1143 repairFailedBackupMergeIfAny(conn, sysTable); 1144 return; 1145 } 1146 backupInfo = list.get(0); 1147 // If this is a cancel exception, then we've already cleaned. 1148 // set the failure timestamp of the overall backup 1149 backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); 1150 // set failure message 1151 backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo); 1152 // set overall backup status: failed 1153 backupInfo.setState(BackupState.FAILED); 1154 // compose the backup failed data 1155 String backupFailedData = "BackupId=" + backupInfo.getBackupId() + ",startts=" 1156 + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" 1157 + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); 1158 System.out.println(backupFailedData); 1159 TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf); 1160 // If backup session is updated to FAILED state - means we 1161 // processed recovery already. 1162 sysTable.updateBackupInfo(backupInfo); 1163 sysTable.finishBackupExclusiveOperation(); 1164 System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo); 1165 } 1166 } 1167 1168 private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable) 1169 throws IOException { 1170 String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation(); 1171 if (backupIds == null || backupIds.length == 0) { 1172 System.out.println("No failed backup DELETE operation found"); 1173 // Delete backup table snapshot if exists 1174 BackupSystemTable.deleteSnapshot(conn); 1175 return; 1176 } 1177 System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds)); 1178 System.out.println("Running DELETE again ..."); 1179 // Restore table from snapshot 1180 BackupSystemTable.restoreFromSnapshot(conn); 1181 // Finish previous failed session 1182 sysTable.finishBackupExclusiveOperation(); 1183 try (BackupAdmin admin = new BackupAdminImpl(conn)) { 1184 admin.deleteBackups(backupIds); 1185 } 1186 System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds)); 1187 } 1188 1189 public static void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable) 1190 throws IOException { 1191 1192 String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation(); 1193 if (backupIds == null || backupIds.length == 0) { 1194 System.out.println("No failed backup MERGE operation found"); 1195 // Delete backup table snapshot if exists 1196 BackupSystemTable.deleteSnapshot(conn); 1197 return; 1198 } 1199 System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds)); 1200 // Check if backup .tmp exists 1201 BackupInfo bInfo = sysTable.readBackupInfo(backupIds[0]); 1202 String backupRoot = bInfo.getBackupRootDir(); 1203 FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration()); 1204 String backupId = BackupUtils.findMostRecentBackupId(backupIds); 1205 Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId); 1206 if (fs.exists(tmpPath)) { 1207 // Move data back 1208 Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId); 1209 if (!fs.delete(destPath, true)) { 1210 System.out.println("Failed to delete " + destPath); 1211 } 1212 boolean res = fs.rename(tmpPath, destPath); 1213 if (!res) { 1214 throw new IOException( 1215 "MERGE repair: failed to rename from " + tmpPath + " to " + destPath); 1216 } 1217 System.out 1218 .println("MERGE repair: renamed from " + tmpPath + " to " + destPath + " res=" + res); 1219 } else { 1220 checkRemoveBackupImages(fs, backupRoot, backupIds); 1221 } 1222 // Restore table from snapshot 1223 BackupSystemTable.restoreFromSnapshot(conn); 1224 // Unlock backup system 1225 sysTable.finishBackupExclusiveOperation(); 1226 // Finish previous failed session 1227 sysTable.finishMergeOperation(); 1228 1229 System.out.println("MERGE repair operation finished OK: " + StringUtils.join(backupIds)); 1230 } 1231 1232 private static void checkRemoveBackupImages(FileSystem fs, String backupRoot, 1233 String[] backupIds) throws IOException { 1234 String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds); 1235 for (String backupId : backupIds) { 1236 if (backupId.equals(mergedBackupId)) { 1237 continue; 1238 } 1239 Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId); 1240 if (fs.exists(path)) { 1241 if (!fs.delete(path, true)) { 1242 System.out.println("MERGE repair removing: " + path + " - FAILED"); 1243 } else { 1244 System.out.println("MERGE repair removing: " + path + " - OK"); 1245 } 1246 } 1247 } 1248 } 1249 1250 @Override 1251 protected void printUsage() { 1252 System.out.println(REPAIR_CMD_USAGE); 1253 } 1254 } 1255 1256 public static class MergeCommand extends Command { 1257 MergeCommand(Configuration conf, CommandLine cmdline) { 1258 super(conf); 1259 this.cmdline = cmdline; 1260 } 1261 1262 @Override 1263 protected boolean requiresNoActiveSession() { 1264 return true; 1265 } 1266 1267 @Override 1268 protected boolean requiresConsistentState() { 1269 return true; 1270 } 1271 1272 @Override 1273 public void execute() throws IOException { 1274 super.execute(); 1275 1276 String[] args = cmdline == null ? null : cmdline.getArgs(); 1277 if (args == null || (args.length != 2)) { 1278 System.err 1279 .println("ERROR: wrong number of arguments: " + (args == null ? null : args.length)); 1280 printUsage(); 1281 throw new IOException(INCORRECT_USAGE); 1282 } 1283 1284 String[] backupIds = args[1].split(","); 1285 if (backupIds.length < 2) { 1286 String msg = "ERROR: can not merge a single backup image. " 1287 + "Number of images must be greater than 1."; 1288 System.err.println(msg); 1289 throw new IOException(msg); 1290 1291 } 1292 Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); 1293 try (final Connection conn = ConnectionFactory.createConnection(conf); 1294 final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1295 admin.mergeBackups(backupIds); 1296 } 1297 } 1298 1299 @Override 1300 protected void printUsage() { 1301 System.out.println(MERGE_CMD_USAGE); 1302 } 1303 } 1304 1305 public static class HistoryCommand extends Command { 1306 private final static int DEFAULT_HISTORY_LENGTH = 10; 1307 1308 HistoryCommand(Configuration conf, CommandLine cmdline) { 1309 super(conf); 1310 this.cmdline = cmdline; 1311 } 1312 1313 @Override 1314 public void execute() throws IOException { 1315 int n = parseHistoryLength(); 1316 final TableName tableName = getTableName(); 1317 final String setName = getTableSetName(); 1318 BackupInfo.Filter tableNameFilter = info -> { 1319 if (tableName == null) { 1320 return true; 1321 } 1322 1323 List<TableName> names = info.getTableNames(); 1324 return names.contains(tableName); 1325 }; 1326 BackupInfo.Filter tableSetFilter = info -> { 1327 if (setName == null) { 1328 return true; 1329 } 1330 1331 String backupId = info.getBackupId(); 1332 return backupId.startsWith(setName); 1333 }; 1334 Path backupRootPath = getBackupRootPath(); 1335 List<BackupInfo> history; 1336 if (backupRootPath == null) { 1337 // Load from backup system table 1338 super.execute(); 1339 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 1340 history = sysTable.getBackupHistory(tableNameFilter, tableSetFilter); 1341 history = history.subList(0, Math.min(n, history.size())); 1342 } 1343 } else { 1344 // load from backup FS 1345 history = 1346 BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter); 1347 } 1348 for (BackupInfo info : history) { 1349 System.out.println(info.getShortDescription()); 1350 } 1351 } 1352 1353 private Path getBackupRootPath() throws IOException { 1354 String value = null; 1355 try { 1356 value = cmdline.getOptionValue(OPTION_PATH); 1357 1358 if (value == null) { 1359 return null; 1360 } 1361 1362 return new Path(value); 1363 } catch (IllegalArgumentException e) { 1364 System.out.println("ERROR: Illegal argument for backup root path: " + value); 1365 printUsage(); 1366 throw new IOException(INCORRECT_USAGE); 1367 } 1368 } 1369 1370 private TableName getTableName() throws IOException { 1371 String value = cmdline.getOptionValue(OPTION_TABLE); 1372 1373 if (value == null) { 1374 return null; 1375 } 1376 1377 try { 1378 return TableName.valueOf(value); 1379 } catch (IllegalArgumentException e) { 1380 System.out.println("Illegal argument for table name: " + value); 1381 printUsage(); 1382 throw new IOException(INCORRECT_USAGE); 1383 } 1384 } 1385 1386 private String getTableSetName() { 1387 return cmdline.getOptionValue(OPTION_SET); 1388 } 1389 1390 private int parseHistoryLength() throws IOException { 1391 String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER); 1392 try { 1393 if (value == null) { 1394 return DEFAULT_HISTORY_LENGTH; 1395 } 1396 1397 return Integer.parseInt(value); 1398 } catch (NumberFormatException e) { 1399 System.out.println("Illegal argument for history length: " + value); 1400 printUsage(); 1401 throw new IOException(INCORRECT_USAGE); 1402 } 1403 } 1404 1405 @Override 1406 protected void printUsage() { 1407 System.out.println(HISTORY_CMD_USAGE); 1408 Options options = new Options(); 1409 options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC); 1410 options.addOption(OPTION_PATH, true, OPTION_PATH_DESC); 1411 options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC); 1412 options.addOption(OPTION_SET, true, OPTION_SET_DESC); 1413 1414 HelpFormatter helpFormatter = new HelpFormatter(); 1415 helpFormatter.setLeftPadding(2); 1416 helpFormatter.setDescPadding(8); 1417 helpFormatter.setWidth(100); 1418 helpFormatter.setSyntaxPrefix("Options:"); 1419 helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); 1420 } 1421 } 1422 1423 public static class BackupSetCommand extends Command { 1424 private final static String SET_ADD_CMD = "add"; 1425 private final static String SET_REMOVE_CMD = "remove"; 1426 private final static String SET_DELETE_CMD = "delete"; 1427 private final static String SET_DESCRIBE_CMD = "describe"; 1428 private final static String SET_LIST_CMD = "list"; 1429 1430 BackupSetCommand(Configuration conf, CommandLine cmdline) { 1431 super(conf); 1432 this.cmdline = cmdline; 1433 } 1434 1435 @Override 1436 public void execute() throws IOException { 1437 // Command-line must have at least one element 1438 if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { 1439 printUsage(); 1440 throw new IOException(INCORRECT_USAGE); 1441 } 1442 1443 String[] args = cmdline.getArgs(); 1444 String cmdStr = args[1]; 1445 BackupCommand cmd = getCommand(cmdStr); 1446 1447 switch (cmd) { 1448 case SET_ADD: 1449 processSetAdd(args); 1450 break; 1451 case SET_REMOVE: 1452 processSetRemove(args); 1453 break; 1454 case SET_DELETE: 1455 processSetDelete(args); 1456 break; 1457 case SET_DESCRIBE: 1458 processSetDescribe(args); 1459 break; 1460 case SET_LIST: 1461 processSetList(); 1462 break; 1463 default: 1464 break; 1465 } 1466 } 1467 1468 private void processSetList() throws IOException { 1469 super.execute(); 1470 1471 // List all backup set names 1472 // does not expect any args 1473 try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1474 List<BackupSet> list = admin.listBackupSets(); 1475 for (BackupSet bs : list) { 1476 System.out.println(bs); 1477 } 1478 } 1479 } 1480 1481 private void processSetDescribe(String[] args) throws IOException { 1482 if (args == null || args.length != 3) { 1483 printUsage(); 1484 throw new IOException(INCORRECT_USAGE); 1485 } 1486 super.execute(); 1487 1488 String setName = args[2]; 1489 try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { 1490 List<TableName> tables = sysTable.describeBackupSet(setName); 1491 BackupSet set = tables == null ? null : new BackupSet(setName, tables); 1492 if (set == null) { 1493 System.out.println("Set '" + setName + "' does not exist."); 1494 } else { 1495 System.out.println(set); 1496 } 1497 } 1498 } 1499 1500 private void processSetDelete(String[] args) throws IOException { 1501 if (args == null || args.length != 3) { 1502 printUsage(); 1503 throw new IOException(INCORRECT_USAGE); 1504 } 1505 super.execute(); 1506 1507 String setName = args[2]; 1508 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1509 boolean result = admin.deleteBackupSet(setName); 1510 if (result) { 1511 System.out.println("Delete set " + setName + " OK."); 1512 } else { 1513 System.out.println("Set " + setName + " does not exist"); 1514 } 1515 } 1516 } 1517 1518 private void processSetRemove(String[] args) throws IOException { 1519 if (args == null || args.length != 4) { 1520 printUsage(); 1521 throw new IOException(INCORRECT_USAGE); 1522 } 1523 super.execute(); 1524 1525 String setName = args[2]; 1526 String[] tables = args[3].split(","); 1527 TableName[] tableNames = toTableNames(tables); 1528 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1529 admin.removeFromBackupSet(setName, tableNames); 1530 } 1531 } 1532 1533 private TableName[] toTableNames(String[] tables) { 1534 TableName[] arr = new TableName[tables.length]; 1535 for (int i = 0; i < tables.length; i++) { 1536 arr[i] = TableName.valueOf(tables[i]); 1537 } 1538 return arr; 1539 } 1540 1541 private void processSetAdd(String[] args) throws IOException { 1542 if (args == null || args.length != 4) { 1543 printUsage(); 1544 throw new IOException(INCORRECT_USAGE); 1545 } 1546 super.execute(); 1547 String setName = args[2]; 1548 TableName[] tableNames = 1549 Splitter.on(',').splitToStream(args[3]).map(TableName::valueOf).toArray(TableName[]::new); 1550 try (final BackupAdminImpl admin = new BackupAdminImpl(conn)) { 1551 admin.addToBackupSet(setName, tableNames); 1552 } 1553 } 1554 1555 private BackupCommand getCommand(String cmdStr) throws IOException { 1556 switch (cmdStr) { 1557 case SET_ADD_CMD: 1558 return BackupCommand.SET_ADD; 1559 case SET_REMOVE_CMD: 1560 return BackupCommand.SET_REMOVE; 1561 case SET_DELETE_CMD: 1562 return BackupCommand.SET_DELETE; 1563 case SET_DESCRIBE_CMD: 1564 return BackupCommand.SET_DESCRIBE; 1565 case SET_LIST_CMD: 1566 return BackupCommand.SET_LIST; 1567 default: 1568 System.out.println("ERROR: Unknown command for 'set' :" + cmdStr); 1569 printUsage(); 1570 throw new IOException(INCORRECT_USAGE); 1571 } 1572 } 1573 1574 @Override 1575 protected void printUsage() { 1576 System.out.println(SET_CMD_USAGE); 1577 } 1578 } 1579}