001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup; 019 020import java.io.IOException; 021import java.io.InputStream; 022import java.util.ArrayList; 023import java.util.Calendar; 024import java.util.Date; 025import java.util.HashMap; 026import java.util.List; 027import java.util.Map; 028import java.util.Map.Entry; 029import java.util.Set; 030import org.apache.commons.lang3.StringUtils; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.backup.util.BackupUtils; 033import org.apache.hadoop.hbase.util.Bytes; 034import org.apache.yetus.audience.InterfaceAudience; 035import org.slf4j.Logger; 036import org.slf4j.LoggerFactory; 037 038import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 039import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; 040 041/** 042 * An object to encapsulate the information for each backup session 043 */ 044@InterfaceAudience.Private 045public class BackupInfo implements Comparable<BackupInfo> { 046 private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class); 047 private static final int MAX_FAILED_MESSAGE_LENGTH = 1024; 048 049 public interface Filter { 050 /** 051 * Filter interface 052 * @param info backup info 053 * @return true if info passes filter, false otherwise 054 */ 055 boolean apply(BackupInfo info); 056 } 057 058 /** 059 * Backup session states 060 */ 061 public enum BackupState { 062 RUNNING, 063 COMPLETE, 064 FAILED, 065 ANY 066 } 067 068 /** 069 * BackupPhase - phases of an ACTIVE backup session (running), when state of a backup session is 070 * BackupState.RUNNING 071 */ 072 public enum BackupPhase { 073 REQUEST, 074 SNAPSHOT, 075 PREPARE_INCREMENTAL, 076 SNAPSHOTCOPY, 077 INCREMENTAL_COPY, 078 STORE_MANIFEST 079 } 080 081 /** 082 * Backup id 083 */ 084 private String backupId; 085 086 /** 087 * Backup type, full or incremental 088 */ 089 private BackupType type; 090 091 /** 092 * Target root directory for storing the backup files 093 */ 094 private String backupRootDir; 095 096 /** 097 * Backup state 098 */ 099 private BackupState state; 100 101 /** 102 * Backup phase 103 */ 104 private BackupPhase phase = BackupPhase.REQUEST; 105 106 /** 107 * Backup failure message 108 */ 109 private String failedMsg; 110 111 /** 112 * Backup status map for all tables 113 */ 114 private Map<TableName, BackupTableInfo> backupTableInfoMap; 115 116 /** 117 * Actual start timestamp of a backup process 118 */ 119 private long startTs; 120 121 /** 122 * Actual end timestamp of the backup process 123 */ 124 private long completeTs; 125 126 /** 127 * Total bytes of incremental logs copied 128 */ 129 private long totalBytesCopied; 130 131 /** 132 * For incremental backup, a location of a backed-up hlogs 133 */ 134 private String hlogTargetDir = null; 135 136 /** 137 * Incremental backup file list 138 */ 139 private List<String> incrBackupFileList; 140 141 /** 142 * New region server log timestamps for table set after distributed log roll key - table name, 143 * value - map of RegionServer hostname -> last log rolled timestamp 144 */ 145 private Map<TableName, Map<String, Long>> tableSetTimestampMap; 146 147 /** 148 * Previous Region server log timestamps for table set after distributed log roll key - table 149 * name, value - map of RegionServer hostname -> last log rolled timestamp 150 */ 151 private Map<TableName, Map<String, Long>> incrTimestampMap; 152 153 /** 154 * Backup progress in %% (0-100) 155 */ 156 private int progress; 157 158 /** 159 * Number of parallel workers. -1 - system defined 160 */ 161 private int workers = -1; 162 163 /** 164 * Bandwidth per worker in MB per sec. -1 - unlimited 165 */ 166 private long bandwidth = -1; 167 168 /** 169 * Do not verify checksum between source snapshot and exported snapshot 170 */ 171 private boolean noChecksumVerify; 172 173 public BackupInfo() { 174 backupTableInfoMap = new HashMap<>(); 175 } 176 177 public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) { 178 this(); 179 this.backupId = backupId; 180 this.type = type; 181 this.backupRootDir = targetRootDir; 182 this.addTables(tables); 183 if (type == BackupType.INCREMENTAL) { 184 setHLogTargetDir(BackupUtils.getLogBackupDir(targetRootDir, backupId)); 185 } 186 this.startTs = 0; 187 this.completeTs = 0; 188 } 189 190 public int getWorkers() { 191 return workers; 192 } 193 194 public void setWorkers(int workers) { 195 this.workers = workers; 196 } 197 198 public long getBandwidth() { 199 return bandwidth; 200 } 201 202 public void setBandwidth(long bandwidth) { 203 this.bandwidth = bandwidth; 204 } 205 206 public void setNoChecksumVerify(boolean noChecksumVerify) { 207 this.noChecksumVerify = noChecksumVerify; 208 } 209 210 public boolean getNoChecksumVerify() { 211 return noChecksumVerify; 212 } 213 214 public void setBackupTableInfoMap(Map<TableName, BackupTableInfo> backupTableInfoMap) { 215 this.backupTableInfoMap = backupTableInfoMap; 216 } 217 218 public Map<TableName, Map<String, Long>> getTableSetTimestampMap() { 219 return tableSetTimestampMap; 220 } 221 222 public void setTableSetTimestampMap(Map<TableName, Map<String, Long>> tableSetTimestampMap) { 223 this.tableSetTimestampMap = tableSetTimestampMap; 224 } 225 226 public void setType(BackupType type) { 227 this.type = type; 228 } 229 230 public void setBackupRootDir(String targetRootDir) { 231 this.backupRootDir = targetRootDir; 232 } 233 234 public void setTotalBytesCopied(long totalBytesCopied) { 235 this.totalBytesCopied = totalBytesCopied; 236 } 237 238 /** 239 * Set progress (0-100%) 240 * @param p progress value 241 */ 242 public void setProgress(int p) { 243 this.progress = p; 244 } 245 246 /** 247 * Get current progress 248 */ 249 public int getProgress() { 250 return progress; 251 } 252 253 public String getBackupId() { 254 return backupId; 255 } 256 257 public void setBackupId(String backupId) { 258 this.backupId = backupId; 259 } 260 261 public BackupTableInfo getBackupTableInfo(TableName table) { 262 return this.backupTableInfoMap.get(table); 263 } 264 265 public String getFailedMsg() { 266 return failedMsg; 267 } 268 269 public void setFailedMsg(String failedMsg) { 270 if (failedMsg.length() > MAX_FAILED_MESSAGE_LENGTH) { 271 failedMsg = failedMsg.substring(0, MAX_FAILED_MESSAGE_LENGTH); 272 } 273 this.failedMsg = failedMsg; 274 } 275 276 public long getStartTs() { 277 return startTs; 278 } 279 280 public void setStartTs(long startTs) { 281 this.startTs = startTs; 282 } 283 284 public long getCompleteTs() { 285 return completeTs; 286 } 287 288 public void setCompleteTs(long endTs) { 289 this.completeTs = endTs; 290 } 291 292 public long getTotalBytesCopied() { 293 return totalBytesCopied; 294 } 295 296 public BackupState getState() { 297 return state; 298 } 299 300 public void setState(BackupState flag) { 301 this.state = flag; 302 } 303 304 public BackupPhase getPhase() { 305 return phase; 306 } 307 308 public void setPhase(BackupPhase phase) { 309 this.phase = phase; 310 } 311 312 public BackupType getType() { 313 return type; 314 } 315 316 public void setSnapshotName(TableName table, String snapshotName) { 317 this.backupTableInfoMap.get(table).setSnapshotName(snapshotName); 318 } 319 320 public String getSnapshotName(TableName table) { 321 return this.backupTableInfoMap.get(table).getSnapshotName(); 322 } 323 324 public List<String> getSnapshotNames() { 325 List<String> snapshotNames = new ArrayList<>(); 326 for (BackupTableInfo backupStatus : this.backupTableInfoMap.values()) { 327 snapshotNames.add(backupStatus.getSnapshotName()); 328 } 329 return snapshotNames; 330 } 331 332 public Set<TableName> getTables() { 333 return this.backupTableInfoMap.keySet(); 334 } 335 336 public List<TableName> getTableNames() { 337 return new ArrayList<>(backupTableInfoMap.keySet()); 338 } 339 340 public void addTables(TableName[] tables) { 341 for (TableName table : tables) { 342 BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId); 343 this.backupTableInfoMap.put(table, backupStatus); 344 } 345 } 346 347 public void setTables(List<TableName> tables) { 348 this.backupTableInfoMap.clear(); 349 for (TableName table : tables) { 350 BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId); 351 this.backupTableInfoMap.put(table, backupStatus); 352 } 353 } 354 355 public String getBackupRootDir() { 356 return backupRootDir; 357 } 358 359 public String getTableBackupDir(TableName tableName) { 360 return BackupUtils.getTableBackupDir(backupRootDir, backupId, tableName); 361 } 362 363 public void setHLogTargetDir(String hlogTagetDir) { 364 this.hlogTargetDir = hlogTagetDir; 365 } 366 367 public String getHLogTargetDir() { 368 return hlogTargetDir; 369 } 370 371 public List<String> getIncrBackupFileList() { 372 return incrBackupFileList; 373 } 374 375 public void setIncrBackupFileList(List<String> incrBackupFileList) { 376 this.incrBackupFileList = incrBackupFileList; 377 } 378 379 /** 380 * Set the new region server log timestamps after distributed log roll 381 * @param prevTableSetTimestampMap table timestamp map 382 */ 383 public void setIncrTimestampMap(Map<TableName, Map<String, Long>> prevTableSetTimestampMap) { 384 this.incrTimestampMap = prevTableSetTimestampMap; 385 } 386 387 /** 388 * Get new region server log timestamps after distributed log roll 389 * @return new region server log timestamps 390 */ 391 public Map<TableName, Map<String, Long>> getIncrTimestampMap() { 392 return this.incrTimestampMap; 393 } 394 395 public TableName getTableBySnapshot(String snapshotName) { 396 for (Entry<TableName, BackupTableInfo> entry : this.backupTableInfoMap.entrySet()) { 397 if (snapshotName.equals(entry.getValue().getSnapshotName())) { 398 return entry.getKey(); 399 } 400 } 401 return null; 402 } 403 404 public BackupProtos.BackupInfo toProtosBackupInfo() { 405 BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); 406 builder.setBackupId(getBackupId()); 407 setBackupTableInfoMap(builder); 408 setTableSetTimestampMap(builder); 409 builder.setCompleteTs(getCompleteTs()); 410 if (getFailedMsg() != null) { 411 builder.setFailedMessage(getFailedMsg()); 412 } 413 if (getState() != null) { 414 builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); 415 } 416 if (getPhase() != null) { 417 builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); 418 } 419 420 builder.setProgress(getProgress()); 421 builder.setStartTs(getStartTs()); 422 builder.setBackupRootDir(getBackupRootDir()); 423 builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name())); 424 builder.setWorkersNumber(workers); 425 builder.setBandwidth(bandwidth); 426 return builder.build(); 427 } 428 429 @Override 430 public int hashCode() { 431 int hash = 33 * type.hashCode() + backupId != null ? backupId.hashCode() : 0; 432 if (backupRootDir != null) { 433 hash = 33 * hash + backupRootDir.hashCode(); 434 } 435 hash = 33 * hash + state.hashCode(); 436 hash = 33 * hash + phase.hashCode(); 437 hash = 33 * hash + (int) (startTs ^ (startTs >>> 32)); 438 hash = 33 * hash + (int) (completeTs ^ (completeTs >>> 32)); 439 hash = 33 * hash + (int) (totalBytesCopied ^ (totalBytesCopied >>> 32)); 440 if (hlogTargetDir != null) { 441 hash = 33 * hash + hlogTargetDir.hashCode(); 442 } 443 return hash; 444 } 445 446 @Override 447 public boolean equals(Object obj) { 448 if (obj instanceof BackupInfo) { 449 BackupInfo other = (BackupInfo) obj; 450 try { 451 return Bytes.equals(toByteArray(), other.toByteArray()); 452 } catch (IOException e) { 453 LOG.error(e.toString(), e); 454 return false; 455 } 456 } else { 457 return false; 458 } 459 } 460 461 @Override 462 public String toString() { 463 return backupId; 464 } 465 466 public byte[] toByteArray() throws IOException { 467 return toProtosBackupInfo().toByteArray(); 468 } 469 470 private void setBackupTableInfoMap(BackupProtos.BackupInfo.Builder builder) { 471 for (Entry<TableName, BackupTableInfo> entry : backupTableInfoMap.entrySet()) { 472 builder.addBackupTableInfo(entry.getValue().toProto()); 473 } 474 } 475 476 private void setTableSetTimestampMap(BackupProtos.BackupInfo.Builder builder) { 477 if (this.getTableSetTimestampMap() != null) { 478 for (Entry<TableName, Map<String, Long>> entry : this.getTableSetTimestampMap().entrySet()) { 479 builder.putTableSetTimestamp(entry.getKey().getNameAsString(), 480 BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue()) 481 .build()); 482 } 483 } 484 } 485 486 public static BackupInfo fromByteArray(byte[] data) throws IOException { 487 return fromProto(BackupProtos.BackupInfo.parseFrom(data)); 488 } 489 490 public static BackupInfo fromStream(final InputStream stream) throws IOException { 491 return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream)); 492 } 493 494 public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { 495 BackupInfo context = new BackupInfo(); 496 context.setBackupId(proto.getBackupId()); 497 context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList())); 498 context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap())); 499 context.setCompleteTs(proto.getCompleteTs()); 500 if (proto.hasFailedMessage()) { 501 context.setFailedMsg(proto.getFailedMessage()); 502 } 503 if (proto.hasBackupState()) { 504 context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name())); 505 } 506 507 context 508 .setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId())); 509 510 if (proto.hasBackupPhase()) { 511 context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name())); 512 } 513 if (proto.hasProgress()) { 514 context.setProgress(proto.getProgress()); 515 } 516 context.setStartTs(proto.getStartTs()); 517 context.setBackupRootDir(proto.getBackupRootDir()); 518 context.setType(BackupType.valueOf(proto.getBackupType().name())); 519 context.setWorkers(proto.getWorkersNumber()); 520 context.setBandwidth(proto.getBandwidth()); 521 return context; 522 } 523 524 private static Map<TableName, BackupTableInfo> toMap(List<BackupProtos.BackupTableInfo> list) { 525 HashMap<TableName, BackupTableInfo> map = new HashMap<>(); 526 for (BackupProtos.BackupTableInfo tbs : list) { 527 map.put(ProtobufUtil.toTableName(tbs.getTableName()), BackupTableInfo.convert(tbs)); 528 } 529 return map; 530 } 531 532 private static Map<TableName, Map<String, Long>> 533 getTableSetTimestampMap(Map<String, BackupProtos.BackupInfo.RSTimestampMap> map) { 534 Map<TableName, Map<String, Long>> tableSetTimestampMap = new HashMap<>(); 535 for (Entry<String, BackupProtos.BackupInfo.RSTimestampMap> entry : map.entrySet()) { 536 tableSetTimestampMap.put(TableName.valueOf(entry.getKey()), 537 entry.getValue().getRsTimestampMap()); 538 } 539 540 return tableSetTimestampMap; 541 } 542 543 public String getShortDescription() { 544 StringBuilder sb = new StringBuilder(); 545 sb.append("{"); 546 sb.append("ID=" + backupId).append(","); 547 sb.append("Type=" + getType()).append(","); 548 sb.append("Tables=" + getTableListAsString()).append(","); 549 sb.append("State=" + getState()).append(","); 550 Calendar cal = Calendar.getInstance(); 551 cal.setTimeInMillis(getStartTs()); 552 Date date = cal.getTime(); 553 sb.append("Start time=" + date).append(","); 554 if (state == BackupState.FAILED) { 555 sb.append("Failed message=" + getFailedMsg()).append(","); 556 } else if (state == BackupState.RUNNING) { 557 sb.append("Phase=" + getPhase()).append(","); 558 } else if (state == BackupState.COMPLETE) { 559 cal = Calendar.getInstance(); 560 cal.setTimeInMillis(getCompleteTs()); 561 date = cal.getTime(); 562 sb.append("End time=" + date).append(","); 563 } 564 sb.append("Progress=" + getProgress() + "%"); 565 sb.append("}"); 566 567 return sb.toString(); 568 } 569 570 public String getStatusAndProgressAsString() { 571 StringBuilder sb = new StringBuilder(); 572 sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) 573 .append(" progress: ").append(getProgress()); 574 return sb.toString(); 575 } 576 577 public String getTableListAsString() { 578 StringBuilder sb = new StringBuilder(); 579 sb.append("{"); 580 sb.append(StringUtils.join(backupTableInfoMap.keySet(), ",")); 581 sb.append("}"); 582 return sb.toString(); 583 } 584 585 /** 586 * We use only time stamps to compare objects during sort operation 587 */ 588 @Override 589 public int compareTo(BackupInfo o) { 590 Long thisTS = 591 Long.valueOf(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1)); 592 Long otherTS = Long.valueOf(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1)); 593 return thisTS.compareTo(otherTS); 594 } 595}