001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.util.ArrayList;
023import java.util.Calendar;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.Map.Entry;
029import java.util.Set;
030import org.apache.commons.lang3.StringUtils;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.backup.util.BackupUtils;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.apache.yetus.audience.InterfaceAudience;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
039import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
040
041/**
042 * An object to encapsulate the information for each backup session
043 */
044@InterfaceAudience.Private
045public class BackupInfo implements Comparable<BackupInfo> {
046  private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class);
047  private static final int MAX_FAILED_MESSAGE_LENGTH = 1024;
048
049  public interface Filter {
050    /**
051     * Filter interface
052     * @param info backup info
053     * @return true if info passes filter, false otherwise
054     */
055    boolean apply(BackupInfo info);
056  }
057
058  /**
059   * Backup session states
060   */
061  public enum BackupState {
062    RUNNING,
063    COMPLETE,
064    FAILED,
065    ANY
066  }
067
068  /**
069   * BackupPhase - phases of an ACTIVE backup session (running), when state of a backup session is
070   * BackupState.RUNNING
071   */
072  public enum BackupPhase {
073    REQUEST,
074    SNAPSHOT,
075    PREPARE_INCREMENTAL,
076    SNAPSHOTCOPY,
077    INCREMENTAL_COPY,
078    STORE_MANIFEST
079  }
080
081  /**
082   * Backup id
083   */
084  private String backupId;
085
086  /**
087   * Backup type, full or incremental
088   */
089  private BackupType type;
090
091  /**
092   * Target root directory for storing the backup files
093   */
094  private String backupRootDir;
095
096  /**
097   * Backup state
098   */
099  private BackupState state;
100
101  /**
102   * Backup phase
103   */
104  private BackupPhase phase = BackupPhase.REQUEST;
105
106  /**
107   * Backup failure message
108   */
109  private String failedMsg;
110
111  /**
112   * Backup status map for all tables
113   */
114  private Map<TableName, BackupTableInfo> backupTableInfoMap;
115
116  /**
117   * Actual start timestamp of a backup process
118   */
119  private long startTs;
120
121  /**
122   * Actual end timestamp of the backup process
123   */
124  private long completeTs;
125
126  /**
127   * Total bytes of incremental logs copied
128   */
129  private long totalBytesCopied;
130
131  /**
132   * For incremental backup, a location of a backed-up hlogs
133   */
134  private String hlogTargetDir = null;
135
136  /**
137   * Incremental backup file list
138   */
139  private List<String> incrBackupFileList;
140
141  /**
142   * New region server log timestamps for table set after distributed log roll key - table name,
143   * value - map of RegionServer hostname -> last log rolled timestamp
144   */
145  private Map<TableName, Map<String, Long>> tableSetTimestampMap;
146
147  /**
148   * Previous Region server log timestamps for table set after distributed log roll key - table
149   * name, value - map of RegionServer hostname -> last log rolled timestamp
150   */
151  private Map<TableName, Map<String, Long>> incrTimestampMap;
152
153  /**
154   * Backup progress in %% (0-100)
155   */
156  private int progress;
157
158  /**
159   * Number of parallel workers. -1 - system defined
160   */
161  private int workers = -1;
162
163  /**
164   * Bandwidth per worker in MB per sec. -1 - unlimited
165   */
166  private long bandwidth = -1;
167
168  /**
169   * Do not verify checksum between source snapshot and exported snapshot
170   */
171  private boolean noChecksumVerify;
172
173  public BackupInfo() {
174    backupTableInfoMap = new HashMap<>();
175  }
176
177  public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) {
178    this();
179    this.backupId = backupId;
180    this.type = type;
181    this.backupRootDir = targetRootDir;
182    this.addTables(tables);
183    if (type == BackupType.INCREMENTAL) {
184      setHLogTargetDir(BackupUtils.getLogBackupDir(targetRootDir, backupId));
185    }
186    this.startTs = 0;
187    this.completeTs = 0;
188  }
189
190  public int getWorkers() {
191    return workers;
192  }
193
194  public void setWorkers(int workers) {
195    this.workers = workers;
196  }
197
198  public long getBandwidth() {
199    return bandwidth;
200  }
201
202  public void setBandwidth(long bandwidth) {
203    this.bandwidth = bandwidth;
204  }
205
206  public void setNoChecksumVerify(boolean noChecksumVerify) {
207    this.noChecksumVerify = noChecksumVerify;
208  }
209
210  public boolean getNoChecksumVerify() {
211    return noChecksumVerify;
212  }
213
214  public void setBackupTableInfoMap(Map<TableName, BackupTableInfo> backupTableInfoMap) {
215    this.backupTableInfoMap = backupTableInfoMap;
216  }
217
218  public Map<TableName, Map<String, Long>> getTableSetTimestampMap() {
219    return tableSetTimestampMap;
220  }
221
222  public void setTableSetTimestampMap(Map<TableName, Map<String, Long>> tableSetTimestampMap) {
223    this.tableSetTimestampMap = tableSetTimestampMap;
224  }
225
226  public void setType(BackupType type) {
227    this.type = type;
228  }
229
230  public void setBackupRootDir(String targetRootDir) {
231    this.backupRootDir = targetRootDir;
232  }
233
234  public void setTotalBytesCopied(long totalBytesCopied) {
235    this.totalBytesCopied = totalBytesCopied;
236  }
237
238  /**
239   * Set progress (0-100%)
240   * @param p progress value
241   */
242  public void setProgress(int p) {
243    this.progress = p;
244  }
245
246  /**
247   * Get current progress
248   */
249  public int getProgress() {
250    return progress;
251  }
252
253  public String getBackupId() {
254    return backupId;
255  }
256
257  public void setBackupId(String backupId) {
258    this.backupId = backupId;
259  }
260
261  public BackupTableInfo getBackupTableInfo(TableName table) {
262    return this.backupTableInfoMap.get(table);
263  }
264
265  public String getFailedMsg() {
266    return failedMsg;
267  }
268
269  public void setFailedMsg(String failedMsg) {
270    if (failedMsg.length() > MAX_FAILED_MESSAGE_LENGTH) {
271      failedMsg = failedMsg.substring(0, MAX_FAILED_MESSAGE_LENGTH);
272    }
273    this.failedMsg = failedMsg;
274  }
275
276  public long getStartTs() {
277    return startTs;
278  }
279
280  public void setStartTs(long startTs) {
281    this.startTs = startTs;
282  }
283
284  public long getCompleteTs() {
285    return completeTs;
286  }
287
288  public void setCompleteTs(long endTs) {
289    this.completeTs = endTs;
290  }
291
292  public long getTotalBytesCopied() {
293    return totalBytesCopied;
294  }
295
296  public BackupState getState() {
297    return state;
298  }
299
300  public void setState(BackupState flag) {
301    this.state = flag;
302  }
303
304  public BackupPhase getPhase() {
305    return phase;
306  }
307
308  public void setPhase(BackupPhase phase) {
309    this.phase = phase;
310  }
311
312  public BackupType getType() {
313    return type;
314  }
315
316  public void setSnapshotName(TableName table, String snapshotName) {
317    this.backupTableInfoMap.get(table).setSnapshotName(snapshotName);
318  }
319
320  public String getSnapshotName(TableName table) {
321    return this.backupTableInfoMap.get(table).getSnapshotName();
322  }
323
324  public List<String> getSnapshotNames() {
325    List<String> snapshotNames = new ArrayList<>();
326    for (BackupTableInfo backupStatus : this.backupTableInfoMap.values()) {
327      snapshotNames.add(backupStatus.getSnapshotName());
328    }
329    return snapshotNames;
330  }
331
332  public Set<TableName> getTables() {
333    return this.backupTableInfoMap.keySet();
334  }
335
336  public List<TableName> getTableNames() {
337    return new ArrayList<>(backupTableInfoMap.keySet());
338  }
339
340  public void addTables(TableName[] tables) {
341    for (TableName table : tables) {
342      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
343      this.backupTableInfoMap.put(table, backupStatus);
344    }
345  }
346
347  public void setTables(List<TableName> tables) {
348    this.backupTableInfoMap.clear();
349    for (TableName table : tables) {
350      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
351      this.backupTableInfoMap.put(table, backupStatus);
352    }
353  }
354
355  public String getBackupRootDir() {
356    return backupRootDir;
357  }
358
359  public String getTableBackupDir(TableName tableName) {
360    return BackupUtils.getTableBackupDir(backupRootDir, backupId, tableName);
361  }
362
363  public void setHLogTargetDir(String hlogTagetDir) {
364    this.hlogTargetDir = hlogTagetDir;
365  }
366
367  public String getHLogTargetDir() {
368    return hlogTargetDir;
369  }
370
371  public List<String> getIncrBackupFileList() {
372    return incrBackupFileList;
373  }
374
375  public void setIncrBackupFileList(List<String> incrBackupFileList) {
376    this.incrBackupFileList = incrBackupFileList;
377  }
378
379  /**
380   * Set the new region server log timestamps after distributed log roll
381   * @param prevTableSetTimestampMap table timestamp map
382   */
383  public void setIncrTimestampMap(Map<TableName, Map<String, Long>> prevTableSetTimestampMap) {
384    this.incrTimestampMap = prevTableSetTimestampMap;
385  }
386
387  /**
388   * Get new region server log timestamps after distributed log roll
389   * @return new region server log timestamps
390   */
391  public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
392    return this.incrTimestampMap;
393  }
394
395  public TableName getTableBySnapshot(String snapshotName) {
396    for (Entry<TableName, BackupTableInfo> entry : this.backupTableInfoMap.entrySet()) {
397      if (snapshotName.equals(entry.getValue().getSnapshotName())) {
398        return entry.getKey();
399      }
400    }
401    return null;
402  }
403
404  public BackupProtos.BackupInfo toProtosBackupInfo() {
405    BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder();
406    builder.setBackupId(getBackupId());
407    setBackupTableInfoMap(builder);
408    setTableSetTimestampMap(builder);
409    builder.setCompleteTs(getCompleteTs());
410    if (getFailedMsg() != null) {
411      builder.setFailedMessage(getFailedMsg());
412    }
413    if (getState() != null) {
414      builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name()));
415    }
416    if (getPhase() != null) {
417      builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name()));
418    }
419
420    builder.setProgress(getProgress());
421    builder.setStartTs(getStartTs());
422    builder.setBackupRootDir(getBackupRootDir());
423    builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name()));
424    builder.setWorkersNumber(workers);
425    builder.setBandwidth(bandwidth);
426    return builder.build();
427  }
428
429  @Override
430  public int hashCode() {
431    int hash = 33 * type.hashCode() + backupId != null ? backupId.hashCode() : 0;
432    if (backupRootDir != null) {
433      hash = 33 * hash + backupRootDir.hashCode();
434    }
435    hash = 33 * hash + state.hashCode();
436    hash = 33 * hash + phase.hashCode();
437    hash = 33 * hash + (int) (startTs ^ (startTs >>> 32));
438    hash = 33 * hash + (int) (completeTs ^ (completeTs >>> 32));
439    hash = 33 * hash + (int) (totalBytesCopied ^ (totalBytesCopied >>> 32));
440    if (hlogTargetDir != null) {
441      hash = 33 * hash + hlogTargetDir.hashCode();
442    }
443    return hash;
444  }
445
446  @Override
447  public boolean equals(Object obj) {
448    if (obj instanceof BackupInfo) {
449      BackupInfo other = (BackupInfo) obj;
450      try {
451        return Bytes.equals(toByteArray(), other.toByteArray());
452      } catch (IOException e) {
453        LOG.error(e.toString(), e);
454        return false;
455      }
456    } else {
457      return false;
458    }
459  }
460
461  @Override
462  public String toString() {
463    return backupId;
464  }
465
466  public byte[] toByteArray() throws IOException {
467    return toProtosBackupInfo().toByteArray();
468  }
469
470  private void setBackupTableInfoMap(BackupProtos.BackupInfo.Builder builder) {
471    for (Entry<TableName, BackupTableInfo> entry : backupTableInfoMap.entrySet()) {
472      builder.addBackupTableInfo(entry.getValue().toProto());
473    }
474  }
475
476  private void setTableSetTimestampMap(BackupProtos.BackupInfo.Builder builder) {
477    if (this.getTableSetTimestampMap() != null) {
478      for (Entry<TableName, Map<String, Long>> entry : this.getTableSetTimestampMap().entrySet()) {
479        builder.putTableSetTimestamp(entry.getKey().getNameAsString(),
480          BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue())
481            .build());
482      }
483    }
484  }
485
486  public static BackupInfo fromByteArray(byte[] data) throws IOException {
487    return fromProto(BackupProtos.BackupInfo.parseFrom(data));
488  }
489
490  public static BackupInfo fromStream(final InputStream stream) throws IOException {
491    return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream));
492  }
493
494  public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
495    BackupInfo context = new BackupInfo();
496    context.setBackupId(proto.getBackupId());
497    context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList()));
498    context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap()));
499    context.setCompleteTs(proto.getCompleteTs());
500    if (proto.hasFailedMessage()) {
501      context.setFailedMsg(proto.getFailedMessage());
502    }
503    if (proto.hasBackupState()) {
504      context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name()));
505    }
506
507    context
508      .setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId()));
509
510    if (proto.hasBackupPhase()) {
511      context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name()));
512    }
513    if (proto.hasProgress()) {
514      context.setProgress(proto.getProgress());
515    }
516    context.setStartTs(proto.getStartTs());
517    context.setBackupRootDir(proto.getBackupRootDir());
518    context.setType(BackupType.valueOf(proto.getBackupType().name()));
519    context.setWorkers(proto.getWorkersNumber());
520    context.setBandwidth(proto.getBandwidth());
521    return context;
522  }
523
524  private static Map<TableName, BackupTableInfo> toMap(List<BackupProtos.BackupTableInfo> list) {
525    HashMap<TableName, BackupTableInfo> map = new HashMap<>();
526    for (BackupProtos.BackupTableInfo tbs : list) {
527      map.put(ProtobufUtil.toTableName(tbs.getTableName()), BackupTableInfo.convert(tbs));
528    }
529    return map;
530  }
531
532  private static Map<TableName, Map<String, Long>>
533    getTableSetTimestampMap(Map<String, BackupProtos.BackupInfo.RSTimestampMap> map) {
534    Map<TableName, Map<String, Long>> tableSetTimestampMap = new HashMap<>();
535    for (Entry<String, BackupProtos.BackupInfo.RSTimestampMap> entry : map.entrySet()) {
536      tableSetTimestampMap.put(TableName.valueOf(entry.getKey()),
537        entry.getValue().getRsTimestampMap());
538    }
539
540    return tableSetTimestampMap;
541  }
542
543  public String getShortDescription() {
544    StringBuilder sb = new StringBuilder();
545    sb.append("{");
546    sb.append("ID=" + backupId).append(",");
547    sb.append("Type=" + getType()).append(",");
548    sb.append("Tables=" + getTableListAsString()).append(",");
549    sb.append("State=" + getState()).append(",");
550    Calendar cal = Calendar.getInstance();
551    cal.setTimeInMillis(getStartTs());
552    Date date = cal.getTime();
553    sb.append("Start time=" + date).append(",");
554    if (state == BackupState.FAILED) {
555      sb.append("Failed message=" + getFailedMsg()).append(",");
556    } else if (state == BackupState.RUNNING) {
557      sb.append("Phase=" + getPhase()).append(",");
558    } else if (state == BackupState.COMPLETE) {
559      cal = Calendar.getInstance();
560      cal.setTimeInMillis(getCompleteTs());
561      date = cal.getTime();
562      sb.append("End time=" + date).append(",");
563    }
564    sb.append("Progress=" + getProgress() + "%");
565    sb.append("}");
566
567    return sb.toString();
568  }
569
570  public String getStatusAndProgressAsString() {
571    StringBuilder sb = new StringBuilder();
572    sb.append("id: ").append(getBackupId()).append(" state: ").append(getState())
573      .append(" progress: ").append(getProgress());
574    return sb.toString();
575  }
576
577  public String getTableListAsString() {
578    StringBuilder sb = new StringBuilder();
579    sb.append("{");
580    sb.append(StringUtils.join(backupTableInfoMap.keySet(), ","));
581    sb.append("}");
582    return sb.toString();
583  }
584
585  /**
586   * We use only time stamps to compare objects during sort operation
587   */
588  @Override
589  public int compareTo(BackupInfo o) {
590    Long thisTS =
591      Long.valueOf(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1));
592    Long otherTS = Long.valueOf(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1));
593    return thisTS.compareTo(otherTS);
594  }
595}