001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.util.ArrayList;
023import java.util.Calendar;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.Map.Entry;
029import java.util.Set;
030import org.apache.commons.lang3.StringUtils;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.backup.util.BackupUtils;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.apache.yetus.audience.InterfaceAudience;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
039import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
040import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos.BackupInfo.Builder;
041
042/**
043 * An object to encapsulate the information for each backup session
044 */
045@InterfaceAudience.Private
046public class BackupInfo implements Comparable<BackupInfo> {
047  private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class);
048
049  public interface Filter {
050    /**
051     * Filter interface
052     * @param info backup info
053     * @return true if info passes filter, false otherwise
054     */
055    boolean apply(BackupInfo info);
056  }
057
058  /**
059   * Backup session states
060   */
061  public enum BackupState {
062    RUNNING,
063    COMPLETE,
064    FAILED,
065    ANY
066  }
067
068  /**
069   * BackupPhase - phases of an ACTIVE backup session (running), when state of a backup session is
070   * BackupState.RUNNING
071   */
072  public enum BackupPhase {
073    REQUEST,
074    SNAPSHOT,
075    PREPARE_INCREMENTAL,
076    SNAPSHOTCOPY,
077    INCREMENTAL_COPY,
078    STORE_MANIFEST
079  }
080
081  /**
082   * Backup id
083   */
084  private String backupId;
085
086  /**
087   * Backup type, full or incremental
088   */
089  private BackupType type;
090
091  /**
092   * Target root directory for storing the backup files
093   */
094  private String backupRootDir;
095
096  /**
097   * Backup state
098   */
099  private BackupState state;
100
101  /**
102   * Backup phase
103   */
104  private BackupPhase phase = BackupPhase.REQUEST;
105
106  /**
107   * Backup failure message
108   */
109  private String failedMsg;
110
111  /**
112   * Backup status map for all tables
113   */
114  private Map<TableName, BackupTableInfo> backupTableInfoMap;
115
116  /**
117   * Actual start timestamp of a backup process
118   */
119  private long startTs;
120
121  /**
122   * Actual end timestamp of the backup process
123   */
124  private long completeTs;
125
126  /**
127   * Total bytes of incremental logs copied
128   */
129  private long totalBytesCopied;
130
131  /**
132   * For incremental backup, a location of a backed-up hlogs
133   */
134  private String hlogTargetDir = null;
135
136  /**
137   * Incremental backup file list
138   */
139  private List<String> incrBackupFileList;
140
141  /**
142   * New region server log timestamps for table set after distributed log roll key - table name,
143   * value - map of RegionServer hostname -> last log rolled timestamp
144   */
145  private Map<TableName, Map<String, Long>> tableSetTimestampMap;
146
147  /**
148   * Previous Region server log timestamps for table set after distributed log roll key - table
149   * name, value - map of RegionServer hostname -> last log rolled timestamp
150   */
151  private Map<TableName, Map<String, Long>> incrTimestampMap;
152
153  /**
154   * Backup progress in %% (0-100)
155   */
156  private int progress;
157
158  /**
159   * Number of parallel workers. -1 - system defined
160   */
161  private int workers = -1;
162
163  /**
164   * Bandwidth per worker in MB per sec. -1 - unlimited
165   */
166  private long bandwidth = -1;
167
168  public BackupInfo() {
169    backupTableInfoMap = new HashMap<>();
170  }
171
172  public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) {
173    this();
174    this.backupId = backupId;
175    this.type = type;
176    this.backupRootDir = targetRootDir;
177    this.addTables(tables);
178    if (type == BackupType.INCREMENTAL) {
179      setHLogTargetDir(BackupUtils.getLogBackupDir(targetRootDir, backupId));
180    }
181    this.startTs = 0;
182    this.completeTs = 0;
183  }
184
185  public int getWorkers() {
186    return workers;
187  }
188
189  public void setWorkers(int workers) {
190    this.workers = workers;
191  }
192
193  public long getBandwidth() {
194    return bandwidth;
195  }
196
197  public void setBandwidth(long bandwidth) {
198    this.bandwidth = bandwidth;
199  }
200
201  public void setBackupTableInfoMap(Map<TableName, BackupTableInfo> backupTableInfoMap) {
202    this.backupTableInfoMap = backupTableInfoMap;
203  }
204
205  public Map<TableName, Map<String, Long>> getTableSetTimestampMap() {
206    return tableSetTimestampMap;
207  }
208
209  public void setTableSetTimestampMap(Map<TableName, Map<String, Long>> tableSetTimestampMap) {
210    this.tableSetTimestampMap = tableSetTimestampMap;
211  }
212
213  public void setType(BackupType type) {
214    this.type = type;
215  }
216
217  public void setBackupRootDir(String targetRootDir) {
218    this.backupRootDir = targetRootDir;
219  }
220
221  public void setTotalBytesCopied(long totalBytesCopied) {
222    this.totalBytesCopied = totalBytesCopied;
223  }
224
225  /**
226   * Set progress (0-100%)
227   * @param p progress value
228   */
229  public void setProgress(int p) {
230    this.progress = p;
231  }
232
233  /**
234   * Get current progress
235   */
236  public int getProgress() {
237    return progress;
238  }
239
240  public String getBackupId() {
241    return backupId;
242  }
243
244  public void setBackupId(String backupId) {
245    this.backupId = backupId;
246  }
247
248  public BackupTableInfo getBackupTableInfo(TableName table) {
249    return this.backupTableInfoMap.get(table);
250  }
251
252  public String getFailedMsg() {
253    return failedMsg;
254  }
255
256  public void setFailedMsg(String failedMsg) {
257    this.failedMsg = failedMsg;
258  }
259
260  public long getStartTs() {
261    return startTs;
262  }
263
264  public void setStartTs(long startTs) {
265    this.startTs = startTs;
266  }
267
268  public long getCompleteTs() {
269    return completeTs;
270  }
271
272  public void setCompleteTs(long endTs) {
273    this.completeTs = endTs;
274  }
275
276  public long getTotalBytesCopied() {
277    return totalBytesCopied;
278  }
279
280  public BackupState getState() {
281    return state;
282  }
283
284  public void setState(BackupState flag) {
285    this.state = flag;
286  }
287
288  public BackupPhase getPhase() {
289    return phase;
290  }
291
292  public void setPhase(BackupPhase phase) {
293    this.phase = phase;
294  }
295
296  public BackupType getType() {
297    return type;
298  }
299
300  public void setSnapshotName(TableName table, String snapshotName) {
301    this.backupTableInfoMap.get(table).setSnapshotName(snapshotName);
302  }
303
304  public String getSnapshotName(TableName table) {
305    return this.backupTableInfoMap.get(table).getSnapshotName();
306  }
307
308  public List<String> getSnapshotNames() {
309    List<String> snapshotNames = new ArrayList<>();
310    for (BackupTableInfo backupStatus : this.backupTableInfoMap.values()) {
311      snapshotNames.add(backupStatus.getSnapshotName());
312    }
313    return snapshotNames;
314  }
315
316  public Set<TableName> getTables() {
317    return this.backupTableInfoMap.keySet();
318  }
319
320  public List<TableName> getTableNames() {
321    return new ArrayList<>(backupTableInfoMap.keySet());
322  }
323
324  public void addTables(TableName[] tables) {
325    for (TableName table : tables) {
326      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
327      this.backupTableInfoMap.put(table, backupStatus);
328    }
329  }
330
331  public void setTables(List<TableName> tables) {
332    this.backupTableInfoMap.clear();
333    for (TableName table : tables) {
334      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
335      this.backupTableInfoMap.put(table, backupStatus);
336    }
337  }
338
339  public String getBackupRootDir() {
340    return backupRootDir;
341  }
342
343  public String getTableBackupDir(TableName tableName) {
344    return BackupUtils.getTableBackupDir(backupRootDir, backupId, tableName);
345  }
346
347  public void setHLogTargetDir(String hlogTagetDir) {
348    this.hlogTargetDir = hlogTagetDir;
349  }
350
351  public String getHLogTargetDir() {
352    return hlogTargetDir;
353  }
354
355  public List<String> getIncrBackupFileList() {
356    return incrBackupFileList;
357  }
358
359  public void setIncrBackupFileList(List<String> incrBackupFileList) {
360    this.incrBackupFileList = incrBackupFileList;
361  }
362
363  /**
364   * Set the new region server log timestamps after distributed log roll
365   * @param prevTableSetTimestampMap table timestamp map
366   */
367  public void setIncrTimestampMap(Map<TableName, Map<String, Long>> prevTableSetTimestampMap) {
368    this.incrTimestampMap = prevTableSetTimestampMap;
369  }
370
371  /**
372   * Get new region server log timestamps after distributed log roll
373   * @return new region server log timestamps
374   */
375  public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
376    return this.incrTimestampMap;
377  }
378
379  public TableName getTableBySnapshot(String snapshotName) {
380    for (Entry<TableName, BackupTableInfo> entry : this.backupTableInfoMap.entrySet()) {
381      if (snapshotName.equals(entry.getValue().getSnapshotName())) {
382        return entry.getKey();
383      }
384    }
385    return null;
386  }
387
388  public BackupProtos.BackupInfo toProtosBackupInfo() {
389    BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder();
390    builder.setBackupId(getBackupId());
391    setBackupTableInfoMap(builder);
392    setTableSetTimestampMap(builder);
393    builder.setCompleteTs(getCompleteTs());
394    if (getFailedMsg() != null) {
395      builder.setFailedMessage(getFailedMsg());
396    }
397    if (getState() != null) {
398      builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name()));
399    }
400    if (getPhase() != null) {
401      builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name()));
402    }
403
404    builder.setProgress(getProgress());
405    builder.setStartTs(getStartTs());
406    builder.setBackupRootDir(getBackupRootDir());
407    builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name()));
408    builder.setWorkersNumber(workers);
409    builder.setBandwidth(bandwidth);
410    return builder.build();
411  }
412
413  @Override
414  public int hashCode() {
415    int hash = 33 * type.hashCode() + backupId != null ? backupId.hashCode() : 0;
416    if (backupRootDir != null) {
417      hash = 33 * hash + backupRootDir.hashCode();
418    }
419    hash = 33 * hash + state.hashCode();
420    hash = 33 * hash + phase.hashCode();
421    hash = 33 * hash + (int) (startTs ^ (startTs >>> 32));
422    hash = 33 * hash + (int) (completeTs ^ (completeTs >>> 32));
423    hash = 33 * hash + (int) (totalBytesCopied ^ (totalBytesCopied >>> 32));
424    if (hlogTargetDir != null) {
425      hash = 33 * hash + hlogTargetDir.hashCode();
426    }
427    return hash;
428  }
429
430  @Override
431  public boolean equals(Object obj) {
432    if (obj instanceof BackupInfo) {
433      BackupInfo other = (BackupInfo) obj;
434      try {
435        return Bytes.equals(toByteArray(), other.toByteArray());
436      } catch (IOException e) {
437        LOG.error(e.toString(), e);
438        return false;
439      }
440    } else {
441      return false;
442    }
443  }
444
445  @Override
446  public String toString() {
447    return backupId;
448  }
449
450  public byte[] toByteArray() throws IOException {
451    return toProtosBackupInfo().toByteArray();
452  }
453
454  private void setBackupTableInfoMap(Builder builder) {
455    for (Entry<TableName, BackupTableInfo> entry : backupTableInfoMap.entrySet()) {
456      builder.addBackupTableInfo(entry.getValue().toProto());
457    }
458  }
459
460  private void setTableSetTimestampMap(Builder builder) {
461    if (this.getTableSetTimestampMap() != null) {
462      for (Entry<TableName, Map<String, Long>> entry : this.getTableSetTimestampMap().entrySet()) {
463        builder.putTableSetTimestamp(entry.getKey().getNameAsString(),
464          BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue())
465            .build());
466      }
467    }
468  }
469
470  public static BackupInfo fromByteArray(byte[] data) throws IOException {
471    return fromProto(BackupProtos.BackupInfo.parseFrom(data));
472  }
473
474  public static BackupInfo fromStream(final InputStream stream) throws IOException {
475    return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream));
476  }
477
478  public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
479    BackupInfo context = new BackupInfo();
480    context.setBackupId(proto.getBackupId());
481    context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList()));
482    context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap()));
483    context.setCompleteTs(proto.getCompleteTs());
484    if (proto.hasFailedMessage()) {
485      context.setFailedMsg(proto.getFailedMessage());
486    }
487    if (proto.hasBackupState()) {
488      context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name()));
489    }
490
491    context
492      .setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId()));
493
494    if (proto.hasBackupPhase()) {
495      context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name()));
496    }
497    if (proto.hasProgress()) {
498      context.setProgress(proto.getProgress());
499    }
500    context.setStartTs(proto.getStartTs());
501    context.setBackupRootDir(proto.getBackupRootDir());
502    context.setType(BackupType.valueOf(proto.getBackupType().name()));
503    context.setWorkers(proto.getWorkersNumber());
504    context.setBandwidth(proto.getBandwidth());
505    return context;
506  }
507
508  private static Map<TableName, BackupTableInfo> toMap(List<BackupProtos.BackupTableInfo> list) {
509    HashMap<TableName, BackupTableInfo> map = new HashMap<>();
510    for (BackupProtos.BackupTableInfo tbs : list) {
511      map.put(ProtobufUtil.toTableName(tbs.getTableName()), BackupTableInfo.convert(tbs));
512    }
513    return map;
514  }
515
516  private static Map<TableName, Map<String, Long>>
517    getTableSetTimestampMap(Map<String, BackupProtos.BackupInfo.RSTimestampMap> map) {
518    Map<TableName, Map<String, Long>> tableSetTimestampMap = new HashMap<>();
519    for (Entry<String, BackupProtos.BackupInfo.RSTimestampMap> entry : map.entrySet()) {
520      tableSetTimestampMap.put(TableName.valueOf(entry.getKey()),
521        entry.getValue().getRsTimestampMap());
522    }
523
524    return tableSetTimestampMap;
525  }
526
527  public String getShortDescription() {
528    StringBuilder sb = new StringBuilder();
529    sb.append("{");
530    sb.append("ID=" + backupId).append(",");
531    sb.append("Type=" + getType()).append(",");
532    sb.append("Tables=" + getTableListAsString()).append(",");
533    sb.append("State=" + getState()).append(",");
534    Date date = null;
535    Calendar cal = Calendar.getInstance();
536    cal.setTimeInMillis(getStartTs());
537    date = cal.getTime();
538    sb.append("Start time=" + date).append(",");
539    if (state == BackupState.FAILED) {
540      sb.append("Failed message=" + getFailedMsg()).append(",");
541    } else if (state == BackupState.RUNNING) {
542      sb.append("Phase=" + getPhase()).append(",");
543    } else if (state == BackupState.COMPLETE) {
544      cal = Calendar.getInstance();
545      cal.setTimeInMillis(getCompleteTs());
546      date = cal.getTime();
547      sb.append("End time=" + date).append(",");
548    }
549    sb.append("Progress=" + getProgress() + "%");
550    sb.append("}");
551
552    return sb.toString();
553  }
554
555  public String getStatusAndProgressAsString() {
556    StringBuilder sb = new StringBuilder();
557    sb.append("id: ").append(getBackupId()).append(" state: ").append(getState())
558      .append(" progress: ").append(getProgress());
559    return sb.toString();
560  }
561
562  public String getTableListAsString() {
563    StringBuffer sb = new StringBuffer();
564    sb.append("{");
565    sb.append(StringUtils.join(backupTableInfoMap.keySet(), ","));
566    sb.append("}");
567    return sb.toString();
568  }
569
570  /**
571   * We use only time stamps to compare objects during sort operation
572   */
573  @Override
574  public int compareTo(BackupInfo o) {
575    Long thisTS =
576      Long.valueOf(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1));
577    Long otherTS = Long.valueOf(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1));
578    return thisTS.compareTo(otherTS);
579  }
580}