001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Map.Entry;
027import java.util.TreeMap;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FSDataInputStream;
030import org.apache.hadoop.fs.FSDataOutputStream;
031import org.apache.hadoop.fs.FileStatus;
032import org.apache.hadoop.fs.FileSystem;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.backup.BackupInfo;
037import org.apache.hadoop.hbase.backup.BackupType;
038import org.apache.hadoop.hbase.backup.HBackupFileSystem;
039import org.apache.hadoop.hbase.backup.util.BackupUtils;
040import org.apache.yetus.audience.InterfaceAudience;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
045import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
046import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
047
048/**
049 * Backup manifest contains all the meta data of a backup image. The manifest info will be bundled
050 * as manifest file together with data. So that each backup image will contain all the info needed
051 * for restore. BackupManifest is a storage container for BackupImage. It is responsible for
052 * storing/reading backup image data and has some additional utility methods.
053 */
054@InterfaceAudience.Private
055public class BackupManifest {
056  private static final Logger LOG = LoggerFactory.getLogger(BackupManifest.class);
057
058  // manifest file name
059  public static final String MANIFEST_FILE_NAME = ".backup.manifest";
060
061  /**
062   * Backup image, the dependency graph is made up by series of backup images BackupImage contains
063   * all the relevant information to restore the backup and is used during restore operation
064   */
065  public static class BackupImage implements Comparable<BackupImage> {
066    static class Builder {
067      BackupImage image;
068
069      Builder() {
070        image = new BackupImage();
071      }
072
073      Builder withBackupId(String backupId) {
074        image.setBackupId(backupId);
075        return this;
076      }
077
078      Builder withType(BackupType type) {
079        image.setType(type);
080        return this;
081      }
082
083      Builder withRootDir(String rootDir) {
084        image.setRootDir(rootDir);
085        return this;
086      }
087
088      Builder withTableList(List<TableName> tableList) {
089        image.setTableList(tableList);
090        return this;
091      }
092
093      Builder withStartTime(long startTime) {
094        image.setStartTs(startTime);
095        return this;
096      }
097
098      Builder withCompleteTime(long completeTime) {
099        image.setCompleteTs(completeTime);
100        return this;
101      }
102
103      BackupImage build() {
104        return image;
105      }
106
107    }
108
109    private String backupId;
110    private BackupType type;
111    private String rootDir;
112    private List<TableName> tableList;
113    private long startTs;
114    private long completeTs;
115    private ArrayList<BackupImage> ancestors;
116    private Map<TableName, Map<String, Long>> incrTimeRanges;
117
118    static Builder newBuilder() {
119      return new Builder();
120    }
121
122    public BackupImage() {
123      super();
124    }
125
126    private BackupImage(String backupId, BackupType type, String rootDir, List<TableName> tableList,
127      long startTs, long completeTs) {
128      this.backupId = backupId;
129      this.type = type;
130      this.rootDir = rootDir;
131      this.tableList = tableList;
132      this.startTs = startTs;
133      this.completeTs = completeTs;
134    }
135
136    static BackupImage fromProto(BackupProtos.BackupImage im) {
137      String backupId = im.getBackupId();
138      String rootDir = im.getBackupRootDir();
139      long startTs = im.getStartTs();
140      long completeTs = im.getCompleteTs();
141      List<HBaseProtos.TableName> tableListList = im.getTableListList();
142      List<TableName> tableList = new ArrayList<>();
143      for (HBaseProtos.TableName tn : tableListList) {
144        tableList.add(ProtobufUtil.toTableName(tn));
145      }
146
147      List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
148
149      BackupType type = im.getBackupType() == BackupProtos.BackupType.FULL
150        ? BackupType.FULL
151        : BackupType.INCREMENTAL;
152
153      BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
154      for (BackupProtos.BackupImage img : ancestorList) {
155        image.addAncestor(fromProto(img));
156      }
157      image.setIncrTimeRanges(loadIncrementalTimestampMap(im));
158      return image;
159    }
160
161    BackupProtos.BackupImage toProto() {
162      BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
163      builder.setBackupId(backupId);
164      builder.setCompleteTs(completeTs);
165      builder.setStartTs(startTs);
166      builder.setBackupRootDir(rootDir);
167      if (type == BackupType.FULL) {
168        builder.setBackupType(BackupProtos.BackupType.FULL);
169      } else {
170        builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
171      }
172
173      for (TableName name : tableList) {
174        builder.addTableList(ProtobufUtil.toProtoTableName(name));
175      }
176
177      if (ancestors != null) {
178        for (BackupImage im : ancestors) {
179          builder.addAncestors(im.toProto());
180        }
181      }
182
183      setIncrementalTimestampMap(builder);
184      return builder.build();
185    }
186
187    private static Map<TableName, Map<String, Long>>
188      loadIncrementalTimestampMap(BackupProtos.BackupImage proto) {
189      List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
190
191      Map<TableName, Map<String, Long>> incrTimeRanges = new HashMap<>();
192
193      if (list == null || list.size() == 0) {
194        return incrTimeRanges;
195      }
196
197      for (BackupProtos.TableServerTimestamp tst : list) {
198        TableName tn = ProtobufUtil.toTableName(tst.getTableName());
199        Map<String, Long> map = incrTimeRanges.get(tn);
200        if (map == null) {
201          map = new HashMap<>();
202          incrTimeRanges.put(tn, map);
203        }
204        List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
205        for (BackupProtos.ServerTimestamp stm : listSt) {
206          ServerName sn = ProtobufUtil.toServerName(stm.getServerName());
207          map.put(sn.getHostname() + ":" + sn.getPort(), stm.getTimestamp());
208        }
209      }
210      return incrTimeRanges;
211    }
212
213    private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) {
214      if (this.incrTimeRanges == null) {
215        return;
216      }
217      for (Entry<TableName, Map<String, Long>> entry : this.incrTimeRanges.entrySet()) {
218        TableName key = entry.getKey();
219        Map<String, Long> value = entry.getValue();
220        BackupProtos.TableServerTimestamp.Builder tstBuilder =
221          BackupProtos.TableServerTimestamp.newBuilder();
222        tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key));
223
224        for (Map.Entry<String, Long> entry2 : value.entrySet()) {
225          String s = entry2.getKey();
226          BackupProtos.ServerTimestamp.Builder stBuilder =
227            BackupProtos.ServerTimestamp.newBuilder();
228          HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
229          ServerName sn = ServerName.parseServerName(s);
230          snBuilder.setHostName(sn.getHostname());
231          snBuilder.setPort(sn.getPort());
232          stBuilder.setServerName(snBuilder.build());
233          stBuilder.setTimestamp(entry2.getValue());
234          tstBuilder.addServerTimestamp(stBuilder.build());
235        }
236        builder.addTstMap(tstBuilder.build());
237      }
238    }
239
240    public String getBackupId() {
241      return backupId;
242    }
243
244    private void setBackupId(String backupId) {
245      this.backupId = backupId;
246    }
247
248    public BackupType getType() {
249      return type;
250    }
251
252    private void setType(BackupType type) {
253      this.type = type;
254    }
255
256    public String getRootDir() {
257      return rootDir;
258    }
259
260    private void setRootDir(String rootDir) {
261      this.rootDir = rootDir;
262    }
263
264    public List<TableName> getTableNames() {
265      return tableList;
266    }
267
268    private void setTableList(List<TableName> tableList) {
269      this.tableList = tableList;
270    }
271
272    public long getStartTs() {
273      return startTs;
274    }
275
276    private void setStartTs(long startTs) {
277      this.startTs = startTs;
278    }
279
280    public long getCompleteTs() {
281      return completeTs;
282    }
283
284    private void setCompleteTs(long completeTs) {
285      this.completeTs = completeTs;
286    }
287
288    public ArrayList<BackupImage> getAncestors() {
289      if (this.ancestors == null) {
290        this.ancestors = new ArrayList<>();
291      }
292      return this.ancestors;
293    }
294
295    public void removeAncestors(List<String> backupIds) {
296      List<BackupImage> toRemove = new ArrayList<>();
297      for (BackupImage im : this.ancestors) {
298        if (backupIds.contains(im.getBackupId())) {
299          toRemove.add(im);
300        }
301      }
302      this.ancestors.removeAll(toRemove);
303    }
304
305    private void addAncestor(BackupImage backupImage) {
306      this.getAncestors().add(backupImage);
307    }
308
309    public boolean hasAncestor(String token) {
310      for (BackupImage image : this.getAncestors()) {
311        if (image.getBackupId().equals(token)) {
312          return true;
313        }
314      }
315      return false;
316    }
317
318    public boolean hasTable(TableName table) {
319      return tableList.contains(table);
320    }
321
322    @Override
323    public int compareTo(BackupImage other) {
324      String thisBackupId = this.getBackupId();
325      String otherBackupId = other.getBackupId();
326      int index1 = thisBackupId.lastIndexOf("_");
327      int index2 = otherBackupId.lastIndexOf("_");
328      String name1 = thisBackupId.substring(0, index1);
329      String name2 = otherBackupId.substring(0, index2);
330      if (name1.equals(name2)) {
331        Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1));
332        Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1));
333        return thisTS.compareTo(otherTS);
334      } else {
335        return name1.compareTo(name2);
336      }
337    }
338
339    @Override
340    public boolean equals(Object obj) {
341      if (obj instanceof BackupImage) {
342        return this.compareTo((BackupImage) obj) == 0;
343      }
344      return false;
345    }
346
347    @Override
348    public int hashCode() {
349      int hash = 33 * this.getBackupId().hashCode() + type.hashCode();
350      hash = 33 * hash + rootDir.hashCode();
351      hash = 33 * hash + Long.valueOf(startTs).hashCode();
352      hash = 33 * hash + Long.valueOf(completeTs).hashCode();
353      for (TableName table : tableList) {
354        hash = 33 * hash + table.hashCode();
355      }
356      return hash;
357    }
358
359    public Map<TableName, Map<String, Long>> getIncrTimeRanges() {
360      return incrTimeRanges;
361    }
362
363    private void setIncrTimeRanges(Map<TableName, Map<String, Long>> incrTimeRanges) {
364      this.incrTimeRanges = incrTimeRanges;
365    }
366  }
367
368  // backup image directory
369  private String tableBackupDir = null;
370  private BackupImage backupImage;
371
372  /**
373   * Construct manifest for a ongoing backup.
374   * @param backup The ongoing backup info
375   */
376  public BackupManifest(BackupInfo backup) {
377    BackupImage.Builder builder = BackupImage.newBuilder();
378    this.backupImage = builder.withBackupId(backup.getBackupId()).withType(backup.getType())
379      .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
380      .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
381  }
382
383  /**
384   * Construct a table level manifest for a backup of the named table.
385   * @param backup The ongoing backup session info
386   */
387  public BackupManifest(BackupInfo backup, TableName table) {
388    this.tableBackupDir = backup.getTableBackupDir(table);
389    List<TableName> tables = new ArrayList<TableName>();
390    tables.add(table);
391    BackupImage.Builder builder = BackupImage.newBuilder();
392    this.backupImage = builder.withBackupId(backup.getBackupId()).withType(backup.getType())
393      .withRootDir(backup.getBackupRootDir()).withTableList(tables)
394      .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
395  }
396
397  /**
398   * Construct manifest from a backup directory.
399   * @param conf       configuration
400   * @param backupPath backup path
401   * @throws IOException if constructing the manifest from the backup directory fails
402   */
403  public BackupManifest(Configuration conf, Path backupPath) throws IOException {
404    this(backupPath.getFileSystem(conf), backupPath);
405  }
406
407  /**
408   * Construct manifest from a backup directory.
409   * @param fs         the FileSystem
410   * @param backupPath backup path
411   * @throws BackupException exception
412   */
413  public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
414    if (LOG.isDebugEnabled()) {
415      LOG.debug("Loading manifest from: " + backupPath.toString());
416    }
417    // The input backupDir may not exactly be the backup table dir.
418    // It could be the backup log dir where there is also a manifest file stored.
419    // This variable's purpose is to keep the correct and original location so
420    // that we can store/persist it.
421    try {
422      FileStatus[] subFiles = BackupUtils.listStatus(fs, backupPath, null);
423      if (subFiles == null) {
424        String errorMsg = backupPath.toString() + " does not exist";
425        LOG.error(errorMsg);
426        throw new IOException(errorMsg);
427      }
428      for (FileStatus subFile : subFiles) {
429        if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
430          // load and set manifest field from file content
431          long len = subFile.getLen();
432          byte[] pbBytes = new byte[(int) len];
433          try (FSDataInputStream in = fs.open(subFile.getPath())) {
434            in.readFully(pbBytes);
435          } catch (IOException e) {
436            throw new BackupException(e.getMessage());
437          }
438          BackupProtos.BackupImage proto = null;
439          try {
440            proto = BackupProtos.BackupImage.parseFrom(pbBytes);
441          } catch (Exception e) {
442            throw new BackupException(e);
443          }
444          this.backupImage = BackupImage.fromProto(proto);
445          LOG.debug("Loaded manifest instance from manifest file: "
446            + BackupUtils.getPath(subFile.getPath()));
447          return;
448        }
449      }
450      String errorMsg = "No manifest file found in: " + backupPath.toString();
451      throw new IOException(errorMsg);
452    } catch (IOException e) {
453      throw new BackupException(e.getMessage());
454    }
455  }
456
457  public BackupType getType() {
458    return backupImage.getType();
459  }
460
461  /**
462   * Get the table set of this image.
463   * @return The table set list
464   */
465  public List<TableName> getTableList() {
466    return backupImage.getTableNames();
467  }
468
469  /**
470   * TODO: fix it. Persist the manifest file.
471   * @throws IOException IOException when storing the manifest file.
472   */
473  public void store(Configuration conf) throws BackupException {
474    byte[] data = backupImage.toProto().toByteArray();
475    // write the file, overwrite if already exist
476    Path manifestFilePath =
477      new Path(HBackupFileSystem.getBackupPath(backupImage.getRootDir(), backupImage.getBackupId()),
478        MANIFEST_FILE_NAME);
479    try (FSDataOutputStream out =
480      manifestFilePath.getFileSystem(conf).create(manifestFilePath, true)) {
481      out.write(data);
482    } catch (IOException e) {
483      throw new BackupException(e.getMessage());
484    }
485
486    LOG.info("Manifest file stored to " + manifestFilePath);
487  }
488
489  /**
490   * Get this backup image.
491   * @return the backup image.
492   */
493  public BackupImage getBackupImage() {
494    return backupImage;
495  }
496
497  /**
498   * Add dependent backup image for this backup.
499   * @param image The direct dependent backup image
500   */
501  public void addDependentImage(BackupImage image) {
502    this.backupImage.addAncestor(image);
503  }
504
505  /**
506   * Set the incremental timestamp map directly.
507   * @param incrTimestampMap timestamp map
508   */
509  public void setIncrTimestampMap(Map<TableName, Map<String, Long>> incrTimestampMap) {
510    this.backupImage.setIncrTimeRanges(incrTimestampMap);
511  }
512
513  public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
514    return backupImage.getIncrTimeRanges();
515  }
516
517  /**
518   * Get the image list of this backup for restore in time order.
519   * @param reverse If true, then output in reverse order, otherwise in time order from old to new
520   * @return the backup image list for restore in time order
521   */
522  public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
523    TreeMap<Long, BackupImage> restoreImages = new TreeMap<>();
524    restoreImages.put(backupImage.startTs, backupImage);
525    for (BackupImage image : backupImage.getAncestors()) {
526      restoreImages.put(Long.valueOf(image.startTs), image);
527    }
528    return new ArrayList<>(
529      reverse ? (restoreImages.descendingMap().values()) : (restoreImages.values()));
530  }
531
532  /**
533   * Get the dependent image list for a specific table of this backup in time order from old to new
534   * if want to restore to this backup image level.
535   * @param table table
536   * @return the backup image list for a table in time order
537   */
538  public ArrayList<BackupImage> getDependentListByTable(TableName table) {
539    ArrayList<BackupImage> tableImageList = new ArrayList<>();
540    ArrayList<BackupImage> imageList = getRestoreDependentList(true);
541    for (BackupImage image : imageList) {
542      if (image.hasTable(table)) {
543        tableImageList.add(image);
544        if (image.getType() == BackupType.FULL) {
545          break;
546        }
547      }
548    }
549    Collections.reverse(tableImageList);
550    return tableImageList;
551  }
552
553  /**
554   * Get the full dependent image list in the whole dependency scope for a specific table of this
555   * backup in time order from old to new.
556   * @param table table
557   * @return the full backup image list for a table in time order in the whole scope of the
558   *         dependency of this image
559   */
560  public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
561    ArrayList<BackupImage> tableImageList = new ArrayList<>();
562    ArrayList<BackupImage> imageList = getRestoreDependentList(false);
563    for (BackupImage image : imageList) {
564      if (image.hasTable(table)) {
565        tableImageList.add(image);
566      }
567    }
568    return tableImageList;
569  }
570
571  /**
572   * Check whether backup image1 could cover backup image2 or not.
573   * @param image1 backup image 1
574   * @param image2 backup image 2
575   * @return true if image1 can cover image2, otherwise false
576   */
577  public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
578    // image1 can cover image2 only when the following conditions are satisfied:
579    // - image1 must not be an incremental image;
580    // - image1 must be taken after image2 has been taken;
581    // - table set of image1 must cover the table set of image2.
582    if (image1.getType() == BackupType.INCREMENTAL) {
583      return false;
584    }
585    if (image1.getStartTs() < image2.getStartTs()) {
586      return false;
587    }
588    List<TableName> image1TableList = image1.getTableNames();
589    List<TableName> image2TableList = image2.getTableNames();
590    boolean found;
591    for (int i = 0; i < image2TableList.size(); i++) {
592      found = false;
593      for (int j = 0; j < image1TableList.size(); j++) {
594        if (image2TableList.get(i).equals(image1TableList.get(j))) {
595          found = true;
596          break;
597        }
598      }
599      if (!found) {
600        return false;
601      }
602    }
603
604    LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
605    return true;
606  }
607
608  /**
609   * Check whether backup image set could cover a backup image or not.
610   * @param fullImages The backup image set
611   * @param image      The target backup image
612   * @return true if fullImages can cover image, otherwise false
613   */
614  public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
615    // fullImages can cover image only when the following conditions are satisfied:
616    // - each image of fullImages must not be an incremental image;
617    // - each image of fullImages must be taken after image has been taken;
618    // - sum table set of fullImages must cover the table set of image.
619    for (BackupImage image1 : fullImages) {
620      if (image1.getType() == BackupType.INCREMENTAL) {
621        return false;
622      }
623      if (image1.getStartTs() < image.getStartTs()) {
624        return false;
625      }
626    }
627
628    ArrayList<String> image1TableList = new ArrayList<>();
629    for (BackupImage image1 : fullImages) {
630      List<TableName> tableList = image1.getTableNames();
631      for (TableName table : tableList) {
632        image1TableList.add(table.getNameAsString());
633      }
634    }
635    ArrayList<String> image2TableList = new ArrayList<>();
636    List<TableName> tableList = image.getTableNames();
637    for (TableName table : tableList) {
638      image2TableList.add(table.getNameAsString());
639    }
640
641    for (int i = 0; i < image2TableList.size(); i++) {
642      if (image1TableList.contains(image2TableList.get(i)) == false) {
643        return false;
644      }
645    }
646
647    LOG.debug("Full image set can cover image " + image.getBackupId());
648    return true;
649  }
650
651  public BackupInfo toBackupInfo() {
652    BackupInfo info = new BackupInfo();
653    info.setType(backupImage.getType());
654    List<TableName> list = backupImage.getTableNames();
655    TableName[] tables = new TableName[list.size()];
656    info.addTables(list.toArray(tables));
657    info.setBackupId(backupImage.getBackupId());
658    info.setStartTs(backupImage.getStartTs());
659    info.setBackupRootDir(backupImage.getRootDir());
660    if (backupImage.getType() == BackupType.INCREMENTAL) {
661      info.setHLogTargetDir(
662        BackupUtils.getLogBackupDir(backupImage.getRootDir(), backupImage.getBackupId()));
663    }
664    return info;
665  }
666}