001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.backup.impl;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.Map.Entry;
028import java.util.TreeMap;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.FSDataInputStream;
032import org.apache.hadoop.fs.FSDataOutputStream;
033import org.apache.hadoop.fs.FileStatus;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.backup.BackupInfo;
039import org.apache.hadoop.hbase.backup.BackupType;
040import org.apache.hadoop.hbase.backup.HBackupFileSystem;
041import org.apache.hadoop.hbase.backup.util.BackupUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
049
050/**
051 * Backup manifest contains all the meta data of a backup image. The manifest info will be bundled
052 * as manifest file together with data. So that each backup image will contain all the info needed
053 * for restore. BackupManifest is a storage container for BackupImage.
054 * It is responsible for storing/reading backup image data and has some additional utility methods.
055 *
056 */
057@InterfaceAudience.Private
058public class BackupManifest {
059  private static final Logger LOG = LoggerFactory.getLogger(BackupManifest.class);
060
061  // manifest file name
062  public static final String MANIFEST_FILE_NAME = ".backup.manifest";
063
064  /**
065   * Backup image, the dependency graph is made up by series of backup images BackupImage contains
066   * all the relevant information to restore the backup and is used during restore operation
067   */
068  public static class BackupImage implements Comparable<BackupImage> {
069    static class Builder {
070      BackupImage image;
071
072      Builder() {
073        image = new BackupImage();
074      }
075
076      Builder withBackupId(String backupId) {
077        image.setBackupId(backupId);
078        return this;
079      }
080
081      Builder withType(BackupType type) {
082        image.setType(type);
083        return this;
084      }
085
086      Builder withRootDir(String rootDir) {
087        image.setRootDir(rootDir);
088        return this;
089      }
090
091      Builder withTableList(List<TableName> tableList) {
092        image.setTableList(tableList);
093        return this;
094      }
095
096      Builder withStartTime(long startTime) {
097        image.setStartTs(startTime);
098        return this;
099      }
100
101      Builder withCompleteTime(long completeTime) {
102        image.setCompleteTs(completeTime);
103        return this;
104      }
105
106      BackupImage build() {
107        return image;
108      }
109
110    }
111
112    private String backupId;
113    private BackupType type;
114    private String rootDir;
115    private List<TableName> tableList;
116    private long startTs;
117    private long completeTs;
118    private ArrayList<BackupImage> ancestors;
119    private HashMap<TableName, HashMap<String, Long>> incrTimeRanges;
120
121    static Builder newBuilder() {
122      return new Builder();
123    }
124
125    public BackupImage() {
126      super();
127    }
128
129    private BackupImage(String backupId, BackupType type, String rootDir,
130        List<TableName> tableList, long startTs, long completeTs) {
131      this.backupId = backupId;
132      this.type = type;
133      this.rootDir = rootDir;
134      this.tableList = tableList;
135      this.startTs = startTs;
136      this.completeTs = completeTs;
137    }
138
139    static BackupImage fromProto(BackupProtos.BackupImage im) {
140      String backupId = im.getBackupId();
141      String rootDir = im.getBackupRootDir();
142      long startTs = im.getStartTs();
143      long completeTs = im.getCompleteTs();
144      List<HBaseProtos.TableName> tableListList = im.getTableListList();
145      List<TableName> tableList = new ArrayList<>();
146      for (HBaseProtos.TableName tn : tableListList) {
147        tableList.add(ProtobufUtil.toTableName(tn));
148      }
149
150      List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
151
152      BackupType type =
153          im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL
154              : BackupType.INCREMENTAL;
155
156      BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
157      for (BackupProtos.BackupImage img : ancestorList) {
158        image.addAncestor(fromProto(img));
159      }
160      image.setIncrTimeRanges(loadIncrementalTimestampMap(im));
161      return image;
162    }
163
164    BackupProtos.BackupImage toProto() {
165      BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
166      builder.setBackupId(backupId);
167      builder.setCompleteTs(completeTs);
168      builder.setStartTs(startTs);
169      builder.setBackupRootDir(rootDir);
170      if (type == BackupType.FULL) {
171        builder.setBackupType(BackupProtos.BackupType.FULL);
172      } else {
173        builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
174      }
175
176      for (TableName name : tableList) {
177        builder.addTableList(ProtobufUtil.toProtoTableName(name));
178      }
179
180      if (ancestors != null) {
181        for (BackupImage im : ancestors) {
182          builder.addAncestors(im.toProto());
183        }
184      }
185
186      setIncrementalTimestampMap(builder);
187      return builder.build();
188    }
189
190    private static HashMap<TableName, HashMap<String, Long>> loadIncrementalTimestampMap(
191        BackupProtos.BackupImage proto) {
192      List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
193
194      HashMap<TableName, HashMap<String, Long>> incrTimeRanges = new HashMap<>();
195
196      if (list == null || list.size() == 0) {
197        return incrTimeRanges;
198      }
199
200      for (BackupProtos.TableServerTimestamp tst : list) {
201        TableName tn = ProtobufUtil.toTableName(tst.getTableName());
202        HashMap<String, Long> map = incrTimeRanges.get(tn);
203        if (map == null) {
204          map = new HashMap<>();
205          incrTimeRanges.put(tn, map);
206        }
207        List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
208        for (BackupProtos.ServerTimestamp stm : listSt) {
209          ServerName sn = ProtobufUtil.toServerName(stm.getServerName());
210          map.put(sn.getHostname() + ":" + sn.getPort(), stm.getTimestamp());
211        }
212      }
213      return incrTimeRanges;
214    }
215
216    private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) {
217      if (this.incrTimeRanges == null) {
218        return;
219      }
220      for (Entry<TableName, HashMap<String, Long>> entry : this.incrTimeRanges.entrySet()) {
221        TableName key = entry.getKey();
222        HashMap<String, Long> value = entry.getValue();
223        BackupProtos.TableServerTimestamp.Builder tstBuilder =
224            BackupProtos.TableServerTimestamp.newBuilder();
225        tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key));
226
227        for (Map.Entry<String, Long> entry2 : value.entrySet()) {
228          String s = entry2.getKey();
229          BackupProtos.ServerTimestamp.Builder stBuilder =
230              BackupProtos.ServerTimestamp.newBuilder();
231          HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
232          ServerName sn = ServerName.parseServerName(s);
233          snBuilder.setHostName(sn.getHostname());
234          snBuilder.setPort(sn.getPort());
235          stBuilder.setServerName(snBuilder.build());
236          stBuilder.setTimestamp(entry2.getValue());
237          tstBuilder.addServerTimestamp(stBuilder.build());
238        }
239        builder.addTstMap(tstBuilder.build());
240      }
241    }
242
243    public String getBackupId() {
244      return backupId;
245    }
246
247    private void setBackupId(String backupId) {
248      this.backupId = backupId;
249    }
250
251    public BackupType getType() {
252      return type;
253    }
254
255    private void setType(BackupType type) {
256      this.type = type;
257    }
258
259    public String getRootDir() {
260      return rootDir;
261    }
262
263    private void setRootDir(String rootDir) {
264      this.rootDir = rootDir;
265    }
266
267    public List<TableName> getTableNames() {
268      return tableList;
269    }
270
271    private void setTableList(List<TableName> tableList) {
272      this.tableList = tableList;
273    }
274
275    public long getStartTs() {
276      return startTs;
277    }
278
279    private void setStartTs(long startTs) {
280      this.startTs = startTs;
281    }
282
283    public long getCompleteTs() {
284      return completeTs;
285    }
286
287    private void setCompleteTs(long completeTs) {
288      this.completeTs = completeTs;
289    }
290
291    public ArrayList<BackupImage> getAncestors() {
292      if (this.ancestors == null) {
293        this.ancestors = new ArrayList<>();
294      }
295      return this.ancestors;
296    }
297
298    public void removeAncestors(List<String> backupIds) {
299      List<BackupImage> toRemove = new ArrayList<>();
300      for (BackupImage im : this.ancestors) {
301        if (backupIds.contains(im.getBackupId())) {
302          toRemove.add(im);
303        }
304      }
305      this.ancestors.removeAll(toRemove);
306    }
307
308    private void addAncestor(BackupImage backupImage) {
309      this.getAncestors().add(backupImage);
310    }
311
312    public boolean hasAncestor(String token) {
313      for (BackupImage image : this.getAncestors()) {
314        if (image.getBackupId().equals(token)) {
315          return true;
316        }
317      }
318      return false;
319    }
320
321    public boolean hasTable(TableName table) {
322      return tableList.contains(table);
323    }
324
325    @Override
326    public int compareTo(BackupImage other) {
327      String thisBackupId = this.getBackupId();
328      String otherBackupId = other.getBackupId();
329      int index1 = thisBackupId.lastIndexOf("_");
330      int index2 = otherBackupId.lastIndexOf("_");
331      String name1 = thisBackupId.substring(0, index1);
332      String name2 = otherBackupId.substring(0, index2);
333      if (name1.equals(name2)) {
334        Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1));
335        Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1));
336        return thisTS.compareTo(otherTS);
337      } else {
338        return name1.compareTo(name2);
339      }
340    }
341
342    @Override
343    public boolean equals(Object obj) {
344      if (obj instanceof BackupImage) {
345        return this.compareTo((BackupImage) obj) == 0;
346      }
347      return false;
348    }
349
350    @Override
351    public int hashCode() {
352      int hash = 33 * this.getBackupId().hashCode() + type.hashCode();
353      hash = 33 * hash + rootDir.hashCode();
354      hash = 33 * hash + Long.valueOf(startTs).hashCode();
355      hash = 33 * hash + Long.valueOf(completeTs).hashCode();
356      for (TableName table : tableList) {
357        hash = 33 * hash + table.hashCode();
358      }
359      return hash;
360    }
361
362    public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() {
363      return incrTimeRanges;
364    }
365
366    private void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) {
367      this.incrTimeRanges = incrTimeRanges;
368    }
369  }
370
371  // backup image directory
372  private String tableBackupDir = null;
373  private BackupImage backupImage;
374
375  /**
376   * Construct manifest for a ongoing backup.
377   * @param backup The ongoing backup info
378   */
379  public BackupManifest(BackupInfo backup) {
380    BackupImage.Builder builder = BackupImage.newBuilder();
381    this.backupImage =
382        builder.withBackupId(backup.getBackupId()).withType(backup.getType())
383            .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
384            .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
385  }
386
387  /**
388   * Construct a table level manifest for a backup of the named table.
389   * @param backup The ongoing backup session info
390   */
391  public BackupManifest(BackupInfo backup, TableName table) {
392    this.tableBackupDir = backup.getTableBackupDir(table);
393    List<TableName> tables = new ArrayList<TableName>();
394    tables.add(table);
395    BackupImage.Builder builder = BackupImage.newBuilder();
396    this.backupImage =
397        builder.withBackupId(backup.getBackupId()).withType(backup.getType())
398            .withRootDir(backup.getBackupRootDir()).withTableList(tables)
399            .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
400  }
401
402  /**
403   * Construct manifest from a backup directory.
404   *
405   * @param conf configuration
406   * @param backupPath backup path
407   * @throws IOException if constructing the manifest from the backup directory fails
408   */
409  public BackupManifest(Configuration conf, Path backupPath) throws IOException {
410    this(backupPath.getFileSystem(conf), backupPath);
411  }
412
413  /**
414   * Construct manifest from a backup directory.
415   * @param fs the FileSystem
416   * @param backupPath backup path
417   * @throws BackupException exception
418   */
419  public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
420    if (LOG.isDebugEnabled()) {
421      LOG.debug("Loading manifest from: " + backupPath.toString());
422    }
423    // The input backupDir may not exactly be the backup table dir.
424    // It could be the backup log dir where there is also a manifest file stored.
425    // This variable's purpose is to keep the correct and original location so
426    // that we can store/persist it.
427    try {
428      FileStatus[] subFiles = BackupUtils.listStatus(fs, backupPath, null);
429      if (subFiles == null) {
430        String errorMsg = backupPath.toString() + " does not exist";
431        LOG.error(errorMsg);
432        throw new IOException(errorMsg);
433      }
434      for (FileStatus subFile : subFiles) {
435        if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
436          // load and set manifest field from file content
437          long len = subFile.getLen();
438          byte[] pbBytes = new byte[(int) len];
439          try (FSDataInputStream in = fs.open(subFile.getPath())) {
440            in.readFully(pbBytes);
441          } catch (IOException e) {
442            throw new BackupException(e.getMessage());
443          }
444          BackupProtos.BackupImage proto = null;
445          try {
446            proto = BackupProtos.BackupImage.parseFrom(pbBytes);
447          } catch (Exception e) {
448            throw new BackupException(e);
449          }
450          this.backupImage = BackupImage.fromProto(proto);
451          LOG.debug("Loaded manifest instance from manifest file: "
452              + BackupUtils.getPath(subFile.getPath()));
453          return;
454        }
455      }
456      String errorMsg = "No manifest file found in: " + backupPath.toString();
457      throw new IOException(errorMsg);
458    } catch (IOException e) {
459      throw new BackupException(e.getMessage());
460    }
461  }
462
463  public BackupType getType() {
464    return backupImage.getType();
465  }
466
467  /**
468   * Get the table set of this image.
469   * @return The table set list
470   */
471  public List<TableName> getTableList() {
472    return backupImage.getTableNames();
473  }
474
475  /**
476   * TODO: fix it. Persist the manifest file.
477   * @throws IOException IOException when storing the manifest file.
478   */
479  public void store(Configuration conf) throws BackupException {
480    byte[] data = backupImage.toProto().toByteArray();
481    // write the file, overwrite if already exist
482    Path manifestFilePath =
483        new Path(HBackupFileSystem.getBackupPath(backupImage.getRootDir(),
484          backupImage.getBackupId()), MANIFEST_FILE_NAME);
485    try (FSDataOutputStream out =
486        manifestFilePath.getFileSystem(conf).create(manifestFilePath, true)) {
487      out.write(data);
488    } catch (IOException e) {
489      throw new BackupException(e.getMessage());
490    }
491
492    LOG.info("Manifest file stored to " + manifestFilePath);
493  }
494
495  /**
496   * Get this backup image.
497   * @return the backup image.
498   */
499  public BackupImage getBackupImage() {
500    return backupImage;
501  }
502
503  /**
504   * Add dependent backup image for this backup.
505   * @param image The direct dependent backup image
506   */
507  public void addDependentImage(BackupImage image) {
508    this.backupImage.addAncestor(image);
509  }
510
511  /**
512   * Set the incremental timestamp map directly.
513   * @param incrTimestampMap timestamp map
514   */
515  public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
516    this.backupImage.setIncrTimeRanges(incrTimestampMap);
517  }
518
519  public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
520    return backupImage.getIncrTimeRanges();
521  }
522
523  /**
524   * Get the image list of this backup for restore in time order.
525   * @param reverse If true, then output in reverse order, otherwise in time order from old to new
526   * @return the backup image list for restore in time order
527   */
528  public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
529    TreeMap<Long, BackupImage> restoreImages = new TreeMap<>();
530    restoreImages.put(backupImage.startTs, backupImage);
531    for (BackupImage image : backupImage.getAncestors()) {
532      restoreImages.put(Long.valueOf(image.startTs), image);
533    }
534    return new ArrayList<>(reverse ? (restoreImages.descendingMap().values())
535        : (restoreImages.values()));
536  }
537
538  /**
539   * Get the dependent image list for a specific table of this backup in time order from old to new
540   * if want to restore to this backup image level.
541   * @param table table
542   * @return the backup image list for a table in time order
543   */
544  public ArrayList<BackupImage> getDependentListByTable(TableName table) {
545    ArrayList<BackupImage> tableImageList = new ArrayList<>();
546    ArrayList<BackupImage> imageList = getRestoreDependentList(true);
547    for (BackupImage image : imageList) {
548      if (image.hasTable(table)) {
549        tableImageList.add(image);
550        if (image.getType() == BackupType.FULL) {
551          break;
552        }
553      }
554    }
555    Collections.reverse(tableImageList);
556    return tableImageList;
557  }
558
559  /**
560   * Get the full dependent image list in the whole dependency scope for a specific table of this
561   * backup in time order from old to new.
562   * @param table table
563   * @return the full backup image list for a table in time order in the whole scope of the
564   *         dependency of this image
565   */
566  public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
567    ArrayList<BackupImage> tableImageList = new ArrayList<>();
568    ArrayList<BackupImage> imageList = getRestoreDependentList(false);
569    for (BackupImage image : imageList) {
570      if (image.hasTable(table)) {
571        tableImageList.add(image);
572      }
573    }
574    return tableImageList;
575  }
576
577  /**
578   * Check whether backup image1 could cover backup image2 or not.
579   * @param image1 backup image 1
580   * @param image2 backup image 2
581   * @return true if image1 can cover image2, otherwise false
582   */
583  public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
584    // image1 can cover image2 only when the following conditions are satisfied:
585    // - image1 must not be an incremental image;
586    // - image1 must be taken after image2 has been taken;
587    // - table set of image1 must cover the table set of image2.
588    if (image1.getType() == BackupType.INCREMENTAL) {
589      return false;
590    }
591    if (image1.getStartTs() < image2.getStartTs()) {
592      return false;
593    }
594    List<TableName> image1TableList = image1.getTableNames();
595    List<TableName> image2TableList = image2.getTableNames();
596    boolean found;
597    for (int i = 0; i < image2TableList.size(); i++) {
598      found = false;
599      for (int j = 0; j < image1TableList.size(); j++) {
600        if (image2TableList.get(i).equals(image1TableList.get(j))) {
601          found = true;
602          break;
603        }
604      }
605      if (!found) {
606        return false;
607      }
608    }
609
610    LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
611    return true;
612  }
613
614  /**
615   * Check whether backup image set could cover a backup image or not.
616   * @param fullImages The backup image set
617   * @param image The target backup image
618   * @return true if fullImages can cover image, otherwise false
619   */
620  public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
621    // fullImages can cover image only when the following conditions are satisfied:
622    // - each image of fullImages must not be an incremental image;
623    // - each image of fullImages must be taken after image has been taken;
624    // - sum table set of fullImages must cover the table set of image.
625    for (BackupImage image1 : fullImages) {
626      if (image1.getType() == BackupType.INCREMENTAL) {
627        return false;
628      }
629      if (image1.getStartTs() < image.getStartTs()) {
630        return false;
631      }
632    }
633
634    ArrayList<String> image1TableList = new ArrayList<>();
635    for (BackupImage image1 : fullImages) {
636      List<TableName> tableList = image1.getTableNames();
637      for (TableName table : tableList) {
638        image1TableList.add(table.getNameAsString());
639      }
640    }
641    ArrayList<String> image2TableList = new ArrayList<>();
642    List<TableName> tableList = image.getTableNames();
643    for (TableName table : tableList) {
644      image2TableList.add(table.getNameAsString());
645    }
646
647    for (int i = 0; i < image2TableList.size(); i++) {
648      if (image1TableList.contains(image2TableList.get(i)) == false) {
649        return false;
650      }
651    }
652
653    LOG.debug("Full image set can cover image " + image.getBackupId());
654    return true;
655  }
656
657  public BackupInfo toBackupInfo() {
658    BackupInfo info = new BackupInfo();
659    info.setType(backupImage.getType());
660    List<TableName> list = backupImage.getTableNames();
661    TableName[] tables = new TableName[list.size()];
662    info.addTables(list.toArray(tables));
663    info.setBackupId(backupImage.getBackupId());
664    info.setStartTs(backupImage.getStartTs());
665    info.setBackupRootDir(backupImage.getRootDir());
666    if (backupImage.getType() == BackupType.INCREMENTAL) {
667      info.setHLogTargetDir(BackupUtils.getLogBackupDir(backupImage.getRootDir(),
668        backupImage.getBackupId()));
669    }
670    return info;
671  }
672}