001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.backup.impl;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.Map.Entry;
028import java.util.TreeMap;
029
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.FSDataInputStream;
032import org.apache.hadoop.fs.FSDataOutputStream;
033import org.apache.hadoop.fs.FileStatus;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.backup.BackupInfo;
039import org.apache.hadoop.hbase.backup.BackupType;
040import org.apache.hadoop.hbase.backup.HBackupFileSystem;
041import org.apache.hadoop.hbase.backup.util.BackupUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
049
050/**
051 * Backup manifest contains all the meta data of a backup image. The manifest info will be bundled
052 * as manifest file together with data. So that each backup image will contain all the info needed
053 * for restore. BackupManifest is a storage container for BackupImage.
054 * It is responsible for storing/reading backup image data and has some additional utility methods.
055 *
056 */
057@InterfaceAudience.Private
058public class BackupManifest {
059  private static final Logger LOG = LoggerFactory.getLogger(BackupManifest.class);
060
061  // manifest file name
062  public static final String MANIFEST_FILE_NAME = ".backup.manifest";
063
064  /**
065   * Backup image, the dependency graph is made up by series of backup images BackupImage contains
066   * all the relevant information to restore the backup and is used during restore operation
067   */
068  public static class BackupImage implements Comparable<BackupImage> {
069    static class Builder {
070      BackupImage image;
071
072      Builder() {
073        image = new BackupImage();
074      }
075
076      Builder withBackupId(String backupId) {
077        image.setBackupId(backupId);
078        return this;
079      }
080
081      Builder withType(BackupType type) {
082        image.setType(type);
083        return this;
084      }
085
086      Builder withRootDir(String rootDir) {
087        image.setRootDir(rootDir);
088        return this;
089      }
090
091      Builder withTableList(List<TableName> tableList) {
092        image.setTableList(tableList);
093        return this;
094      }
095
096      Builder withStartTime(long startTime) {
097        image.setStartTs(startTime);
098        return this;
099      }
100
101      Builder withCompleteTime(long completeTime) {
102        image.setCompleteTs(completeTime);
103        return this;
104      }
105
106      BackupImage build() {
107        return image;
108      }
109
110    }
111
112    private String backupId;
113    private BackupType type;
114    private String rootDir;
115    private List<TableName> tableList;
116    private long startTs;
117    private long completeTs;
118    private ArrayList<BackupImage> ancestors;
119    private HashMap<TableName, HashMap<String, Long>> incrTimeRanges;
120
121    static Builder newBuilder() {
122      return new Builder();
123    }
124
125    public BackupImage() {
126      super();
127    }
128
129    private BackupImage(String backupId, BackupType type, String rootDir,
130        List<TableName> tableList, long startTs, long completeTs) {
131      this.backupId = backupId;
132      this.type = type;
133      this.rootDir = rootDir;
134      this.tableList = tableList;
135      this.startTs = startTs;
136      this.completeTs = completeTs;
137    }
138
139    static BackupImage fromProto(BackupProtos.BackupImage im) {
140      String backupId = im.getBackupId();
141      String rootDir = im.getBackupRootDir();
142      long startTs = im.getStartTs();
143      long completeTs = im.getCompleteTs();
144      List<HBaseProtos.TableName> tableListList = im.getTableListList();
145      List<TableName> tableList = new ArrayList<>();
146      for (HBaseProtos.TableName tn : tableListList) {
147        tableList.add(ProtobufUtil.toTableName(tn));
148      }
149
150      List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
151
152      BackupType type =
153          im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL
154              : BackupType.INCREMENTAL;
155
156      BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
157      for (BackupProtos.BackupImage img : ancestorList) {
158        image.addAncestor(fromProto(img));
159      }
160      image.setIncrTimeRanges(loadIncrementalTimestampMap(im));
161      return image;
162    }
163
164    BackupProtos.BackupImage toProto() {
165      BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
166      builder.setBackupId(backupId);
167      builder.setCompleteTs(completeTs);
168      builder.setStartTs(startTs);
169      builder.setBackupRootDir(rootDir);
170      if (type == BackupType.FULL) {
171        builder.setBackupType(BackupProtos.BackupType.FULL);
172      } else {
173        builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
174      }
175
176      for (TableName name : tableList) {
177        builder.addTableList(ProtobufUtil.toProtoTableName(name));
178      }
179
180      if (ancestors != null) {
181        for (BackupImage im : ancestors) {
182          builder.addAncestors(im.toProto());
183        }
184      }
185
186      setIncrementalTimestampMap(builder);
187      return builder.build();
188    }
189
190    private static HashMap<TableName, HashMap<String, Long>> loadIncrementalTimestampMap(
191        BackupProtos.BackupImage proto) {
192      List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
193
194      HashMap<TableName, HashMap<String, Long>> incrTimeRanges = new HashMap<>();
195
196      if (list == null || list.size() == 0) {
197        return incrTimeRanges;
198      }
199
200      for (BackupProtos.TableServerTimestamp tst : list) {
201        TableName tn = ProtobufUtil.toTableName(tst.getTableName());
202        HashMap<String, Long> map = incrTimeRanges.get(tn);
203        if (map == null) {
204          map = new HashMap<>();
205          incrTimeRanges.put(tn, map);
206        }
207        List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
208        for (BackupProtos.ServerTimestamp stm : listSt) {
209          ServerName sn = ProtobufUtil.toServerName(stm.getServerName());
210          map.put(sn.getHostname() + ":" + sn.getPort(), stm.getTimestamp());
211        }
212      }
213      return incrTimeRanges;
214    }
215
216    private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) {
217      if (this.incrTimeRanges == null) {
218        return;
219      }
220      for (Entry<TableName, HashMap<String, Long>> entry : this.incrTimeRanges.entrySet()) {
221        TableName key = entry.getKey();
222        HashMap<String, Long> value = entry.getValue();
223        BackupProtos.TableServerTimestamp.Builder tstBuilder =
224            BackupProtos.TableServerTimestamp.newBuilder();
225        tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key));
226
227        for (Map.Entry<String, Long> entry2 : value.entrySet()) {
228          String s = entry2.getKey();
229          BackupProtos.ServerTimestamp.Builder stBuilder =
230              BackupProtos.ServerTimestamp.newBuilder();
231          HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
232          ServerName sn = ServerName.parseServerName(s);
233          snBuilder.setHostName(sn.getHostname());
234          snBuilder.setPort(sn.getPort());
235          stBuilder.setServerName(snBuilder.build());
236          stBuilder.setTimestamp(entry2.getValue());
237          tstBuilder.addServerTimestamp(stBuilder.build());
238        }
239        builder.addTstMap(tstBuilder.build());
240      }
241    }
242
243    public String getBackupId() {
244      return backupId;
245    }
246
247    private void setBackupId(String backupId) {
248      this.backupId = backupId;
249    }
250
251    public BackupType getType() {
252      return type;
253    }
254
255    private void setType(BackupType type) {
256      this.type = type;
257    }
258
259    public String getRootDir() {
260      return rootDir;
261    }
262
263    private void setRootDir(String rootDir) {
264      this.rootDir = rootDir;
265    }
266
267    public List<TableName> getTableNames() {
268      return tableList;
269    }
270
271    private void setTableList(List<TableName> tableList) {
272      this.tableList = tableList;
273    }
274
275    public long getStartTs() {
276      return startTs;
277    }
278
279    private void setStartTs(long startTs) {
280      this.startTs = startTs;
281    }
282
283    public long getCompleteTs() {
284      return completeTs;
285    }
286
287    private void setCompleteTs(long completeTs) {
288      this.completeTs = completeTs;
289    }
290
291    public ArrayList<BackupImage> getAncestors() {
292      if (this.ancestors == null) {
293        this.ancestors = new ArrayList<>();
294      }
295      return this.ancestors;
296    }
297
298    public void removeAncestors(List<String> backupIds) {
299      List<BackupImage> toRemove = new ArrayList<>();
300      for (BackupImage im : this.ancestors) {
301        if (backupIds.contains(im.getBackupId())) {
302          toRemove.add(im);
303        }
304      }
305      this.ancestors.removeAll(toRemove);
306    }
307
308    private void addAncestor(BackupImage backupImage) {
309      this.getAncestors().add(backupImage);
310    }
311
312    public boolean hasAncestor(String token) {
313      for (BackupImage image : this.getAncestors()) {
314        if (image.getBackupId().equals(token)) {
315          return true;
316        }
317      }
318      return false;
319    }
320
321    public boolean hasTable(TableName table) {
322      return tableList.contains(table);
323    }
324
325    @Override
326    public int compareTo(BackupImage other) {
327      String thisBackupId = this.getBackupId();
328      String otherBackupId = other.getBackupId();
329      int index1 = thisBackupId.lastIndexOf("_");
330      int index2 = otherBackupId.lastIndexOf("_");
331      String name1 = thisBackupId.substring(0, index1);
332      String name2 = otherBackupId.substring(0, index2);
333      if (name1.equals(name2)) {
334        Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1));
335        Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1));
336        return thisTS.compareTo(otherTS);
337      } else {
338        return name1.compareTo(name2);
339      }
340    }
341
342    @Override
343    public boolean equals(Object obj) {
344      if (obj instanceof BackupImage) {
345        return this.compareTo((BackupImage) obj) == 0;
346      }
347      return false;
348    }
349
350    @Override
351    public int hashCode() {
352      int hash = 33 * this.getBackupId().hashCode() + type.hashCode();
353      hash = 33 * hash + rootDir.hashCode();
354      hash = 33 * hash + Long.valueOf(startTs).hashCode();
355      hash = 33 * hash + Long.valueOf(completeTs).hashCode();
356      for (TableName table : tableList) {
357        hash = 33 * hash + table.hashCode();
358      }
359      return hash;
360    }
361
362    public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() {
363      return incrTimeRanges;
364    }
365
366    private void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) {
367      this.incrTimeRanges = incrTimeRanges;
368    }
369  }
370
371  // backup image directory
372  private String tableBackupDir = null;
373  private BackupImage backupImage;
374
375  /**
376   * Construct manifest for a ongoing backup.
377   * @param backup The ongoing backup info
378   */
379  public BackupManifest(BackupInfo backup) {
380    BackupImage.Builder builder = BackupImage.newBuilder();
381    this.backupImage =
382        builder.withBackupId(backup.getBackupId()).withType(backup.getType())
383            .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
384            .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
385  }
386
387  /**
388   * Construct a table level manifest for a backup of the named table.
389   * @param backup The ongoing backup session info
390   */
391  public BackupManifest(BackupInfo backup, TableName table) {
392    this.tableBackupDir = backup.getTableBackupDir(table);
393    List<TableName> tables = new ArrayList<TableName>();
394    tables.add(table);
395    BackupImage.Builder builder = BackupImage.newBuilder();
396    this.backupImage =
397        builder.withBackupId(backup.getBackupId()).withType(backup.getType())
398            .withRootDir(backup.getBackupRootDir()).withTableList(tables)
399            .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
400  }
401
402  /**
403   * Construct manifest from a backup directory.
404   *
405   * @param conf configuration
406   * @param backupPath backup path
407   * @throws IOException if constructing the manifest from the backup directory fails
408   */
409  public BackupManifest(Configuration conf, Path backupPath) throws IOException {
410    this(backupPath.getFileSystem(conf), backupPath);
411  }
412
413  /**
414   * Construct manifest from a backup directory.
415   * @param fs the FileSystem
416   * @param backupPath backup path
417   * @throws BackupException exception
418   */
419  public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
420    if (LOG.isDebugEnabled()) {
421      LOG.debug("Loading manifest from: " + backupPath.toString());
422    }
423    // The input backupDir may not exactly be the backup table dir.
424    // It could be the backup log dir where there is also a manifest file stored.
425    // This variable's purpose is to keep the correct and original location so
426    // that we can store/persist it.
427    try {
428      FileStatus[] subFiles = BackupUtils.listStatus(fs, backupPath, null);
429      if (subFiles == null) {
430        String errorMsg = backupPath.toString() + " does not exist";
431        LOG.error(errorMsg);
432        throw new IOException(errorMsg);
433      }
434      for (FileStatus subFile : subFiles) {
435        if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
436          // load and set manifest field from file content
437          FSDataInputStream in = fs.open(subFile.getPath());
438          long len = subFile.getLen();
439          byte[] pbBytes = new byte[(int) len];
440          in.readFully(pbBytes);
441          BackupProtos.BackupImage proto = null;
442          try {
443            proto = BackupProtos.BackupImage.parseFrom(pbBytes);
444          } catch (Exception e) {
445            throw new BackupException(e);
446          }
447          this.backupImage = BackupImage.fromProto(proto);
448          LOG.debug("Loaded manifest instance from manifest file: "
449              + BackupUtils.getPath(subFile.getPath()));
450          return;
451        }
452      }
453      String errorMsg = "No manifest file found in: " + backupPath.toString();
454      throw new IOException(errorMsg);
455    } catch (IOException e) {
456      throw new BackupException(e.getMessage());
457    }
458  }
459
460  public BackupType getType() {
461    return backupImage.getType();
462  }
463
464  /**
465   * Get the table set of this image.
466   * @return The table set list
467   */
468  public List<TableName> getTableList() {
469    return backupImage.getTableNames();
470  }
471
472  /**
473   * TODO: fix it. Persist the manifest file.
474   * @throws IOException IOException when storing the manifest file.
475   */
476  public void store(Configuration conf) throws BackupException {
477    byte[] data = backupImage.toProto().toByteArray();
478    // write the file, overwrite if already exist
479    Path manifestFilePath =
480        new Path(HBackupFileSystem.getBackupPath(backupImage.getRootDir(),
481          backupImage.getBackupId()), MANIFEST_FILE_NAME);
482    try (FSDataOutputStream out =
483        manifestFilePath.getFileSystem(conf).create(manifestFilePath, true)) {
484      out.write(data);
485    } catch (IOException e) {
486      throw new BackupException(e.getMessage());
487    }
488
489    LOG.info("Manifest file stored to " + manifestFilePath);
490  }
491
492  /**
493   * Get this backup image.
494   * @return the backup image.
495   */
496  public BackupImage getBackupImage() {
497    return backupImage;
498  }
499
500  /**
501   * Add dependent backup image for this backup.
502   * @param image The direct dependent backup image
503   */
504  public void addDependentImage(BackupImage image) {
505    this.backupImage.addAncestor(image);
506  }
507
508  /**
509   * Set the incremental timestamp map directly.
510   * @param incrTimestampMap timestamp map
511   */
512  public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
513    this.backupImage.setIncrTimeRanges(incrTimestampMap);
514  }
515
516  public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
517    return backupImage.getIncrTimeRanges();
518  }
519
520  /**
521   * Get the image list of this backup for restore in time order.
522   * @param reverse If true, then output in reverse order, otherwise in time order from old to new
523   * @return the backup image list for restore in time order
524   */
525  public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
526    TreeMap<Long, BackupImage> restoreImages = new TreeMap<>();
527    restoreImages.put(backupImage.startTs, backupImage);
528    for (BackupImage image : backupImage.getAncestors()) {
529      restoreImages.put(Long.valueOf(image.startTs), image);
530    }
531    return new ArrayList<>(reverse ? (restoreImages.descendingMap().values())
532        : (restoreImages.values()));
533  }
534
535  /**
536   * Get the dependent image list for a specific table of this backup in time order from old to new
537   * if want to restore to this backup image level.
538   * @param table table
539   * @return the backup image list for a table in time order
540   */
541  public ArrayList<BackupImage> getDependentListByTable(TableName table) {
542    ArrayList<BackupImage> tableImageList = new ArrayList<>();
543    ArrayList<BackupImage> imageList = getRestoreDependentList(true);
544    for (BackupImage image : imageList) {
545      if (image.hasTable(table)) {
546        tableImageList.add(image);
547        if (image.getType() == BackupType.FULL) {
548          break;
549        }
550      }
551    }
552    Collections.reverse(tableImageList);
553    return tableImageList;
554  }
555
556  /**
557   * Get the full dependent image list in the whole dependency scope for a specific table of this
558   * backup in time order from old to new.
559   * @param table table
560   * @return the full backup image list for a table in time order in the whole scope of the
561   *         dependency of this image
562   */
563  public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
564    ArrayList<BackupImage> tableImageList = new ArrayList<>();
565    ArrayList<BackupImage> imageList = getRestoreDependentList(false);
566    for (BackupImage image : imageList) {
567      if (image.hasTable(table)) {
568        tableImageList.add(image);
569      }
570    }
571    return tableImageList;
572  }
573
574  /**
575   * Check whether backup image1 could cover backup image2 or not.
576   * @param image1 backup image 1
577   * @param image2 backup image 2
578   * @return true if image1 can cover image2, otherwise false
579   */
580  public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
581    // image1 can cover image2 only when the following conditions are satisfied:
582    // - image1 must not be an incremental image;
583    // - image1 must be taken after image2 has been taken;
584    // - table set of image1 must cover the table set of image2.
585    if (image1.getType() == BackupType.INCREMENTAL) {
586      return false;
587    }
588    if (image1.getStartTs() < image2.getStartTs()) {
589      return false;
590    }
591    List<TableName> image1TableList = image1.getTableNames();
592    List<TableName> image2TableList = image2.getTableNames();
593    boolean found;
594    for (int i = 0; i < image2TableList.size(); i++) {
595      found = false;
596      for (int j = 0; j < image1TableList.size(); j++) {
597        if (image2TableList.get(i).equals(image1TableList.get(j))) {
598          found = true;
599          break;
600        }
601      }
602      if (!found) {
603        return false;
604      }
605    }
606
607    LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
608    return true;
609  }
610
611  /**
612   * Check whether backup image set could cover a backup image or not.
613   * @param fullImages The backup image set
614   * @param image The target backup image
615   * @return true if fullImages can cover image, otherwise false
616   */
617  public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
618    // fullImages can cover image only when the following conditions are satisfied:
619    // - each image of fullImages must not be an incremental image;
620    // - each image of fullImages must be taken after image has been taken;
621    // - sum table set of fullImages must cover the table set of image.
622    for (BackupImage image1 : fullImages) {
623      if (image1.getType() == BackupType.INCREMENTAL) {
624        return false;
625      }
626      if (image1.getStartTs() < image.getStartTs()) {
627        return false;
628      }
629    }
630
631    ArrayList<String> image1TableList = new ArrayList<>();
632    for (BackupImage image1 : fullImages) {
633      List<TableName> tableList = image1.getTableNames();
634      for (TableName table : tableList) {
635        image1TableList.add(table.getNameAsString());
636      }
637    }
638    ArrayList<String> image2TableList = new ArrayList<>();
639    List<TableName> tableList = image.getTableNames();
640    for (TableName table : tableList) {
641      image2TableList.add(table.getNameAsString());
642    }
643
644    for (int i = 0; i < image2TableList.size(); i++) {
645      if (image1TableList.contains(image2TableList.get(i)) == false) {
646        return false;
647      }
648    }
649
650    LOG.debug("Full image set can cover image " + image.getBackupId());
651    return true;
652  }
653
654  public BackupInfo toBackupInfo() {
655    BackupInfo info = new BackupInfo();
656    info.setType(backupImage.getType());
657    List<TableName> list = backupImage.getTableNames();
658    TableName[] tables = new TableName[list.size()];
659    info.addTables(list.toArray(tables));
660    info.setBackupId(backupImage.getBackupId());
661    info.setStartTs(backupImage.getStartTs());
662    info.setBackupRootDir(backupImage.getRootDir());
663    if (backupImage.getType() == BackupType.INCREMENTAL) {
664      info.setHLogTargetDir(BackupUtils.getLogBackupDir(backupImage.getRootDir(),
665        backupImage.getBackupId()));
666    }
667    return info;
668  }
669}