001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import java.io.IOException;
021import java.io.InterruptedIOException;
022import java.util.ArrayList;
023import java.util.List;
024import java.util.concurrent.Callable;
025import java.util.concurrent.ExecutionException;
026import java.util.concurrent.Executor;
027import java.util.concurrent.ExecutorCompletionService;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FSDataInputStream;
030import org.apache.hadoop.fs.FSDataOutputStream;
031import org.apache.hadoop.fs.FileStatus;
032import org.apache.hadoop.fs.FileSystem;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.fs.PathFilter;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
037import org.apache.hadoop.hbase.util.CommonFSUtils;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042import org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream;
043import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
044import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
045
046import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
049
050/**
051 * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}. Snapshot v2 layout format - Single Manifest
052 * file containing all the information of regions - In the online-snapshot case each region will
053 * write a "region manifest" /snapshotName/manifest.regionName
054 */
055@InterfaceAudience.Private
056public final class SnapshotManifestV2 {
057  private static final Logger LOG = LoggerFactory.getLogger(SnapshotManifestV2.class);
058
059  public static final int DESCRIPTOR_VERSION = 2;
060
061  public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
062
063  private SnapshotManifestV2() {
064  }
065
066  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
067    SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
068    private final Configuration conf;
069    private final Path snapshotDir;
070    private final FileSystem rootFs;
071
072    public ManifestBuilder(final Configuration conf, final FileSystem rootFs,
073      final Path snapshotDir) {
074      this.snapshotDir = snapshotDir;
075      this.conf = conf;
076      this.rootFs = rootFs;
077    }
078
079    @Override
080    public SnapshotRegionManifest.Builder regionOpen(final RegionInfo regionInfo) {
081      SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
082      manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
083      return manifest;
084    }
085
086    @Override
087    public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
088      // we should ensure the snapshot dir exist, maybe it has been deleted by master
089      // see HBASE-16464
090      FileSystem workingDirFs = snapshotDir.getFileSystem(this.conf);
091      if (workingDirFs.exists(snapshotDir)) {
092        SnapshotRegionManifest manifest = region.build();
093        try (FSDataOutputStream stream =
094          workingDirFs.create(getRegionManifestPath(snapshotDir, manifest))) {
095          manifest.writeTo(stream);
096        }
097      } else {
098        LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
099      }
100    }
101
102    @Override
103    public SnapshotRegionManifest.FamilyFiles.Builder
104      familyOpen(final SnapshotRegionManifest.Builder region, final byte[] familyName) {
105      SnapshotRegionManifest.FamilyFiles.Builder family =
106        SnapshotRegionManifest.FamilyFiles.newBuilder();
107      family.setFamilyName(UnsafeByteOperations.unsafeWrap(familyName));
108      return family;
109    }
110
111    @Override
112    public void familyClose(final SnapshotRegionManifest.Builder region,
113      final SnapshotRegionManifest.FamilyFiles.Builder family) {
114      region.addFamilyFiles(family.build());
115    }
116
117    @Override
118    public void storeFile(final SnapshotRegionManifest.Builder region,
119      final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
120      throws IOException {
121      SnapshotRegionManifest.StoreFile.Builder sfManifest =
122        SnapshotRegionManifest.StoreFile.newBuilder();
123      sfManifest.setName(storeFile.getPath().getName());
124      if (storeFile.isReference()) {
125        sfManifest.setReference(storeFile.getReference().convert());
126      }
127      if (!storeFile.isReference() && !storeFile.isLink()) {
128        sfManifest.setFileSize(storeFile.getSize());
129      } else {
130        sfManifest.setFileSize(storeFile.getReferencedFileStatus(rootFs).getLen());
131      }
132      family.addStoreFiles(sfManifest.build());
133    }
134  }
135
136  static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
137    final Executor executor, final FileSystem fs, final Path snapshotDir,
138    final SnapshotDescription desc, final int manifestSizeLimit) throws IOException {
139    FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
140      @Override
141      public boolean accept(Path path) {
142        return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
143      }
144    });
145
146    if (manifestFiles == null || manifestFiles.length == 0) return null;
147
148    final ExecutorCompletionService<SnapshotRegionManifest> completionService =
149      new ExecutorCompletionService<>(executor);
150    for (final FileStatus st : manifestFiles) {
151      completionService.submit(new Callable<SnapshotRegionManifest>() {
152        @Override
153        public SnapshotRegionManifest call() throws IOException {
154          try (FSDataInputStream stream = fs.open(st.getPath())) {
155            CodedInputStream cin = CodedInputStream.newInstance(stream);
156            cin.setSizeLimit(manifestSizeLimit);
157            return SnapshotRegionManifest.parseFrom(cin);
158          }
159        }
160      });
161    }
162
163    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(manifestFiles.length);
164    try {
165      for (int i = 0; i < manifestFiles.length; ++i) {
166        regionsManifest.add(completionService.take().get());
167      }
168    } catch (InterruptedException e) {
169      throw new InterruptedIOException(e.getMessage());
170    } catch (ExecutionException e) {
171      Throwable t = e.getCause();
172
173      if (t instanceof InvalidProtocolBufferException) {
174        throw (InvalidProtocolBufferException) t;
175      } else {
176        throw new IOException("ExecutionException", e.getCause());
177      }
178    }
179    return regionsManifest;
180  }
181
182  static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
183    final SnapshotRegionManifest manifest) throws IOException {
184    fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
185  }
186
187  private static Path getRegionManifestPath(final Path snapshotDir,
188    final SnapshotRegionManifest manifest) {
189    String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
190    return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
191  }
192}