001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.snapshot;
020
021import java.io.IOException;
022import java.io.InterruptedIOException;
023import java.util.ArrayList;
024import java.util.List;
025import java.util.concurrent.Callable;
026import java.util.concurrent.ExecutionException;
027import java.util.concurrent.Executor;
028import java.util.concurrent.ExecutorCompletionService;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FSDataInputStream;
031import org.apache.hadoop.fs.FSDataOutputStream;
032import org.apache.hadoop.fs.FileStatus;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.fs.PathFilter;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
038import org.apache.hadoop.hbase.util.CommonFSUtils;
039import org.apache.yetus.audience.InterfaceAudience;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043import org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream;
044import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
045import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
046
047import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
049import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
050
051/**
052 * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
053 *
054 * Snapshot v2 layout format
055 *  - Single Manifest file containing all the information of regions
056 *  - In the online-snapshot case each region will write a "region manifest"
057 *      /snapshotName/manifest.regionName
058 */
059@InterfaceAudience.Private
060public final class SnapshotManifestV2 {
061  private static final Logger LOG = LoggerFactory.getLogger(SnapshotManifestV2.class);
062
063  public static final int DESCRIPTOR_VERSION = 2;
064
065  public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
066
067  private SnapshotManifestV2() {}
068
069  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
070                    SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
071    private final Configuration conf;
072    private final Path snapshotDir;
073    private final FileSystem rootFs;
074
075    public ManifestBuilder(final Configuration conf, final FileSystem rootFs,
076        final Path snapshotDir) {
077      this.snapshotDir = snapshotDir;
078      this.conf = conf;
079      this.rootFs = rootFs;
080    }
081
082    @Override
083    public SnapshotRegionManifest.Builder regionOpen(final RegionInfo regionInfo) {
084      SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
085      manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
086      return manifest;
087    }
088
089    @Override
090    public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
091      // we should ensure the snapshot dir exist, maybe it has been deleted by master
092      // see HBASE-16464
093      FileSystem workingDirFs = snapshotDir.getFileSystem(this.conf);
094      if (workingDirFs.exists(snapshotDir)) {
095        SnapshotRegionManifest manifest = region.build();
096        FSDataOutputStream stream = workingDirFs.create(
097            getRegionManifestPath(snapshotDir, manifest));
098        try {
099          manifest.writeTo(stream);
100        } finally {
101          stream.close();
102        }
103      } else {
104        LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
105      }
106    }
107
108    @Override
109    public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
110        final SnapshotRegionManifest.Builder region, final byte[] familyName) {
111      SnapshotRegionManifest.FamilyFiles.Builder family =
112          SnapshotRegionManifest.FamilyFiles.newBuilder();
113      family.setFamilyName(UnsafeByteOperations.unsafeWrap(familyName));
114      return family;
115    }
116
117    @Override
118    public void familyClose(final SnapshotRegionManifest.Builder region,
119        final SnapshotRegionManifest.FamilyFiles.Builder family) {
120      region.addFamilyFiles(family.build());
121    }
122
123    @Override
124    public void storeFile(final SnapshotRegionManifest.Builder region,
125        final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
126        throws IOException {
127      SnapshotRegionManifest.StoreFile.Builder sfManifest =
128            SnapshotRegionManifest.StoreFile.newBuilder();
129      sfManifest.setName(storeFile.getPath().getName());
130      if (storeFile.isReference()) {
131        sfManifest.setReference(storeFile.getReference().convert());
132      }
133      if (!storeFile.isReference() && !storeFile.isLink()) {
134        sfManifest.setFileSize(storeFile.getSize());
135      } else {
136        sfManifest.setFileSize(storeFile.getReferencedFileStatus(rootFs).getLen());
137      }
138      family.addStoreFiles(sfManifest.build());
139    }
140  }
141
142  static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
143      final Executor executor, final FileSystem fs, final Path snapshotDir,
144      final SnapshotDescription desc, final int manifestSizeLimit) throws IOException {
145    FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
146      @Override
147      public boolean accept(Path path) {
148        return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
149      }
150    });
151
152    if (manifestFiles == null || manifestFiles.length == 0) return null;
153
154    final ExecutorCompletionService<SnapshotRegionManifest> completionService =
155      new ExecutorCompletionService<>(executor);
156    for (final FileStatus st: manifestFiles) {
157      completionService.submit(new Callable<SnapshotRegionManifest>() {
158        @Override
159        public SnapshotRegionManifest call() throws IOException {
160          FSDataInputStream stream = fs.open(st.getPath());
161          CodedInputStream cin = CodedInputStream.newInstance(stream);
162          cin.setSizeLimit(manifestSizeLimit);
163
164          try {
165            return SnapshotRegionManifest.parseFrom(cin);
166          } finally {
167            stream.close();
168          }
169        }
170      });
171    }
172
173    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(manifestFiles.length);
174    try {
175      for (int i = 0; i < manifestFiles.length; ++i) {
176        regionsManifest.add(completionService.take().get());
177      }
178    } catch (InterruptedException e) {
179      throw new InterruptedIOException(e.getMessage());
180    } catch (ExecutionException e) {
181      Throwable t = e.getCause();
182
183      if(t instanceof InvalidProtocolBufferException) {
184        throw (InvalidProtocolBufferException)t;
185      } else {
186        throw new IOException("ExecutionException", e.getCause());
187      }
188    }
189    return regionsManifest;
190  }
191
192  static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
193      final SnapshotRegionManifest manifest) throws IOException {
194    fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
195  }
196
197  private static Path getRegionManifestPath(final Path snapshotDir,
198      final SnapshotRegionManifest manifest) {
199    String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
200    return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
201  }
202}