1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import com.google.protobuf.InvalidProtocolBufferException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
44 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
45 import org.apache.hadoop.hbase.util.ByteStringer;
46 import org.apache.hadoop.hbase.util.FSUtils;
47
48
49
50
51
52
53
54
55
56 @InterfaceAudience.Private
57 public class SnapshotManifestV2 {
58 private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
59
60 public static final int DESCRIPTOR_VERSION = 2;
61
62 public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
63
64 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
65 SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
66 private final Configuration conf;
67 private final Path snapshotDir;
68 private final FileSystem fs;
69
70 public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
71 this.snapshotDir = snapshotDir;
72 this.conf = conf;
73 this.fs = fs;
74 }
75
76 public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
77 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
78 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
79 return manifest;
80 }
81
82 public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
83
84
85 if (fs.exists(snapshotDir)) {
86 SnapshotRegionManifest manifest = region.build();
87 FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
88 try {
89 manifest.writeTo(stream);
90 } finally {
91 stream.close();
92 }
93 } else {
94 LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
95 }
96 }
97
98 public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
99 final SnapshotRegionManifest.Builder region, final byte[] familyName) {
100 SnapshotRegionManifest.FamilyFiles.Builder family =
101 SnapshotRegionManifest.FamilyFiles.newBuilder();
102 family.setFamilyName(ByteStringer.wrap(familyName));
103 return family;
104 }
105
106 public void familyClose(final SnapshotRegionManifest.Builder region,
107 final SnapshotRegionManifest.FamilyFiles.Builder family) {
108 region.addFamilyFiles(family.build());
109 }
110
111 public void storeFile(final SnapshotRegionManifest.Builder region,
112 final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
113 throws IOException {
114 SnapshotRegionManifest.StoreFile.Builder sfManifest =
115 SnapshotRegionManifest.StoreFile.newBuilder();
116 sfManifest.setName(storeFile.getPath().getName());
117 if (storeFile.isReference()) {
118 sfManifest.setReference(storeFile.getReference().convert());
119 }
120 sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
121 family.addStoreFiles(sfManifest.build());
122 }
123 }
124
125 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
126 final Executor executor,final FileSystem fs, final Path snapshotDir,
127 final SnapshotDescription desc) throws IOException {
128 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
129 @Override
130 public boolean accept(Path path) {
131 return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
132 }
133 });
134
135 if (manifestFiles == null || manifestFiles.length == 0) return null;
136
137 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
138 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
139 for (final FileStatus st: manifestFiles) {
140 completionService.submit(new Callable<SnapshotRegionManifest>() {
141 @Override
142 public SnapshotRegionManifest call() throws IOException {
143 FSDataInputStream stream = fs.open(st.getPath());
144 try {
145 return SnapshotRegionManifest.parseFrom(stream);
146 } finally {
147 stream.close();
148 }
149 }
150 });
151 }
152
153 ArrayList<SnapshotRegionManifest> regionsManifest =
154 new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
155 try {
156 for (int i = 0; i < manifestFiles.length; ++i) {
157 regionsManifest.add(completionService.take().get());
158 }
159 } catch (InterruptedException e) {
160 throw new InterruptedIOException(e.getMessage());
161 } catch (ExecutionException e) {
162 Throwable t = e.getCause();
163
164 if(t instanceof InvalidProtocolBufferException) {
165 throw (InvalidProtocolBufferException)t;
166 } else {
167 IOException ex = new IOException("ExecutionException");
168 ex.initCause(e.getCause());
169 throw ex;
170 }
171 }
172 return regionsManifest;
173 }
174
175 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
176 final SnapshotRegionManifest manifest) throws IOException {
177 fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
178 }
179
180 private static Path getRegionManifestPath(final Path snapshotDir,
181 final SnapshotRegionManifest manifest) {
182 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
183 return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
184 }
185 }