1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
40 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
41 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
42 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
43 import org.apache.hadoop.hbase.util.ByteStringer;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSUtils;
46
47
48
49
50
51
52
53
54
55
56 @InterfaceAudience.Private
57 public class SnapshotManifestV1 {
58 private static final Log LOG = LogFactory.getLog(SnapshotManifestV1.class);
59
60 public static final int DESCRIPTOR_VERSION = 0;
61
62 private SnapshotManifestV1() {
63 }
64
65 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
66 HRegionFileSystem, Path> {
67 private final Configuration conf;
68 private final Path snapshotDir;
69 private final FileSystem fs;
70
71 public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
72 this.snapshotDir = snapshotDir;
73 this.conf = conf;
74 this.fs = fs;
75 }
76
77 public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
78 HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
79 fs, snapshotDir, regionInfo);
80 return snapshotRegionFs;
81 }
82
83 public void regionClose(final HRegionFileSystem region) {
84 }
85
86 public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
87 Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
88 return familyDir;
89 }
90
91 public void familyClose(final HRegionFileSystem region, final Path family) {
92 }
93
94 public void storeFile(final HRegionFileSystem region, final Path familyDir,
95 final StoreFileInfo storeFile) throws IOException {
96 Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
97 boolean success = true;
98 if (storeFile.isReference()) {
99
100 storeFile.getReference().write(fs, referenceFile);
101 } else {
102
103
104
105
106 success = fs.createNewFile(referenceFile);
107 }
108 if (!success) {
109 throw new IOException("Failed to create reference file:" + referenceFile);
110 }
111 }
112 }
113
114 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
115 final Executor executor,final FileSystem fs, final Path snapshotDir,
116 final SnapshotDescription desc) throws IOException {
117 FileStatus[] regions = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
118 if (regions == null) {
119 LOG.debug("No regions under directory:" + snapshotDir);
120 return null;
121 }
122
123 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
124 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
125 for (final FileStatus region: regions) {
126 completionService.submit(new Callable<SnapshotRegionManifest>() {
127 @Override
128 public SnapshotRegionManifest call() throws IOException {
129 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
130 return buildManifestFromDisk(conf, fs, snapshotDir, hri);
131 }
132 });
133 }
134
135 ArrayList<SnapshotRegionManifest> regionsManifest =
136 new ArrayList<SnapshotRegionManifest>(regions.length);
137 try {
138 for (int i = 0; i < regions.length; ++i) {
139 regionsManifest.add(completionService.take().get());
140 }
141 } catch (InterruptedException e) {
142 throw new InterruptedIOException(e.getMessage());
143 } catch (ExecutionException e) {
144 IOException ex = new IOException();
145 ex.initCause(e.getCause());
146 throw ex;
147 }
148 return regionsManifest;
149 }
150
151 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
152 final SnapshotRegionManifest manifest) throws IOException {
153 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
154 fs.delete(new Path(snapshotDir, regionName), true);
155 }
156
157 static SnapshotRegionManifest buildManifestFromDisk (final Configuration conf,
158 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
159 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
160 tableDir, regionInfo, true);
161 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
162
163
164 LOG.debug("Storing region-info for snapshot.");
165 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
166
167
168 LOG.debug("Creating references for hfiles");
169
170
171
172
173
174
175 Collection<String> familyNames = regionFs.getFamilies();
176 if (familyNames != null) {
177 for (String familyName: familyNames) {
178 Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
179 if (storeFiles == null) {
180 LOG.debug("No files under family: " + familyName);
181 continue;
182 }
183
184
185 SnapshotRegionManifest.FamilyFiles.Builder family =
186 SnapshotRegionManifest.FamilyFiles.newBuilder();
187 family.setFamilyName(ByteStringer.wrap(Bytes.toBytes(familyName)));
188
189 if (LOG.isDebugEnabled()) {
190 LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
191 }
192
193
194 int i = 0;
195 int sz = storeFiles.size();
196 for (StoreFileInfo storeFile: storeFiles) {
197
198 LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
199 SnapshotRegionManifest.StoreFile.Builder sfManifest =
200 SnapshotRegionManifest.StoreFile.newBuilder();
201 sfManifest.setName(storeFile.getPath().getName());
202 family.addStoreFiles(sfManifest.build());
203 }
204 manifest.addFamilyFiles(family.build());
205 }
206 }
207 return manifest.build();
208 }
209 }