1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.regex.Matcher;
25 import java.util.regex.Pattern;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.HFileLink;
37 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
38 import org.apache.hadoop.hbase.io.Reference;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54
55
56 private static final Pattern HFILE_NAME_PATTERN =
57 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58
59
60
61
62
63
64
65
66 private static final Pattern REF_NAME_PATTERN =
67 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69
70
71 private Configuration conf;
72
73
74 private final FileSystem fs;
75
76
77 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
78
79
80 private final Reference reference;
81
82
83 private final HFileLink link;
84
85 private final Path initialPath;
86
87 private RegionCoprocessorHost coprocessorHost;
88
89
90
91
92
93
94
95 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath)
96 throws IOException {
97 assert fs != null;
98 assert initialPath != null;
99 assert conf != null;
100
101 this.fs = fs;
102 this.conf = conf;
103 this.initialPath = initialPath;
104 Path p = initialPath;
105 if (HFileLink.isHFileLink(p)) {
106
107 this.reference = null;
108 this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
109 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
110 } else if (isReference(p)) {
111 this.reference = Reference.read(fs, p);
112 Path referencePath = getReferredToFile(p);
113 if (HFileLink.isHFileLink(referencePath)) {
114
115 this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
116 } else {
117
118 this.link = null;
119 }
120 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
121 " reference to " + referencePath);
122 } else if (isHFile(p)) {
123
124 this.reference = null;
125 this.link = null;
126 } else {
127 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
128 }
129 }
130
131
132
133
134
135
136
137 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
138 throws IOException {
139 this(conf, fs, fileStatus.getPath());
140 }
141
142
143
144
145
146
147
148 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
149 final HFileLink link)
150 throws IOException {
151 this.fs = fs;
152 this.conf = conf;
153
154 this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
155
156 this.reference = null;
157 this.link = link;
158 }
159
160
161
162
163
164
165
166
167
168 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
169 final Reference reference)
170 throws IOException {
171 this.fs = fs;
172 this.conf = conf;
173 this.initialPath = fileStatus.getPath();
174 this.reference = reference;
175 this.link = null;
176 }
177
178
179
180
181
182 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
183 this.coprocessorHost = coprocessorHost;
184 }
185
186
187
188
189
190 public Reference getReference() {
191 return this.reference;
192 }
193
194
195 public boolean isReference() {
196 return this.reference != null;
197 }
198
199
200 public boolean isTopReference() {
201 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
202 }
203
204
205 public boolean isLink() {
206 return this.link != null && this.reference == null;
207 }
208
209
210 public HDFSBlocksDistribution getHDFSBlockDistribution() {
211 return this.hdfsBlocksDistribution;
212 }
213
214
215
216
217
218
219
220 public StoreFile.Reader open(final FileSystem fs,
221 final CacheConfig cacheConf) throws IOException {
222 FSDataInputStreamWrapper in;
223 FileStatus status;
224
225 if (this.link != null) {
226
227 in = new FSDataInputStreamWrapper(fs, this.link);
228 status = this.link.getFileStatus(fs);
229 } else if (this.reference != null) {
230
231 Path referencePath = getReferredToFile(this.getPath());
232 in = new FSDataInputStreamWrapper(fs, referencePath);
233 status = fs.getFileStatus(referencePath);
234 } else {
235 in = new FSDataInputStreamWrapper(fs, this.getPath());
236 status = fs.getFileStatus(initialPath);
237 }
238 long length = status.getLen();
239 hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
240
241 StoreFile.Reader reader = null;
242 if (this.coprocessorHost != null) {
243 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
244 cacheConf, reference);
245 }
246 if (reader == null) {
247 if (this.reference != null) {
248 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
249 conf);
250 } else {
251 reader = new StoreFile.Reader(fs, status.getPath(), in, length, cacheConf, conf);
252 }
253 }
254 if (this.coprocessorHost != null) {
255 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
256 cacheConf, reference, reader);
257 }
258 return reader;
259 }
260
261
262
263
264 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
265 throws IOException {
266
267
268
269 if (this.link != null) {
270 FileNotFoundException exToThrow = null;
271 for (int i = 0; i < this.link.getLocations().length; i++) {
272 try {
273 return computeHDFSBlocksDistributionInternal(fs);
274 } catch (FileNotFoundException ex) {
275
276 exToThrow = ex;
277 }
278 }
279 throw exToThrow;
280 } else {
281 return computeHDFSBlocksDistributionInternal(fs);
282 }
283 }
284
285 private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
286 throws IOException {
287 FileStatus status = getReferencedFileStatus(fs);
288 if (this.reference != null) {
289 return computeRefFileHDFSBlockDistribution(fs, reference, status);
290 } else {
291 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
292 }
293 }
294
295
296
297
298
299
300 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
301 FileStatus status;
302 if (this.reference != null) {
303 if (this.link != null) {
304 FileNotFoundException exToThrow = null;
305 for (int i = 0; i < this.link.getLocations().length; i++) {
306
307 try {
308 return link.getFileStatus(fs);
309 } catch (FileNotFoundException ex) {
310
311 exToThrow = ex;
312 }
313 }
314 throw exToThrow;
315 } else {
316
317 Path referencePath = getReferredToFile(this.getPath());
318 status = fs.getFileStatus(referencePath);
319 }
320 } else {
321 if (this.link != null) {
322 FileNotFoundException exToThrow = null;
323 for (int i = 0; i < this.link.getLocations().length; i++) {
324
325 try {
326 return link.getFileStatus(fs);
327 } catch (FileNotFoundException ex) {
328
329 exToThrow = ex;
330 }
331 }
332 throw exToThrow;
333 } else {
334 status = fs.getFileStatus(initialPath);
335 }
336 }
337 return status;
338 }
339
340
341 public Path getPath() {
342 return initialPath;
343 }
344
345
346 public FileStatus getFileStatus() throws IOException {
347 return getReferencedFileStatus(fs);
348 }
349
350
351 public long getModificationTime() throws IOException {
352 return getFileStatus().getModificationTime();
353 }
354
355 @Override
356 public String toString() {
357 return this.getPath() +
358 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
359 }
360
361
362
363
364
365 public static boolean isHFile(final Path path) {
366 return isHFile(path.getName());
367 }
368
369 public static boolean isHFile(final String fileName) {
370 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
371 return m.matches() && m.groupCount() > 0;
372 }
373
374
375
376
377
378 public static boolean isReference(final Path path) {
379 return isReference(path.getName());
380 }
381
382
383
384
385
386 public static boolean isReference(final String name) {
387 Matcher m = REF_NAME_PATTERN.matcher(name);
388 return m.matches() && m.groupCount() > 1;
389 }
390
391
392
393
394
395
396
397
398 public static Path getReferredToFile(final Path p) {
399 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
400 if (m == null || !m.matches()) {
401 LOG.warn("Failed match of store file name " + p.toString());
402 throw new IllegalArgumentException("Failed match of store file name " +
403 p.toString());
404 }
405
406
407 String otherRegion = m.group(2);
408
409 Path tableDir = p.getParent().getParent().getParent();
410 String nameStrippedOfSuffix = m.group(1);
411 if (LOG.isDebugEnabled()) {
412 LOG.debug("reference '" + p + "' to region=" + otherRegion
413 + " hfile=" + nameStrippedOfSuffix);
414 }
415
416
417
418 return new Path(new Path(new Path(tableDir, otherRegion),
419 p.getParent().getName()), nameStrippedOfSuffix);
420 }
421
422
423
424
425
426
427 public static boolean validateStoreFileName(final String fileName) {
428 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
429 return(true);
430 return !fileName.contains("-");
431 }
432
433
434
435
436
437
438 public static boolean isValid(final FileStatus fileStatus)
439 throws IOException {
440 final Path p = fileStatus.getPath();
441
442 if (fileStatus.isDirectory())
443 return false;
444
445
446
447
448 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
449 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
450 return false;
451 }
452
453 return validateStoreFileName(p.getName());
454 }
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
470 final FileSystem fs, final Reference reference, final FileStatus status)
471 throws IOException {
472 if (status == null) {
473 return null;
474 }
475
476 long start = 0;
477 long length = 0;
478
479 if (Reference.isTopFileRegion(reference.getFileRegion())) {
480 start = status.getLen()/2;
481 length = status.getLen() - status.getLen()/2;
482 } else {
483 start = 0;
484 length = status.getLen()/2;
485 }
486 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
487 }
488
489 @Override
490 public boolean equals(Object that) {
491 if (this == that) return true;
492 if (that == null) return false;
493
494 if (!(that instanceof StoreFileInfo)) return false;
495
496 StoreFileInfo o = (StoreFileInfo)that;
497 if (initialPath != null && o.initialPath == null) return false;
498 if (initialPath == null && o.initialPath != null) return false;
499 if (initialPath != o.initialPath && initialPath != null
500 && !initialPath.equals(o.initialPath)) return false;
501
502 if (reference != null && o.reference == null) return false;
503 if (reference == null && o.reference != null) return false;
504 if (reference != o.reference && reference != null
505 && !reference.equals(o.reference)) return false;
506
507 if (link != null && o.link == null) return false;
508 if (link == null && o.link != null) return false;
509 if (link != o.link && link != null && !link.equals(o.link)) return false;
510
511 return true;
512 };
513
514
515 @Override
516 public int hashCode() {
517 int hash = 17;
518 hash = hash * 31 + ((reference == null) ? 0 : reference.hashCode());
519 hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
520 hash = hash * 31 + ((link == null) ? 0 : link.hashCode());
521 return hash;
522 }
523 }