1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReentrantLock;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.fs.PathFilter;
38 import org.apache.hadoop.fs.permission.FsPermission;
39 import org.apache.hadoop.hbase.ClusterId;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
46 import org.apache.hadoop.hbase.RemoteExceptionHandler;
47 import org.apache.hadoop.hbase.Server;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.backup.HFileArchiver;
50 import org.apache.hadoop.hbase.exceptions.DeserializationException;
51 import org.apache.hadoop.hbase.fs.HFileSystem;
52 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
53 import org.apache.hadoop.hbase.regionserver.HRegion;
54 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
55 import org.apache.hadoop.hbase.wal.WALSplitter;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
58 import org.apache.hadoop.hbase.util.FSTableDescriptors;
59 import org.apache.hadoop.hbase.util.FSUtils;
60
61
62
63
64
65
66 @InterfaceAudience.Private
67 public class MasterFileSystem {
68 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
69
70 Configuration conf;
71
72 Server master;
73
74 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
75
76 private ClusterId clusterId;
77
78 private final FileSystem fs;
79
80 private volatile boolean fsOk = true;
81
82 private final Path oldLogDir;
83
84 private final Path rootdir;
85
86 private final Path tempdir;
87
88 final Lock splitLogLock = new ReentrantLock();
89 final boolean distributedLogReplay;
90 final SplitLogManager splitLogManager;
91 private final MasterServices services;
92
93 final static PathFilter META_FILTER = new PathFilter() {
94 @Override
95 public boolean accept(Path p) {
96 return DefaultWALProvider.isMetaFile(p);
97 }
98 };
99
100 final static PathFilter NON_META_FILTER = new PathFilter() {
101 @Override
102 public boolean accept(Path p) {
103 return !DefaultWALProvider.isMetaFile(p);
104 }
105 };
106
107 public MasterFileSystem(Server master, MasterServices services)
108 throws IOException {
109 this.conf = master.getConfiguration();
110 this.master = master;
111 this.services = services;
112
113
114
115
116 this.rootdir = FSUtils.getRootDir(conf);
117 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
118
119
120 this.fs = this.rootdir.getFileSystem(conf);
121 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
122
123 fs.setConf(conf);
124
125
126 this.oldLogDir = createInitialFileSystemLayout();
127 HFileSystem.addLocationsOrderInterceptor(conf);
128 this.splitLogManager =
129 new SplitLogManager(master, master.getConfiguration(), master, services,
130 master.getServerName());
131 this.distributedLogReplay = this.splitLogManager.isLogReplaying();
132 }
133
134
135
136
137
138
139
140
141
142
143
144 private Path createInitialFileSystemLayout() throws IOException {
145
146 checkRootDir(this.rootdir, conf, this.fs);
147
148
149 checkTempDir(this.tempdir, conf, this.fs);
150
151 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
152
153
154 if(!this.fs.exists(oldLogDir)) {
155 this.fs.mkdirs(oldLogDir);
156 }
157
158 return oldLogDir;
159 }
160
161 public FileSystem getFileSystem() {
162 return this.fs;
163 }
164
165
166
167
168
169 public Path getOldLogDir() {
170 return this.oldLogDir;
171 }
172
173
174
175
176
177
178 public boolean checkFileSystem() {
179 if (this.fsOk) {
180 try {
181 FSUtils.checkFileSystemAvailable(this.fs);
182 FSUtils.checkDfsSafeMode(this.conf);
183 } catch (IOException e) {
184 master.abort("Shutting down HBase cluster: file system not available", e);
185 this.fsOk = false;
186 }
187 }
188 return this.fsOk;
189 }
190
191
192
193
194 public Path getRootDir() {
195 return this.rootdir;
196 }
197
198
199
200
201 public Path getTempDir() {
202 return this.tempdir;
203 }
204
205
206
207
208 public ClusterId getClusterId() {
209 return clusterId;
210 }
211
212
213
214
215
216 Set<ServerName> getFailedServersFromLogFolders() {
217 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
218 WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
219
220 Set<ServerName> serverNames = new HashSet<ServerName>();
221 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
222
223 do {
224 if (master.isStopped()) {
225 LOG.warn("Master stopped while trying to get failed servers.");
226 break;
227 }
228 try {
229 if (!this.fs.exists(logsDirPath)) return serverNames;
230 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
231
232
233 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
234 .keySet();
235
236 if (logFolders == null || logFolders.length == 0) {
237 LOG.debug("No log files to split, proceeding...");
238 return serverNames;
239 }
240 for (FileStatus status : logFolders) {
241 FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
242 if (curLogFiles == null || curLogFiles.length == 0) {
243
244 continue;
245 }
246 final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
247 status.getPath());
248 if (null == serverName) {
249 LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
250 "region server name; leaving in place. If you see later errors about missing " +
251 "write ahead logs they may be saved in this location.");
252 } else if (!onlineServers.contains(serverName)) {
253 LOG.info("Log folder " + status.getPath() + " doesn't belong "
254 + "to a known region server, splitting");
255 serverNames.add(serverName);
256 } else {
257 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
258 }
259 }
260 retrySplitting = false;
261 } catch (IOException ioe) {
262 LOG.warn("Failed getting failed servers to be recovered.", ioe);
263 if (!checkFileSystem()) {
264 LOG.warn("Bad Filesystem, exiting");
265 Runtime.getRuntime().halt(1);
266 }
267 try {
268 if (retrySplitting) {
269 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
270 }
271 } catch (InterruptedException e) {
272 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
273 Thread.currentThread().interrupt();
274 retrySplitting = false;
275 Runtime.getRuntime().halt(1);
276 }
277 }
278 } while (retrySplitting);
279
280 return serverNames;
281 }
282
283 public void splitLog(final ServerName serverName) throws IOException {
284 Set<ServerName> serverNames = new HashSet<ServerName>();
285 serverNames.add(serverName);
286 splitLog(serverNames);
287 }
288
289
290
291
292
293
294 public void splitMetaLog(final ServerName serverName) throws IOException {
295 Set<ServerName> serverNames = new HashSet<ServerName>();
296 serverNames.add(serverName);
297 splitMetaLog(serverNames);
298 }
299
300
301
302
303
304
305 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
306 splitLog(serverNames, META_FILTER);
307 }
308
309 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
310 "We only release this lock when we set it. Updates to code that uses it should verify use " +
311 "of the guard boolean.")
312 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
313 List<Path> logDirs = new ArrayList<Path>();
314 boolean needReleaseLock = false;
315 if (!this.services.isInitialized()) {
316
317 this.splitLogLock.lock();
318 needReleaseLock = true;
319 }
320 try {
321 for (ServerName serverName : serverNames) {
322 Path logDir = new Path(this.rootdir,
323 DefaultWALProvider.getWALDirectoryName(serverName.toString()));
324 Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
325
326 if (fs.exists(logDir)) {
327 if (!this.fs.rename(logDir, splitDir)) {
328 throw new IOException("Failed fs.rename for log split: " + logDir);
329 }
330 logDir = splitDir;
331 LOG.debug("Renamed region directory: " + splitDir);
332 } else if (!fs.exists(splitDir)) {
333 LOG.info("Log dir for server " + serverName + " does not exist");
334 continue;
335 }
336 logDirs.add(splitDir);
337 }
338 } finally {
339 if (needReleaseLock) {
340 this.splitLogLock.unlock();
341 }
342 }
343 return logDirs;
344 }
345
346
347
348
349
350
351
352 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
353 if (!this.distributedLogReplay) {
354 return;
355 }
356
357 if (regions == null || regions.isEmpty()) {
358 return;
359 }
360 this.splitLogManager.markRegionsRecovering(serverName, regions);
361 }
362
363 public void splitLog(final Set<ServerName> serverNames) throws IOException {
364 splitLog(serverNames, NON_META_FILTER);
365 }
366
367
368
369
370
371
372 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
373 throws IOException, InterruptedIOException {
374 this.splitLogManager.removeStaleRecoveringRegions(failedServers);
375 }
376
377
378
379
380
381
382
383
384
385 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
386 long splitTime = 0, splitLogSize = 0;
387 List<Path> logDirs = getLogDirs(serverNames);
388
389 splitLogManager.handleDeadWorkers(serverNames);
390 splitTime = EnvironmentEdgeManager.currentTime();
391 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
392 splitTime = EnvironmentEdgeManager.currentTime() - splitTime;
393
394 if (this.metricsMasterFilesystem != null) {
395 if (filter == META_FILTER) {
396 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
397 } else {
398 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
399 }
400 }
401 }
402
403
404
405
406
407
408
409
410
411
412 @SuppressWarnings("deprecation")
413 private Path checkRootDir(final Path rd, final Configuration c,
414 final FileSystem fs)
415 throws IOException {
416
417 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
418
419 boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
420 FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
421
422
423 try {
424 if (!fs.exists(rd)) {
425 if (isSecurityEnabled) {
426 fs.mkdirs(rd, rootDirPerms);
427 } else {
428 fs.mkdirs(rd);
429 }
430
431
432
433
434
435
436
437 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
438 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
439 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
440 } else {
441 if (!fs.isDirectory(rd)) {
442 throw new IllegalArgumentException(rd.toString() + " is not a directory");
443 }
444 if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
445
446 LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
447 + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
448 + " and \"hbase.rootdir.perms\" configured as "
449 + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
450 + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
451 + "and restarting the master");
452 fs.setPermission(rd, rootDirPerms);
453 }
454
455 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
456 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
457 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
458 }
459 } catch (DeserializationException de) {
460 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
461 IOException ioe = new IOException();
462 ioe.initCause(de);
463 throw ioe;
464 } catch (IllegalArgumentException iae) {
465 LOG.fatal("Please fix invalid configuration for "
466 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
467 throw iae;
468 }
469
470 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
471 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
472 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
473 }
474 clusterId = FSUtils.getClusterId(fs, rd);
475
476
477 if (!FSUtils.metaRegionExists(fs, rd)) {
478 bootstrap(rd, c);
479 } else {
480
481 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
482 .migrateFSTableDescriptorsIfNecessary(fs, rd);
483 }
484
485
486
487
488
489 FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
490 fsd.createTableDescriptor(
491 new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
492
493 return rd;
494 }
495
496
497
498
499
500 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
501 throws IOException {
502
503 if (fs.exists(tmpdir)) {
504
505
506 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
507 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
508 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
509 }
510 }
511 if (!fs.delete(tmpdir, true)) {
512 throw new IOException("Unable to clean the temp directory: " + tmpdir);
513 }
514 }
515
516
517 if (!fs.mkdirs(tmpdir)) {
518 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
519 }
520 }
521
522 private static void bootstrap(final Path rd, final Configuration c)
523 throws IOException {
524 LOG.info("BOOTSTRAP: creating hbase:meta region");
525 try {
526
527
528
529
530 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
531 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
532 setInfoFamilyCachingForMeta(metaDescriptor, false);
533 HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null, true, true);
534 setInfoFamilyCachingForMeta(metaDescriptor, true);
535 HRegion.closeHRegion(meta);
536 } catch (IOException e) {
537 e = RemoteExceptionHandler.checkIOException(e);
538 LOG.error("bootstrap", e);
539 throw e;
540 }
541 }
542
543
544
545
546 public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
547 final boolean b) {
548 for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
549 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
550 hcd.setBlockCacheEnabled(b);
551 hcd.setInMemory(b);
552 }
553 }
554 }
555
556 public void deleteRegion(HRegionInfo region) throws IOException {
557 HFileArchiver.archiveRegion(conf, fs, region);
558 }
559
560 public void deleteTable(TableName tableName) throws IOException {
561 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
562 }
563
564
565
566
567
568
569
570 public Path moveTableToTemp(TableName tableName) throws IOException {
571 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
572 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
573
574
575 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
576 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
577 }
578
579 if (!fs.rename(srcPath, tempPath)) {
580 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
581 }
582
583 return tempPath;
584 }
585
586 public void updateRegionInfo(HRegionInfo region) {
587
588
589
590 }
591
592 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
593 throws IOException {
594
595 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
596 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
597
598
599 Path familyDir = new Path(tableDir,
600 new Path(region.getEncodedName(), Bytes.toString(familyName)));
601 if (fs.delete(familyDir, true) == false) {
602 if (fs.exists(familyDir)) {
603 throw new IOException("Could not delete family "
604 + Bytes.toString(familyName) + " from FileSystem for region "
605 + region.getRegionNameAsString() + "(" + region.getEncodedName()
606 + ")");
607 }
608 }
609 }
610
611 public void stop() {
612 if (splitLogManager != null) {
613 this.splitLogManager.stop();
614 }
615 }
616
617
618
619
620
621
622
623
624 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
625 throws IOException {
626 LOG.info("DeleteColumn. Table = " + tableName
627 + " family = " + Bytes.toString(familyName));
628 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
629 htd.removeFamily(familyName);
630 this.services.getTableDescriptors().add(htd);
631 return htd;
632 }
633
634
635
636
637
638
639
640
641 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
642 throws IOException {
643 LOG.info("AddModifyColumn. Table = " + tableName
644 + " HCD = " + hcd.toString());
645
646 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
647 byte [] familyName = hcd.getName();
648 if(!htd.hasFamily(familyName)) {
649 throw new InvalidFamilyOperationException("Family '" +
650 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
651 }
652 htd.modifyFamily(hcd);
653 this.services.getTableDescriptors().add(htd);
654 return htd;
655 }
656
657
658
659
660
661
662
663
664 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
665 throws IOException {
666 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
667 hcd.toString());
668 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
669 if (htd == null) {
670 throw new InvalidFamilyOperationException("Family '" +
671 hcd.getNameAsString() + "' cannot be modified as HTD is null");
672 }
673 htd.addFamily(hcd);
674 this.services.getTableDescriptors().add(htd);
675 return htd;
676 }
677
678
679
680
681
682
683 public void setLogRecoveryMode() throws IOException {
684 this.splitLogManager.setRecoveryMode(false);
685 }
686
687 public RecoveryMode getLogRecoveryMode() {
688 return this.splitLogManager.getRecoveryMode();
689 }
690 }