001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase.tool; 020 021import java.io.IOException; 022import java.util.Collection; 023import java.util.List; 024import java.util.concurrent.ExecutorService; 025import java.util.concurrent.Executors; 026import java.util.concurrent.TimeUnit; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.hbase.HBaseInterfaceAudience; 031import org.apache.hadoop.hbase.HConstants; 032import org.apache.hadoop.hbase.util.AbstractHBaseTool; 033import org.apache.hadoop.hbase.util.CommonFSUtils; 034import org.apache.hadoop.hbase.util.FSUtils; 035import org.apache.hadoop.hbase.util.Threads; 036import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; 037import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; 038import org.apache.yetus.audience.InterfaceAudience; 039import org.slf4j.Logger; 040import org.slf4j.LoggerFactory; 041 042import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 043 044@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) 045public class HFileContentValidator extends AbstractHBaseTool { 046 047 private static final Logger LOG = LoggerFactory.getLogger(HFileContentValidator.class); 048 049 /** 050 * Check HFile contents are readable by HBase 2. 051 * 052 * @param conf used configuration 053 * @return number of HFiles corrupted HBase 054 * @throws IOException if a remote or network exception occurs 055 */ 056 private boolean validateHFileContent(Configuration conf) throws IOException { 057 FileSystem fileSystem = CommonFSUtils.getCurrentFileSystem(conf); 058 059 ExecutorService threadPool = createThreadPool(conf); 060 HFileCorruptionChecker checker; 061 062 try { 063 checker = new HFileCorruptionChecker(conf, threadPool, false); 064 065 Path rootDir = CommonFSUtils.getRootDir(conf); 066 LOG.info("Validating HFile contents under {}", rootDir); 067 068 Collection<Path> tableDirs = FSUtils.getTableDirs(fileSystem, rootDir); 069 checker.checkTables(tableDirs); 070 071 Path archiveRootDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); 072 LOG.info("Validating HFile contents under {}", archiveRootDir); 073 074 List<Path> archiveTableDirs = FSUtils.getTableDirs(fileSystem, archiveRootDir); 075 checker.checkTables(archiveTableDirs); 076 } finally { 077 threadPool.shutdown(); 078 079 try { 080 threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); 081 } catch (InterruptedException e) { 082 Thread.currentThread().interrupt(); 083 } 084 } 085 086 int checkedFiles = checker.getHFilesChecked(); 087 Collection<Path> corrupted = checker.getCorrupted(); 088 089 if (corrupted.isEmpty()) { 090 LOG.info("Checked {} HFiles, none of them are corrupted.", checkedFiles); 091 LOG.info("There are no incompatible HFiles."); 092 093 return true; 094 } else { 095 LOG.info("Checked {} HFiles, {} are corrupted.", checkedFiles, corrupted.size()); 096 097 for (Path path : corrupted) { 098 LOG.info("Corrupted file: {}", path); 099 } 100 101 LOG.info("Change data block encodings before upgrading. " 102 + "Check https://s.apache.org/prefixtree for instructions."); 103 104 return false; 105 } 106 } 107 108 private ExecutorService createThreadPool(Configuration conf) { 109 int availableProcessors = Runtime.getRuntime().availableProcessors(); 110 int numThreads = conf.getInt("hfilevalidator.numthreads", availableProcessors); 111 return Executors.newFixedThreadPool(numThreads, 112 new ThreadFactoryBuilder().setNameFormat("hfile-validator-pool-%d").setDaemon(true) 113 .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build()); 114 } 115 116 @Override 117 protected void addOptions() { 118 } 119 120 @Override 121 protected void processOptions(CommandLine cmd) { 122 } 123 124 @Override 125 protected int doWork() throws Exception { 126 return (validateHFileContent(getConf())) ? EXIT_SUCCESS : EXIT_FAILURE; 127 } 128}