001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNull; 022 023import java.io.IOException; 024import java.util.ArrayList; 025import java.util.Arrays; 026import java.util.Collections; 027import java.util.List; 028 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileStatus; 031import org.apache.hadoop.fs.FileSystem; 032import org.apache.hadoop.fs.Path; 033import org.apache.hadoop.hbase.HBaseTestingUtility; 034import org.apache.hadoop.hbase.regionserver.HRegion; 035import org.apache.hadoop.hbase.regionserver.Store; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039/** 040 * Test helper for testing archiving of HFiles 041 */ 042public class HFileArchiveTestingUtil { 043 044 private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class); 045 046 private HFileArchiveTestingUtil() { 047 // NOOP private ctor since this is just a utility class 048 } 049 050 public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived, 051 FileSystem fs, boolean hasTimedBackup) { 052 053 List<List<String>> lists = getFileLists(previous, archived); 054 List<String> original = lists.get(0); 055 Collections.sort(original); 056 057 List<String> currentFiles = lists.get(1); 058 Collections.sort(currentFiles); 059 060 List<String> backedup = lists.get(2); 061 Collections.sort(backedup); 062 063 // check the backed up files versus the current (should match up, less the 064 // backup time in the name) 065 if (!hasTimedBackup == (backedup.size() > 0)) { 066 LOG.debug("backedup files doesn't match expected."); 067 return false; 068 } 069 String msg = null; 070 if (hasTimedBackup) { 071 msg = assertArchiveEquality(original, backedup); 072 if (msg != null) { 073 LOG.debug(msg); 074 return false; 075 } 076 } 077 msg = assertArchiveEquality(original, currentFiles); 078 if (msg != null) { 079 LOG.debug(msg); 080 return false; 081 } 082 return true; 083 } 084 085 /** 086 * Compare the archived files to the files in the original directory 087 * @param expected original files that should have been archived 088 * @param actual files that were archived 089 * @param fs filessystem on which the archiving took place 090 * @throws IOException 091 */ 092 public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual, 093 FileSystem fs) throws IOException { 094 assertArchiveEqualToOriginal(expected, actual, fs, false); 095 } 096 097 /** 098 * Compare the archived files to the files in the original directory 099 * @param expected original files that should have been archived 100 * @param actual files that were archived 101 * @param fs {@link FileSystem} on which the archiving took place 102 * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a 103 * copy of the files in the archive directory (and the original files). 104 * @throws IOException 105 */ 106 public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual, 107 FileSystem fs, boolean hasTimedBackup) throws IOException { 108 109 List<List<String>> lists = getFileLists(expected, actual); 110 List<String> original = lists.get(0); 111 Collections.sort(original); 112 113 List<String> currentFiles = lists.get(1); 114 Collections.sort(currentFiles); 115 116 List<String> backedup = lists.get(2); 117 Collections.sort(backedup); 118 119 // check the backed up files versus the current (should match up, less the 120 // backup time in the name) 121 assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup, 122 backedup.size() > 0); 123 String msg = null; 124 if (hasTimedBackup) { 125 assertArchiveEquality(original, backedup); 126 assertNull(msg, msg); 127 } 128 129 // do the rest of the comparison 130 msg = assertArchiveEquality(original, currentFiles); 131 assertNull(msg, msg); 132 } 133 134 private static String assertArchiveEquality(List<String> expected, List<String> archived) { 135 String compare = compareFileLists(expected, archived); 136 if (!(expected.size() == archived.size())) return "Not the same number of current files\n" 137 + compare; 138 if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare; 139 return null; 140 } 141 142 /** 143 * @return <expected, gotten, backup>, where each is sorted 144 */ 145 private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) { 146 List<List<String>> files = new ArrayList<>(3); 147 148 // copy over the original files 149 List<String> originalFileNames = convertToString(previous); 150 files.add(originalFileNames); 151 152 List<String> currentFiles = new ArrayList<>(previous.length); 153 List<FileStatus> backedupFiles = new ArrayList<>(previous.length); 154 for (FileStatus f : archived) { 155 String name = f.getPath().getName(); 156 // if the file has been backed up 157 if (name.contains(".")) { 158 Path parent = f.getPath().getParent(); 159 String shortName = name.split("[.]")[0]; 160 Path modPath = new Path(parent, shortName); 161 FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(), 162 f.getBlockSize(), f.getModificationTime(), modPath); 163 backedupFiles.add(file); 164 } else { 165 // otherwise, add it to the list to compare to the original store files 166 currentFiles.add(name); 167 } 168 } 169 170 files.add(currentFiles); 171 files.add(convertToString(backedupFiles)); 172 return files; 173 } 174 175 private static List<String> convertToString(FileStatus[] files) { 176 return convertToString(Arrays.asList(files)); 177 } 178 179 private static List<String> convertToString(List<FileStatus> files) { 180 List<String> originalFileNames = new ArrayList<>(files.size()); 181 for (FileStatus f : files) { 182 originalFileNames.add(f.getPath().getName()); 183 } 184 return originalFileNames; 185 } 186 187 /* Get a pretty representation of the differences */ 188 private static String compareFileLists(List<String> expected, List<String> gotten) { 189 StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten (" 190 + gotten.size() + "):\n"); 191 List<String> notFound = new ArrayList<>(); 192 for (String s : expected) { 193 if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n"); 194 else notFound.add(s); 195 } 196 sb.append("Not Found:\n"); 197 for (String s : notFound) { 198 sb.append(s + "\n"); 199 } 200 sb.append("\nExtra:\n"); 201 for (String s : gotten) { 202 if (!expected.contains(s)) sb.append(s + "\n"); 203 } 204 return sb.toString(); 205 } 206 207 /** 208 * Helper method to get the archive directory for the specified region 209 * @param conf {@link Configuration} to check for the name of the archive directory 210 * @param region region that is being archived 211 * @return {@link Path} to the archive directory for the given region 212 */ 213 public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException { 214 return HFileArchiveUtil.getRegionArchiveDir( 215 FSUtils.getRootDir(conf), 216 region.getTableDescriptor().getTableName(), 217 region.getRegionInfo().getEncodedName()); 218 } 219 220 /** 221 * Helper method to get the store archive directory for the specified region 222 * @param conf {@link Configuration} to check for the name of the archive directory 223 * @param region region that is being archived 224 * @param store store that is archiving files 225 * @return {@link Path} to the store archive directory for the given region 226 */ 227 public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store) 228 throws IOException { 229 return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(), 230 region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName()); 231 } 232 233 public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName, 234 byte[] storeName) throws IOException { 235 byte[] table = Bytes.toBytes(tableName); 236 // get the RS and region serving our table 237 List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table); 238 HRegion region = servingRegions.get(0); 239 240 // check that we actually have some store files that were archived 241 Store store = region.getStore(storeName); 242 return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store); 243 } 244}