001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collections;
027import java.util.List;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.regionserver.HRegion;
035import org.apache.hadoop.hbase.regionserver.Store;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039/**
040 * Test helper for testing archiving of HFiles
041 */
042public class HFileArchiveTestingUtil {
043
044  private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class);
045
046  private HFileArchiveTestingUtil() {
047    // NOOP private ctor since this is just a utility class
048  }
049
050  public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
051      FileSystem fs, boolean hasTimedBackup) {
052
053    List<List<String>> lists = getFileLists(previous, archived);
054    List<String> original = lists.get(0);
055    Collections.sort(original);
056
057    List<String> currentFiles = lists.get(1);
058    Collections.sort(currentFiles);
059
060    List<String> backedup = lists.get(2);
061    Collections.sort(backedup);
062
063    // check the backed up files versus the current (should match up, less the
064    // backup time in the name)
065    if (!hasTimedBackup == (backedup.size() > 0)) {
066      LOG.debug("backedup files doesn't match expected.");
067      return false;
068    }
069    String msg = null;
070    if (hasTimedBackup) {
071      msg = assertArchiveEquality(original, backedup);
072      if (msg != null) {
073        LOG.debug(msg);
074        return false;
075      }
076    }
077    msg = assertArchiveEquality(original, currentFiles);
078    if (msg != null) {
079      LOG.debug(msg);
080      return false;
081    }
082    return true;
083  }
084
085  /**
086   * Compare the archived files to the files in the original directory
087   * @param expected original files that should have been archived
088   * @param actual files that were archived
089   * @param fs filessystem on which the archiving took place
090   * @throws IOException
091   */
092  public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
093      FileSystem fs) throws IOException {
094    assertArchiveEqualToOriginal(expected, actual, fs, false);
095  }
096
097  /**
098   * Compare the archived files to the files in the original directory
099   * @param expected original files that should have been archived
100   * @param actual files that were archived
101   * @param fs {@link FileSystem} on which the archiving took place
102   * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
103   *          copy of the files in the archive directory (and the original files).
104   * @throws IOException
105   */
106  public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
107      FileSystem fs, boolean hasTimedBackup) throws IOException {
108
109    List<List<String>> lists = getFileLists(expected, actual);
110    List<String> original = lists.get(0);
111    Collections.sort(original);
112
113    List<String> currentFiles = lists.get(1);
114    Collections.sort(currentFiles);
115
116    List<String> backedup = lists.get(2);
117    Collections.sort(backedup);
118
119    // check the backed up files versus the current (should match up, less the
120    // backup time in the name)
121    assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
122      backedup.size() > 0);
123    String msg = null;
124    if (hasTimedBackup) {
125      assertArchiveEquality(original, backedup);
126      assertNull(msg, msg);
127    }
128
129    // do the rest of the comparison
130    msg = assertArchiveEquality(original, currentFiles);
131    assertNull(msg, msg);
132  }
133
134  private static String assertArchiveEquality(List<String> expected, List<String> archived) {
135    String compare = compareFileLists(expected, archived);
136    if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
137        + compare;
138    if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
139    return null;
140  }
141
142  /**
143   * @return &lt;expected, gotten, backup&gt;, where each is sorted
144   */
145  private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
146    List<List<String>> files = new ArrayList<>(3);
147
148    // copy over the original files
149    List<String> originalFileNames = convertToString(previous);
150    files.add(originalFileNames);
151
152    List<String> currentFiles = new ArrayList<>(previous.length);
153    List<FileStatus> backedupFiles = new ArrayList<>(previous.length);
154    for (FileStatus f : archived) {
155      String name = f.getPath().getName();
156      // if the file has been backed up
157      if (name.contains(".")) {
158        Path parent = f.getPath().getParent();
159        String shortName = name.split("[.]")[0];
160        Path modPath = new Path(parent, shortName);
161        FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
162            f.getBlockSize(), f.getModificationTime(), modPath);
163        backedupFiles.add(file);
164      } else {
165        // otherwise, add it to the list to compare to the original store files
166        currentFiles.add(name);
167      }
168    }
169
170    files.add(currentFiles);
171    files.add(convertToString(backedupFiles));
172    return files;
173  }
174
175  private static List<String> convertToString(FileStatus[] files) {
176    return convertToString(Arrays.asList(files));
177  }
178
179  private static List<String> convertToString(List<FileStatus> files) {
180    List<String> originalFileNames = new ArrayList<>(files.size());
181    for (FileStatus f : files) {
182      originalFileNames.add(f.getPath().getName());
183    }
184    return originalFileNames;
185  }
186
187  /* Get a pretty representation of the differences */
188  private static String compareFileLists(List<String> expected, List<String> gotten) {
189    StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
190        + gotten.size() + "):\n");
191    List<String> notFound = new ArrayList<>();
192    for (String s : expected) {
193      if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
194      else notFound.add(s);
195    }
196    sb.append("Not Found:\n");
197    for (String s : notFound) {
198      sb.append(s + "\n");
199    }
200    sb.append("\nExtra:\n");
201    for (String s : gotten) {
202      if (!expected.contains(s)) sb.append(s + "\n");
203    }
204    return sb.toString();
205  }
206
207  /**
208   * Helper method to get the archive directory for the specified region
209   * @param conf {@link Configuration} to check for the name of the archive directory
210   * @param region region that is being archived
211   * @return {@link Path} to the archive directory for the given region
212   */
213  public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
214    return HFileArchiveUtil.getRegionArchiveDir(
215        FSUtils.getRootDir(conf),
216        region.getTableDescriptor().getTableName(),
217        region.getRegionInfo().getEncodedName());
218  }
219
220  /**
221   * Helper method to get the store archive directory for the specified region
222   * @param conf {@link Configuration} to check for the name of the archive directory
223   * @param region region that is being archived
224   * @param store store that is archiving files
225   * @return {@link Path} to the store archive directory for the given region
226   */
227  public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
228      throws IOException {
229    return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
230        region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName());
231  }
232
233  public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
234      byte[] storeName) throws IOException {
235    byte[] table = Bytes.toBytes(tableName);
236    // get the RS and region serving our table
237    List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
238    HRegion region = servingRegions.get(0);
239
240    // check that we actually have some store files that were archived
241    Store store = region.getStore(storeName);
242    return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
243  }
244}