001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collections;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.regionserver.HRegion;
034import org.apache.hadoop.hbase.regionserver.Store;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038/**
039 * Test helper for testing archiving of HFiles
040 */
041public class HFileArchiveTestingUtil {
042
043  private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class);
044
045  private HFileArchiveTestingUtil() {
046    // NOOP private ctor since this is just a utility class
047  }
048
049  public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
050    FileSystem fs, boolean hasTimedBackup) {
051
052    List<List<String>> lists = getFileLists(previous, archived);
053    List<String> original = lists.get(0);
054    Collections.sort(original);
055
056    List<String> currentFiles = lists.get(1);
057    Collections.sort(currentFiles);
058
059    List<String> backedup = lists.get(2);
060    Collections.sort(backedup);
061
062    // check the backed up files versus the current (should match up, less the
063    // backup time in the name)
064    if (!hasTimedBackup == (backedup.size() > 0)) {
065      LOG.debug("backedup files doesn't match expected.");
066      return false;
067    }
068    String msg = null;
069    if (hasTimedBackup) {
070      msg = assertArchiveEquality(original, backedup);
071      if (msg != null) {
072        LOG.debug(msg);
073        return false;
074      }
075    }
076    msg = assertArchiveEquality(original, currentFiles);
077    if (msg != null) {
078      LOG.debug(msg);
079      return false;
080    }
081    return true;
082  }
083
084  /**
085   * Compare the archived files to the files in the original directory
086   * @param expected original files that should have been archived
087   * @param actual   files that were archived
088   * @param fs       filessystem on which the archiving took place n
089   */
090  public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
091    FileSystem fs) throws IOException {
092    assertArchiveEqualToOriginal(expected, actual, fs, false);
093  }
094
095  /**
096   * Compare the archived files to the files in the original directory
097   * @param expected       original files that should have been archived
098   * @param actual         files that were archived
099   * @param fs             {@link FileSystem} on which the archiving took place
100   * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
101   *                       copy of the files in the archive directory (and the original files). n
102   */
103  public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
104    FileSystem fs, boolean hasTimedBackup) throws IOException {
105
106    List<List<String>> lists = getFileLists(expected, actual);
107    List<String> original = lists.get(0);
108    Collections.sort(original);
109
110    List<String> currentFiles = lists.get(1);
111    Collections.sort(currentFiles);
112
113    List<String> backedup = lists.get(2);
114    Collections.sort(backedup);
115
116    // check the backed up files versus the current (should match up, less the
117    // backup time in the name)
118    assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
119      backedup.size() > 0);
120    String msg = null;
121    if (hasTimedBackup) {
122      assertArchiveEquality(original, backedup);
123      assertNull(msg, msg);
124    }
125
126    // do the rest of the comparison
127    msg = assertArchiveEquality(original, currentFiles);
128    assertNull(msg, msg);
129  }
130
131  private static String assertArchiveEquality(List<String> expected, List<String> archived) {
132    String compare = compareFileLists(expected, archived);
133    if (!(expected.size() == archived.size()))
134      return "Not the same number of current files\n" + compare;
135    if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
136    return null;
137  }
138
139  /**
140   * @return &lt;expected, gotten, backup&gt;, where each is sorted
141   */
142  private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
143    List<List<String>> files = new ArrayList<>(3);
144
145    // copy over the original files
146    List<String> originalFileNames = convertToString(previous);
147    files.add(originalFileNames);
148
149    List<String> currentFiles = new ArrayList<>(previous.length);
150    List<FileStatus> backedupFiles = new ArrayList<>(previous.length);
151    for (FileStatus f : archived) {
152      String name = f.getPath().getName();
153      // if the file has been backed up
154      if (name.contains(".")) {
155        Path parent = f.getPath().getParent();
156        String shortName = name.split("[.]")[0];
157        Path modPath = new Path(parent, shortName);
158        FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
159          f.getBlockSize(), f.getModificationTime(), modPath);
160        backedupFiles.add(file);
161      } else {
162        // otherwise, add it to the list to compare to the original store files
163        currentFiles.add(name);
164      }
165    }
166
167    files.add(currentFiles);
168    files.add(convertToString(backedupFiles));
169    return files;
170  }
171
172  private static List<String> convertToString(FileStatus[] files) {
173    return convertToString(Arrays.asList(files));
174  }
175
176  private static List<String> convertToString(List<FileStatus> files) {
177    List<String> originalFileNames = new ArrayList<>(files.size());
178    for (FileStatus f : files) {
179      originalFileNames.add(f.getPath().getName());
180    }
181    return originalFileNames;
182  }
183
184  /* Get a pretty representation of the differences */
185  private static String compareFileLists(List<String> expected, List<String> gotten) {
186    StringBuilder sb = new StringBuilder(
187      "Expected (" + expected.size() + "): \t\t Gotten (" + gotten.size() + "):\n");
188    List<String> notFound = new ArrayList<>();
189    for (String s : expected) {
190      if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
191      else notFound.add(s);
192    }
193    sb.append("Not Found:\n");
194    for (String s : notFound) {
195      sb.append(s + "\n");
196    }
197    sb.append("\nExtra:\n");
198    for (String s : gotten) {
199      if (!expected.contains(s)) sb.append(s + "\n");
200    }
201    return sb.toString();
202  }
203
204  /**
205   * Helper method to get the archive directory for the specified region
206   * @param conf   {@link Configuration} to check for the name of the archive directory
207   * @param region region that is being archived
208   * @return {@link Path} to the archive directory for the given region
209   */
210  public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
211    return HFileArchiveUtil.getRegionArchiveDir(CommonFSUtils.getRootDir(conf),
212      region.getTableDescriptor().getTableName(), region.getRegionInfo().getEncodedName());
213  }
214
215  /**
216   * Helper method to get the store archive directory for the specified region
217   * @param conf   {@link Configuration} to check for the name of the archive directory
218   * @param region region that is being archived
219   * @param store  store that is archiving files
220   * @return {@link Path} to the store archive directory for the given region
221   */
222  public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
223    throws IOException {
224    return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
225      region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName());
226  }
227
228  public static Path getStoreArchivePath(HBaseTestingUtil util, String tableName, byte[] storeName)
229    throws IOException {
230    byte[] table = Bytes.toBytes(tableName);
231    // get the RS and region serving our table
232    List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
233    HRegion region = servingRegions.get(0);
234
235    // check that we actually have some store files that were archived
236    Store store = region.getStore(storeName);
237    return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
238  }
239}