001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.fs.PathFilter;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.TableNotEnabledException;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.BufferedMutator;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.RegionReplicaUtil;
051import org.apache.hadoop.hbase.client.SnapshotDescription;
052import org.apache.hadoop.hbase.client.SnapshotType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
057import org.apache.hadoop.hbase.io.HFileLink;
058import org.apache.hadoop.hbase.master.HMaster;
059import org.apache.hadoop.hbase.master.MasterFileSystem;
060import org.apache.hadoop.hbase.mob.MobUtils;
061import org.apache.hadoop.hbase.regionserver.HRegion;
062import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
063import org.apache.hadoop.hbase.regionserver.HRegionServer;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.FSTableDescriptors;
068import org.apache.hadoop.hbase.util.FSVisitor;
069import org.apache.hadoop.hbase.util.MD5Hash;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.junit.Assert;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
079
080/**
081 * Utilities class for snapshots
082 */
083@InterfaceAudience.Private
084public final class SnapshotTestingUtils {
085  private static final Logger LOG = LoggerFactory.getLogger(SnapshotTestingUtils.class);
086
087  // default number of regions (and keys) given by getSplitKeys() and createTable()
088  private static byte[] KEYS = Bytes.toBytes("0123456");
089
090  private SnapshotTestingUtils() {
091    // private constructor for utility class
092  }
093
094  /**
095   * Assert that we don't have any snapshots lists
096   *
097   * @throws IOException
098   *           if the admin operation fails
099   */
100  public static void assertNoSnapshots(Admin admin) throws IOException {
101    assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
102        .size());
103  }
104
105  /**
106   * Make sure that there is only one snapshot returned from the master and its
107   * name and table match the passed in parameters.
108   */
109  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
110      Admin admin, String snapshotName, TableName tableName)
111      throws IOException {
112    // list the snapshot
113    List<SnapshotDescription> snapshots = admin.listSnapshots();
114
115    List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
116    for (SnapshotDescription sd : snapshots) {
117      if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
118        returnedSnapshots.add(sd);
119      }
120    }
121
122    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
123    return returnedSnapshots;
124  }
125
126  /**
127   * Make sure that there is only one snapshot returned from the master
128   */
129  public static void assertOneSnapshotThatMatches(Admin admin,
130      SnapshotProtos.SnapshotDescription snapshot) throws IOException {
131    assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable()));
132  }
133
134  /**
135   * Make sure that there is only one snapshot returned from the master and its
136   * name and table match the passed in parameters.
137   */
138  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
139      Admin admin, String snapshotName, TableName tableName)
140      throws IOException {
141    // list the snapshot
142    List<SnapshotDescription> snapshots = admin.listSnapshots();
143
144    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
145    assertEquals(snapshotName, snapshots.get(0).getName());
146    assertEquals(tableName, snapshots.get(0).getTableName());
147
148    return snapshots;
149  }
150
151  /**
152   * Make sure that there is only one snapshot returned from the master and its
153   * name and table match the passed in parameters.
154   */
155  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
156      Admin admin, byte[] snapshot, TableName tableName) throws IOException {
157    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
158        tableName);
159  }
160
161  public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
162      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
163      throws IOException {
164    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
165    confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(),
166      testUtil.getAdmin(), mfs.getFileSystem());
167  }
168
169  /**
170   * Confirm that the snapshot contains references to all the files that should
171   * be in the snapshot.
172   */
173  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
174      TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
175      throws IOException {
176    ArrayList nonEmptyTestFamilies = new ArrayList(1);
177    nonEmptyTestFamilies.add(testFamily);
178    confirmSnapshotValid(snapshotDescriptor, tableName,
179      nonEmptyTestFamilies, null, rootDir, admin, fs);
180  }
181
182  /**
183   * Confirm that the snapshot has no references files but only metadata.
184   */
185  public static void confirmEmptySnapshotValid(
186      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
187      byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
188      throws IOException {
189    ArrayList emptyTestFamilies = new ArrayList(1);
190    emptyTestFamilies.add(testFamily);
191    confirmSnapshotValid(snapshotDescriptor, tableName,
192      null, emptyTestFamilies, rootDir, admin, fs);
193  }
194
195  /**
196   * Confirm that the snapshot contains references to all the files that should
197   * be in the snapshot. This method also perform some redundant check like
198   * the existence of the snapshotinfo or the regioninfo which are done always
199   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
200   */
201  public static void confirmSnapshotValid(
202      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
203      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
204      Path rootDir, Admin admin, FileSystem fs) throws IOException {
205    final Configuration conf = admin.getConfiguration();
206
207    // check snapshot dir
208    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
209        snapshotDescriptor, rootDir);
210    assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir));
211
212    SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
213
214    // Extract regions and families with store files
215    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
216
217    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
218    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
219    for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
220      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
221          new SnapshotReferenceUtil.StoreFileVisitor() {
222        @Override
223        public void storeFile(final RegionInfo regionInfo, final String family,
224              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
225          snapshotFamilies.add(Bytes.toBytes(family));
226        }
227      });
228    }
229    // Verify that there are store files in the specified families
230    if (nonEmptyTestFamilies != null) {
231      for (final byte[] familyName: nonEmptyTestFamilies) {
232        assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
233      }
234    }
235
236    // Verify that there are no store files in the specified families
237    if (emptyTestFamilies != null) {
238      for (final byte[] familyName: emptyTestFamilies) {
239        assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
240      }
241    }
242
243    // check the region snapshot for all the regions
244    List<RegionInfo> regions = admin.getRegions(tableName);
245    // remove the non-default regions
246    RegionReplicaUtil.removeNonDefaultRegions(regions);
247    boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
248        .getEncodedName());
249    if (hasMob) {
250      assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
251    } else {
252      // if create snapshot when table splitting, parent region will be included to the snapshot
253      // region manifest. we should exclude the parent regions.
254      int regionCountExclusiveSplitParent = 0;
255      for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
256        RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
257        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
258          continue;
259        }
260        regionCountExclusiveSplitParent++;
261      }
262      assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
263    }
264
265    // Verify Regions (redundant check, see MasterSnapshotVerifier)
266    for (RegionInfo info : regions) {
267      String regionName = info.getEncodedName();
268      assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
269    }
270  }
271
272  /*
273   * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
274   * except for the last CorruptedSnapshotException
275   */
276  public static void snapshot(Admin admin, final String snapshotName, final TableName tableName,
277      final SnapshotType type, final int numTries) throws IOException {
278    int tries = 0;
279    CorruptedSnapshotException lastEx = null;
280    while (tries++ < numTries) {
281      try {
282        admin.snapshot(snapshotName, tableName, type);
283        return;
284      } catch (CorruptedSnapshotException cse) {
285        LOG.warn("Got CorruptedSnapshotException", cse);
286        lastEx = cse;
287      }
288    }
289    throw lastEx;
290  }
291
292  public static void cleanupSnapshot(Admin admin, byte[] tableName)
293      throws IOException {
294    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
295  }
296
297  public static void cleanupSnapshot(Admin admin, String snapshotName)
298      throws IOException {
299    // delete the taken snapshot
300    admin.deleteSnapshot(snapshotName);
301    assertNoSnapshots(admin);
302  }
303
304  /**
305   * Expect the snapshot to throw an error when checking if the snapshot is
306   * complete
307   *
308   * @param master master to check
309   * @param snapshot the {@link SnapshotDescription} request to pass to the master
310   * @param clazz expected exception from the master
311   */
312  public static void expectSnapshotDoneException(HMaster master,
313      IsSnapshotDoneRequest snapshot,
314      Class<? extends HBaseSnapshotException> clazz) {
315    try {
316      master.getMasterRpcServices().isSnapshotDone(null, snapshot);
317      Assert.fail("didn't fail to lookup a snapshot");
318    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) {
319      try {
320        throw ProtobufUtil.handleRemoteException(se);
321      } catch (HBaseSnapshotException e) {
322        assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
323      } catch (Throwable t) {
324        Assert.fail("Threw an unexpected exception:" + t);
325      }
326    }
327  }
328
329  /**
330   * List all the HFiles in the given table
331   *
332   * @param fs FileSystem where the table lives
333   * @param tableDir directory of the table
334   * @return array of the current HFiles in the table (could be a zero-length array)
335   * @throws IOException on unexecpted error reading the FS
336   */
337  public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
338      throws IOException {
339    final ArrayList<String> hfiles = new ArrayList<>();
340    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
341      @Override
342      public void storeFile(final String region, final String family, final String hfileName)
343          throws IOException {
344        hfiles.add(hfileName);
345      }
346    });
347    Collections.sort(hfiles);
348    return hfiles;
349  }
350
351  /**
352   * Take a snapshot of the specified table and verify that the given family is
353   * not empty. Note that this will leave the table disabled
354   * in the case of an offline snapshot.
355   */
356  public static void createSnapshotAndValidate(Admin admin,
357      TableName tableName, String familyName, String snapshotNameString,
358      Path rootDir, FileSystem fs, boolean onlineSnapshot)
359      throws Exception {
360    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
361    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
362    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
363                              snapshotNameString, rootDir, fs, onlineSnapshot);
364  }
365
366  /**
367   * Take a snapshot of the specified table and verify the given families.
368   * Note that this will leave the table disabled in the case of an offline snapshot.
369   */
370  public static void createSnapshotAndValidate(Admin admin,
371      TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
372      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
373        throws Exception {
374    if (!onlineSnapshot) {
375      try {
376        LOG.info("prepping for offline snapshot.");
377        admin.disableTable(tableName);
378      } catch (TableNotEnabledException tne) {
379        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
380            "already disabled.");
381      }
382    }
383    LOG.info("taking snapshot.");
384    admin.snapshot(snapshotNameString, tableName);
385
386    LOG.info("Confirming snapshot exists.");
387    List<SnapshotDescription> snapshots =
388        SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
389    if (snapshots == null || snapshots.size() != 1) {
390      Assert.fail("Incorrect number of snapshots for table " + tableName);
391    }
392
393    LOG.info("validating snapshot.");
394    SnapshotTestingUtils.confirmSnapshotValid(
395      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
396      emptyFamilyNames, rootDir, admin, fs);
397  }
398
399  /**
400   * Corrupt the specified snapshot by deleting some files.
401   *
402   * @param util {@link HBaseTestingUtility}
403   * @param snapshotName name of the snapshot to corrupt
404   * @return array of the corrupted HFiles
405   * @throws IOException on unexecpted error reading the FS
406   */
407  public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
408      throws IOException {
409    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
410    final FileSystem fs = mfs.getFileSystem();
411
412    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
413                                                                        mfs.getRootDir());
414    SnapshotProtos.SnapshotDescription snapshotDesc =
415        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
416    final TableName table = TableName.valueOf(snapshotDesc.getTable());
417
418    final ArrayList corruptedFiles = new ArrayList();
419    final Configuration conf = util.getConfiguration();
420    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
421        new SnapshotReferenceUtil.StoreFileVisitor() {
422      @Override
423      public void storeFile(final RegionInfo regionInfo, final String family,
424            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
425        String region = regionInfo.getEncodedName();
426        String hfile = storeFile.getName();
427        HFileLink link = HFileLink.build(conf, table, region, family, hfile);
428        if (corruptedFiles.size() % 2 == 0) {
429          fs.delete(link.getAvailablePath(fs), true);
430          corruptedFiles.add(hfile);
431        }
432      }
433    });
434
435    assertTrue(corruptedFiles.size() > 0);
436    return corruptedFiles;
437  }
438
439  // ==========================================================================
440  //  Snapshot Mock
441  // ==========================================================================
442  public static class SnapshotMock {
443    protected final static String TEST_FAMILY = "cf";
444    public final static int TEST_NUM_REGIONS = 4;
445
446    private final Configuration conf;
447    private final FileSystem fs;
448    private final Path rootDir;
449
450    static class RegionData {
451      public RegionInfo hri;
452      public Path tableDir;
453      public Path[] files;
454
455      public RegionData(final Path tableDir, final RegionInfo hri, final int nfiles) {
456        this.tableDir = tableDir;
457        this.hri = hri;
458        this.files = new Path[nfiles];
459      }
460    }
461
462    public static class SnapshotBuilder {
463      private final RegionData[] tableRegions;
464      private final SnapshotProtos.SnapshotDescription desc;
465      private final TableDescriptor htd;
466      private final Configuration conf;
467      private final FileSystem fs;
468      private final Path rootDir;
469      private Path snapshotDir;
470      private int snapshotted = 0;
471
472      public SnapshotBuilder(final Configuration conf, final FileSystem fs,
473          final Path rootDir, final TableDescriptor htd,
474          final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions)
475          throws IOException {
476        this.fs = fs;
477        this.conf = conf;
478        this.rootDir = rootDir;
479        this.htd = htd;
480        this.desc = desc;
481        this.tableRegions = tableRegions;
482        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
483        new FSTableDescriptors(conf)
484          .createTableDescriptorForTableDirectory(this.snapshotDir.getFileSystem(conf),
485            snapshotDir, htd, false);
486      }
487
488      public TableDescriptor getTableDescriptor() {
489        return this.htd;
490      }
491
492      public SnapshotProtos.SnapshotDescription getSnapshotDescription() {
493        return this.desc;
494      }
495
496      public Path getSnapshotsDir() {
497        return this.snapshotDir;
498      }
499
500      public Path[] addRegion() throws IOException {
501        return addRegion(desc);
502      }
503
504      public Path[] addRegionV1() throws IOException {
505        return addRegion(desc.toBuilder()
506                          .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
507                          .build());
508      }
509
510      public Path[] addRegionV2() throws IOException {
511        return addRegion(desc.toBuilder()
512                          .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
513                          .build());
514      }
515
516      private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException {
517        if (this.snapshotted == tableRegions.length) {
518          throw new UnsupportedOperationException("No more regions in the table");
519        }
520
521        RegionData regionData = tableRegions[this.snapshotted++];
522        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
523        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
524        manifest.addRegion(regionData.tableDir, regionData.hri);
525        return regionData.files;
526      }
527
528      private void corruptFile(Path p) throws IOException {
529        String manifestName = p.getName();
530
531        // Rename the original region-manifest file
532        Path newP = new Path(p.getParent(), manifestName + "1");
533        fs.rename(p, newP);
534
535        // Create a new region-manifest file
536        FSDataOutputStream out = fs.create(p);
537
538        //Copy the first 25 bytes of the original region-manifest into the new one,
539        //make it a corrupted region-manifest file.
540        FSDataInputStream input = fs.open(newP);
541        byte[] buffer = new byte[25];
542        int len = input.read(0, buffer, 0, 25);
543        if (len > 1) {
544          out.write(buffer, 0, len - 1);
545        }
546        out.close();
547
548        // Delete the original region-manifest
549        fs.delete(newP);
550      }
551
552      /**
553       * Corrupt one region-manifest file
554       *
555       * @throws IOException on unexecpted error from the FS
556       */
557      public void corruptOneRegionManifest() throws IOException {
558        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
559          @Override public boolean accept(Path path) {
560            return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
561          }
562        });
563
564        if (manifestFiles.length == 0) return;
565
566        // Just choose the first one
567        Path p = manifestFiles[0].getPath();
568        corruptFile(p);
569      }
570
571      public void missOneRegionSnapshotFile() throws IOException {
572        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir);
573        for (FileStatus fileStatus : manifestFiles) {
574          String fileName = fileStatus.getPath().getName();
575          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
576            || fileName.endsWith(".tabledesc")
577            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
578              fs.delete(fileStatus.getPath(), true);
579          }
580        }
581      }
582
583      /**
584       * Corrupt data-manifest file
585       *
586       * @throws IOException on unexecpted error from the FS
587       */
588      public void corruptDataManifest() throws IOException {
589        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
590          @Override
591          public boolean accept(Path path) {
592            return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
593          }
594        });
595
596        if (manifestFiles.length == 0) return;
597
598        // Just choose the first one
599        Path p = manifestFiles[0].getPath();
600        corruptFile(p);
601      }
602
603      public Path commit() throws IOException {
604        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
605        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
606        manifest.addTableDescriptor(htd);
607        manifest.consolidate();
608        Path finishedDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
609        SnapshotDescriptionUtils.completeSnapshot(finishedDir, snapshotDir, fs,
610          snapshotDir.getFileSystem(conf), conf);
611        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
612        return snapshotDir;
613      }
614
615      public void consolidate() throws IOException {
616        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
617        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
618        manifest.addTableDescriptor(htd);
619        manifest.consolidate();
620      }
621    }
622
623    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
624      this.fs = fs;
625      this.conf = conf;
626      this.rootDir = rootDir;
627    }
628
629    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
630        throws IOException {
631      return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
632    }
633
634    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
635        final int numRegions) throws IOException {
636      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
637    }
638
639    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
640        throws IOException {
641      return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
642    }
643
644    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
645        final int numRegions) throws IOException {
646      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
647    }
648
649    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
650        final int numRegions, final long ttl) throws IOException {
651      return createSnapshot(snapshotName, tableName, numRegions,
652          SnapshotManifestV2.DESCRIPTOR_VERSION, ttl);
653    }
654
655    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
656        final int version) throws IOException {
657      return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
658    }
659
660    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
661        final int numRegions, final int version) throws IOException {
662      TableDescriptor htd = createHtd(tableName);
663      RegionData[] regions = createTable(htd, numRegions);
664
665      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
666        .setTable(htd.getTableName().getNameAsString())
667        .setName(snapshotName)
668        .setVersion(version)
669        .build();
670
671      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
672      FileSystem workingFs = workingDir.getFileSystem(conf);
673      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs);
674      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
675    }
676
677    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
678        final int numRegions, final int version, final long ttl) throws IOException {
679      TableDescriptor htd = createHtd(tableName);
680      RegionData[] regions = createTable(htd, numRegions);
681      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
682          .setTable(htd.getTableName().getNameAsString())
683          .setName(snapshotName)
684          .setVersion(version)
685          .setCreationTime(EnvironmentEdgeManager.currentTime())
686          .setTtl(ttl)
687          .build();
688      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
689      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
690      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
691    }
692
693    public TableDescriptor createHtd(final String tableName) {
694      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
695              .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY))
696              .build();
697    }
698
699    private RegionData[] createTable(final TableDescriptor htd, final int nregions)
700        throws IOException {
701      Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
702      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
703
704      assertTrue(nregions % 2 == 0);
705      RegionData[] regions = new RegionData[nregions];
706      for (int i = 0; i < regions.length; i += 2) {
707        byte[] startKey = Bytes.toBytes(0 + i * 2);
708        byte[] endKey = Bytes.toBytes(1 + i * 2);
709
710        // First region, simple with one plain hfile.
711        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
712            .setStartKey(startKey)
713            .setEndKey(endKey)
714            .build();
715        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
716        regions[i] = new RegionData(tableDir, hri, 3);
717        for (int j = 0; j < regions[i].files.length; ++j) {
718          Path storeFile = createStoreFile(rfs.createTempName());
719          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
720        }
721
722        // Second region, used to test the split case.
723        // This region contains a reference to the hfile in the first region.
724        startKey = Bytes.toBytes(2 + i * 2);
725        endKey = Bytes.toBytes(3 + i * 2);
726        hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
727        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
728        regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
729        for (int j = 0; j < regions[i].files.length; ++j) {
730          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
731          Path refFile = createStoreFile(new Path(rootDir, refName));
732          regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
733        }
734      }
735      return regions;
736    }
737
738    private Path createStoreFile(final Path storeFile)
739        throws IOException {
740      FSDataOutputStream out = fs.create(storeFile);
741      try {
742        out.write(Bytes.toBytes(storeFile.toString()));
743      } finally {
744        out.close();
745      }
746      return storeFile;
747    }
748  }
749
750  // ==========================================================================
751  //  Table Helpers
752  // ==========================================================================
753  public static void waitForTableToBeOnline(final HBaseTestingUtility util,
754                                            final TableName tableName)
755      throws IOException, InterruptedException {
756    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
757    List<HRegion> onlineRegions = rs.getRegions(tableName);
758    for (HRegion region : onlineRegions) {
759      region.waitForFlushesAndCompactions();
760    }
761    // Wait up to 60 seconds for a table to be available.
762    util.waitFor(60000, util.predicateTableAvailable(tableName));
763  }
764
765  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
766      int regionReplication, int nRegions, final byte[]... families)
767      throws IOException, InterruptedException {
768    TableDescriptorBuilder builder
769      = TableDescriptorBuilder
770          .newBuilder(tableName)
771          .setRegionReplication(regionReplication);
772    for (byte[] family : families) {
773      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
774    }
775    byte[][] splitKeys = getSplitKeys(nRegions);
776    util.createTable(builder.build(), splitKeys);
777    assertEquals((splitKeys.length + 1) * regionReplication,
778        util.getAdmin().getRegions(tableName).size());
779  }
780
781  public static byte[][] getSplitKeys() {
782    return getSplitKeys(KEYS.length);
783  }
784
785  public static byte[][] getSplitKeys(int nRegions) {
786    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
787    final byte[][] splitKeys = new byte[nRegions-1][];
788    final int step = KEYS.length / nRegions;
789    int keyIndex = 1;
790    for (int i = 0; i < splitKeys.length; ++i) {
791      splitKeys[i] = new byte[] { KEYS[keyIndex] };
792      keyIndex += step;
793    }
794    return splitKeys;
795  }
796
797  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
798      final byte[]... families) throws IOException, InterruptedException {
799    createTable(util, tableName, 1, families);
800  }
801
802  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
803      final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
804    createTable(util, tableName, regionReplication, KEYS.length, families);
805  }
806
807  public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName,
808      final int nRegions, final byte[]... families) throws IOException, InterruptedException {
809    createTable(util, tableName, 1, nRegions, families);
810  }
811
812  public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
813      byte[]... families) throws IOException, InterruptedException {
814    BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
815    loadData(util, mutator, rows, families);
816  }
817
818  public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
819      byte[]... families) throws IOException, InterruptedException {
820    // Ensure one row per region
821    assertTrue(rows >= KEYS.length);
822    for (byte k0: KEYS) {
823      byte[] k = new byte[] { k0 };
824      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
825      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
826      final byte[][] families1 = families;
827      final byte[] key1 = key;
828      final byte[] value1 = value;
829      mutator.mutate(createPut(families1, key1, value1));
830      rows--;
831    }
832
833    // Add other extra rows. more rows, more files
834    while (rows-- > 0) {
835      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
836      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
837      final byte[][] families1 = families;
838      final byte[] key1 = key;
839      final byte[] value1 = value;
840      mutator.mutate(createPut(families1, key1, value1));
841    }
842    mutator.flush();
843
844    waitForTableToBeOnline(util, mutator.getName());
845  }
846
847  private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
848    byte[] q = Bytes.toBytes("q");
849    Put put = new Put(key);
850    put.setDurability(Durability.SKIP_WAL);
851    for (byte[] family: families) {
852      put.addColumn(family, q, value);
853    }
854    return put;
855  }
856
857  public static void deleteAllSnapshots(final Admin admin)
858      throws IOException {
859    // Delete all the snapshots
860    for (SnapshotDescription snapshot: admin.listSnapshots()) {
861      admin.deleteSnapshot(snapshot.getName());
862    }
863    SnapshotTestingUtils.assertNoSnapshots(admin);
864  }
865
866  public static void deleteArchiveDirectory(final HBaseTestingUtility util)
867      throws IOException {
868    // Ensure the archiver to be empty
869    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
870    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
871    mfs.getFileSystem().delete(archiveDir, true);
872  }
873
874  public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
875      long expectedRows) throws IOException {
876    Table table = util.getConnection().getTable(tableName);
877    try {
878      assertEquals(expectedRows, util.countRows(table));
879    } finally {
880      table.close();
881    }
882  }
883
884  public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
885      int regionReplication) throws IOException {
886    List<RegionInfo> regions = admin.getRegions(tableName);
887    HashSet<RegionInfo> set = new HashSet<>();
888    for (RegionInfo hri : regions) {
889      set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
890      for (int i = 0; i < regionReplication; i++) {
891        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
892        if (!regions.contains(replica)) {
893          Assert.fail(replica + " is not contained in the list of online regions");
894        }
895      }
896    }
897    assertEquals(getSplitKeys().length + 1, set.size());
898  }
899}