001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import com.google.protobuf.ServiceException;
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Collections;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Set;
032import java.util.TreeSet;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FSDataInputStream;
035import org.apache.hadoop.fs.FSDataOutputStream;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.fs.PathFilter;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.TableNotEnabledException;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.BufferedMutator;
046import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
047import org.apache.hadoop.hbase.client.Durability;
048import org.apache.hadoop.hbase.client.Put;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.client.RegionInfoBuilder;
051import org.apache.hadoop.hbase.client.RegionReplicaUtil;
052import org.apache.hadoop.hbase.client.SnapshotDescription;
053import org.apache.hadoop.hbase.client.SnapshotType;
054import org.apache.hadoop.hbase.client.Table;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
058import org.apache.hadoop.hbase.io.HFileLink;
059import org.apache.hadoop.hbase.master.HMaster;
060import org.apache.hadoop.hbase.master.MasterFileSystem;
061import org.apache.hadoop.hbase.mob.MobUtils;
062import org.apache.hadoop.hbase.regionserver.HRegion;
063import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
064import org.apache.hadoop.hbase.regionserver.HRegionServer;
065import org.apache.hadoop.hbase.util.Bytes;
066import org.apache.hadoop.hbase.util.FSTableDescriptors;
067import org.apache.hadoop.hbase.util.FSUtils;
068import org.apache.hadoop.hbase.util.FSVisitor;
069import org.apache.hadoop.hbase.util.MD5Hash;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.junit.Assert;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
079import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
080
081/**
082 * Utilities class for snapshots
083 */
084@InterfaceAudience.Private
085public final class SnapshotTestingUtils {
086  private static final Logger LOG = LoggerFactory.getLogger(SnapshotTestingUtils.class);
087
088  // default number of regions (and keys) given by getSplitKeys() and createTable()
089  private static byte[] KEYS = Bytes.toBytes("0123456");
090
091  private SnapshotTestingUtils() {
092    // private constructor for utility class
093  }
094
095  /**
096   * Assert that we don't have any snapshots lists
097   *
098   * @throws IOException
099   *           if the admin operation fails
100   */
101  public static void assertNoSnapshots(Admin admin) throws IOException {
102    assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
103        .size());
104  }
105
106  /**
107   * Make sure that there is only one snapshot returned from the master and its
108   * name and table match the passed in parameters.
109   */
110  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
111      Admin admin, String snapshotName, TableName tableName)
112      throws IOException {
113    // list the snapshot
114    List<SnapshotDescription> snapshots = admin.listSnapshots();
115
116    List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
117    for (SnapshotDescription sd : snapshots) {
118      if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
119        returnedSnapshots.add(sd);
120      }
121    }
122
123    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
124    return returnedSnapshots;
125  }
126
127  /**
128   * Make sure that there is only one snapshot returned from the master
129   */
130  public static void assertOneSnapshotThatMatches(Admin admin,
131      SnapshotProtos.SnapshotDescription snapshot) throws IOException {
132    assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable()));
133  }
134
135  /**
136   * Make sure that there is only one snapshot returned from the master and its
137   * name and table match the passed in parameters.
138   */
139  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
140      Admin admin, String snapshotName, TableName tableName)
141      throws IOException {
142    // list the snapshot
143    List<SnapshotDescription> snapshots = admin.listSnapshots();
144
145    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
146    assertEquals(snapshotName, snapshots.get(0).getName());
147    assertEquals(tableName, snapshots.get(0).getTableName());
148
149    return snapshots;
150  }
151
152  /**
153   * Make sure that there is only one snapshot returned from the master and its
154   * name and table match the passed in parameters.
155   */
156  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
157      Admin admin, byte[] snapshot, TableName tableName) throws IOException {
158    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
159        tableName);
160  }
161
162  public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
163      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
164      throws IOException {
165    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
166    confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(),
167      testUtil.getAdmin(), mfs.getFileSystem());
168  }
169
170  /**
171   * Confirm that the snapshot contains references to all the files that should
172   * be in the snapshot.
173   */
174  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
175      TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
176      throws IOException {
177    ArrayList nonEmptyTestFamilies = new ArrayList(1);
178    nonEmptyTestFamilies.add(testFamily);
179    confirmSnapshotValid(snapshotDescriptor, tableName,
180      nonEmptyTestFamilies, null, rootDir, admin, fs);
181  }
182
183  /**
184   * Confirm that the snapshot has no references files but only metadata.
185   */
186  public static void confirmEmptySnapshotValid(
187      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
188      byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
189      throws IOException {
190    ArrayList emptyTestFamilies = new ArrayList(1);
191    emptyTestFamilies.add(testFamily);
192    confirmSnapshotValid(snapshotDescriptor, tableName,
193      null, emptyTestFamilies, rootDir, admin, fs);
194  }
195
196  /**
197   * Confirm that the snapshot contains references to all the files that should
198   * be in the snapshot. This method also perform some redundant check like
199   * the existence of the snapshotinfo or the regioninfo which are done always
200   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
201   */
202  public static void confirmSnapshotValid(
203      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
204      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
205      Path rootDir, Admin admin, FileSystem fs) throws IOException {
206    final Configuration conf = admin.getConfiguration();
207
208    // check snapshot dir
209    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
210        snapshotDescriptor, rootDir);
211    assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir));
212
213    SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
214
215    // Extract regions and families with store files
216    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
217
218    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
219    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
220    for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
221      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
222          new SnapshotReferenceUtil.StoreFileVisitor() {
223        @Override
224        public void storeFile(final RegionInfo regionInfo, final String family,
225              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
226          snapshotFamilies.add(Bytes.toBytes(family));
227        }
228      });
229    }
230    // Verify that there are store files in the specified families
231    if (nonEmptyTestFamilies != null) {
232      for (final byte[] familyName: nonEmptyTestFamilies) {
233        assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
234      }
235    }
236
237    // Verify that there are no store files in the specified families
238    if (emptyTestFamilies != null) {
239      for (final byte[] familyName: emptyTestFamilies) {
240        assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
241      }
242    }
243
244    // check the region snapshot for all the regions
245    List<RegionInfo> regions = admin.getRegions(tableName);
246    // remove the non-default regions
247    RegionReplicaUtil.removeNonDefaultRegions(regions);
248    boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
249        .getEncodedName());
250    if (hasMob) {
251      assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
252    } else {
253      // if create snapshot when table splitting, parent region will be included to the snapshot
254      // region manifest. we should exclude the parent regions.
255      int regionCountExclusiveSplitParent = 0;
256      for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
257        RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
258        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
259          continue;
260        }
261        regionCountExclusiveSplitParent++;
262      }
263      assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
264    }
265
266    // Verify Regions (redundant check, see MasterSnapshotVerifier)
267    for (RegionInfo info : regions) {
268      String regionName = info.getEncodedName();
269      assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
270    }
271  }
272
273  /**
274   * Helper method for testing async snapshot operations. Just waits for the
275   * given snapshot to complete on the server by repeatedly checking the master.
276   *
277   * @param master: the master running the snapshot
278   * @param snapshot: the snapshot to check
279   * @param sleep: amount to sleep between checks to see if the snapshot is done
280   * @throws ServiceException if the snapshot fails
281   * @throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
282   */
283  public static void waitForSnapshotToComplete(HMaster master,
284      SnapshotProtos.SnapshotDescription snapshot, long sleep)
285          throws org.apache.hbase.thirdparty.com.google.protobuf.ServiceException {
286    final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
287        .setSnapshot(snapshot).build();
288    IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
289        .buildPartial();
290    while (!done.getDone()) {
291      done = master.getMasterRpcServices().isSnapshotDone(null, request);
292      try {
293        Thread.sleep(sleep);
294      } catch (InterruptedException e) {
295        throw new org.apache.hbase.thirdparty.com.google.protobuf.ServiceException(e);
296      }
297    }
298  }
299
300  /*
301   * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
302   * except for the last CorruptedSnapshotException
303   */
304  public static void snapshot(Admin admin, final String snapshotName, final TableName tableName,
305      final SnapshotType type, final int numTries) throws IOException {
306    int tries = 0;
307    CorruptedSnapshotException lastEx = null;
308    while (tries++ < numTries) {
309      try {
310        admin.snapshot(snapshotName, tableName, type);
311        return;
312      } catch (CorruptedSnapshotException cse) {
313        LOG.warn("Got CorruptedSnapshotException", cse);
314        lastEx = cse;
315      }
316    }
317    throw lastEx;
318  }
319
320  public static void cleanupSnapshot(Admin admin, byte[] tableName)
321      throws IOException {
322    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
323  }
324
325  public static void cleanupSnapshot(Admin admin, String snapshotName)
326      throws IOException {
327    // delete the taken snapshot
328    admin.deleteSnapshot(snapshotName);
329    assertNoSnapshots(admin);
330  }
331
332  /**
333   * Expect the snapshot to throw an error when checking if the snapshot is
334   * complete
335   *
336   * @param master master to check
337   * @param snapshot the {@link SnapshotDescription} request to pass to the master
338   * @param clazz expected exception from the master
339   */
340  public static void expectSnapshotDoneException(HMaster master,
341      IsSnapshotDoneRequest snapshot,
342      Class<? extends HBaseSnapshotException> clazz) {
343    try {
344      master.getMasterRpcServices().isSnapshotDone(null, snapshot);
345      Assert.fail("didn't fail to lookup a snapshot");
346    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) {
347      try {
348        throw ProtobufUtil.handleRemoteException(se);
349      } catch (HBaseSnapshotException e) {
350        assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
351      } catch (Throwable t) {
352        Assert.fail("Threw an unexpected exception:" + t);
353      }
354    }
355  }
356
357  /**
358   * List all the HFiles in the given table
359   *
360   * @param fs: FileSystem where the table lives
361   * @param tableDir directory of the table
362   * @return array of the current HFiles in the table (could be a zero-length array)
363   * @throws IOException on unexecpted error reading the FS
364   */
365  public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
366      throws IOException {
367    final ArrayList<String> hfiles = new ArrayList<>();
368    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
369      @Override
370      public void storeFile(final String region, final String family, final String hfileName)
371          throws IOException {
372        hfiles.add(hfileName);
373      }
374    });
375    Collections.sort(hfiles);
376    return hfiles;
377  }
378
379  /**
380   * Take a snapshot of the specified table and verify that the given family is
381   * not empty. Note that this will leave the table disabled
382   * in the case of an offline snapshot.
383   */
384  public static void createSnapshotAndValidate(Admin admin,
385      TableName tableName, String familyName, String snapshotNameString,
386      Path rootDir, FileSystem fs, boolean onlineSnapshot)
387      throws Exception {
388    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
389    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
390    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
391                              snapshotNameString, rootDir, fs, onlineSnapshot);
392  }
393
394  /**
395   * Take a snapshot of the specified table and verify the given families.
396   * Note that this will leave the table disabled in the case of an offline snapshot.
397   */
398  public static void createSnapshotAndValidate(Admin admin,
399      TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
400      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
401        throws Exception {
402    if (!onlineSnapshot) {
403      try {
404        LOG.info("prepping for offline snapshot.");
405        admin.disableTable(tableName);
406      } catch (TableNotEnabledException tne) {
407        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
408            "already disabled.");
409      }
410    }
411    LOG.info("taking snapshot.");
412    admin.snapshot(snapshotNameString, tableName);
413
414    LOG.info("Confirming snapshot exists.");
415    List<SnapshotDescription> snapshots =
416        SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
417    if (snapshots == null || snapshots.size() != 1) {
418      Assert.fail("Incorrect number of snapshots for table " + tableName);
419    }
420
421    LOG.info("validating snapshot.");
422    SnapshotTestingUtils.confirmSnapshotValid(
423      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
424      emptyFamilyNames, rootDir, admin, fs);
425  }
426
427  /**
428   * Corrupt the specified snapshot by deleting some files.
429   *
430   * @param util {@link HBaseTestingUtility}
431   * @param snapshotName name of the snapshot to corrupt
432   * @return array of the corrupted HFiles
433   * @throws IOException on unexecpted error reading the FS
434   */
435  public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
436      throws IOException {
437    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
438    final FileSystem fs = mfs.getFileSystem();
439
440    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
441                                                                        mfs.getRootDir());
442    SnapshotProtos.SnapshotDescription snapshotDesc =
443        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
444    final TableName table = TableName.valueOf(snapshotDesc.getTable());
445
446    final ArrayList corruptedFiles = new ArrayList();
447    final Configuration conf = util.getConfiguration();
448    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
449        new SnapshotReferenceUtil.StoreFileVisitor() {
450      @Override
451      public void storeFile(final RegionInfo regionInfo, final String family,
452            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
453        String region = regionInfo.getEncodedName();
454        String hfile = storeFile.getName();
455        HFileLink link = HFileLink.build(conf, table, region, family, hfile);
456        if (corruptedFiles.size() % 2 == 0) {
457          fs.delete(link.getAvailablePath(fs), true);
458          corruptedFiles.add(hfile);
459        }
460      }
461    });
462
463    assertTrue(corruptedFiles.size() > 0);
464    return corruptedFiles;
465  }
466
467  // ==========================================================================
468  //  Snapshot Mock
469  // ==========================================================================
470  public static class SnapshotMock {
471    protected final static String TEST_FAMILY = "cf";
472    public final static int TEST_NUM_REGIONS = 4;
473
474    private final Configuration conf;
475    private final FileSystem fs;
476    private final Path rootDir;
477
478    static class RegionData {
479      public RegionInfo hri;
480      public Path tableDir;
481      public Path[] files;
482
483      public RegionData(final Path tableDir, final RegionInfo hri, final int nfiles) {
484        this.tableDir = tableDir;
485        this.hri = hri;
486        this.files = new Path[nfiles];
487      }
488    }
489
490    public static class SnapshotBuilder {
491      private final RegionData[] tableRegions;
492      private final SnapshotProtos.SnapshotDescription desc;
493      private final TableDescriptor htd;
494      private final Configuration conf;
495      private final FileSystem fs;
496      private final Path rootDir;
497      private Path snapshotDir;
498      private int snapshotted = 0;
499
500      public SnapshotBuilder(final Configuration conf, final FileSystem fs,
501          final Path rootDir, final TableDescriptor htd,
502          final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions)
503          throws IOException {
504        this.fs = fs;
505        this.conf = conf;
506        this.rootDir = rootDir;
507        this.htd = htd;
508        this.desc = desc;
509        this.tableRegions = tableRegions;
510        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
511        new FSTableDescriptors(conf)
512          .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
513      }
514
515      public TableDescriptor getTableDescriptor() {
516        return this.htd;
517      }
518
519      public SnapshotProtos.SnapshotDescription getSnapshotDescription() {
520        return this.desc;
521      }
522
523      public Path getSnapshotsDir() {
524        return this.snapshotDir;
525      }
526
527      public Path[] addRegion() throws IOException {
528        return addRegion(desc);
529      }
530
531      public Path[] addRegionV1() throws IOException {
532        return addRegion(desc.toBuilder()
533                          .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
534                          .build());
535      }
536
537      public Path[] addRegionV2() throws IOException {
538        return addRegion(desc.toBuilder()
539                          .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
540                          .build());
541      }
542
543      private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException {
544        if (this.snapshotted == tableRegions.length) {
545          throw new UnsupportedOperationException("No more regions in the table");
546        }
547
548        RegionData regionData = tableRegions[this.snapshotted++];
549        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
550        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
551        manifest.addRegion(regionData.tableDir, regionData.hri);
552        return regionData.files;
553      }
554
555      private void corruptFile(Path p) throws IOException {
556        String manifestName = p.getName();
557
558        // Rename the original region-manifest file
559        Path newP = new Path(p.getParent(), manifestName + "1");
560        fs.rename(p, newP);
561
562        // Create a new region-manifest file
563        FSDataOutputStream out = fs.create(p);
564
565        //Copy the first 25 bytes of the original region-manifest into the new one,
566        //make it a corrupted region-manifest file.
567        FSDataInputStream input = fs.open(newP);
568        byte[] buffer = new byte[25];
569        int len = input.read(0, buffer, 0, 25);
570        if (len > 1) {
571          out.write(buffer, 0, len - 1);
572        }
573        out.close();
574
575        // Delete the original region-manifest
576        fs.delete(newP);
577      }
578
579      /**
580       * Corrupt one region-manifest file
581       *
582       * @throws IOException on unexecpted error from the FS
583       */
584      public void corruptOneRegionManifest() throws IOException {
585        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
586          @Override public boolean accept(Path path) {
587            return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
588          }
589        });
590
591        if (manifestFiles.length == 0) return;
592
593        // Just choose the first one
594        Path p = manifestFiles[0].getPath();
595        corruptFile(p);
596      }
597
598      public void missOneRegionSnapshotFile() throws IOException {
599        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
600        for (FileStatus fileStatus : manifestFiles) {
601          String fileName = fileStatus.getPath().getName();
602          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
603            || fileName.endsWith(".tabledesc")
604            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
605              fs.delete(fileStatus.getPath(), true);
606          }
607        }
608      }
609
610      /**
611       * Corrupt data-manifest file
612       *
613       * @throws IOException on unexecpted error from the FS
614       */
615      public void corruptDataManifest() throws IOException {
616        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
617          @Override
618          public boolean accept(Path path) {
619            return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
620          }
621        });
622
623        if (manifestFiles.length == 0) return;
624
625        // Just choose the first one
626        Path p = manifestFiles[0].getPath();
627        corruptFile(p);
628      }
629
630      public Path commit() throws IOException {
631        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
632        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
633        manifest.addTableDescriptor(htd);
634        manifest.consolidate();
635        SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
636        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
637        return snapshotDir;
638      }
639
640      public void consolidate() throws IOException {
641        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
642        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
643        manifest.addTableDescriptor(htd);
644        manifest.consolidate();
645      }
646    }
647
648    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
649      this.fs = fs;
650      this.conf = conf;
651      this.rootDir = rootDir;
652    }
653
654    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
655        throws IOException {
656      return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
657    }
658
659    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
660        final int numRegions) throws IOException {
661      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
662    }
663
664    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
665        throws IOException {
666      return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
667    }
668
669    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
670        final int numRegions) throws IOException {
671      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
672    }
673
674    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
675        final int version) throws IOException {
676      return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
677    }
678
679    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
680        final int numRegions, final int version) throws IOException {
681      TableDescriptor htd = createHtd(tableName);
682      RegionData[] regions = createTable(htd, numRegions);
683
684      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
685        .setTable(htd.getTableName().getNameAsString())
686        .setName(snapshotName)
687        .setVersion(version)
688        .build();
689
690      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
691      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
692      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
693    }
694
695    public TableDescriptor createHtd(final String tableName) {
696      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
697              .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY))
698              .build();
699    }
700
701    private RegionData[] createTable(final TableDescriptor htd, final int nregions)
702        throws IOException {
703      Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
704      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
705
706      assertTrue(nregions % 2 == 0);
707      RegionData[] regions = new RegionData[nregions];
708      for (int i = 0; i < regions.length; i += 2) {
709        byte[] startKey = Bytes.toBytes(0 + i * 2);
710        byte[] endKey = Bytes.toBytes(1 + i * 2);
711
712        // First region, simple with one plain hfile.
713        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
714            .setStartKey(startKey)
715            .setEndKey(endKey)
716            .build();
717        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
718        regions[i] = new RegionData(tableDir, hri, 3);
719        for (int j = 0; j < regions[i].files.length; ++j) {
720          Path storeFile = createStoreFile(rfs.createTempName());
721          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
722        }
723
724        // Second region, used to test the split case.
725        // This region contains a reference to the hfile in the first region.
726        startKey = Bytes.toBytes(2 + i * 2);
727        endKey = Bytes.toBytes(3 + i * 2);
728        hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
729        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
730        regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
731        for (int j = 0; j < regions[i].files.length; ++j) {
732          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
733          Path refFile = createStoreFile(new Path(rootDir, refName));
734          regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
735        }
736      }
737      return regions;
738    }
739
740    private Path createStoreFile(final Path storeFile)
741        throws IOException {
742      FSDataOutputStream out = fs.create(storeFile);
743      try {
744        out.write(Bytes.toBytes(storeFile.toString()));
745      } finally {
746        out.close();
747      }
748      return storeFile;
749    }
750  }
751
752  // ==========================================================================
753  //  Table Helpers
754  // ==========================================================================
755  public static void waitForTableToBeOnline(final HBaseTestingUtility util,
756                                            final TableName tableName)
757      throws IOException, InterruptedException {
758    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
759    List<HRegion> onlineRegions = rs.getRegions(tableName);
760    for (HRegion region : onlineRegions) {
761      region.waitForFlushesAndCompactions();
762    }
763    // Wait up to 60 seconds for a table to be available.
764    util.waitFor(60000, util.predicateTableAvailable(tableName));
765  }
766
767  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
768      int regionReplication, int nRegions, final byte[]... families)
769      throws IOException, InterruptedException {
770    TableDescriptorBuilder builder
771      = TableDescriptorBuilder
772          .newBuilder(tableName)
773          .setRegionReplication(regionReplication);
774    for (byte[] family : families) {
775      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
776    }
777    byte[][] splitKeys = getSplitKeys(nRegions);
778    util.createTable(builder.build(), splitKeys);
779    assertEquals((splitKeys.length + 1) * regionReplication,
780        util.getAdmin().getTableRegions(tableName).size());
781  }
782
783  public static byte[][] getSplitKeys() {
784    return getSplitKeys(KEYS.length);
785  }
786
787  public static byte[][] getSplitKeys(int nRegions) {
788    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
789    final byte[][] splitKeys = new byte[nRegions-1][];
790    final int step = KEYS.length / nRegions;
791    int keyIndex = 1;
792    for (int i = 0; i < splitKeys.length; ++i) {
793      splitKeys[i] = new byte[] { KEYS[keyIndex] };
794      keyIndex += step;
795    }
796    return splitKeys;
797  }
798
799  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
800      final byte[]... families) throws IOException, InterruptedException {
801    createTable(util, tableName, 1, families);
802  }
803
804  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
805      final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
806    createTable(util, tableName, regionReplication, KEYS.length, families);
807  }
808
809  public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName,
810      final int nRegions, final byte[]... families) throws IOException, InterruptedException {
811    createTable(util, tableName, 1, nRegions, families);
812  }
813
814  public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
815      byte[]... families) throws IOException, InterruptedException {
816    BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
817    loadData(util, mutator, rows, families);
818  }
819
820  public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
821      byte[]... families) throws IOException, InterruptedException {
822    // Ensure one row per region
823    assertTrue(rows >= KEYS.length);
824    for (byte k0: KEYS) {
825      byte[] k = new byte[] { k0 };
826      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
827      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
828      final byte[][] families1 = families;
829      final byte[] key1 = key;
830      final byte[] value1 = value;
831      mutator.mutate(createPut(families1, key1, value1));
832      rows--;
833    }
834
835    // Add other extra rows. more rows, more files
836    while (rows-- > 0) {
837      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
838      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
839      final byte[][] families1 = families;
840      final byte[] key1 = key;
841      final byte[] value1 = value;
842      mutator.mutate(createPut(families1, key1, value1));
843    }
844    mutator.flush();
845
846    waitForTableToBeOnline(util, mutator.getName());
847  }
848
849  private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
850    byte[] q = Bytes.toBytes("q");
851    Put put = new Put(key);
852    put.setDurability(Durability.SKIP_WAL);
853    for (byte[] family: families) {
854      put.addColumn(family, q, value);
855    }
856    return put;
857  }
858
859  public static void deleteAllSnapshots(final Admin admin)
860      throws IOException {
861    // Delete all the snapshots
862    for (SnapshotDescription snapshot: admin.listSnapshots()) {
863      admin.deleteSnapshot(snapshot.getName());
864    }
865    SnapshotTestingUtils.assertNoSnapshots(admin);
866  }
867
868  public static void deleteArchiveDirectory(final HBaseTestingUtility util)
869      throws IOException {
870    // Ensure the archiver to be empty
871    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
872    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
873    mfs.getFileSystem().delete(archiveDir, true);
874  }
875
876  public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
877      long expectedRows) throws IOException {
878    Table table = util.getConnection().getTable(tableName);
879    try {
880      assertEquals(expectedRows, util.countRows(table));
881    } finally {
882      table.close();
883    }
884  }
885
886  public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
887      int regionReplication) throws IOException {
888    List<RegionInfo> regions = admin.getRegions(tableName);
889    HashSet<RegionInfo> set = new HashSet<>();
890    for (RegionInfo hri : regions) {
891      set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
892      for (int i = 0; i < regionReplication; i++) {
893        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
894        if (!regions.contains(replica)) {
895          Assert.fail(replica + " is not contained in the list of online regions");
896        }
897      }
898    }
899    assertEquals(getSplitKeys().length + 1, set.size());
900  }
901}