001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.fs.PathFilter;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.TableNotEnabledException;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.BufferedMutator;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.RegionReplicaUtil;
051import org.apache.hadoop.hbase.client.SnapshotDescription;
052import org.apache.hadoop.hbase.client.SnapshotType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
057import org.apache.hadoop.hbase.io.HFileLink;
058import org.apache.hadoop.hbase.master.HMaster;
059import org.apache.hadoop.hbase.master.MasterFileSystem;
060import org.apache.hadoop.hbase.mob.MobUtils;
061import org.apache.hadoop.hbase.regionserver.HRegion;
062import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
063import org.apache.hadoop.hbase.regionserver.HRegionServer;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.FSTableDescriptors;
068import org.apache.hadoop.hbase.util.FSVisitor;
069import org.apache.hadoop.hbase.util.MD5Hash;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.junit.Assert;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
079
080/**
081 * Utilities class for snapshots
082 */
083@InterfaceAudience.Private
084public final class SnapshotTestingUtils {
085  private static final Logger LOG = LoggerFactory.getLogger(SnapshotTestingUtils.class);
086
087  // default number of regions (and keys) given by getSplitKeys() and createTable()
088  private static byte[] KEYS = Bytes.toBytes("0123456");
089
090  private SnapshotTestingUtils() {
091    // private constructor for utility class
092  }
093
094  /**
095   * Assert that we don't have any snapshots lists
096   *
097   * @throws IOException
098   *           if the admin operation fails
099   */
100  public static void assertNoSnapshots(Admin admin) throws IOException {
101    assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
102        .size());
103  }
104
105  /**
106   * Make sure that there is only one snapshot returned from the master and its
107   * name and table match the passed in parameters.
108   */
109  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
110      Admin admin, String snapshotName, TableName tableName)
111      throws IOException {
112    // list the snapshot
113    List<SnapshotDescription> snapshots = admin.listSnapshots();
114
115    List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
116    for (SnapshotDescription sd : snapshots) {
117      if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
118        returnedSnapshots.add(sd);
119      }
120    }
121
122    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
123    return returnedSnapshots;
124  }
125
126  /**
127   * Make sure that there is only one snapshot returned from the master
128   */
129  public static void assertOneSnapshotThatMatches(Admin admin,
130      SnapshotProtos.SnapshotDescription snapshot) throws IOException {
131    assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable()));
132  }
133
134  /**
135   * Make sure that there is only one snapshot returned from the master and its
136   * name and table match the passed in parameters.
137   */
138  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
139      Admin admin, String snapshotName, TableName tableName)
140      throws IOException {
141    // list the snapshot
142    List<SnapshotDescription> snapshots = admin.listSnapshots();
143
144    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
145    assertEquals(snapshotName, snapshots.get(0).getName());
146    assertEquals(tableName, snapshots.get(0).getTableName());
147
148    return snapshots;
149  }
150
151  /**
152   * Make sure that there is only one snapshot returned from the master and its
153   * name and table match the passed in parameters.
154   */
155  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
156      Admin admin, byte[] snapshot, TableName tableName) throws IOException {
157    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
158        tableName);
159  }
160
161  public static void confirmSnapshotValid(HBaseTestingUtil testUtil,
162      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
163      throws IOException {
164    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
165    confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(),
166      testUtil.getAdmin(), mfs.getFileSystem());
167  }
168
169  /**
170   * Confirm that the snapshot contains references to all the files that should
171   * be in the snapshot.
172   */
173  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
174      TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
175      throws IOException {
176    ArrayList nonEmptyTestFamilies = new ArrayList(1);
177    nonEmptyTestFamilies.add(testFamily);
178    confirmSnapshotValid(snapshotDescriptor, tableName,
179      nonEmptyTestFamilies, null, rootDir, admin, fs);
180  }
181
182  /**
183   * Confirm that the snapshot has no references files but only metadata.
184   */
185  public static void confirmEmptySnapshotValid(
186      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
187      byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
188      throws IOException {
189    ArrayList emptyTestFamilies = new ArrayList(1);
190    emptyTestFamilies.add(testFamily);
191    confirmSnapshotValid(snapshotDescriptor, tableName,
192      null, emptyTestFamilies, rootDir, admin, fs);
193  }
194
195  /**
196   * Confirm that the snapshot contains references to all the files that should
197   * be in the snapshot. This method also perform some redundant check like
198   * the existence of the snapshotinfo or the regioninfo which are done always
199   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
200   */
201  public static void confirmSnapshotValid(
202      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
203      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
204      Path rootDir, Admin admin, FileSystem fs) throws IOException {
205    final Configuration conf = admin.getConfiguration();
206
207    // check snapshot dir
208    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
209        snapshotDescriptor, rootDir);
210    assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir));
211
212    SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
213
214    // Extract regions and families with store files
215    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
216
217    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
218    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
219    for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
220      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
221          new SnapshotReferenceUtil.StoreFileVisitor() {
222        @Override
223        public void storeFile(final RegionInfo regionInfo, final String family,
224              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
225          snapshotFamilies.add(Bytes.toBytes(family));
226        }
227      });
228    }
229    // Verify that there are store files in the specified families
230    if (nonEmptyTestFamilies != null) {
231      for (final byte[] familyName: nonEmptyTestFamilies) {
232        assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
233      }
234    }
235
236    // Verify that there are no store files in the specified families
237    if (emptyTestFamilies != null) {
238      for (final byte[] familyName: emptyTestFamilies) {
239        assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
240      }
241    }
242
243    // check the region snapshot for all the regions
244    List<RegionInfo> regions = admin.getRegions(tableName);
245    // remove the non-default regions
246    RegionReplicaUtil.removeNonDefaultRegions(regions);
247    boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
248        .getEncodedName());
249    if (hasMob) {
250      assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
251    } else {
252      // if create snapshot when table splitting, parent region will be included to the snapshot
253      // region manifest. we should exclude the parent regions.
254      int regionCountExclusiveSplitParent = 0;
255      for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
256        RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
257        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
258          continue;
259        }
260        regionCountExclusiveSplitParent++;
261      }
262      assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
263    }
264
265    // Verify Regions (redundant check, see MasterSnapshotVerifier)
266    for (RegionInfo info : regions) {
267      String regionName = info.getEncodedName();
268      assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
269    }
270  }
271
272  /*
273   * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
274   * except for the last CorruptedSnapshotException
275   */
276  public static void snapshot(Admin admin, final String snapshotName, final TableName tableName,
277      final SnapshotType type, final int numTries) throws IOException {
278    int tries = 0;
279    CorruptedSnapshotException lastEx = null;
280    while (tries++ < numTries) {
281      try {
282        admin.snapshot(snapshotName, tableName, type);
283        return;
284      } catch (CorruptedSnapshotException cse) {
285        LOG.warn("Got CorruptedSnapshotException", cse);
286        lastEx = cse;
287      }
288    }
289    throw lastEx;
290  }
291
292  public static void cleanupSnapshot(Admin admin, byte[] tableName)
293      throws IOException {
294    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
295  }
296
297  public static void cleanupSnapshot(Admin admin, String snapshotName)
298      throws IOException {
299    // delete the taken snapshot
300    admin.deleteSnapshot(snapshotName);
301    assertNoSnapshots(admin);
302  }
303
304  /**
305   * Expect the snapshot to throw an error when checking if the snapshot is
306   * complete
307   *
308   * @param master master to check
309   * @param snapshot the {@link SnapshotDescription} request to pass to the master
310   * @param clazz expected exception from the master
311   */
312  public static void expectSnapshotDoneException(HMaster master,
313      IsSnapshotDoneRequest snapshot,
314      Class<? extends HBaseSnapshotException> clazz) {
315    try {
316      master.getMasterRpcServices().isSnapshotDone(null, snapshot);
317      Assert.fail("didn't fail to lookup a snapshot");
318    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) {
319      try {
320        throw ProtobufUtil.handleRemoteException(se);
321      } catch (HBaseSnapshotException e) {
322        assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
323      } catch (Throwable t) {
324        Assert.fail("Threw an unexpected exception:" + t);
325      }
326    }
327  }
328
329  /**
330   * List all the HFiles in the given table
331   *
332   * @param fs FileSystem where the table lives
333   * @param tableDir directory of the table
334   * @return array of the current HFiles in the table (could be a zero-length array)
335   * @throws IOException on unexecpted error reading the FS
336   */
337  public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
338      throws IOException {
339    final ArrayList<String> hfiles = new ArrayList<>();
340    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
341      @Override
342      public void storeFile(final String region, final String family, final String hfileName)
343          throws IOException {
344        hfiles.add(hfileName);
345      }
346    });
347    Collections.sort(hfiles);
348    return hfiles;
349  }
350
351  /**
352   * Take a snapshot of the specified table and verify that the given family is
353   * not empty. Note that this will leave the table disabled
354   * in the case of an offline snapshot.
355   */
356  public static void createSnapshotAndValidate(Admin admin,
357      TableName tableName, String familyName, String snapshotNameString,
358      Path rootDir, FileSystem fs, boolean onlineSnapshot)
359      throws Exception {
360    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
361    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
362    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
363                              snapshotNameString, rootDir, fs, onlineSnapshot);
364  }
365
366  /**
367   * Take a snapshot of the specified table and verify the given families.
368   * Note that this will leave the table disabled in the case of an offline snapshot.
369   */
370  public static void createSnapshotAndValidate(Admin admin,
371      TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
372      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
373        throws Exception {
374    if (!onlineSnapshot) {
375      try {
376        LOG.info("prepping for offline snapshot.");
377        admin.disableTable(tableName);
378      } catch (TableNotEnabledException tne) {
379        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
380            "already disabled.");
381      }
382    }
383    LOG.info("taking snapshot.");
384    admin.snapshot(snapshotNameString, tableName);
385
386    LOG.info("Confirming snapshot exists.");
387    List<SnapshotDescription> snapshots =
388        SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
389    if (snapshots == null || snapshots.size() != 1) {
390      Assert.fail("Incorrect number of snapshots for table " + tableName);
391    }
392
393    LOG.info("validating snapshot.");
394    SnapshotTestingUtils.confirmSnapshotValid(
395      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
396      emptyFamilyNames, rootDir, admin, fs);
397  }
398
399  /**
400   * Corrupt the specified snapshot by deleting some files.
401   *
402   * @param util {@link HBaseTestingUtil}
403   * @param snapshotName name of the snapshot to corrupt
404   * @return array of the corrupted HFiles
405   * @throws IOException on unexecpted error reading the FS
406   */
407  public static ArrayList corruptSnapshot(final HBaseTestingUtil util, final String snapshotName)
408      throws IOException {
409    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
410    final FileSystem fs = mfs.getFileSystem();
411
412    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
413                                                                        mfs.getRootDir());
414    SnapshotProtos.SnapshotDescription snapshotDesc =
415        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
416    final TableName table = TableName.valueOf(snapshotDesc.getTable());
417
418    final ArrayList corruptedFiles = new ArrayList();
419    final Configuration conf = util.getConfiguration();
420    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
421        new SnapshotReferenceUtil.StoreFileVisitor() {
422      @Override
423      public void storeFile(final RegionInfo regionInfo, final String family,
424            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
425        String region = regionInfo.getEncodedName();
426        String hfile = storeFile.getName();
427        HFileLink link = HFileLink.build(conf, table, region, family, hfile);
428        if (corruptedFiles.size() % 2 == 0) {
429          fs.delete(link.getAvailablePath(fs), true);
430          corruptedFiles.add(hfile);
431        }
432      }
433    });
434
435    assertTrue(corruptedFiles.size() > 0);
436    return corruptedFiles;
437  }
438
439  // ==========================================================================
440  //  Snapshot Mock
441  // ==========================================================================
442  public static class SnapshotMock {
443    protected final static String TEST_FAMILY = "cf";
444    public final static int TEST_NUM_REGIONS = 4;
445
446    private final Configuration conf;
447    private final FileSystem fs;
448    private final Path rootDir;
449
450    static class RegionData {
451      public RegionInfo hri;
452      public Path tableDir;
453      public Path[] files;
454
455      public RegionData(final Path tableDir, final RegionInfo hri, final int nfiles) {
456        this.tableDir = tableDir;
457        this.hri = hri;
458        this.files = new Path[nfiles];
459      }
460    }
461
462    public static class SnapshotBuilder {
463      private final RegionData[] tableRegions;
464      private final SnapshotProtos.SnapshotDescription desc;
465      private final TableDescriptor htd;
466      private final Configuration conf;
467      private final FileSystem fs;
468      private final Path rootDir;
469      private Path snapshotDir;
470      private int snapshotted = 0;
471
472      public SnapshotBuilder(final Configuration conf, final FileSystem fs,
473          final Path rootDir, final TableDescriptor htd,
474          final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions)
475          throws IOException {
476        this.fs = fs;
477        this.conf = conf;
478        this.rootDir = rootDir;
479        this.htd = htd;
480        this.desc = desc;
481        this.tableRegions = tableRegions;
482        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
483        FSTableDescriptors.createTableDescriptorForTableDirectory(
484          this.snapshotDir.getFileSystem(conf), snapshotDir, htd, false);
485      }
486
487      public TableDescriptor getTableDescriptor() {
488        return this.htd;
489      }
490
491      public SnapshotProtos.SnapshotDescription getSnapshotDescription() {
492        return this.desc;
493      }
494
495      public Path getSnapshotsDir() {
496        return this.snapshotDir;
497      }
498
499      public Path[] addRegion() throws IOException {
500        return addRegion(desc);
501      }
502
503      public Path[] addRegionV1() throws IOException {
504        return addRegion(
505          desc.toBuilder().setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build());
506      }
507
508      public Path[] addRegionV2() throws IOException {
509        return addRegion(
510          desc.toBuilder().setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build());
511      }
512
513      private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException {
514        if (this.snapshotted == tableRegions.length) {
515          throw new UnsupportedOperationException("No more regions in the table");
516        }
517
518        RegionData regionData = tableRegions[this.snapshotted++];
519        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
520        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
521        manifest.addTableDescriptor(htd);
522        manifest.addRegion(regionData.tableDir, regionData.hri);
523        return regionData.files;
524      }
525
526      private void corruptFile(Path p) throws IOException {
527        String manifestName = p.getName();
528
529        // Rename the original region-manifest file
530        Path newP = new Path(p.getParent(), manifestName + "1");
531        fs.rename(p, newP);
532
533        // Create a new region-manifest file
534        FSDataOutputStream out = fs.create(p);
535
536        //Copy the first 25 bytes of the original region-manifest into the new one,
537        //make it a corrupted region-manifest file.
538        FSDataInputStream input = fs.open(newP);
539        byte[] buffer = new byte[25];
540        int len = input.read(0, buffer, 0, 25);
541        if (len > 1) {
542          out.write(buffer, 0, len - 1);
543        }
544        out.close();
545
546        // Delete the original region-manifest
547        fs.delete(newP);
548      }
549
550      /**
551       * Corrupt one region-manifest file
552       *
553       * @throws IOException on unexecpted error from the FS
554       */
555      public void corruptOneRegionManifest() throws IOException {
556        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
557          @Override public boolean accept(Path path) {
558            return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
559          }
560        });
561
562        if (manifestFiles.length == 0) return;
563
564        // Just choose the first one
565        Path p = manifestFiles[0].getPath();
566        corruptFile(p);
567      }
568
569      public void missOneRegionSnapshotFile() throws IOException {
570        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir);
571        for (FileStatus fileStatus : manifestFiles) {
572          String fileName = fileStatus.getPath().getName();
573          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
574            || fileName.endsWith(".tabledesc")
575            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
576              fs.delete(fileStatus.getPath(), true);
577          }
578        }
579      }
580
581      /**
582       * Corrupt data-manifest file
583       *
584       * @throws IOException on unexecpted error from the FS
585       */
586      public void corruptDataManifest() throws IOException {
587        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
588          @Override
589          public boolean accept(Path path) {
590            return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
591          }
592        });
593
594        if (manifestFiles.length == 0) return;
595
596        // Just choose the first one
597        Path p = manifestFiles[0].getPath();
598        corruptFile(p);
599      }
600
601      public Path commit() throws IOException {
602        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
603        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
604        manifest.addTableDescriptor(htd);
605        manifest.consolidate();
606        Path finishedDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
607        SnapshotDescriptionUtils.completeSnapshot(finishedDir, snapshotDir, fs,
608          snapshotDir.getFileSystem(conf), conf);
609        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
610        return snapshotDir;
611      }
612
613      public void consolidate() throws IOException {
614        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
615        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
616        manifest.addTableDescriptor(htd);
617        manifest.consolidate();
618      }
619    }
620
621    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
622      this.fs = fs;
623      this.conf = conf;
624      this.rootDir = rootDir;
625    }
626
627    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
628        throws IOException {
629      return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
630    }
631
632    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
633        final int numRegions) throws IOException {
634      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
635    }
636
637    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
638        throws IOException {
639      return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
640    }
641
642    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
643        final int numRegions) throws IOException {
644      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
645    }
646
647    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
648        final int numRegions, final long ttl) throws IOException {
649      return createSnapshot(snapshotName, tableName, numRegions,
650          SnapshotManifestV2.DESCRIPTOR_VERSION, ttl);
651    }
652
653    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
654        final int version) throws IOException {
655      return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
656    }
657
658    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
659        final int numRegions, final int version) throws IOException {
660      TableDescriptor htd = createHtd(tableName);
661      RegionData[] regions = createTable(htd, numRegions);
662
663      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
664        .setTable(htd.getTableName().getNameAsString())
665        .setName(snapshotName)
666        .setVersion(version)
667        .build();
668
669      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
670      FileSystem workingFs = workingDir.getFileSystem(conf);
671      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs);
672      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
673    }
674
675    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
676        final int numRegions, final int version, final long ttl) throws IOException {
677      TableDescriptor htd = createHtd(tableName);
678      RegionData[] regions = createTable(htd, numRegions);
679      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
680          .setTable(htd.getTableName().getNameAsString())
681          .setName(snapshotName)
682          .setVersion(version)
683          .setCreationTime(EnvironmentEdgeManager.currentTime())
684          .setTtl(ttl)
685          .build();
686      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
687      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
688      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
689    }
690
691    public TableDescriptor createHtd(final String tableName) {
692      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
693              .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY))
694              .build();
695    }
696
697    private RegionData[] createTable(final TableDescriptor htd, final int nregions)
698        throws IOException {
699      Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
700      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
701
702      assertTrue(nregions % 2 == 0);
703      RegionData[] regions = new RegionData[nregions];
704      for (int i = 0; i < regions.length; i += 2) {
705        byte[] startKey = Bytes.toBytes(0 + i * 2);
706        byte[] endKey = Bytes.toBytes(1 + i * 2);
707
708        // First region, simple with one plain hfile.
709        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
710            .setStartKey(startKey)
711            .setEndKey(endKey)
712            .build();
713        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
714        regions[i] = new RegionData(tableDir, hri, 3);
715        for (int j = 0; j < regions[i].files.length; ++j) {
716          Path storeFile = createStoreFile(rfs.createTempName());
717          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
718        }
719
720        // Second region, used to test the split case.
721        // This region contains a reference to the hfile in the first region.
722        startKey = Bytes.toBytes(2 + i * 2);
723        endKey = Bytes.toBytes(3 + i * 2);
724        hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
725        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
726        regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
727        for (int j = 0; j < regions[i].files.length; ++j) {
728          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
729          Path refFile = createStoreFile(new Path(rootDir, refName));
730          regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
731        }
732      }
733      return regions;
734    }
735
736    private Path createStoreFile(final Path storeFile)
737        throws IOException {
738      FSDataOutputStream out = fs.create(storeFile);
739      try {
740        out.write(Bytes.toBytes(storeFile.toString()));
741      } finally {
742        out.close();
743      }
744      return storeFile;
745    }
746  }
747
748  // ==========================================================================
749  //  Table Helpers
750  // ==========================================================================
751  public static void waitForTableToBeOnline(final HBaseTestingUtil util,
752                                            final TableName tableName)
753      throws IOException, InterruptedException {
754    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
755    List<HRegion> onlineRegions = rs.getRegions(tableName);
756    for (HRegion region : onlineRegions) {
757      region.waitForFlushesAndCompactions();
758    }
759    // Wait up to 60 seconds for a table to be available.
760    util.waitFor(60000, util.predicateTableAvailable(tableName));
761  }
762
763  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
764      int regionReplication, int nRegions, final byte[]... families)
765      throws IOException, InterruptedException {
766    TableDescriptorBuilder builder
767      = TableDescriptorBuilder
768          .newBuilder(tableName)
769          .setRegionReplication(regionReplication);
770    for (byte[] family : families) {
771      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
772    }
773    byte[][] splitKeys = getSplitKeys(nRegions);
774    util.createTable(builder.build(), splitKeys);
775    assertEquals((splitKeys.length + 1) * regionReplication,
776        util.getAdmin().getRegions(tableName).size());
777  }
778
779  public static byte[][] getSplitKeys() {
780    return getSplitKeys(KEYS.length);
781  }
782
783  public static byte[][] getSplitKeys(int nRegions) {
784    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
785    final byte[][] splitKeys = new byte[nRegions-1][];
786    final int step = KEYS.length / nRegions;
787    int keyIndex = 1;
788    for (int i = 0; i < splitKeys.length; ++i) {
789      splitKeys[i] = new byte[] { KEYS[keyIndex] };
790      keyIndex += step;
791    }
792    return splitKeys;
793  }
794
795  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
796      final byte[]... families) throws IOException, InterruptedException {
797    createTable(util, tableName, 1, families);
798  }
799
800  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
801      final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
802    createTable(util, tableName, regionReplication, KEYS.length, families);
803  }
804
805  public static void createPreSplitTable(final HBaseTestingUtil util, final TableName tableName,
806      final int nRegions, final byte[]... families) throws IOException, InterruptedException {
807    createTable(util, tableName, 1, nRegions, families);
808  }
809
810  public static void loadData(final HBaseTestingUtil util, final TableName tableName, int rows,
811      byte[]... families) throws IOException, InterruptedException {
812    BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
813    loadData(util, mutator, rows, families);
814  }
815
816  public static void loadData(final HBaseTestingUtil util, final BufferedMutator mutator,
817      int rows, byte[]... families) throws IOException, InterruptedException {
818    // Ensure one row per region
819    assertTrue(rows >= KEYS.length);
820    for (byte k0: KEYS) {
821      byte[] k = new byte[] { k0 };
822      byte[] value = Bytes.add(Bytes.toBytes(EnvironmentEdgeManager.currentTime()), k);
823      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
824      final byte[][] families1 = families;
825      final byte[] key1 = key;
826      final byte[] value1 = value;
827      mutator.mutate(createPut(families1, key1, value1));
828      rows--;
829    }
830
831    // Add other extra rows. more rows, more files
832    while (rows-- > 0) {
833      byte[] value = Bytes.add(Bytes.toBytes(EnvironmentEdgeManager.currentTime()),
834        Bytes.toBytes(rows));
835      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
836      final byte[][] families1 = families;
837      final byte[] key1 = key;
838      final byte[] value1 = value;
839      mutator.mutate(createPut(families1, key1, value1));
840    }
841    mutator.flush();
842
843    waitForTableToBeOnline(util, mutator.getName());
844  }
845
846  private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
847    byte[] q = Bytes.toBytes("q");
848    Put put = new Put(key);
849    put.setDurability(Durability.SKIP_WAL);
850    for (byte[] family: families) {
851      put.addColumn(family, q, value);
852    }
853    return put;
854  }
855
856  public static void deleteAllSnapshots(final Admin admin)
857      throws IOException {
858    // Delete all the snapshots
859    for (SnapshotDescription snapshot: admin.listSnapshots()) {
860      admin.deleteSnapshot(snapshot.getName());
861    }
862    SnapshotTestingUtils.assertNoSnapshots(admin);
863  }
864
865  public static void deleteArchiveDirectory(final HBaseTestingUtil util)
866      throws IOException {
867    // Ensure the archiver to be empty
868    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
869    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
870    mfs.getFileSystem().delete(archiveDir, true);
871  }
872
873  public static void verifyRowCount(final HBaseTestingUtil util, final TableName tableName,
874      long expectedRows) throws IOException {
875    Table table = util.getConnection().getTable(tableName);
876    try {
877      assertEquals(expectedRows, util.countRows(table));
878    } finally {
879      table.close();
880    }
881  }
882
883  public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
884      int regionReplication) throws IOException {
885    List<RegionInfo> regions = admin.getRegions(tableName);
886    HashSet<RegionInfo> set = new HashSet<>();
887    for (RegionInfo hri : regions) {
888      set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
889      for (int i = 0; i < regionReplication; i++) {
890        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
891        if (!regions.contains(replica)) {
892          Assert.fail(replica + " is not contained in the list of online regions");
893        }
894      }
895    }
896    assertEquals(getSplitKeys().length + 1, set.size());
897  }
898}