001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.fs.PathFilter;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.TableNotEnabledException;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.BufferedMutator;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.RegionReplicaUtil;
051import org.apache.hadoop.hbase.client.SnapshotDescription;
052import org.apache.hadoop.hbase.client.SnapshotType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
057import org.apache.hadoop.hbase.io.HFileLink;
058import org.apache.hadoop.hbase.master.HMaster;
059import org.apache.hadoop.hbase.master.MasterFileSystem;
060import org.apache.hadoop.hbase.mob.MobUtils;
061import org.apache.hadoop.hbase.regionserver.HRegion;
062import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
063import org.apache.hadoop.hbase.regionserver.HRegionServer;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.FSTableDescriptors;
066import org.apache.hadoop.hbase.util.FSUtils;
067import org.apache.hadoop.hbase.util.FSVisitor;
068import org.apache.hadoop.hbase.util.MD5Hash;
069import org.apache.yetus.audience.InterfaceAudience;
070import org.junit.Assert;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
075import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
078
079/**
080 * Utilities class for snapshots
081 */
082@InterfaceAudience.Private
083public final class SnapshotTestingUtils {
084  private static final Logger LOG = LoggerFactory.getLogger(SnapshotTestingUtils.class);
085
086  // default number of regions (and keys) given by getSplitKeys() and createTable()
087  private static byte[] KEYS = Bytes.toBytes("0123456");
088
089  private SnapshotTestingUtils() {
090    // private constructor for utility class
091  }
092
093  /**
094   * Assert that we don't have any snapshots lists
095   *
096   * @throws IOException
097   *           if the admin operation fails
098   */
099  public static void assertNoSnapshots(Admin admin) throws IOException {
100    assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
101        .size());
102  }
103
104  /**
105   * Make sure that there is only one snapshot returned from the master and its
106   * name and table match the passed in parameters.
107   */
108  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
109      Admin admin, String snapshotName, TableName tableName)
110      throws IOException {
111    // list the snapshot
112    List<SnapshotDescription> snapshots = admin.listSnapshots();
113
114    List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
115    for (SnapshotDescription sd : snapshots) {
116      if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
117        returnedSnapshots.add(sd);
118      }
119    }
120
121    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
122    return returnedSnapshots;
123  }
124
125  /**
126   * Make sure that there is only one snapshot returned from the master
127   */
128  public static void assertOneSnapshotThatMatches(Admin admin,
129      SnapshotProtos.SnapshotDescription snapshot) throws IOException {
130    assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable()));
131  }
132
133  /**
134   * Make sure that there is only one snapshot returned from the master and its
135   * name and table match the passed in parameters.
136   */
137  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
138      Admin admin, String snapshotName, TableName tableName)
139      throws IOException {
140    // list the snapshot
141    List<SnapshotDescription> snapshots = admin.listSnapshots();
142
143    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
144    assertEquals(snapshotName, snapshots.get(0).getName());
145    assertEquals(tableName, snapshots.get(0).getTableName());
146
147    return snapshots;
148  }
149
150  /**
151   * Make sure that there is only one snapshot returned from the master and its
152   * name and table match the passed in parameters.
153   */
154  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
155      Admin admin, byte[] snapshot, TableName tableName) throws IOException {
156    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
157        tableName);
158  }
159
160  public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
161      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
162      throws IOException {
163    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
164    confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(),
165      testUtil.getAdmin(), mfs.getFileSystem());
166  }
167
168  /**
169   * Confirm that the snapshot contains references to all the files that should
170   * be in the snapshot.
171   */
172  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
173      TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
174      throws IOException {
175    ArrayList nonEmptyTestFamilies = new ArrayList(1);
176    nonEmptyTestFamilies.add(testFamily);
177    confirmSnapshotValid(snapshotDescriptor, tableName,
178      nonEmptyTestFamilies, null, rootDir, admin, fs);
179  }
180
181  /**
182   * Confirm that the snapshot has no references files but only metadata.
183   */
184  public static void confirmEmptySnapshotValid(
185      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
186      byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
187      throws IOException {
188    ArrayList emptyTestFamilies = new ArrayList(1);
189    emptyTestFamilies.add(testFamily);
190    confirmSnapshotValid(snapshotDescriptor, tableName,
191      null, emptyTestFamilies, rootDir, admin, fs);
192  }
193
194  /**
195   * Confirm that the snapshot contains references to all the files that should
196   * be in the snapshot. This method also perform some redundant check like
197   * the existence of the snapshotinfo or the regioninfo which are done always
198   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
199   */
200  public static void confirmSnapshotValid(
201      SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
202      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
203      Path rootDir, Admin admin, FileSystem fs) throws IOException {
204    final Configuration conf = admin.getConfiguration();
205
206    // check snapshot dir
207    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
208        snapshotDescriptor, rootDir);
209    assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir));
210
211    SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
212
213    // Extract regions and families with store files
214    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
215
216    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
217    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
218    for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
219      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
220          new SnapshotReferenceUtil.StoreFileVisitor() {
221        @Override
222        public void storeFile(final RegionInfo regionInfo, final String family,
223              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
224          snapshotFamilies.add(Bytes.toBytes(family));
225        }
226      });
227    }
228    // Verify that there are store files in the specified families
229    if (nonEmptyTestFamilies != null) {
230      for (final byte[] familyName: nonEmptyTestFamilies) {
231        assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
232      }
233    }
234
235    // Verify that there are no store files in the specified families
236    if (emptyTestFamilies != null) {
237      for (final byte[] familyName: emptyTestFamilies) {
238        assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
239      }
240    }
241
242    // check the region snapshot for all the regions
243    List<RegionInfo> regions = admin.getRegions(tableName);
244    // remove the non-default regions
245    RegionReplicaUtil.removeNonDefaultRegions(regions);
246    boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
247        .getEncodedName());
248    if (hasMob) {
249      assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
250    } else {
251      // if create snapshot when table splitting, parent region will be included to the snapshot
252      // region manifest. we should exclude the parent regions.
253      int regionCountExclusiveSplitParent = 0;
254      for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
255        RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
256        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
257          continue;
258        }
259        regionCountExclusiveSplitParent++;
260      }
261      assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
262    }
263
264    // Verify Regions (redundant check, see MasterSnapshotVerifier)
265    for (RegionInfo info : regions) {
266      String regionName = info.getEncodedName();
267      assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
268    }
269  }
270
271  /*
272   * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
273   * except for the last CorruptedSnapshotException
274   */
275  public static void snapshot(Admin admin, final String snapshotName, final TableName tableName,
276      final SnapshotType type, final int numTries) throws IOException {
277    int tries = 0;
278    CorruptedSnapshotException lastEx = null;
279    while (tries++ < numTries) {
280      try {
281        admin.snapshot(snapshotName, tableName, type);
282        return;
283      } catch (CorruptedSnapshotException cse) {
284        LOG.warn("Got CorruptedSnapshotException", cse);
285        lastEx = cse;
286      }
287    }
288    throw lastEx;
289  }
290
291  public static void cleanupSnapshot(Admin admin, byte[] tableName)
292      throws IOException {
293    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
294  }
295
296  public static void cleanupSnapshot(Admin admin, String snapshotName)
297      throws IOException {
298    // delete the taken snapshot
299    admin.deleteSnapshot(snapshotName);
300    assertNoSnapshots(admin);
301  }
302
303  /**
304   * Expect the snapshot to throw an error when checking if the snapshot is
305   * complete
306   *
307   * @param master master to check
308   * @param snapshot the {@link SnapshotDescription} request to pass to the master
309   * @param clazz expected exception from the master
310   */
311  public static void expectSnapshotDoneException(HMaster master,
312      IsSnapshotDoneRequest snapshot,
313      Class<? extends HBaseSnapshotException> clazz) {
314    try {
315      master.getMasterRpcServices().isSnapshotDone(null, snapshot);
316      Assert.fail("didn't fail to lookup a snapshot");
317    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) {
318      try {
319        throw ProtobufUtil.handleRemoteException(se);
320      } catch (HBaseSnapshotException e) {
321        assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
322      } catch (Throwable t) {
323        Assert.fail("Threw an unexpected exception:" + t);
324      }
325    }
326  }
327
328  /**
329   * List all the HFiles in the given table
330   *
331   * @param fs FileSystem where the table lives
332   * @param tableDir directory of the table
333   * @return array of the current HFiles in the table (could be a zero-length array)
334   * @throws IOException on unexecpted error reading the FS
335   */
336  public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
337      throws IOException {
338    final ArrayList<String> hfiles = new ArrayList<>();
339    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
340      @Override
341      public void storeFile(final String region, final String family, final String hfileName)
342          throws IOException {
343        hfiles.add(hfileName);
344      }
345    });
346    Collections.sort(hfiles);
347    return hfiles;
348  }
349
350  /**
351   * Take a snapshot of the specified table and verify that the given family is
352   * not empty. Note that this will leave the table disabled
353   * in the case of an offline snapshot.
354   */
355  public static void createSnapshotAndValidate(Admin admin,
356      TableName tableName, String familyName, String snapshotNameString,
357      Path rootDir, FileSystem fs, boolean onlineSnapshot)
358      throws Exception {
359    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
360    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
361    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
362                              snapshotNameString, rootDir, fs, onlineSnapshot);
363  }
364
365  /**
366   * Take a snapshot of the specified table and verify the given families.
367   * Note that this will leave the table disabled in the case of an offline snapshot.
368   */
369  public static void createSnapshotAndValidate(Admin admin,
370      TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
371      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
372        throws Exception {
373    if (!onlineSnapshot) {
374      try {
375        LOG.info("prepping for offline snapshot.");
376        admin.disableTable(tableName);
377      } catch (TableNotEnabledException tne) {
378        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
379            "already disabled.");
380      }
381    }
382    LOG.info("taking snapshot.");
383    admin.snapshot(snapshotNameString, tableName);
384
385    LOG.info("Confirming snapshot exists.");
386    List<SnapshotDescription> snapshots =
387        SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
388    if (snapshots == null || snapshots.size() != 1) {
389      Assert.fail("Incorrect number of snapshots for table " + tableName);
390    }
391
392    LOG.info("validating snapshot.");
393    SnapshotTestingUtils.confirmSnapshotValid(
394      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
395      emptyFamilyNames, rootDir, admin, fs);
396  }
397
398  /**
399   * Corrupt the specified snapshot by deleting some files.
400   *
401   * @param util {@link HBaseTestingUtility}
402   * @param snapshotName name of the snapshot to corrupt
403   * @return array of the corrupted HFiles
404   * @throws IOException on unexecpted error reading the FS
405   */
406  public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
407      throws IOException {
408    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
409    final FileSystem fs = mfs.getFileSystem();
410
411    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
412                                                                        mfs.getRootDir());
413    SnapshotProtos.SnapshotDescription snapshotDesc =
414        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
415    final TableName table = TableName.valueOf(snapshotDesc.getTable());
416
417    final ArrayList corruptedFiles = new ArrayList();
418    final Configuration conf = util.getConfiguration();
419    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
420        new SnapshotReferenceUtil.StoreFileVisitor() {
421      @Override
422      public void storeFile(final RegionInfo regionInfo, final String family,
423            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
424        String region = regionInfo.getEncodedName();
425        String hfile = storeFile.getName();
426        HFileLink link = HFileLink.build(conf, table, region, family, hfile);
427        if (corruptedFiles.size() % 2 == 0) {
428          fs.delete(link.getAvailablePath(fs), true);
429          corruptedFiles.add(hfile);
430        }
431      }
432    });
433
434    assertTrue(corruptedFiles.size() > 0);
435    return corruptedFiles;
436  }
437
438  // ==========================================================================
439  //  Snapshot Mock
440  // ==========================================================================
441  public static class SnapshotMock {
442    protected final static String TEST_FAMILY = "cf";
443    public final static int TEST_NUM_REGIONS = 4;
444
445    private final Configuration conf;
446    private final FileSystem fs;
447    private final Path rootDir;
448
449    static class RegionData {
450      public RegionInfo hri;
451      public Path tableDir;
452      public Path[] files;
453
454      public RegionData(final Path tableDir, final RegionInfo hri, final int nfiles) {
455        this.tableDir = tableDir;
456        this.hri = hri;
457        this.files = new Path[nfiles];
458      }
459    }
460
461    public static class SnapshotBuilder {
462      private final RegionData[] tableRegions;
463      private final SnapshotProtos.SnapshotDescription desc;
464      private final TableDescriptor htd;
465      private final Configuration conf;
466      private final FileSystem fs;
467      private final Path rootDir;
468      private Path snapshotDir;
469      private int snapshotted = 0;
470
471      public SnapshotBuilder(final Configuration conf, final FileSystem fs,
472          final Path rootDir, final TableDescriptor htd,
473          final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions)
474          throws IOException {
475        this.fs = fs;
476        this.conf = conf;
477        this.rootDir = rootDir;
478        this.htd = htd;
479        this.desc = desc;
480        this.tableRegions = tableRegions;
481        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
482        new FSTableDescriptors(conf)
483          .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
484      }
485
486      public TableDescriptor getTableDescriptor() {
487        return this.htd;
488      }
489
490      public SnapshotProtos.SnapshotDescription getSnapshotDescription() {
491        return this.desc;
492      }
493
494      public Path getSnapshotsDir() {
495        return this.snapshotDir;
496      }
497
498      public Path[] addRegion() throws IOException {
499        return addRegion(desc);
500      }
501
502      public Path[] addRegionV1() throws IOException {
503        return addRegion(desc.toBuilder()
504                          .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
505                          .build());
506      }
507
508      public Path[] addRegionV2() throws IOException {
509        return addRegion(desc.toBuilder()
510                          .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
511                          .build());
512      }
513
514      private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException {
515        if (this.snapshotted == tableRegions.length) {
516          throw new UnsupportedOperationException("No more regions in the table");
517        }
518
519        RegionData regionData = tableRegions[this.snapshotted++];
520        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
521        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
522        manifest.addRegion(regionData.tableDir, regionData.hri);
523        return regionData.files;
524      }
525
526      private void corruptFile(Path p) throws IOException {
527        String manifestName = p.getName();
528
529        // Rename the original region-manifest file
530        Path newP = new Path(p.getParent(), manifestName + "1");
531        fs.rename(p, newP);
532
533        // Create a new region-manifest file
534        FSDataOutputStream out = fs.create(p);
535
536        //Copy the first 25 bytes of the original region-manifest into the new one,
537        //make it a corrupted region-manifest file.
538        FSDataInputStream input = fs.open(newP);
539        byte[] buffer = new byte[25];
540        int len = input.read(0, buffer, 0, 25);
541        if (len > 1) {
542          out.write(buffer, 0, len - 1);
543        }
544        out.close();
545
546        // Delete the original region-manifest
547        fs.delete(newP);
548      }
549
550      /**
551       * Corrupt one region-manifest file
552       *
553       * @throws IOException on unexecpted error from the FS
554       */
555      public void corruptOneRegionManifest() throws IOException {
556        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
557          @Override public boolean accept(Path path) {
558            return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
559          }
560        });
561
562        if (manifestFiles.length == 0) return;
563
564        // Just choose the first one
565        Path p = manifestFiles[0].getPath();
566        corruptFile(p);
567      }
568
569      public void missOneRegionSnapshotFile() throws IOException {
570        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
571        for (FileStatus fileStatus : manifestFiles) {
572          String fileName = fileStatus.getPath().getName();
573          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
574            || fileName.endsWith(".tabledesc")
575            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
576              fs.delete(fileStatus.getPath(), true);
577          }
578        }
579      }
580
581      /**
582       * Corrupt data-manifest file
583       *
584       * @throws IOException on unexecpted error from the FS
585       */
586      public void corruptDataManifest() throws IOException {
587        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
588          @Override
589          public boolean accept(Path path) {
590            return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
591          }
592        });
593
594        if (manifestFiles.length == 0) return;
595
596        // Just choose the first one
597        Path p = manifestFiles[0].getPath();
598        corruptFile(p);
599      }
600
601      public Path commit() throws IOException {
602        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
603        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
604        manifest.addTableDescriptor(htd);
605        manifest.consolidate();
606        SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
607        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
608        return snapshotDir;
609      }
610
611      public void consolidate() throws IOException {
612        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
613        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
614        manifest.addTableDescriptor(htd);
615        manifest.consolidate();
616      }
617    }
618
619    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
620      this.fs = fs;
621      this.conf = conf;
622      this.rootDir = rootDir;
623    }
624
625    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
626        throws IOException {
627      return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
628    }
629
630    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
631        final int numRegions) throws IOException {
632      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
633    }
634
635    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
636        throws IOException {
637      return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
638    }
639
640    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
641        final int numRegions) throws IOException {
642      return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
643    }
644
645    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
646        final int version) throws IOException {
647      return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
648    }
649
650    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
651        final int numRegions, final int version) throws IOException {
652      TableDescriptor htd = createHtd(tableName);
653      RegionData[] regions = createTable(htd, numRegions);
654
655      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
656        .setTable(htd.getTableName().getNameAsString())
657        .setName(snapshotName)
658        .setVersion(version)
659        .build();
660
661      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
662      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
663      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
664    }
665
666    public TableDescriptor createHtd(final String tableName) {
667      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
668              .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY))
669              .build();
670    }
671
672    private RegionData[] createTable(final TableDescriptor htd, final int nregions)
673        throws IOException {
674      Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
675      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
676
677      assertTrue(nregions % 2 == 0);
678      RegionData[] regions = new RegionData[nregions];
679      for (int i = 0; i < regions.length; i += 2) {
680        byte[] startKey = Bytes.toBytes(0 + i * 2);
681        byte[] endKey = Bytes.toBytes(1 + i * 2);
682
683        // First region, simple with one plain hfile.
684        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
685            .setStartKey(startKey)
686            .setEndKey(endKey)
687            .build();
688        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
689        regions[i] = new RegionData(tableDir, hri, 3);
690        for (int j = 0; j < regions[i].files.length; ++j) {
691          Path storeFile = createStoreFile(rfs.createTempName());
692          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
693        }
694
695        // Second region, used to test the split case.
696        // This region contains a reference to the hfile in the first region.
697        startKey = Bytes.toBytes(2 + i * 2);
698        endKey = Bytes.toBytes(3 + i * 2);
699        hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
700        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
701        regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
702        for (int j = 0; j < regions[i].files.length; ++j) {
703          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
704          Path refFile = createStoreFile(new Path(rootDir, refName));
705          regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
706        }
707      }
708      return regions;
709    }
710
711    private Path createStoreFile(final Path storeFile)
712        throws IOException {
713      FSDataOutputStream out = fs.create(storeFile);
714      try {
715        out.write(Bytes.toBytes(storeFile.toString()));
716      } finally {
717        out.close();
718      }
719      return storeFile;
720    }
721  }
722
723  // ==========================================================================
724  //  Table Helpers
725  // ==========================================================================
726  public static void waitForTableToBeOnline(final HBaseTestingUtility util,
727                                            final TableName tableName)
728      throws IOException, InterruptedException {
729    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
730    List<HRegion> onlineRegions = rs.getRegions(tableName);
731    for (HRegion region : onlineRegions) {
732      region.waitForFlushesAndCompactions();
733    }
734    // Wait up to 60 seconds for a table to be available.
735    util.waitFor(60000, util.predicateTableAvailable(tableName));
736  }
737
738  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
739      int regionReplication, int nRegions, final byte[]... families)
740      throws IOException, InterruptedException {
741    TableDescriptorBuilder builder
742      = TableDescriptorBuilder
743          .newBuilder(tableName)
744          .setRegionReplication(regionReplication);
745    for (byte[] family : families) {
746      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
747    }
748    byte[][] splitKeys = getSplitKeys(nRegions);
749    util.createTable(builder.build(), splitKeys);
750    assertEquals((splitKeys.length + 1) * regionReplication,
751        util.getAdmin().getRegions(tableName).size());
752  }
753
754  public static byte[][] getSplitKeys() {
755    return getSplitKeys(KEYS.length);
756  }
757
758  public static byte[][] getSplitKeys(int nRegions) {
759    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
760    final byte[][] splitKeys = new byte[nRegions-1][];
761    final int step = KEYS.length / nRegions;
762    int keyIndex = 1;
763    for (int i = 0; i < splitKeys.length; ++i) {
764      splitKeys[i] = new byte[] { KEYS[keyIndex] };
765      keyIndex += step;
766    }
767    return splitKeys;
768  }
769
770  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
771      final byte[]... families) throws IOException, InterruptedException {
772    createTable(util, tableName, 1, families);
773  }
774
775  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
776      final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
777    createTable(util, tableName, regionReplication, KEYS.length, families);
778  }
779
780  public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName,
781      final int nRegions, final byte[]... families) throws IOException, InterruptedException {
782    createTable(util, tableName, 1, nRegions, families);
783  }
784
785  public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
786      byte[]... families) throws IOException, InterruptedException {
787    BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
788    loadData(util, mutator, rows, families);
789  }
790
791  public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
792      byte[]... families) throws IOException, InterruptedException {
793    // Ensure one row per region
794    assertTrue(rows >= KEYS.length);
795    for (byte k0: KEYS) {
796      byte[] k = new byte[] { k0 };
797      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
798      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
799      final byte[][] families1 = families;
800      final byte[] key1 = key;
801      final byte[] value1 = value;
802      mutator.mutate(createPut(families1, key1, value1));
803      rows--;
804    }
805
806    // Add other extra rows. more rows, more files
807    while (rows-- > 0) {
808      byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
809      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
810      final byte[][] families1 = families;
811      final byte[] key1 = key;
812      final byte[] value1 = value;
813      mutator.mutate(createPut(families1, key1, value1));
814    }
815    mutator.flush();
816
817    waitForTableToBeOnline(util, mutator.getName());
818  }
819
820  private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
821    byte[] q = Bytes.toBytes("q");
822    Put put = new Put(key);
823    put.setDurability(Durability.SKIP_WAL);
824    for (byte[] family: families) {
825      put.addColumn(family, q, value);
826    }
827    return put;
828  }
829
830  public static void deleteAllSnapshots(final Admin admin)
831      throws IOException {
832    // Delete all the snapshots
833    for (SnapshotDescription snapshot: admin.listSnapshots()) {
834      admin.deleteSnapshot(snapshot.getName());
835    }
836    SnapshotTestingUtils.assertNoSnapshots(admin);
837  }
838
839  public static void deleteArchiveDirectory(final HBaseTestingUtility util)
840      throws IOException {
841    // Ensure the archiver to be empty
842    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
843    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
844    mfs.getFileSystem().delete(archiveDir, true);
845  }
846
847  public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
848      long expectedRows) throws IOException {
849    Table table = util.getConnection().getTable(tableName);
850    try {
851      assertEquals(expectedRows, util.countRows(table));
852    } finally {
853      table.close();
854    }
855  }
856
857  public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
858      int regionReplication) throws IOException {
859    List<RegionInfo> regions = admin.getRegions(tableName);
860    HashSet<RegionInfo> set = new HashSet<>();
861    for (RegionInfo hri : regions) {
862      set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
863      for (int i = 0; i < regionReplication; i++) {
864        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
865        if (!regions.contains(replica)) {
866          Assert.fail(replica + " is not contained in the list of online regions");
867        }
868      }
869    }
870    assertEquals(getSplitKeys().length + 1, set.size());
871  }
872}