001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.fs.PathFilter;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.TableNotEnabledException;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.BufferedMutator;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.RegionInfoBuilder;
050import org.apache.hadoop.hbase.client.RegionReplicaUtil;
051import org.apache.hadoop.hbase.client.SnapshotDescription;
052import org.apache.hadoop.hbase.client.SnapshotType;
053import org.apache.hadoop.hbase.client.Table;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
057import org.apache.hadoop.hbase.io.HFileLink;
058import org.apache.hadoop.hbase.master.HMaster;
059import org.apache.hadoop.hbase.master.MasterFileSystem;
060import org.apache.hadoop.hbase.mob.MobUtils;
061import org.apache.hadoop.hbase.regionserver.HRegion;
062import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
063import org.apache.hadoop.hbase.regionserver.HRegionServer;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.FSTableDescriptors;
068import org.apache.hadoop.hbase.util.FSVisitor;
069import org.apache.hadoop.hbase.util.MD5Hash;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.junit.Assert;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
077import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
078import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
079
080/**
081 * Utilities class for snapshots
082 */
083@InterfaceAudience.Private
084public final class SnapshotTestingUtils {
085  private static final Logger LOG = LoggerFactory.getLogger(SnapshotTestingUtils.class);
086
087  // default number of regions (and keys) given by getSplitKeys() and createTable()
088  private static byte[] KEYS = Bytes.toBytes("0123456");
089
090  private SnapshotTestingUtils() {
091    // private constructor for utility class
092  }
093
094  /**
095   * Assert that we don't have any snapshots lists n * if the admin operation fails
096   */
097  public static void assertNoSnapshots(Admin admin) throws IOException {
098    assertEquals("Have some previous snapshots", 0, admin.listSnapshots().size());
099  }
100
101  /**
102   * Make sure that there is only one snapshot returned from the master and its name and table match
103   * the passed in parameters.
104   */
105  public static List<SnapshotDescription> assertExistsMatchingSnapshot(Admin admin,
106    String snapshotName, TableName tableName) throws IOException {
107    // list the snapshot
108    List<SnapshotDescription> snapshots = admin.listSnapshots();
109
110    List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
111    for (SnapshotDescription sd : snapshots) {
112      if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
113        returnedSnapshots.add(sd);
114      }
115    }
116
117    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size() > 0);
118    return returnedSnapshots;
119  }
120
121  /**
122   * Make sure that there is only one snapshot returned from the master
123   */
124  public static void assertOneSnapshotThatMatches(Admin admin,
125    SnapshotProtos.SnapshotDescription snapshot) throws IOException {
126    assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable()));
127  }
128
129  /**
130   * Make sure that there is only one snapshot returned from the master and its name and table match
131   * the passed in parameters.
132   */
133  public static List<SnapshotDescription> assertOneSnapshotThatMatches(Admin admin,
134    String snapshotName, TableName tableName) throws IOException {
135    // list the snapshot
136    List<SnapshotDescription> snapshots = admin.listSnapshots();
137
138    assertEquals("Should only have 1 snapshot", 1, snapshots.size());
139    assertEquals(snapshotName, snapshots.get(0).getName());
140    assertEquals(tableName, snapshots.get(0).getTableName());
141
142    return snapshots;
143  }
144
145  /**
146   * Make sure that there is only one snapshot returned from the master and its name and table match
147   * the passed in parameters.
148   */
149  public static List<SnapshotDescription> assertOneSnapshotThatMatches(Admin admin, byte[] snapshot,
150    TableName tableName) throws IOException {
151    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), tableName);
152  }
153
154  public static void confirmSnapshotValid(HBaseTestingUtil testUtil,
155    SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
156    throws IOException {
157    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
158    confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(),
159      testUtil.getAdmin(), mfs.getFileSystem());
160  }
161
162  /**
163   * Confirm that the snapshot contains references to all the files that should be in the snapshot.
164   */
165  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
166    TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
167    throws IOException {
168    ArrayList nonEmptyTestFamilies = new ArrayList(1);
169    nonEmptyTestFamilies.add(testFamily);
170    confirmSnapshotValid(snapshotDescriptor, tableName, nonEmptyTestFamilies, null, rootDir, admin,
171      fs);
172  }
173
174  /**
175   * Confirm that the snapshot has no references files but only metadata.
176   */
177  public static void confirmEmptySnapshotValid(
178    SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] testFamily,
179    Path rootDir, Admin admin, FileSystem fs) throws IOException {
180    ArrayList emptyTestFamilies = new ArrayList(1);
181    emptyTestFamilies.add(testFamily);
182    confirmSnapshotValid(snapshotDescriptor, tableName, null, emptyTestFamilies, rootDir, admin,
183      fs);
184  }
185
186  /**
187   * Confirm that the snapshot contains references to all the files that should be in the snapshot.
188   * This method also perform some redundant check like the existence of the snapshotinfo or the
189   * regioninfo which are done always by the MasterSnapshotVerifier, at the end of the snapshot
190   * operation.
191   */
192  public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor,
193    TableName tableName, List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
194    Path rootDir, Admin admin, FileSystem fs) throws IOException {
195    final Configuration conf = admin.getConfiguration();
196
197    // check snapshot dir
198    Path snapshotDir =
199      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDescriptor, rootDir);
200    assertTrue("target snapshot directory, '" + snapshotDir + "', doesn't exist.",
201      fs.exists(snapshotDir));
202
203    SnapshotProtos.SnapshotDescription desc =
204      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
205
206    // Extract regions and families with store files
207    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
208
209    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
210    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
211    for (SnapshotRegionManifest regionManifest : regionManifests.values()) {
212      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
213        new SnapshotReferenceUtil.StoreFileVisitor() {
214          @Override
215          public void storeFile(final RegionInfo regionInfo, final String family,
216            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
217            snapshotFamilies.add(Bytes.toBytes(family));
218          }
219        });
220    }
221    // Verify that there are store files in the specified families
222    if (nonEmptyTestFamilies != null) {
223      for (final byte[] familyName : nonEmptyTestFamilies) {
224        assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName)
225          + "', but it does not.", snapshotFamilies.contains(familyName));
226      }
227    }
228
229    // Verify that there are no store files in the specified families
230    if (emptyTestFamilies != null) {
231      for (final byte[] familyName : emptyTestFamilies) {
232        assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName)
233          + "', but it is present.", snapshotFamilies.contains(familyName));
234      }
235    }
236
237    // check the region snapshot for all the regions
238    List<RegionInfo> regions = admin.getRegions(tableName);
239    // remove the non-default regions
240    RegionReplicaUtil.removeNonDefaultRegions(regions);
241    boolean hasMob =
242      regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName).getEncodedName());
243    if (hasMob) {
244      assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
245    } else {
246      // if create snapshot when table splitting, parent region will be included to the snapshot
247      // region manifest. we should exclude the parent regions.
248      int regionCountExclusiveSplitParent = 0;
249      for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
250        RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
251        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
252          continue;
253        }
254        regionCountExclusiveSplitParent++;
255      }
256      assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
257    }
258
259    // Verify Regions (redundant check, see MasterSnapshotVerifier)
260    for (RegionInfo info : regions) {
261      String regionName = info.getEncodedName();
262      assertTrue("Missing region name: '" + regionName + "'",
263        regionManifests.containsKey(regionName));
264    }
265  }
266
267  /*
268   * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException except for
269   * the last CorruptedSnapshotException
270   */
271  public static void snapshot(Admin admin, final String snapshotName, final TableName tableName,
272    final SnapshotType type, final int numTries) throws IOException {
273    int tries = 0;
274    CorruptedSnapshotException lastEx = null;
275    while (tries++ < numTries) {
276      try {
277        admin.snapshot(snapshotName, tableName, type);
278        return;
279      } catch (CorruptedSnapshotException cse) {
280        LOG.warn("Got CorruptedSnapshotException", cse);
281        lastEx = cse;
282      }
283    }
284    throw lastEx;
285  }
286
287  public static void cleanupSnapshot(Admin admin, byte[] tableName) throws IOException {
288    SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
289  }
290
291  public static void cleanupSnapshot(Admin admin, String snapshotName) throws IOException {
292    // delete the taken snapshot
293    admin.deleteSnapshot(snapshotName);
294    assertNoSnapshots(admin);
295  }
296
297  /**
298   * Expect the snapshot to throw an error when checking if the snapshot is complete
299   * @param master   master to check
300   * @param snapshot the {@link SnapshotDescription} request to pass to the master
301   * @param clazz    expected exception from the master
302   */
303  public static void expectSnapshotDoneException(HMaster master, IsSnapshotDoneRequest snapshot,
304    Class<? extends HBaseSnapshotException> clazz) {
305    try {
306      master.getMasterRpcServices().isSnapshotDone(null, snapshot);
307      Assert.fail("didn't fail to lookup a snapshot");
308    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) {
309      try {
310        throw ProtobufUtil.handleRemoteException(se);
311      } catch (HBaseSnapshotException e) {
312        assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
313      } catch (Throwable t) {
314        Assert.fail("Threw an unexpected exception:" + t);
315      }
316    }
317  }
318
319  /**
320   * List all the HFiles in the given table
321   * @param fs       FileSystem where the table lives
322   * @param tableDir directory of the table
323   * @return array of the current HFiles in the table (could be a zero-length array)
324   * @throws IOException on unexecpted error reading the FS
325   */
326  public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
327    throws IOException {
328    final ArrayList<String> hfiles = new ArrayList<>();
329    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
330      @Override
331      public void storeFile(final String region, final String family, final String hfileName)
332        throws IOException {
333        hfiles.add(hfileName);
334      }
335    });
336    Collections.sort(hfiles);
337    return hfiles;
338  }
339
340  /**
341   * Take a snapshot of the specified table and verify that the given family is not empty. Note that
342   * this will leave the table disabled in the case of an offline snapshot.
343   */
344  public static void createSnapshotAndValidate(Admin admin, TableName tableName, String familyName,
345    String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
346    throws Exception {
347    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
348    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
349    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
350      snapshotNameString, rootDir, fs, onlineSnapshot);
351  }
352
353  /**
354   * Take a snapshot of the specified table and verify the given families. Note that this will leave
355   * the table disabled in the case of an offline snapshot.
356   */
357  public static void createSnapshotAndValidate(Admin admin, TableName tableName,
358    List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames, String snapshotNameString,
359    Path rootDir, FileSystem fs, boolean onlineSnapshot) throws Exception {
360    if (!onlineSnapshot) {
361      try {
362        LOG.info("prepping for offline snapshot.");
363        admin.disableTable(tableName);
364      } catch (TableNotEnabledException tne) {
365        LOG.info("In attempting to disable " + tableName + " it turns out that the this table is "
366          + "already disabled.");
367      }
368    }
369    LOG.info("taking snapshot.");
370    admin.snapshot(snapshotNameString, tableName);
371
372    LOG.info("Confirming snapshot exists.");
373    List<SnapshotDescription> snapshots =
374      SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
375    if (snapshots == null || snapshots.size() != 1) {
376      Assert.fail("Incorrect number of snapshots for table " + tableName);
377    }
378
379    LOG.info("validating snapshot.");
380    SnapshotTestingUtils.confirmSnapshotValid(
381      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
382      emptyFamilyNames, rootDir, admin, fs);
383  }
384
385  /**
386   * Corrupt the specified snapshot by deleting some files.
387   * @param util         {@link HBaseTestingUtil}
388   * @param snapshotName name of the snapshot to corrupt
389   * @return array of the corrupted HFiles
390   * @throws IOException on unexecpted error reading the FS
391   */
392  public static ArrayList corruptSnapshot(final HBaseTestingUtil util, final String snapshotName)
393    throws IOException {
394    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
395    final FileSystem fs = mfs.getFileSystem();
396
397    Path snapshotDir =
398      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir());
399    SnapshotProtos.SnapshotDescription snapshotDesc =
400      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
401    final TableName table = TableName.valueOf(snapshotDesc.getTable());
402
403    final ArrayList corruptedFiles = new ArrayList();
404    final Configuration conf = util.getConfiguration();
405    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
406      new SnapshotReferenceUtil.StoreFileVisitor() {
407        @Override
408        public void storeFile(final RegionInfo regionInfo, final String family,
409          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
410          String region = regionInfo.getEncodedName();
411          String hfile = storeFile.getName();
412          HFileLink link = HFileLink.build(conf, table, region, family, hfile);
413          if (corruptedFiles.size() % 2 == 0) {
414            fs.delete(link.getAvailablePath(fs), true);
415            corruptedFiles.add(hfile);
416          }
417        }
418      });
419
420    assertTrue(corruptedFiles.size() > 0);
421    return corruptedFiles;
422  }
423
424  // ==========================================================================
425  // Snapshot Mock
426  // ==========================================================================
427  public static class SnapshotMock {
428    protected final static String TEST_FAMILY = "cf";
429    public final static int TEST_NUM_REGIONS = 4;
430
431    private final Configuration conf;
432    private final FileSystem fs;
433    private final Path rootDir;
434
435    static class RegionData {
436      public RegionInfo hri;
437      public Path tableDir;
438      public Path[] files;
439
440      public RegionData(final Path tableDir, final RegionInfo hri, final int nfiles) {
441        this.tableDir = tableDir;
442        this.hri = hri;
443        this.files = new Path[nfiles];
444      }
445    }
446
447    public static class SnapshotBuilder {
448      private final RegionData[] tableRegions;
449      private final SnapshotProtos.SnapshotDescription desc;
450      private final TableDescriptor htd;
451      private final Configuration conf;
452      private final FileSystem fs;
453      private final Path rootDir;
454      private Path snapshotDir;
455      private int snapshotted = 0;
456
457      public SnapshotBuilder(final Configuration conf, final FileSystem fs, final Path rootDir,
458        final TableDescriptor htd, final SnapshotProtos.SnapshotDescription desc,
459        final RegionData[] tableRegions) throws IOException {
460        this.fs = fs;
461        this.conf = conf;
462        this.rootDir = rootDir;
463        this.htd = htd;
464        this.desc = desc;
465        this.tableRegions = tableRegions;
466        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
467        FSTableDescriptors.createTableDescriptorForTableDirectory(
468          this.snapshotDir.getFileSystem(conf), snapshotDir, htd, false);
469      }
470
471      public TableDescriptor getTableDescriptor() {
472        return this.htd;
473      }
474
475      public SnapshotProtos.SnapshotDescription getSnapshotDescription() {
476        return this.desc;
477      }
478
479      public Path getSnapshotsDir() {
480        return this.snapshotDir;
481      }
482
483      public Path[] addRegion() throws IOException {
484        return addRegion(desc);
485      }
486
487      public Path[] addRegionV1() throws IOException {
488        return addRegion(
489          desc.toBuilder().setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build());
490      }
491
492      public Path[] addRegionV2() throws IOException {
493        return addRegion(
494          desc.toBuilder().setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build());
495      }
496
497      private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException {
498        if (this.snapshotted == tableRegions.length) {
499          throw new UnsupportedOperationException("No more regions in the table");
500        }
501
502        RegionData regionData = tableRegions[this.snapshotted++];
503        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
504        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
505        manifest.addTableDescriptor(htd);
506        manifest.addRegion(regionData.tableDir, regionData.hri);
507        return regionData.files;
508      }
509
510      private void corruptFile(Path p) throws IOException {
511        String manifestName = p.getName();
512
513        // Rename the original region-manifest file
514        Path newP = new Path(p.getParent(), manifestName + "1");
515        fs.rename(p, newP);
516
517        // Create a new region-manifest file
518        FSDataOutputStream out = fs.create(p);
519
520        // Copy the first 25 bytes of the original region-manifest into the new one,
521        // make it a corrupted region-manifest file.
522        FSDataInputStream input = fs.open(newP);
523        byte[] buffer = new byte[25];
524        int len = input.read(0, buffer, 0, 25);
525        if (len > 1) {
526          out.write(buffer, 0, len - 1);
527        }
528        out.close();
529
530        // Delete the original region-manifest
531        fs.delete(newP);
532      }
533
534      /**
535       * Corrupt one region-manifest file
536       * @throws IOException on unexecpted error from the FS
537       */
538      public void corruptOneRegionManifest() throws IOException {
539        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
540          @Override
541          public boolean accept(Path path) {
542            return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
543          }
544        });
545
546        if (manifestFiles.length == 0) return;
547
548        // Just choose the first one
549        Path p = manifestFiles[0].getPath();
550        corruptFile(p);
551      }
552
553      public void missOneRegionSnapshotFile() throws IOException {
554        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir);
555        for (FileStatus fileStatus : manifestFiles) {
556          String fileName = fileStatus.getPath().getName();
557          if (
558            fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
559              || fileName.endsWith(".tabledesc")
560              || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)
561          ) {
562            fs.delete(fileStatus.getPath(), true);
563          }
564        }
565      }
566
567      /**
568       * Corrupt data-manifest file
569       * @throws IOException on unexecpted error from the FS
570       */
571      public void corruptDataManifest() throws IOException {
572        FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {
573          @Override
574          public boolean accept(Path path) {
575            return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
576          }
577        });
578
579        if (manifestFiles.length == 0) return;
580
581        // Just choose the first one
582        Path p = manifestFiles[0].getPath();
583        corruptFile(p);
584      }
585
586      public Path commit() throws IOException {
587        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
588        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
589        manifest.addTableDescriptor(htd);
590        manifest.consolidate();
591        Path finishedDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
592        SnapshotDescriptionUtils.completeSnapshot(finishedDir, snapshotDir, fs,
593          snapshotDir.getFileSystem(conf), conf);
594        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
595        return snapshotDir;
596      }
597
598      public void consolidate() throws IOException {
599        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
600        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
601        manifest.addTableDescriptor(htd);
602        manifest.consolidate();
603      }
604    }
605
606    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
607      this.fs = fs;
608      this.conf = conf;
609      this.rootDir = rootDir;
610    }
611
612    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
613      throws IOException {
614      return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
615    }
616
617    public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
618      final int numRegions) throws IOException {
619      return createSnapshot(snapshotName, tableName, numRegions,
620        SnapshotManifestV1.DESCRIPTOR_VERSION);
621    }
622
623    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
624      throws IOException {
625      return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
626    }
627
628    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
629      final int numRegions) throws IOException {
630      return createSnapshot(snapshotName, tableName, numRegions,
631        SnapshotManifestV2.DESCRIPTOR_VERSION);
632    }
633
634    public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
635      final int numRegions, final long ttl) throws IOException {
636      return createSnapshot(snapshotName, tableName, numRegions,
637        SnapshotManifestV2.DESCRIPTOR_VERSION, ttl);
638    }
639
640    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
641      final int version) throws IOException {
642      return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
643    }
644
645    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
646      final int numRegions, final int version) throws IOException {
647      TableDescriptor htd = createHtd(tableName);
648      RegionData[] regions = createTable(htd, numRegions);
649
650      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
651        .setTable(htd.getTableName().getNameAsString()).setName(snapshotName).setVersion(version)
652        .build();
653
654      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
655      FileSystem workingFs = workingDir.getFileSystem(conf);
656      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs);
657      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
658    }
659
660    private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
661      final int numRegions, final int version, final long ttl) throws IOException {
662      TableDescriptor htd = createHtd(tableName);
663      RegionData[] regions = createTable(htd, numRegions);
664      SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
665        .setTable(htd.getTableName().getNameAsString()).setName(snapshotName).setVersion(version)
666        .setCreationTime(EnvironmentEdgeManager.currentTime()).setTtl(ttl).build();
667      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
668      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
669      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
670    }
671
672    public TableDescriptor createHtd(final String tableName) {
673      return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
674        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY)).build();
675    }
676
677    private RegionData[] createTable(final TableDescriptor htd, final int nregions)
678      throws IOException {
679      Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
680      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
681
682      assertTrue(nregions % 2 == 0);
683      RegionData[] regions = new RegionData[nregions];
684      for (int i = 0; i < regions.length; i += 2) {
685        byte[] startKey = Bytes.toBytes(0 + i * 2);
686        byte[] endKey = Bytes.toBytes(1 + i * 2);
687
688        // First region, simple with one plain hfile.
689        RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey)
690          .setEndKey(endKey).build();
691        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
692        regions[i] = new RegionData(tableDir, hri, 3);
693        for (int j = 0; j < regions[i].files.length; ++j) {
694          Path storeFile = createStoreFile(rfs.createTempName());
695          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
696        }
697
698        // Second region, used to test the split case.
699        // This region contains a reference to the hfile in the first region.
700        startKey = Bytes.toBytes(2 + i * 2);
701        endKey = Bytes.toBytes(3 + i * 2);
702        hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
703        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
704        regions[i + 1] = new RegionData(tableDir, hri, regions[i].files.length);
705        for (int j = 0; j < regions[i].files.length; ++j) {
706          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
707          Path refFile = createStoreFile(new Path(rootDir, refName));
708          regions[i + 1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
709        }
710      }
711      return regions;
712    }
713
714    private Path createStoreFile(final Path storeFile) throws IOException {
715      FSDataOutputStream out = fs.create(storeFile);
716      try {
717        out.write(Bytes.toBytes(storeFile.toString()));
718      } finally {
719        out.close();
720      }
721      return storeFile;
722    }
723  }
724
725  // ==========================================================================
726  // Table Helpers
727  // ==========================================================================
728  public static void waitForTableToBeOnline(final HBaseTestingUtil util, final TableName tableName)
729    throws IOException, InterruptedException {
730    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
731    List<HRegion> onlineRegions = rs.getRegions(tableName);
732    for (HRegion region : onlineRegions) {
733      region.waitForFlushesAndCompactions();
734    }
735    // Wait up to 60 seconds for a table to be available.
736    util.waitFor(60000, util.predicateTableAvailable(tableName));
737  }
738
739  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
740    int regionReplication, int nRegions, final byte[]... families)
741    throws IOException, InterruptedException {
742    TableDescriptorBuilder builder =
743      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication);
744    for (byte[] family : families) {
745      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
746    }
747    byte[][] splitKeys = getSplitKeys(nRegions);
748    util.createTable(builder.build(), splitKeys);
749    assertEquals((splitKeys.length + 1) * regionReplication,
750      util.getAdmin().getRegions(tableName).size());
751  }
752
753  public static byte[][] getSplitKeys() {
754    return getSplitKeys(KEYS.length);
755  }
756
757  public static byte[][] getSplitKeys(int nRegions) {
758    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
759    final byte[][] splitKeys = new byte[nRegions - 1][];
760    final int step = KEYS.length / nRegions;
761    int keyIndex = 1;
762    for (int i = 0; i < splitKeys.length; ++i) {
763      splitKeys[i] = new byte[] { KEYS[keyIndex] };
764      keyIndex += step;
765    }
766    return splitKeys;
767  }
768
769  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
770    final byte[]... families) throws IOException, InterruptedException {
771    createTable(util, tableName, 1, families);
772  }
773
774  public static void createTable(final HBaseTestingUtil util, final TableName tableName,
775    final int regionReplication, final byte[]... families)
776    throws IOException, InterruptedException {
777    createTable(util, tableName, regionReplication, KEYS.length, families);
778  }
779
780  public static void createPreSplitTable(final HBaseTestingUtil util, final TableName tableName,
781    final int nRegions, final byte[]... families) throws IOException, InterruptedException {
782    createTable(util, tableName, 1, nRegions, families);
783  }
784
785  public static void loadData(final HBaseTestingUtil util, final TableName tableName, int rows,
786    byte[]... families) throws IOException, InterruptedException {
787    BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
788    loadData(util, mutator, rows, families);
789  }
790
791  public static void loadData(final HBaseTestingUtil util, final BufferedMutator mutator, int rows,
792    byte[]... families) throws IOException, InterruptedException {
793    // Ensure one row per region
794    assertTrue(rows >= KEYS.length);
795    for (byte k0 : KEYS) {
796      byte[] k = new byte[] { k0 };
797      byte[] value = Bytes.add(Bytes.toBytes(EnvironmentEdgeManager.currentTime()), k);
798      byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
799      final byte[][] families1 = families;
800      final byte[] key1 = key;
801      final byte[] value1 = value;
802      mutator.mutate(createPut(families1, key1, value1));
803      rows--;
804    }
805
806    // Add other extra rows. more rows, more files
807    while (rows-- > 0) {
808      byte[] value =
809        Bytes.add(Bytes.toBytes(EnvironmentEdgeManager.currentTime()), Bytes.toBytes(rows));
810      byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
811      final byte[][] families1 = families;
812      final byte[] key1 = key;
813      final byte[] value1 = value;
814      mutator.mutate(createPut(families1, key1, value1));
815    }
816    mutator.flush();
817
818    waitForTableToBeOnline(util, mutator.getName());
819  }
820
821  private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
822    byte[] q = Bytes.toBytes("q");
823    Put put = new Put(key);
824    put.setDurability(Durability.SKIP_WAL);
825    for (byte[] family : families) {
826      put.addColumn(family, q, value);
827    }
828    return put;
829  }
830
831  public static void deleteAllSnapshots(final Admin admin) throws IOException {
832    // Delete all the snapshots
833    for (SnapshotDescription snapshot : admin.listSnapshots()) {
834      admin.deleteSnapshot(snapshot.getName());
835    }
836    SnapshotTestingUtils.assertNoSnapshots(admin);
837  }
838
839  public static void deleteArchiveDirectory(final HBaseTestingUtil util) throws IOException {
840    // Ensure the archiver to be empty
841    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
842    Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
843    mfs.getFileSystem().delete(archiveDir, true);
844  }
845
846  public static void verifyRowCount(final HBaseTestingUtil util, final TableName tableName,
847    long expectedRows) throws IOException {
848    Table table = util.getConnection().getTable(tableName);
849    try {
850      assertEquals(expectedRows, util.countRows(table));
851    } finally {
852      table.close();
853    }
854  }
855
856  public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
857    int regionReplication) throws IOException {
858    List<RegionInfo> regions = admin.getRegions(tableName);
859    HashSet<RegionInfo> set = new HashSet<>();
860    for (RegionInfo hri : regions) {
861      set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
862      for (int i = 0; i < regionReplication; i++) {
863        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
864        if (!regions.contains(replica)) {
865          Assert.fail(replica + " is not contained in the list of online regions");
866        }
867      }
868    }
869    assertEquals(getSplitKeys().length + 1, set.size());
870  }
871}