Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import java.util.regex.Pattern;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.TableNameTestRule;
038import org.apache.hadoop.hbase.TableNotFoundException;
039import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
040import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
041import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
042import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
043import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
044import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
045import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
046import org.apache.hadoop.hbase.testclassification.ClientTests;
047import org.apache.hadoop.hbase.testclassification.LargeTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.CommonFSUtils;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Rule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.runner.RunWith;
059import org.junit.runners.Parameterized;
060import org.junit.runners.Parameterized.Parameter;
061import org.junit.runners.Parameterized.Parameters;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068
069/**
070 * Test create/using/deleting snapshots from the client
071 * <p>
072 * This is an end-to-end test for the snapshot utility
073 */
074@RunWith(Parameterized.class)
075@Category({ LargeTests.class, ClientTests.class })
076public class TestSnapshotFromClient {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE =
080    HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
081
082  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
083
084  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
085  protected static final int NUM_RS = 2;
086  protected static final String STRING_TABLE_NAME = "test";
087  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
088  protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
089  private static final Pattern MATCH_ALL = Pattern.compile(".*");
090
091  @Rule
092  public TableNameTestRule name = new TableNameTestRule();
093
094  @Parameter
095  public StoreFileTrackerFactory.Trackers trackerImpl;
096
097  @Parameters(name = "{index}: tracker={0}")
098  public static List<Object[]> params() {
099    return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
100      new Object[] { StoreFileTrackerFactory.Trackers.FILE });
101  }
102
103  /**
104   * Setup the config for the cluster
105   * @throws Exception on failure
106   */
107  @BeforeClass
108  public static void setupCluster() throws Exception {
109    setupConf(UTIL.getConfiguration());
110    UTIL.startMiniCluster(NUM_RS);
111  }
112
113  protected static void setupConf(Configuration conf) {
114    // disable the ui
115    conf.setInt("hbase.regionsever.info.port", -1);
116    // change the flush size to a small amount, regulating number of store files
117    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
118    // so make sure we get a compaction when doing a load, but keep around some
119    // files in the store
120    conf.setInt("hbase.hstore.compaction.min", 10);
121    conf.setInt("hbase.hstore.compactionThreshold", 10);
122    // block writes if we get to 12 store files
123    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
124    // Enable snapshot
125    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
126    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
127      ConstantSizeRegionSplitPolicy.class.getName());
128  }
129
130  @Before
131  public void setup() throws Exception {
132    createTable();
133  }
134
135  protected void createTable() throws Exception {
136    TableDescriptor htd =
137      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
138        .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
139    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
140  }
141
142  protected int getNumReplicas() {
143    return 1;
144  }
145
146  @After
147  public void tearDown() throws Exception {
148    UTIL.deleteTable(TABLE_NAME);
149    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
150    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
151  }
152
153  @AfterClass
154  public static void cleanupTest() throws Exception {
155    try {
156      UTIL.shutdownMiniCluster();
157    } catch (Exception e) {
158      LOG.warn("failure shutting down cluster", e);
159    }
160  }
161
162  /**
163   * Test snapshotting not allowed hbase:meta and -ROOT- n
164   */
165  @Test
166  public void testMetaTablesSnapshot() throws Exception {
167    Admin admin = UTIL.getAdmin();
168    byte[] snapshotName = Bytes.toBytes("metaSnapshot");
169
170    try {
171      admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
172      fail("taking a snapshot of hbase:meta should not be allowed");
173    } catch (IllegalArgumentException e) {
174      // expected
175    }
176  }
177
178  /**
179   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
180   * n
181   */
182  @Test
183  public void testSnapshotDeletionWithRegex() throws Exception {
184    Admin admin = UTIL.getAdmin();
185    // make sure we don't fail on listing snapshots
186    SnapshotTestingUtils.assertNoSnapshots(admin);
187
188    // put some stuff in the table
189    Table table = UTIL.getConnection().getTable(TABLE_NAME);
190    UTIL.loadTable(table, TEST_FAM);
191    table.close();
192
193    byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
194    admin.snapshot(snapshot1, TABLE_NAME);
195    LOG.debug("Snapshot1 completed.");
196
197    byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
198    admin.snapshot(snapshot2, TABLE_NAME);
199    LOG.debug("Snapshot2 completed.");
200
201    String snapshot3 = "3rdTableSnapshot";
202    admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
203    LOG.debug(snapshot3 + " completed.");
204
205    // delete the first two snapshots
206    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
207    List<SnapshotDescription> snapshots = admin.listSnapshots();
208    assertEquals(1, snapshots.size());
209    assertEquals(snapshot3, snapshots.get(0).getName());
210
211    admin.deleteSnapshot(snapshot3);
212    admin.close();
213  }
214
215  /**
216   * Test snapshotting a table that is offline n
217   */
218  @Test
219  public void testOfflineTableSnapshot() throws Exception {
220    Admin admin = UTIL.getAdmin();
221    // make sure we don't fail on listing snapshots
222    SnapshotTestingUtils.assertNoSnapshots(admin);
223
224    // put some stuff in the table
225    Table table = UTIL.getConnection().getTable(TABLE_NAME);
226    UTIL.loadTable(table, TEST_FAM, false);
227
228    LOG.debug("FS state before disable:");
229    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
230      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
231    // XXX if this is flakey, might want to consider using the async version and looping as
232    // disableTable can succeed and still timeout.
233    admin.disableTable(TABLE_NAME);
234
235    LOG.debug("FS state before snapshot:");
236    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
237      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
238
239    // take a snapshot of the disabled table
240    final String SNAPSHOT_NAME = "offlineTableSnapshot";
241    byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
242
243    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.DISABLED, null,
244      -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
245    LOG.debug("Snapshot completed.");
246
247    // make sure we have the snapshot
248    List<SnapshotDescription> snapshots =
249      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
250
251    // make sure its a valid snapshot
252    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
253    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
254    LOG.debug("FS state after snapshot:");
255    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
256      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
257    SnapshotTestingUtils.confirmSnapshotValid(
258      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, rootDir,
259      admin, fs);
260
261    admin.deleteSnapshot(snapshot);
262    snapshots = admin.listSnapshots();
263    SnapshotTestingUtils.assertNoSnapshots(admin);
264  }
265
266  @Test
267  public void testSnapshotFailsOnNonExistantTable() throws Exception {
268    Admin admin = UTIL.getAdmin();
269    // make sure we don't fail on listing snapshots
270    SnapshotTestingUtils.assertNoSnapshots(admin);
271    String tableName = "_not_a_table";
272
273    // make sure the table doesn't exist
274    boolean fail = false;
275    do {
276      try {
277        admin.getTableDescriptor(TableName.valueOf(tableName));
278        fail = true;
279        LOG.error("Table:" + tableName + " already exists, checking a new name");
280        tableName = tableName + "!";
281      } catch (TableNotFoundException e) {
282        fail = false;
283      }
284    } while (fail);
285
286    // snapshot the non-existant table
287    try {
288      admin.snapshot("fail", TableName.valueOf(tableName));
289      fail("Snapshot succeeded even though there is not table.");
290    } catch (SnapshotCreationException e) {
291      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
292    }
293  }
294
295  @Test
296  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
297    // test with an empty table with one region
298
299    Admin admin = UTIL.getAdmin();
300    // make sure we don't fail on listing snapshots
301    SnapshotTestingUtils.assertNoSnapshots(admin);
302
303    LOG.debug("FS state before disable:");
304    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
305      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
306    admin.disableTable(TABLE_NAME);
307
308    LOG.debug("FS state before snapshot:");
309    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
310      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
311
312    // take a snapshot of the disabled table
313    byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
314    admin.snapshot(snapshot, TABLE_NAME);
315    LOG.debug("Snapshot completed.");
316
317    // make sure we have the snapshot
318    List<SnapshotDescription> snapshots =
319      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
320
321    // make sure its a valid snapshot
322    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
323    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
324    LOG.debug("FS state after snapshot:");
325    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
326      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
327
328    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
329    List<byte[]> nonEmptyCfs = Lists.newArrayList();
330    SnapshotTestingUtils.confirmSnapshotValid(
331      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
332      emptyCfs, rootDir, admin, fs);
333
334    admin.deleteSnapshot(snapshot);
335    snapshots = admin.listSnapshots();
336    SnapshotTestingUtils.assertNoSnapshots(admin);
337  }
338
339  @Test
340  public void testListTableSnapshots() throws Exception {
341    Admin admin = null;
342    final TableName tableName = name.getTableName();
343    try {
344      admin = UTIL.getAdmin();
345
346      HTableDescriptor htd = new HTableDescriptor(tableName);
347      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
348
349      String table1Snapshot1 = "Table1Snapshot1";
350      admin.snapshot(table1Snapshot1, TABLE_NAME);
351      LOG.debug("Snapshot1 completed.");
352
353      String table1Snapshot2 = "Table1Snapshot2";
354      admin.snapshot(table1Snapshot2, TABLE_NAME);
355      LOG.debug("Snapshot2 completed.");
356
357      String table2Snapshot1 = "Table2Snapshot1";
358      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
359      LOG.debug(table2Snapshot1 + " completed.");
360
361      List<SnapshotDescription> listTableSnapshots =
362        admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
363      List<String> listTableSnapshotNames = new ArrayList<>();
364      assertEquals(3, listTableSnapshots.size());
365      for (SnapshotDescription s : listTableSnapshots) {
366        listTableSnapshotNames.add(s.getName());
367      }
368      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
369      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
370      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
371    } finally {
372      if (admin != null) {
373        try {
374          admin.deleteSnapshots(Pattern.compile("Table.*"));
375        } catch (SnapshotDoesNotExistException ignore) {
376        }
377        if (admin.tableExists(tableName)) {
378          UTIL.deleteTable(tableName);
379        }
380        admin.close();
381      }
382    }
383  }
384
385  @Test
386  public void testListTableSnapshotsWithRegex() throws Exception {
387    Admin admin = null;
388    try {
389      admin = UTIL.getAdmin();
390
391      String table1Snapshot1 = "Table1Snapshot1";
392      admin.snapshot(table1Snapshot1, TABLE_NAME);
393      LOG.debug("Snapshot1 completed.");
394
395      String table1Snapshot2 = "Table1Snapshot2";
396      admin.snapshot(table1Snapshot2, TABLE_NAME);
397      LOG.debug("Snapshot2 completed.");
398
399      String table2Snapshot1 = "Table2Snapshot1";
400      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
401      LOG.debug(table2Snapshot1 + " completed.");
402
403      List<SnapshotDescription> listTableSnapshots =
404        admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
405      List<String> listTableSnapshotNames = new ArrayList<>();
406      assertEquals(2, listTableSnapshots.size());
407      for (SnapshotDescription s : listTableSnapshots) {
408        listTableSnapshotNames.add(s.getName());
409      }
410      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
411      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
412      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
413    } finally {
414      if (admin != null) {
415        try {
416          admin.deleteSnapshots(Pattern.compile("Table.*"));
417        } catch (SnapshotDoesNotExistException ignore) {
418        }
419        admin.close();
420      }
421    }
422  }
423
424  @Test
425  public void testDeleteTableSnapshots() throws Exception {
426    Admin admin = null;
427    final TableName tableName = name.getTableName();
428    try {
429      admin = UTIL.getAdmin();
430
431      HTableDescriptor htd = new HTableDescriptor(tableName);
432      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
433
434      String table1Snapshot1 = "Table1Snapshot1";
435      admin.snapshot(table1Snapshot1, TABLE_NAME);
436      LOG.debug("Snapshot1 completed.");
437
438      String table1Snapshot2 = "Table1Snapshot2";
439      admin.snapshot(table1Snapshot2, TABLE_NAME);
440      LOG.debug("Snapshot2 completed.");
441
442      String table2Snapshot1 = "Table2Snapshot1";
443      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
444      LOG.debug(table2Snapshot1 + " completed.");
445
446      Pattern tableNamePattern = Pattern.compile("test.*");
447      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
448      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
449    } finally {
450      if (admin != null) {
451        if (admin.tableExists(tableName)) {
452          UTIL.deleteTable(tableName);
453        }
454        admin.close();
455      }
456    }
457  }
458
459  @Test
460  public void testDeleteTableSnapshotsWithRegex() throws Exception {
461    Admin admin = null;
462    Pattern tableNamePattern = Pattern.compile("test.*");
463    try {
464      admin = UTIL.getAdmin();
465
466      String table1Snapshot1 = "Table1Snapshot1";
467      admin.snapshot(table1Snapshot1, TABLE_NAME);
468      LOG.debug("Snapshot1 completed.");
469
470      String table1Snapshot2 = "Table1Snapshot2";
471      admin.snapshot(table1Snapshot2, TABLE_NAME);
472      LOG.debug("Snapshot2 completed.");
473
474      String table2Snapshot1 = "Table2Snapshot1";
475      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
476      LOG.debug(table2Snapshot1 + " completed.");
477
478      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
479      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
480    } finally {
481      if (admin != null) {
482        try {
483          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
484        } catch (SnapshotDoesNotExistException ignore) {
485        }
486        admin.close();
487      }
488    }
489  }
490}