Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import java.util.regex.Pattern;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.TableNameTestRule;
037import org.apache.hadoop.hbase.TableNotFoundException;
038import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
039import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
040import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
041import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
042import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
043import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
044import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
045import org.apache.hadoop.hbase.testclassification.ClientTests;
046import org.apache.hadoop.hbase.testclassification.LargeTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.junit.After;
050import org.junit.AfterClass;
051import org.junit.Before;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.runner.RunWith;
058import org.junit.runners.Parameterized;
059import org.junit.runners.Parameterized.Parameter;
060import org.junit.runners.Parameterized.Parameters;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
065
066import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
067
068/**
069 * Test create/using/deleting snapshots from the client
070 * <p>
071 * This is an end-to-end test for the snapshot utility
072 */
073@RunWith(Parameterized.class)
074@Category({ LargeTests.class, ClientTests.class })
075public class TestSnapshotFromClient {
076
077  @ClassRule
078  public static final HBaseClassTestRule CLASS_RULE =
079    HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
080
081  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
082
083  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
084  protected static final int NUM_RS = 2;
085  protected static final String STRING_TABLE_NAME = "test";
086  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
087  protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
088  private static final Pattern MATCH_ALL = Pattern.compile(".*");
089
090  @Rule
091  public TableNameTestRule name = new TableNameTestRule();
092
093  @Parameter
094  public StoreFileTrackerFactory.Trackers trackerImpl;
095
096  @Parameters(name = "{index}: tracker={0}")
097  public static List<Object[]> params() {
098    return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
099      new Object[] { StoreFileTrackerFactory.Trackers.FILE });
100  }
101
102  /**
103   * Setup the config for the cluster
104   * @throws Exception on failure
105   */
106  @BeforeClass
107  public static void setupCluster() throws Exception {
108    setupConf(UTIL.getConfiguration());
109    UTIL.startMiniCluster(NUM_RS);
110  }
111
112  protected static void setupConf(Configuration conf) {
113    // disable the ui
114    conf.setInt("hbase.regionsever.info.port", -1);
115    // change the flush size to a small amount, regulating number of store files
116    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
117    // so make sure we get a compaction when doing a load, but keep around some
118    // files in the store
119    conf.setInt("hbase.hstore.compaction.min", 10);
120    conf.setInt("hbase.hstore.compactionThreshold", 10);
121    // block writes if we get to 12 store files
122    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
123    // Enable snapshot
124    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
125    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
126      ConstantSizeRegionSplitPolicy.class.getName());
127  }
128
129  @Before
130  public void setup() throws Exception {
131    createTable();
132  }
133
134  protected void createTable() throws Exception {
135    TableDescriptor htd =
136      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
137        .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
138    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
139  }
140
141  protected int getNumReplicas() {
142    return 1;
143  }
144
145  @After
146  public void tearDown() throws Exception {
147    UTIL.deleteTable(TABLE_NAME);
148    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
149    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
150  }
151
152  @AfterClass
153  public static void cleanupTest() throws Exception {
154    try {
155      UTIL.shutdownMiniCluster();
156    } catch (Exception e) {
157      LOG.warn("failure shutting down cluster", e);
158    }
159  }
160
161  /**
162   * Test snapshotting not allowed hbase:meta and -ROOT-
163   */
164  @Test(expected = IllegalArgumentException.class)
165  public void testMetaTablesSnapshot() throws Exception {
166    UTIL.getAdmin().snapshot("metaSnapshot", TableName.META_TABLE_NAME);
167  }
168
169  /**
170   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
171   */
172  @Test
173  public void testSnapshotDeletionWithRegex() throws Exception {
174    Admin admin = UTIL.getAdmin();
175    // make sure we don't fail on listing snapshots
176    SnapshotTestingUtils.assertNoSnapshots(admin);
177
178    // put some stuff in the table
179    Table table = UTIL.getConnection().getTable(TABLE_NAME);
180    UTIL.loadTable(table, TEST_FAM);
181    table.close();
182
183    String snapshot1 = "TableSnapshot1";
184    admin.snapshot(snapshot1, TABLE_NAME);
185    LOG.debug("Snapshot1 completed.");
186
187    String snapshot2 = "TableSnapshot2";
188    admin.snapshot(snapshot2, TABLE_NAME);
189    LOG.debug("Snapshot2 completed.");
190
191    String snapshot3 = "3rdTableSnapshot";
192    admin.snapshot(snapshot3, TABLE_NAME);
193    LOG.debug(snapshot3 + " completed.");
194
195    // delete the first two snapshots
196    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
197    List<SnapshotDescription> snapshots = admin.listSnapshots();
198    assertEquals(1, snapshots.size());
199    assertEquals(snapshot3, snapshots.get(0).getName());
200
201    admin.deleteSnapshot(snapshot3);
202    admin.close();
203  }
204
205  /**
206   * Test snapshotting a table that is offline
207   */
208  @Test
209  public void testOfflineTableSnapshot() throws Exception {
210    Admin admin = UTIL.getAdmin();
211    // make sure we don't fail on listing snapshots
212    SnapshotTestingUtils.assertNoSnapshots(admin);
213
214    // put some stuff in the table
215    Table table = UTIL.getConnection().getTable(TABLE_NAME);
216    UTIL.loadTable(table, TEST_FAM, false);
217
218    LOG.debug("FS state before disable:");
219    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
220      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
221    // XXX if this is flakey, might want to consider using the async version and looping as
222    // disableTable can succeed and still timeout.
223    admin.disableTable(TABLE_NAME);
224
225    LOG.debug("FS state before snapshot:");
226    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
227      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
228
229    // take a snapshot of the disabled table
230    final String SNAPSHOT_NAME = "offlineTableSnapshot";
231    String snapshot = SNAPSHOT_NAME;
232
233    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.DISABLED, null,
234      -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
235    LOG.debug("Snapshot completed.");
236
237    // make sure we have the snapshot
238    List<SnapshotDescription> snapshots =
239      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
240
241    // make sure its a valid snapshot
242    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
243    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
244    LOG.debug("FS state after snapshot:");
245    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
246      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
247    SnapshotTestingUtils.confirmSnapshotValid(
248      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, rootDir,
249      admin, fs);
250
251    admin.deleteSnapshot(snapshot);
252    snapshots = admin.listSnapshots();
253    SnapshotTestingUtils.assertNoSnapshots(admin);
254  }
255
256  @Test
257  public void testSnapshotFailsOnNonExistantTable() throws Exception {
258    Admin admin = UTIL.getAdmin();
259    // make sure we don't fail on listing snapshots
260    SnapshotTestingUtils.assertNoSnapshots(admin);
261    String tableName = "_not_a_table";
262
263    // make sure the table doesn't exist
264    boolean fail = false;
265    do {
266      try {
267        admin.getDescriptor(TableName.valueOf(tableName));
268        fail = true;
269        LOG.error("Table:" + tableName + " already exists, checking a new name");
270        tableName = tableName + "!";
271      } catch (TableNotFoundException e) {
272        fail = false;
273      }
274    } while (fail);
275
276    // snapshot the non-existant table
277    try {
278      admin.snapshot("fail", TableName.valueOf(tableName));
279      fail("Snapshot succeeded even though there is not table.");
280    } catch (SnapshotCreationException e) {
281      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
282    }
283  }
284
285  @Test
286  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
287    // test with an empty table with one region
288
289    Admin admin = UTIL.getAdmin();
290    // make sure we don't fail on listing snapshots
291    SnapshotTestingUtils.assertNoSnapshots(admin);
292
293    LOG.debug("FS state before disable:");
294    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
295      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
296    admin.disableTable(TABLE_NAME);
297
298    LOG.debug("FS state before snapshot:");
299    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
300      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
301
302    // take a snapshot of the disabled table
303    String snapshot = "testOfflineTableSnapshotWithEmptyRegions";
304    admin.snapshot(snapshot, TABLE_NAME);
305    LOG.debug("Snapshot completed.");
306
307    // make sure we have the snapshot
308    List<SnapshotDescription> snapshots =
309      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
310
311    // make sure its a valid snapshot
312    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
313    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
314    LOG.debug("FS state after snapshot:");
315    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
316      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
317
318    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
319    List<byte[]> nonEmptyCfs = Lists.newArrayList();
320    SnapshotTestingUtils.confirmSnapshotValid(
321      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
322      emptyCfs, rootDir, admin, fs);
323
324    admin.deleteSnapshot(snapshot);
325    snapshots = admin.listSnapshots();
326    SnapshotTestingUtils.assertNoSnapshots(admin);
327  }
328
329  @Test
330  public void testListTableSnapshots() throws Exception {
331    Admin admin = null;
332    final TableName tableName = name.getTableName();
333    try {
334      admin = UTIL.getAdmin();
335
336      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
337      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
338
339      String table1Snapshot1 = "Table1Snapshot1";
340      admin.snapshot(table1Snapshot1, TABLE_NAME);
341      LOG.debug("Snapshot1 completed.");
342
343      String table1Snapshot2 = "Table1Snapshot2";
344      admin.snapshot(table1Snapshot2, TABLE_NAME);
345      LOG.debug("Snapshot2 completed.");
346
347      String table2Snapshot1 = "Table2Snapshot1";
348      admin.snapshot(table2Snapshot1, tableName);
349      LOG.debug(table2Snapshot1 + " completed.");
350
351      List<SnapshotDescription> listTableSnapshots =
352        admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
353      List<String> listTableSnapshotNames = new ArrayList<>();
354      assertEquals(3, listTableSnapshots.size());
355      for (SnapshotDescription s : listTableSnapshots) {
356        listTableSnapshotNames.add(s.getName());
357      }
358      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
359      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
360      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
361    } finally {
362      if (admin != null) {
363        try {
364          admin.deleteSnapshots(Pattern.compile("Table.*"));
365        } catch (SnapshotDoesNotExistException ignore) {
366        }
367        if (admin.tableExists(tableName)) {
368          UTIL.deleteTable(tableName);
369        }
370        admin.close();
371      }
372    }
373  }
374
375  @Test
376  public void testListTableSnapshotsWithRegex() throws Exception {
377    Admin admin = null;
378    try {
379      admin = UTIL.getAdmin();
380
381      String table1Snapshot1 = "Table1Snapshot1";
382      admin.snapshot(table1Snapshot1, TABLE_NAME);
383      LOG.debug("Snapshot1 completed.");
384
385      String table1Snapshot2 = "Table1Snapshot2";
386      admin.snapshot(table1Snapshot2, TABLE_NAME);
387      LOG.debug("Snapshot2 completed.");
388
389      String table2Snapshot1 = "Table2Snapshot1";
390      admin.snapshot(table2Snapshot1, TABLE_NAME);
391      LOG.debug(table2Snapshot1 + " completed.");
392
393      List<SnapshotDescription> listTableSnapshots =
394        admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
395      List<String> listTableSnapshotNames = new ArrayList<>();
396      assertEquals(2, listTableSnapshots.size());
397      for (SnapshotDescription s : listTableSnapshots) {
398        listTableSnapshotNames.add(s.getName());
399      }
400      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
401      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
402      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
403    } finally {
404      if (admin != null) {
405        try {
406          admin.deleteSnapshots(Pattern.compile("Table.*"));
407        } catch (SnapshotDoesNotExistException ignore) {
408        }
409        admin.close();
410      }
411    }
412  }
413
414  @Test
415  public void testDeleteTableSnapshots() throws Exception {
416    Admin admin = null;
417    final TableName tableName = name.getTableName();
418    try {
419      admin = UTIL.getAdmin();
420
421      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
422      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
423
424      String table1Snapshot1 = "Table1Snapshot1";
425      admin.snapshot(table1Snapshot1, TABLE_NAME);
426      LOG.debug("Snapshot1 completed.");
427
428      String table1Snapshot2 = "Table1Snapshot2";
429      admin.snapshot(table1Snapshot2, TABLE_NAME);
430      LOG.debug("Snapshot2 completed.");
431
432      String table2Snapshot1 = "Table2Snapshot1";
433      admin.snapshot(table2Snapshot1, tableName);
434      LOG.debug(table2Snapshot1 + " completed.");
435
436      Pattern tableNamePattern = Pattern.compile("test.*");
437      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
438      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
439    } finally {
440      if (admin != null) {
441        if (admin.tableExists(tableName)) {
442          UTIL.deleteTable(tableName);
443        }
444        admin.close();
445      }
446    }
447  }
448
449  @Test
450  public void testDeleteTableSnapshotsWithRegex() throws Exception {
451    Admin admin = null;
452    Pattern tableNamePattern = Pattern.compile("test.*");
453    try {
454      admin = UTIL.getAdmin();
455
456      String table1Snapshot1 = "Table1Snapshot1";
457      admin.snapshot(table1Snapshot1, TABLE_NAME);
458      LOG.debug("Snapshot1 completed.");
459
460      String table1Snapshot2 = "Table1Snapshot2";
461      admin.snapshot(table1Snapshot2, TABLE_NAME);
462      LOG.debug("Snapshot2 completed.");
463
464      String table2Snapshot1 = "Table2Snapshot1";
465      admin.snapshot(table2Snapshot1, TABLE_NAME);
466      LOG.debug(table2Snapshot1 + " completed.");
467
468      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
469      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
470    } finally {
471      if (admin != null) {
472        try {
473          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
474        } catch (SnapshotDoesNotExistException ignore) {
475        }
476        admin.close();
477      }
478    }
479  }
480}