001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import java.util.regex.Pattern;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.TableNameTestRule;
037import org.apache.hadoop.hbase.TableNotFoundException;
038import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
039import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
040import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
041import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
042import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
043import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
044import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
045import org.apache.hadoop.hbase.testclassification.ClientTests;
046import org.apache.hadoop.hbase.testclassification.LargeTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.junit.After;
050import org.junit.AfterClass;
051import org.junit.Before;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.runner.RunWith;
058import org.junit.runners.Parameterized;
059import org.junit.runners.Parameterized.Parameter;
060import org.junit.runners.Parameterized.Parameters;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
065
066import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
067
068/**
069 * Test create/using/deleting snapshots from the client
070 * <p>
071 * This is an end-to-end test for the snapshot utility
072 */
073@RunWith(Parameterized.class)
074@Category({ LargeTests.class, ClientTests.class })
075public class TestSnapshotFromClient {
076
077  @ClassRule
078  public static final HBaseClassTestRule CLASS_RULE =
079    HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
080
081  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
082
083  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
084  protected static final int NUM_RS = 2;
085  protected static final String STRING_TABLE_NAME = "test";
086  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
087  protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
088  private static final Pattern MATCH_ALL = Pattern.compile(".*");
089
090  @Rule
091  public TableNameTestRule name = new TableNameTestRule();
092
093  @Parameter
094  public StoreFileTrackerFactory.Trackers trackerImpl;
095
096  @Parameters(name = "{index}: tracker={0}")
097  public static List<Object[]> params() {
098    return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
099      new Object[] { StoreFileTrackerFactory.Trackers.FILE });
100  }
101
102  /**
103   * Setup the config for the cluster
104   * @throws Exception on failure
105   */
106  @BeforeClass
107  public static void setupCluster() throws Exception {
108    setupConf(UTIL.getConfiguration());
109    UTIL.startMiniCluster(NUM_RS);
110  }
111
112  protected static void setupConf(Configuration conf) {
113    // disable the ui
114    conf.setInt("hbase.regionsever.info.port", -1);
115    // change the flush size to a small amount, regulating number of store files
116    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
117    // so make sure we get a compaction when doing a load, but keep around some
118    // files in the store
119    conf.setInt("hbase.hstore.compaction.min", 10);
120    conf.setInt("hbase.hstore.compactionThreshold", 10);
121    // block writes if we get to 12 store files
122    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
123    // Enable snapshot
124    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
125    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
126      ConstantSizeRegionSplitPolicy.class.getName());
127  }
128
129  @Before
130  public void setup() throws Exception {
131    createTable();
132  }
133
134  protected void createTable() throws Exception {
135    TableDescriptor htd =
136      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
137        .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
138    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
139  }
140
141  protected int getNumReplicas() {
142    return 1;
143  }
144
145  @After
146  public void tearDown() throws Exception {
147    UTIL.deleteTable(TABLE_NAME);
148    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
149    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
150  }
151
152  @AfterClass
153  public static void cleanupTest() throws Exception {
154    try {
155      UTIL.shutdownMiniCluster();
156    } catch (Exception e) {
157      LOG.warn("failure shutting down cluster", e);
158    }
159  }
160
161  /**
162   * Test snapshotting not allowed hbase:meta and -ROOT-
163   */
164  @Test(expected = IllegalArgumentException.class)
165  public void testMetaTablesSnapshot() throws Exception {
166    UTIL.getAdmin().snapshot("metaSnapshot", TableName.META_TABLE_NAME);
167  }
168
169  /**
170   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
171   * n
172   */
173  @Test
174  public void testSnapshotDeletionWithRegex() throws Exception {
175    Admin admin = UTIL.getAdmin();
176    // make sure we don't fail on listing snapshots
177    SnapshotTestingUtils.assertNoSnapshots(admin);
178
179    // put some stuff in the table
180    Table table = UTIL.getConnection().getTable(TABLE_NAME);
181    UTIL.loadTable(table, TEST_FAM);
182    table.close();
183
184    String snapshot1 = "TableSnapshot1";
185    admin.snapshot(snapshot1, TABLE_NAME);
186    LOG.debug("Snapshot1 completed.");
187
188    String snapshot2 = "TableSnapshot2";
189    admin.snapshot(snapshot2, TABLE_NAME);
190    LOG.debug("Snapshot2 completed.");
191
192    String snapshot3 = "3rdTableSnapshot";
193    admin.snapshot(snapshot3, TABLE_NAME);
194    LOG.debug(snapshot3 + " completed.");
195
196    // delete the first two snapshots
197    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
198    List<SnapshotDescription> snapshots = admin.listSnapshots();
199    assertEquals(1, snapshots.size());
200    assertEquals(snapshot3, snapshots.get(0).getName());
201
202    admin.deleteSnapshot(snapshot3);
203    admin.close();
204  }
205
206  /**
207   * Test snapshotting a table that is offline n
208   */
209  @Test
210  public void testOfflineTableSnapshot() throws Exception {
211    Admin admin = UTIL.getAdmin();
212    // make sure we don't fail on listing snapshots
213    SnapshotTestingUtils.assertNoSnapshots(admin);
214
215    // put some stuff in the table
216    Table table = UTIL.getConnection().getTable(TABLE_NAME);
217    UTIL.loadTable(table, TEST_FAM, false);
218
219    LOG.debug("FS state before disable:");
220    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
221      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
222    // XXX if this is flakey, might want to consider using the async version and looping as
223    // disableTable can succeed and still timeout.
224    admin.disableTable(TABLE_NAME);
225
226    LOG.debug("FS state before snapshot:");
227    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
228      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
229
230    // take a snapshot of the disabled table
231    final String SNAPSHOT_NAME = "offlineTableSnapshot";
232    String snapshot = SNAPSHOT_NAME;
233
234    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.DISABLED, null,
235      -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
236    LOG.debug("Snapshot completed.");
237
238    // make sure we have the snapshot
239    List<SnapshotDescription> snapshots =
240      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
241
242    // make sure its a valid snapshot
243    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
244    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
245    LOG.debug("FS state after snapshot:");
246    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
247      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
248    SnapshotTestingUtils.confirmSnapshotValid(
249      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, rootDir,
250      admin, fs);
251
252    admin.deleteSnapshot(snapshot);
253    snapshots = admin.listSnapshots();
254    SnapshotTestingUtils.assertNoSnapshots(admin);
255  }
256
257  @Test
258  public void testSnapshotFailsOnNonExistantTable() throws Exception {
259    Admin admin = UTIL.getAdmin();
260    // make sure we don't fail on listing snapshots
261    SnapshotTestingUtils.assertNoSnapshots(admin);
262    String tableName = "_not_a_table";
263
264    // make sure the table doesn't exist
265    boolean fail = false;
266    do {
267      try {
268        admin.getDescriptor(TableName.valueOf(tableName));
269        fail = true;
270        LOG.error("Table:" + tableName + " already exists, checking a new name");
271        tableName = tableName + "!";
272      } catch (TableNotFoundException e) {
273        fail = false;
274      }
275    } while (fail);
276
277    // snapshot the non-existant table
278    try {
279      admin.snapshot("fail", TableName.valueOf(tableName));
280      fail("Snapshot succeeded even though there is not table.");
281    } catch (SnapshotCreationException e) {
282      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
283    }
284  }
285
286  @Test
287  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
288    // test with an empty table with one region
289
290    Admin admin = UTIL.getAdmin();
291    // make sure we don't fail on listing snapshots
292    SnapshotTestingUtils.assertNoSnapshots(admin);
293
294    LOG.debug("FS state before disable:");
295    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
296      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
297    admin.disableTable(TABLE_NAME);
298
299    LOG.debug("FS state before snapshot:");
300    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
301      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
302
303    // take a snapshot of the disabled table
304    String snapshot = "testOfflineTableSnapshotWithEmptyRegions";
305    admin.snapshot(snapshot, TABLE_NAME);
306    LOG.debug("Snapshot completed.");
307
308    // make sure we have the snapshot
309    List<SnapshotDescription> snapshots =
310      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
311
312    // make sure its a valid snapshot
313    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
314    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
315    LOG.debug("FS state after snapshot:");
316    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
317      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
318
319    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
320    List<byte[]> nonEmptyCfs = Lists.newArrayList();
321    SnapshotTestingUtils.confirmSnapshotValid(
322      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
323      emptyCfs, rootDir, admin, fs);
324
325    admin.deleteSnapshot(snapshot);
326    snapshots = admin.listSnapshots();
327    SnapshotTestingUtils.assertNoSnapshots(admin);
328  }
329
330  @Test
331  public void testListTableSnapshots() throws Exception {
332    Admin admin = null;
333    final TableName tableName = name.getTableName();
334    try {
335      admin = UTIL.getAdmin();
336
337      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
338      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
339
340      String table1Snapshot1 = "Table1Snapshot1";
341      admin.snapshot(table1Snapshot1, TABLE_NAME);
342      LOG.debug("Snapshot1 completed.");
343
344      String table1Snapshot2 = "Table1Snapshot2";
345      admin.snapshot(table1Snapshot2, TABLE_NAME);
346      LOG.debug("Snapshot2 completed.");
347
348      String table2Snapshot1 = "Table2Snapshot1";
349      admin.snapshot(table2Snapshot1, tableName);
350      LOG.debug(table2Snapshot1 + " completed.");
351
352      List<SnapshotDescription> listTableSnapshots =
353        admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
354      List<String> listTableSnapshotNames = new ArrayList<>();
355      assertEquals(3, listTableSnapshots.size());
356      for (SnapshotDescription s : listTableSnapshots) {
357        listTableSnapshotNames.add(s.getName());
358      }
359      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
360      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
361      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
362    } finally {
363      if (admin != null) {
364        try {
365          admin.deleteSnapshots(Pattern.compile("Table.*"));
366        } catch (SnapshotDoesNotExistException ignore) {
367        }
368        if (admin.tableExists(tableName)) {
369          UTIL.deleteTable(tableName);
370        }
371        admin.close();
372      }
373    }
374  }
375
376  @Test
377  public void testListTableSnapshotsWithRegex() throws Exception {
378    Admin admin = null;
379    try {
380      admin = UTIL.getAdmin();
381
382      String table1Snapshot1 = "Table1Snapshot1";
383      admin.snapshot(table1Snapshot1, TABLE_NAME);
384      LOG.debug("Snapshot1 completed.");
385
386      String table1Snapshot2 = "Table1Snapshot2";
387      admin.snapshot(table1Snapshot2, TABLE_NAME);
388      LOG.debug("Snapshot2 completed.");
389
390      String table2Snapshot1 = "Table2Snapshot1";
391      admin.snapshot(table2Snapshot1, TABLE_NAME);
392      LOG.debug(table2Snapshot1 + " completed.");
393
394      List<SnapshotDescription> listTableSnapshots =
395        admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
396      List<String> listTableSnapshotNames = new ArrayList<>();
397      assertEquals(2, listTableSnapshots.size());
398      for (SnapshotDescription s : listTableSnapshots) {
399        listTableSnapshotNames.add(s.getName());
400      }
401      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
402      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
403      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
404    } finally {
405      if (admin != null) {
406        try {
407          admin.deleteSnapshots(Pattern.compile("Table.*"));
408        } catch (SnapshotDoesNotExistException ignore) {
409        }
410        admin.close();
411      }
412    }
413  }
414
415  @Test
416  public void testDeleteTableSnapshots() throws Exception {
417    Admin admin = null;
418    final TableName tableName = name.getTableName();
419    try {
420      admin = UTIL.getAdmin();
421
422      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
423      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
424
425      String table1Snapshot1 = "Table1Snapshot1";
426      admin.snapshot(table1Snapshot1, TABLE_NAME);
427      LOG.debug("Snapshot1 completed.");
428
429      String table1Snapshot2 = "Table1Snapshot2";
430      admin.snapshot(table1Snapshot2, TABLE_NAME);
431      LOG.debug("Snapshot2 completed.");
432
433      String table2Snapshot1 = "Table2Snapshot1";
434      admin.snapshot(table2Snapshot1, tableName);
435      LOG.debug(table2Snapshot1 + " completed.");
436
437      Pattern tableNamePattern = Pattern.compile("test.*");
438      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
439      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
440    } finally {
441      if (admin != null) {
442        if (admin.tableExists(tableName)) {
443          UTIL.deleteTable(tableName);
444        }
445        admin.close();
446      }
447    }
448  }
449
450  @Test
451  public void testDeleteTableSnapshotsWithRegex() throws Exception {
452    Admin admin = null;
453    Pattern tableNamePattern = Pattern.compile("test.*");
454    try {
455      admin = UTIL.getAdmin();
456
457      String table1Snapshot1 = "Table1Snapshot1";
458      admin.snapshot(table1Snapshot1, TABLE_NAME);
459      LOG.debug("Snapshot1 completed.");
460
461      String table1Snapshot2 = "Table1Snapshot2";
462      admin.snapshot(table1Snapshot2, TABLE_NAME);
463      LOG.debug("Snapshot2 completed.");
464
465      String table2Snapshot1 = "Table2Snapshot1";
466      admin.snapshot(table2Snapshot1, TABLE_NAME);
467      LOG.debug(table2Snapshot1 + " completed.");
468
469      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
470      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
471    } finally {
472      if (admin != null) {
473        try {
474          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
475        } catch (SnapshotDoesNotExistException ignore) {
476        }
477        admin.close();
478      }
479    }
480  }
481}