001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.regex.Pattern;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.TableNotFoundException;
037import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
038import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
039import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
040import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
041import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
042import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
043import org.apache.hadoop.hbase.testclassification.ClientTests;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.FSUtils;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
060
061import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
062
063/**
064 * Test create/using/deleting snapshots from the client
065 * <p>
066 * This is an end-to-end test for the snapshot utility
067 */
068@Category({LargeTests.class, ClientTests.class})
069public class TestSnapshotFromClient {
070
071  @ClassRule
072  public static final HBaseClassTestRule CLASS_RULE =
073      HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
074
075  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
076
077  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
078  protected static final int NUM_RS = 2;
079  protected static final String STRING_TABLE_NAME = "test";
080  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
081  protected static final TableName TABLE_NAME =
082      TableName.valueOf(STRING_TABLE_NAME);
083  private static final Pattern MATCH_ALL = Pattern.compile(".*");
084
085  @Rule
086  public TestName name = new TestName();
087
088  /**
089   * Setup the config for the cluster
090   * @throws Exception on failure
091   */
092  @BeforeClass
093  public static void setupCluster() throws Exception {
094    setupConf(UTIL.getConfiguration());
095    UTIL.startMiniCluster(NUM_RS);
096  }
097
098  protected static void setupConf(Configuration conf) {
099    // disable the ui
100    conf.setInt("hbase.regionsever.info.port", -1);
101    // change the flush size to a small amount, regulating number of store files
102    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
103    // so make sure we get a compaction when doing a load, but keep around some
104    // files in the store
105    conf.setInt("hbase.hstore.compaction.min", 10);
106    conf.setInt("hbase.hstore.compactionThreshold", 10);
107    // block writes if we get to 12 store files
108    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
109    // Enable snapshot
110    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
111    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
112        ConstantSizeRegionSplitPolicy.class.getName());
113
114  }
115
116  @Before
117  public void setup() throws Exception {
118    createTable();
119  }
120
121  protected void createTable() throws Exception {
122    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
123    htd.setRegionReplication(getNumReplicas());
124    UTIL.createTable(htd, new byte[][]{TEST_FAM}, null);
125  }
126
127  protected int getNumReplicas() {
128    return 1;
129  }
130
131  @After
132  public void tearDown() throws Exception {
133    UTIL.deleteTable(TABLE_NAME);
134    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
135    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
136  }
137
138  @AfterClass
139  public static void cleanupTest() throws Exception {
140    try {
141      UTIL.shutdownMiniCluster();
142    } catch (Exception e) {
143      LOG.warn("failure shutting down cluster", e);
144    }
145  }
146
147  /**
148   * Test snapshotting not allowed hbase:meta and -ROOT-
149   * @throws Exception
150   */
151  @Test
152  public void testMetaTablesSnapshot() throws Exception {
153    Admin admin = UTIL.getAdmin();
154    byte[] snapshotName = Bytes.toBytes("metaSnapshot");
155
156    try {
157      admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
158      fail("taking a snapshot of hbase:meta should not be allowed");
159    } catch (IllegalArgumentException e) {
160      // expected
161    }
162  }
163
164  /**
165   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
166   *
167   * @throws Exception
168   */
169  @Test
170  public void testSnapshotDeletionWithRegex() throws Exception {
171    Admin admin = UTIL.getAdmin();
172    // make sure we don't fail on listing snapshots
173    SnapshotTestingUtils.assertNoSnapshots(admin);
174
175    // put some stuff in the table
176    Table table = UTIL.getConnection().getTable(TABLE_NAME);
177    UTIL.loadTable(table, TEST_FAM);
178    table.close();
179
180    byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
181    admin.snapshot(snapshot1, TABLE_NAME);
182    LOG.debug("Snapshot1 completed.");
183
184    byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
185    admin.snapshot(snapshot2, TABLE_NAME);
186    LOG.debug("Snapshot2 completed.");
187
188    String snapshot3 = "3rdTableSnapshot";
189    admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
190    LOG.debug(snapshot3 + " completed.");
191
192    // delete the first two snapshots
193    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
194    List<SnapshotDescription> snapshots = admin.listSnapshots();
195    assertEquals(1, snapshots.size());
196    assertEquals(snapshot3, snapshots.get(0).getName());
197
198    admin.deleteSnapshot(snapshot3);
199    admin.close();
200  }
201  /**
202   * Test snapshotting a table that is offline
203   * @throws Exception
204   */
205  @Test
206  public void testOfflineTableSnapshot() throws Exception {
207    Admin admin = UTIL.getAdmin();
208    // make sure we don't fail on listing snapshots
209    SnapshotTestingUtils.assertNoSnapshots(admin);
210
211    // put some stuff in the table
212    Table table = UTIL.getConnection().getTable(TABLE_NAME);
213    UTIL.loadTable(table, TEST_FAM, false);
214
215    LOG.debug("FS state before disable:");
216    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
217      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
218    // XXX if this is flakey, might want to consider using the async version and looping as
219    // disableTable can succeed and still timeout.
220    admin.disableTable(TABLE_NAME);
221
222    LOG.debug("FS state before snapshot:");
223    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
224      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
225
226    // take a snapshot of the disabled table
227    final String SNAPSHOT_NAME = "offlineTableSnapshot";
228    byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
229
230    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME,
231        SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION));
232    LOG.debug("Snapshot completed.");
233
234    // make sure we have the snapshot
235    List<SnapshotDescription> snapshots =
236        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
237
238    // make sure its a valid snapshot
239    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
240    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
241    LOG.debug("FS state after snapshot:");
242    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
243      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
244    SnapshotTestingUtils.confirmSnapshotValid(
245      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
246      rootDir, admin, fs);
247
248    admin.deleteSnapshot(snapshot);
249    snapshots = admin.listSnapshots();
250    SnapshotTestingUtils.assertNoSnapshots(admin);
251  }
252
253  @Test
254  public void testSnapshotFailsOnNonExistantTable() throws Exception {
255    Admin admin = UTIL.getAdmin();
256    // make sure we don't fail on listing snapshots
257    SnapshotTestingUtils.assertNoSnapshots(admin);
258    String tableName = "_not_a_table";
259
260    // make sure the table doesn't exist
261    boolean fail = false;
262    do {
263    try {
264      admin.getTableDescriptor(TableName.valueOf(tableName));
265      fail = true;
266          LOG.error("Table:" + tableName + " already exists, checking a new name");
267      tableName = tableName+"!";
268    } catch (TableNotFoundException e) {
269      fail = false;
270      }
271    } while (fail);
272
273    // snapshot the non-existant table
274    try {
275      admin.snapshot("fail", TableName.valueOf(tableName));
276      fail("Snapshot succeeded even though there is not table.");
277    } catch (SnapshotCreationException e) {
278      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
279    }
280  }
281
282  @Test
283  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
284    // test with an empty table with one region
285
286    Admin admin = UTIL.getAdmin();
287    // make sure we don't fail on listing snapshots
288    SnapshotTestingUtils.assertNoSnapshots(admin);
289
290    LOG.debug("FS state before disable:");
291    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
292      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
293    admin.disableTable(TABLE_NAME);
294
295    LOG.debug("FS state before snapshot:");
296    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
297      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
298
299    // take a snapshot of the disabled table
300    byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
301    admin.snapshot(snapshot, TABLE_NAME);
302    LOG.debug("Snapshot completed.");
303
304    // make sure we have the snapshot
305    List<SnapshotDescription> snapshots =
306        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
307
308    // make sure its a valid snapshot
309    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
310    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
311    LOG.debug("FS state after snapshot:");
312    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
313      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
314
315    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
316    List<byte[]> nonEmptyCfs = Lists.newArrayList();
317    SnapshotTestingUtils.confirmSnapshotValid(
318      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
319      emptyCfs, rootDir, admin, fs);
320
321    admin.deleteSnapshot(snapshot);
322    snapshots = admin.listSnapshots();
323    SnapshotTestingUtils.assertNoSnapshots(admin);
324  }
325
326  @Test
327  public void testListTableSnapshots() throws Exception {
328    Admin admin = null;
329    final TableName tableName = TableName.valueOf(name.getMethodName());
330    try {
331      admin = UTIL.getAdmin();
332
333      HTableDescriptor htd = new HTableDescriptor(tableName);
334      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
335
336      String table1Snapshot1 = "Table1Snapshot1";
337      admin.snapshot(table1Snapshot1, TABLE_NAME);
338      LOG.debug("Snapshot1 completed.");
339
340      String table1Snapshot2 = "Table1Snapshot2";
341      admin.snapshot(table1Snapshot2, TABLE_NAME);
342      LOG.debug("Snapshot2 completed.");
343
344      String table2Snapshot1 = "Table2Snapshot1";
345      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
346      LOG.debug(table2Snapshot1 + " completed.");
347
348      List<SnapshotDescription> listTableSnapshots =
349          admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
350      List<String> listTableSnapshotNames = new ArrayList<>();
351      assertEquals(3, listTableSnapshots.size());
352      for (SnapshotDescription s : listTableSnapshots) {
353        listTableSnapshotNames.add(s.getName());
354      }
355      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
356      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
357      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
358    } finally {
359      if (admin != null) {
360        try {
361          admin.deleteSnapshots(Pattern.compile("Table.*"));
362        } catch (SnapshotDoesNotExistException ignore) {
363        }
364        if (admin.tableExists(tableName)) {
365          UTIL.deleteTable(tableName);
366        }
367        admin.close();
368      }
369    }
370  }
371
372  @Test
373  public void testListTableSnapshotsWithRegex() throws Exception {
374    Admin admin = null;
375    try {
376      admin = UTIL.getAdmin();
377
378      String table1Snapshot1 = "Table1Snapshot1";
379      admin.snapshot(table1Snapshot1, TABLE_NAME);
380      LOG.debug("Snapshot1 completed.");
381
382      String table1Snapshot2 = "Table1Snapshot2";
383      admin.snapshot(table1Snapshot2, TABLE_NAME);
384      LOG.debug("Snapshot2 completed.");
385
386      String table2Snapshot1 = "Table2Snapshot1";
387      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
388      LOG.debug(table2Snapshot1 + " completed.");
389
390      List<SnapshotDescription> listTableSnapshots =
391          admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
392      List<String> listTableSnapshotNames = new ArrayList<>();
393      assertEquals(2, listTableSnapshots.size());
394      for (SnapshotDescription s : listTableSnapshots) {
395        listTableSnapshotNames.add(s.getName());
396      }
397      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
398      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
399      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
400    } finally {
401      if (admin != null) {
402        try {
403          admin.deleteSnapshots(Pattern.compile("Table.*"));
404        } catch (SnapshotDoesNotExistException ignore) {
405        }
406        admin.close();
407      }
408    }
409  }
410
411  @Test
412  public void testDeleteTableSnapshots() throws Exception {
413    Admin admin = null;
414    final TableName tableName = TableName.valueOf(name.getMethodName());
415    try {
416      admin = UTIL.getAdmin();
417
418      HTableDescriptor htd = new HTableDescriptor(tableName);
419      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
420
421      String table1Snapshot1 = "Table1Snapshot1";
422      admin.snapshot(table1Snapshot1, TABLE_NAME);
423      LOG.debug("Snapshot1 completed.");
424
425      String table1Snapshot2 = "Table1Snapshot2";
426      admin.snapshot(table1Snapshot2, TABLE_NAME);
427      LOG.debug("Snapshot2 completed.");
428
429      String table2Snapshot1 = "Table2Snapshot1";
430      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
431      LOG.debug(table2Snapshot1 + " completed.");
432
433      Pattern tableNamePattern = Pattern.compile("test.*");
434      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
435      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
436    } finally {
437      if (admin != null) {
438        if (admin.tableExists(tableName)) {
439          UTIL.deleteTable(tableName);
440        }
441        admin.close();
442      }
443    }
444  }
445
446  @Test
447  public void testDeleteTableSnapshotsWithRegex() throws Exception {
448    Admin admin = null;
449    Pattern tableNamePattern = Pattern.compile("test.*");
450    try {
451      admin = UTIL.getAdmin();
452
453      String table1Snapshot1 = "Table1Snapshot1";
454      admin.snapshot(table1Snapshot1, TABLE_NAME);
455      LOG.debug("Snapshot1 completed.");
456
457      String table1Snapshot2 = "Table1Snapshot2";
458      admin.snapshot(table1Snapshot2, TABLE_NAME);
459      LOG.debug("Snapshot2 completed.");
460
461      String table2Snapshot1 = "Table2Snapshot1";
462      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
463      LOG.debug(table2Snapshot1 + " completed.");
464
465      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
466      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
467    } finally {
468      if (admin != null) {
469        try {
470          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
471        } catch (SnapshotDoesNotExistException ignore) {
472        }
473        admin.close();
474      }
475    }
476  }
477}