001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.regex.Pattern;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.TableNotFoundException;
036import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
037import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
038import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
039import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
040import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
041import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
042import org.apache.hadoop.hbase.testclassification.ClientTests;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
059
060import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
061
062/**
063 * Test create/using/deleting snapshots from the client
064 * <p>
065 * This is an end-to-end test for the snapshot utility
066 */
067@Category({LargeTests.class, ClientTests.class})
068public class TestSnapshotFromClient {
069
070  @ClassRule
071  public static final HBaseClassTestRule CLASS_RULE =
072      HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
073
074  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
075
076  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
077  protected static final int NUM_RS = 2;
078  protected static final String STRING_TABLE_NAME = "test";
079  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
080  protected static final TableName TABLE_NAME =
081      TableName.valueOf(STRING_TABLE_NAME);
082  private static final Pattern MATCH_ALL = Pattern.compile(".*");
083
084  @Rule
085  public TestName name = new TestName();
086
087  /**
088   * Setup the config for the cluster
089   * @throws Exception on failure
090   */
091  @BeforeClass
092  public static void setupCluster() throws Exception {
093    setupConf(UTIL.getConfiguration());
094    UTIL.startMiniCluster(NUM_RS);
095  }
096
097  protected static void setupConf(Configuration conf) {
098    // disable the ui
099    conf.setInt("hbase.regionsever.info.port", -1);
100    // change the flush size to a small amount, regulating number of store files
101    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
102    // so make sure we get a compaction when doing a load, but keep around some
103    // files in the store
104    conf.setInt("hbase.hstore.compaction.min", 10);
105    conf.setInt("hbase.hstore.compactionThreshold", 10);
106    // block writes if we get to 12 store files
107    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
108    // Enable snapshot
109    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
110    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
111        ConstantSizeRegionSplitPolicy.class.getName());
112
113  }
114
115  @Before
116  public void setup() throws Exception {
117    createTable();
118  }
119
120  protected void createTable() throws Exception {
121    TableDescriptor htd =
122      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()).build();
123    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
124  }
125
126  protected int getNumReplicas() {
127    return 1;
128  }
129
130  @After
131  public void tearDown() throws Exception {
132    UTIL.deleteTable(TABLE_NAME);
133    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
134    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
135  }
136
137  @AfterClass
138  public static void cleanupTest() throws Exception {
139    try {
140      UTIL.shutdownMiniCluster();
141    } catch (Exception e) {
142      LOG.warn("failure shutting down cluster", e);
143    }
144  }
145
146  /**
147   * Test snapshotting not allowed hbase:meta and -ROOT-
148   */
149  @Test(expected = IllegalArgumentException.class)
150  public void testMetaTablesSnapshot() throws Exception {
151    UTIL.getAdmin().snapshot("metaSnapshot", TableName.META_TABLE_NAME);
152  }
153
154  /**
155   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
156   *
157   * @throws Exception
158   */
159  @Test
160  public void testSnapshotDeletionWithRegex() throws Exception {
161    Admin admin = UTIL.getAdmin();
162    // make sure we don't fail on listing snapshots
163    SnapshotTestingUtils.assertNoSnapshots(admin);
164
165    // put some stuff in the table
166    Table table = UTIL.getConnection().getTable(TABLE_NAME);
167    UTIL.loadTable(table, TEST_FAM);
168    table.close();
169
170    String snapshot1 = "TableSnapshot1";
171    admin.snapshot(snapshot1, TABLE_NAME);
172    LOG.debug("Snapshot1 completed.");
173
174    String snapshot2 = "TableSnapshot2";
175    admin.snapshot(snapshot2, TABLE_NAME);
176    LOG.debug("Snapshot2 completed.");
177
178    String snapshot3 = "3rdTableSnapshot";
179    admin.snapshot(snapshot3, TABLE_NAME);
180    LOG.debug(snapshot3 + " completed.");
181
182    // delete the first two snapshots
183    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
184    List<SnapshotDescription> snapshots = admin.listSnapshots();
185    assertEquals(1, snapshots.size());
186    assertEquals(snapshot3, snapshots.get(0).getName());
187
188    admin.deleteSnapshot(snapshot3);
189    admin.close();
190  }
191  /**
192   * Test snapshotting a table that is offline
193   * @throws Exception
194   */
195  @Test
196  public void testOfflineTableSnapshot() throws Exception {
197    Admin admin = UTIL.getAdmin();
198    // make sure we don't fail on listing snapshots
199    SnapshotTestingUtils.assertNoSnapshots(admin);
200
201    // put some stuff in the table
202    Table table = UTIL.getConnection().getTable(TABLE_NAME);
203    UTIL.loadTable(table, TEST_FAM, false);
204
205    LOG.debug("FS state before disable:");
206    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
207      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
208    // XXX if this is flakey, might want to consider using the async version and looping as
209    // disableTable can succeed and still timeout.
210    admin.disableTable(TABLE_NAME);
211
212    LOG.debug("FS state before snapshot:");
213    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
214      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
215
216    // take a snapshot of the disabled table
217    final String SNAPSHOT_NAME = "offlineTableSnapshot";
218    String snapshot = SNAPSHOT_NAME;
219
220    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME,
221        SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
222    LOG.debug("Snapshot completed.");
223
224    // make sure we have the snapshot
225    List<SnapshotDescription> snapshots =
226        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
227
228    // make sure its a valid snapshot
229    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
230    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
231    LOG.debug("FS state after snapshot:");
232    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
233      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
234    SnapshotTestingUtils.confirmSnapshotValid(
235      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
236      rootDir, admin, fs);
237
238    admin.deleteSnapshot(snapshot);
239    snapshots = admin.listSnapshots();
240    SnapshotTestingUtils.assertNoSnapshots(admin);
241  }
242
243  @Test
244  public void testSnapshotFailsOnNonExistantTable() throws Exception {
245    Admin admin = UTIL.getAdmin();
246    // make sure we don't fail on listing snapshots
247    SnapshotTestingUtils.assertNoSnapshots(admin);
248    String tableName = "_not_a_table";
249
250    // make sure the table doesn't exist
251    boolean fail = false;
252    do {
253      try {
254        admin.getDescriptor(TableName.valueOf(tableName));
255        fail = true;
256        LOG.error("Table:" + tableName + " already exists, checking a new name");
257        tableName = tableName + "!";
258      } catch (TableNotFoundException e) {
259        fail = false;
260      }
261    } while (fail);
262
263    // snapshot the non-existant table
264    try {
265      admin.snapshot("fail", TableName.valueOf(tableName));
266      fail("Snapshot succeeded even though there is not table.");
267    } catch (SnapshotCreationException e) {
268      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
269    }
270  }
271
272  @Test
273  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
274    // test with an empty table with one region
275
276    Admin admin = UTIL.getAdmin();
277    // make sure we don't fail on listing snapshots
278    SnapshotTestingUtils.assertNoSnapshots(admin);
279
280    LOG.debug("FS state before disable:");
281    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
282      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
283    admin.disableTable(TABLE_NAME);
284
285    LOG.debug("FS state before snapshot:");
286    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
287      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
288
289    // take a snapshot of the disabled table
290    String snapshot = "testOfflineTableSnapshotWithEmptyRegions";
291    admin.snapshot(snapshot, TABLE_NAME);
292    LOG.debug("Snapshot completed.");
293
294    // make sure we have the snapshot
295    List<SnapshotDescription> snapshots =
296        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
297
298    // make sure its a valid snapshot
299    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
300    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
301    LOG.debug("FS state after snapshot:");
302    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
303      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
304
305    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
306    List<byte[]> nonEmptyCfs = Lists.newArrayList();
307    SnapshotTestingUtils.confirmSnapshotValid(
308      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
309      emptyCfs, rootDir, admin, fs);
310
311    admin.deleteSnapshot(snapshot);
312    snapshots = admin.listSnapshots();
313    SnapshotTestingUtils.assertNoSnapshots(admin);
314  }
315
316  @Test
317  public void testListTableSnapshots() throws Exception {
318    Admin admin = null;
319    final TableName tableName = TableName.valueOf(name.getMethodName());
320    try {
321      admin = UTIL.getAdmin();
322
323      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
324      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
325
326      String table1Snapshot1 = "Table1Snapshot1";
327      admin.snapshot(table1Snapshot1, TABLE_NAME);
328      LOG.debug("Snapshot1 completed.");
329
330      String table1Snapshot2 = "Table1Snapshot2";
331      admin.snapshot(table1Snapshot2, TABLE_NAME);
332      LOG.debug("Snapshot2 completed.");
333
334      String table2Snapshot1 = "Table2Snapshot1";
335      admin.snapshot(table2Snapshot1, tableName);
336      LOG.debug(table2Snapshot1 + " completed.");
337
338      List<SnapshotDescription> listTableSnapshots =
339          admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
340      List<String> listTableSnapshotNames = new ArrayList<>();
341      assertEquals(3, listTableSnapshots.size());
342      for (SnapshotDescription s : listTableSnapshots) {
343        listTableSnapshotNames.add(s.getName());
344      }
345      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
346      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
347      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
348    } finally {
349      if (admin != null) {
350        try {
351          admin.deleteSnapshots(Pattern.compile("Table.*"));
352        } catch (SnapshotDoesNotExistException ignore) {
353        }
354        if (admin.tableExists(tableName)) {
355          UTIL.deleteTable(tableName);
356        }
357        admin.close();
358      }
359    }
360  }
361
362  @Test
363  public void testListTableSnapshotsWithRegex() throws Exception {
364    Admin admin = null;
365    try {
366      admin = UTIL.getAdmin();
367
368      String table1Snapshot1 = "Table1Snapshot1";
369      admin.snapshot(table1Snapshot1, TABLE_NAME);
370      LOG.debug("Snapshot1 completed.");
371
372      String table1Snapshot2 = "Table1Snapshot2";
373      admin.snapshot(table1Snapshot2, TABLE_NAME);
374      LOG.debug("Snapshot2 completed.");
375
376      String table2Snapshot1 = "Table2Snapshot1";
377      admin.snapshot(table2Snapshot1, TABLE_NAME);
378      LOG.debug(table2Snapshot1 + " completed.");
379
380      List<SnapshotDescription> listTableSnapshots =
381          admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
382      List<String> listTableSnapshotNames = new ArrayList<>();
383      assertEquals(2, listTableSnapshots.size());
384      for (SnapshotDescription s : listTableSnapshots) {
385        listTableSnapshotNames.add(s.getName());
386      }
387      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
388      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
389      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
390    } finally {
391      if (admin != null) {
392        try {
393          admin.deleteSnapshots(Pattern.compile("Table.*"));
394        } catch (SnapshotDoesNotExistException ignore) {
395        }
396        admin.close();
397      }
398    }
399  }
400
401  @Test
402  public void testDeleteTableSnapshots() throws Exception {
403    Admin admin = null;
404    final TableName tableName = TableName.valueOf(name.getMethodName());
405    try {
406      admin = UTIL.getAdmin();
407
408      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
409      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
410
411      String table1Snapshot1 = "Table1Snapshot1";
412      admin.snapshot(table1Snapshot1, TABLE_NAME);
413      LOG.debug("Snapshot1 completed.");
414
415      String table1Snapshot2 = "Table1Snapshot2";
416      admin.snapshot(table1Snapshot2, TABLE_NAME);
417      LOG.debug("Snapshot2 completed.");
418
419      String table2Snapshot1 = "Table2Snapshot1";
420      admin.snapshot(table2Snapshot1, tableName);
421      LOG.debug(table2Snapshot1 + " completed.");
422
423      Pattern tableNamePattern = Pattern.compile("test.*");
424      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
425      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
426    } finally {
427      if (admin != null) {
428        if (admin.tableExists(tableName)) {
429          UTIL.deleteTable(tableName);
430        }
431        admin.close();
432      }
433    }
434  }
435
436  @Test
437  public void testDeleteTableSnapshotsWithRegex() throws Exception {
438    Admin admin = null;
439    Pattern tableNamePattern = Pattern.compile("test.*");
440    try {
441      admin = UTIL.getAdmin();
442
443      String table1Snapshot1 = "Table1Snapshot1";
444      admin.snapshot(table1Snapshot1, TABLE_NAME);
445      LOG.debug("Snapshot1 completed.");
446
447      String table1Snapshot2 = "Table1Snapshot2";
448      admin.snapshot(table1Snapshot2, TABLE_NAME);
449      LOG.debug("Snapshot2 completed.");
450
451      String table2Snapshot1 = "Table2Snapshot1";
452      admin.snapshot(table2Snapshot1, TABLE_NAME);
453      LOG.debug(table2Snapshot1 + " completed.");
454
455      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
456      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
457    } finally {
458      if (admin != null) {
459        try {
460          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
461        } catch (SnapshotDoesNotExistException ignore) {
462        }
463        admin.close();
464      }
465    }
466  }
467}