Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertThrows;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.junit.jupiter.api.Assertions.fail;
025
026import java.util.ArrayList;
027import java.util.List;
028import java.util.regex.Pattern;
029import java.util.stream.Stream;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.TableNameTestExtension;
037import org.apache.hadoop.hbase.TableNotFoundException;
038import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
039import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
040import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
041import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
042import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
043import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
044import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.CommonFSUtils;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.TestTemplate;
051import org.junit.jupiter.api.extension.RegisterExtension;
052import org.junit.jupiter.params.provider.Arguments;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
057
058import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
059
060/**
061 * Test create/using/deleting snapshots from the client
062 * <p>
063 * This is an end-to-end test for the snapshot utility
064 */
065public abstract class SnapshotFromClientTestBase {
066
067  private static final Logger LOG = LoggerFactory.getLogger(SnapshotFromClientTestBase.class);
068
069  protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
070  protected static final int NUM_RS = 2;
071  protected static final String STRING_TABLE_NAME = "test";
072  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
073  protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
074  private static final Pattern MATCH_ALL = Pattern.compile(".*");
075
076  @RegisterExtension
077  protected TableNameTestExtension name = new TableNameTestExtension();
078
079  protected final StoreFileTrackerFactory.Trackers trackerImpl;
080
081  public SnapshotFromClientTestBase(StoreFileTrackerFactory.Trackers trackerImpl) {
082    this.trackerImpl = trackerImpl;
083  }
084
085  public static Stream<Arguments> parameters() {
086    return Stream.of(Arguments.of(StoreFileTrackerFactory.Trackers.DEFAULT),
087      Arguments.of(StoreFileTrackerFactory.Trackers.FILE));
088  }
089
090  protected static void setupConf(Configuration conf) {
091    // disable the ui
092    conf.setInt("hbase.regionsever.info.port", -1);
093    // change the flush size to a small amount, regulating number of store files
094    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
095    // so make sure we get a compaction when doing a load, but keep around some
096    // files in the store
097    conf.setInt("hbase.hstore.compaction.min", 10);
098    conf.setInt("hbase.hstore.compactionThreshold", 10);
099    // block writes if we get to 12 store files
100    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
101    // Enable snapshot
102    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
103    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
104      ConstantSizeRegionSplitPolicy.class.getName());
105  }
106
107  @BeforeEach
108  public void setup() throws Exception {
109    createTable();
110  }
111
112  protected void createTable() throws Exception {
113    TableDescriptor htd =
114      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
115        .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
116    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
117  }
118
119  protected int getNumReplicas() {
120    return 1;
121  }
122
123  @AfterEach
124  public void tearDown() throws Exception {
125    UTIL.deleteTable(TABLE_NAME);
126    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
127    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
128  }
129
130  @AfterAll
131  public static void cleanupTest() throws Exception {
132    try {
133      UTIL.shutdownMiniCluster();
134    } catch (Exception e) {
135      LOG.warn("failure shutting down cluster", e);
136    }
137  }
138
139  /**
140   * Test snapshotting not allowed hbase:meta and -ROOT-
141   */
142  @TestTemplate
143  public void testMetaTablesSnapshot() throws Exception {
144    assertThrows(IllegalArgumentException.class,
145      () -> UTIL.getAdmin().snapshot("metaSnapshot", TableName.META_TABLE_NAME));
146  }
147
148  /**
149   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
150   */
151  @TestTemplate
152  public void testSnapshotDeletionWithRegex() throws Exception {
153    Admin admin = UTIL.getAdmin();
154    // make sure we don't fail on listing snapshots
155    SnapshotTestingUtils.assertNoSnapshots(admin);
156
157    // put some stuff in the table
158    Table table = UTIL.getConnection().getTable(TABLE_NAME);
159    UTIL.loadTable(table, TEST_FAM);
160    table.close();
161
162    String snapshot1 = "TableSnapshot1";
163    admin.snapshot(snapshot1, TABLE_NAME);
164    LOG.debug("Snapshot1 completed.");
165
166    String snapshot2 = "TableSnapshot2";
167    admin.snapshot(snapshot2, TABLE_NAME);
168    LOG.debug("Snapshot2 completed.");
169
170    String snapshot3 = "3rdTableSnapshot";
171    admin.snapshot(snapshot3, TABLE_NAME);
172    LOG.debug(snapshot3 + " completed.");
173
174    // delete the first two snapshots
175    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
176    List<SnapshotDescription> snapshots = admin.listSnapshots();
177    assertEquals(1, snapshots.size());
178    assertEquals(snapshot3, snapshots.get(0).getName());
179
180    admin.deleteSnapshot(snapshot3);
181    admin.close();
182  }
183
184  /**
185   * Test snapshotting a table that is offline
186   */
187  @TestTemplate
188  public void testOfflineTableSnapshot() throws Exception {
189    Admin admin = UTIL.getAdmin();
190    // make sure we don't fail on listing snapshots
191    SnapshotTestingUtils.assertNoSnapshots(admin);
192
193    // put some stuff in the table
194    Table table = UTIL.getConnection().getTable(TABLE_NAME);
195    UTIL.loadTable(table, TEST_FAM, false);
196
197    LOG.debug("FS state before disable:");
198    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
199      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
200    // XXX if this is flakey, might want to consider using the async version and looping as
201    // disableTable can succeed and still timeout.
202    admin.disableTable(TABLE_NAME);
203
204    LOG.debug("FS state before snapshot:");
205    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
206      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
207
208    // take a snapshot of the disabled table
209    final String SNAPSHOT_NAME = "offlineTableSnapshot";
210    String snapshot = SNAPSHOT_NAME;
211
212    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.DISABLED, null,
213      -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
214    LOG.debug("Snapshot completed.");
215
216    // make sure we have the snapshot
217    List<SnapshotDescription> snapshots =
218      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
219
220    // make sure its a valid snapshot
221    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
222    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
223    LOG.debug("FS state after snapshot:");
224    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
225      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
226    SnapshotTestingUtils.confirmSnapshotValid(
227      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, rootDir,
228      admin, fs);
229
230    admin.deleteSnapshot(snapshot);
231    snapshots = admin.listSnapshots();
232    SnapshotTestingUtils.assertNoSnapshots(admin);
233  }
234
235  @TestTemplate
236  public void testSnapshotFailsOnNonExistantTable() throws Exception {
237    Admin admin = UTIL.getAdmin();
238    // make sure we don't fail on listing snapshots
239    SnapshotTestingUtils.assertNoSnapshots(admin);
240    String tableName = "_not_a_table";
241
242    // make sure the table doesn't exist
243    boolean fail = false;
244    do {
245      try {
246        admin.getDescriptor(TableName.valueOf(tableName));
247        fail = true;
248        LOG.error("Table:" + tableName + " already exists, checking a new name");
249        tableName = tableName + "!";
250      } catch (TableNotFoundException e) {
251        fail = false;
252      }
253    } while (fail);
254
255    // snapshot the non-existant table
256    try {
257      admin.snapshot("fail", TableName.valueOf(tableName));
258      fail("Snapshot succeeded even though there is not table.");
259    } catch (SnapshotCreationException e) {
260      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
261    }
262  }
263
264  @TestTemplate
265  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
266    // test with an empty table with one region
267
268    Admin admin = UTIL.getAdmin();
269    // make sure we don't fail on listing snapshots
270    SnapshotTestingUtils.assertNoSnapshots(admin);
271
272    LOG.debug("FS state before disable:");
273    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
274      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
275    admin.disableTable(TABLE_NAME);
276
277    LOG.debug("FS state before snapshot:");
278    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
279      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
280
281    // take a snapshot of the disabled table
282    String snapshot = "testOfflineTableSnapshotWithEmptyRegions";
283    admin.snapshot(snapshot, TABLE_NAME);
284    LOG.debug("Snapshot completed.");
285
286    // make sure we have the snapshot
287    List<SnapshotDescription> snapshots =
288      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
289
290    // make sure its a valid snapshot
291    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
292    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
293    LOG.debug("FS state after snapshot:");
294    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
295      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
296
297    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
298    List<byte[]> nonEmptyCfs = Lists.newArrayList();
299    SnapshotTestingUtils.confirmSnapshotValid(
300      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
301      emptyCfs, rootDir, admin, fs);
302
303    admin.deleteSnapshot(snapshot);
304    snapshots = admin.listSnapshots();
305    SnapshotTestingUtils.assertNoSnapshots(admin);
306  }
307
308  @TestTemplate
309  public void testListTableSnapshots() throws Exception {
310    Admin admin = null;
311    final TableName tableName = name.getTableName();
312    try {
313      admin = UTIL.getAdmin();
314
315      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
316      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
317
318      String table1Snapshot1 = "Table1Snapshot1";
319      admin.snapshot(table1Snapshot1, TABLE_NAME);
320      LOG.debug("Snapshot1 completed.");
321
322      String table1Snapshot2 = "Table1Snapshot2";
323      admin.snapshot(table1Snapshot2, TABLE_NAME);
324      LOG.debug("Snapshot2 completed.");
325
326      String table2Snapshot1 = "Table2Snapshot1";
327      admin.snapshot(table2Snapshot1, tableName);
328      LOG.debug(table2Snapshot1 + " completed.");
329
330      List<SnapshotDescription> listTableSnapshots =
331        admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
332      List<String> listTableSnapshotNames = new ArrayList<>();
333      assertEquals(3, listTableSnapshots.size());
334      for (SnapshotDescription s : listTableSnapshots) {
335        listTableSnapshotNames.add(s.getName());
336      }
337      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
338      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
339      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
340    } finally {
341      if (admin != null) {
342        try {
343          admin.deleteSnapshots(Pattern.compile("Table.*"));
344        } catch (SnapshotDoesNotExistException ignore) {
345        }
346        if (admin.tableExists(tableName)) {
347          UTIL.deleteTable(tableName);
348        }
349        admin.close();
350      }
351    }
352  }
353
354  @TestTemplate
355  public void testListTableSnapshotsWithRegex() throws Exception {
356    Admin admin = null;
357    try {
358      admin = UTIL.getAdmin();
359
360      String table1Snapshot1 = "Table1Snapshot1";
361      admin.snapshot(table1Snapshot1, TABLE_NAME);
362      LOG.debug("Snapshot1 completed.");
363
364      String table1Snapshot2 = "Table1Snapshot2";
365      admin.snapshot(table1Snapshot2, TABLE_NAME);
366      LOG.debug("Snapshot2 completed.");
367
368      String table2Snapshot1 = "Table2Snapshot1";
369      admin.snapshot(table2Snapshot1, TABLE_NAME);
370      LOG.debug(table2Snapshot1 + " completed.");
371
372      List<SnapshotDescription> listTableSnapshots =
373        admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
374      List<String> listTableSnapshotNames = new ArrayList<>();
375      assertEquals(2, listTableSnapshots.size());
376      for (SnapshotDescription s : listTableSnapshots) {
377        listTableSnapshotNames.add(s.getName());
378      }
379      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
380      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
381      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
382    } finally {
383      if (admin != null) {
384        try {
385          admin.deleteSnapshots(Pattern.compile("Table.*"));
386        } catch (SnapshotDoesNotExistException ignore) {
387        }
388        admin.close();
389      }
390    }
391  }
392
393  @TestTemplate
394  public void testDeleteTableSnapshots() throws Exception {
395    Admin admin = null;
396    final TableName tableName = name.getTableName();
397    try {
398      admin = UTIL.getAdmin();
399
400      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
401      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
402
403      String table1Snapshot1 = "Table1Snapshot1";
404      admin.snapshot(table1Snapshot1, TABLE_NAME);
405      LOG.debug("Snapshot1 completed.");
406
407      String table1Snapshot2 = "Table1Snapshot2";
408      admin.snapshot(table1Snapshot2, TABLE_NAME);
409      LOG.debug("Snapshot2 completed.");
410
411      String table2Snapshot1 = "Table2Snapshot1";
412      admin.snapshot(table2Snapshot1, tableName);
413      LOG.debug(table2Snapshot1 + " completed.");
414
415      Pattern tableNamePattern = Pattern.compile("test.*");
416      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
417      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
418    } finally {
419      if (admin != null) {
420        if (admin.tableExists(tableName)) {
421          UTIL.deleteTable(tableName);
422        }
423        admin.close();
424      }
425    }
426  }
427
428  @TestTemplate
429  public void testDeleteTableSnapshotsWithRegex() throws Exception {
430    Admin admin = null;
431    Pattern tableNamePattern = Pattern.compile("test.*");
432    try {
433      admin = UTIL.getAdmin();
434
435      String table1Snapshot1 = "Table1Snapshot1";
436      admin.snapshot(table1Snapshot1, TABLE_NAME);
437      LOG.debug("Snapshot1 completed.");
438
439      String table1Snapshot2 = "Table1Snapshot2";
440      admin.snapshot(table1Snapshot2, TABLE_NAME);
441      LOG.debug("Snapshot2 completed.");
442
443      String table2Snapshot1 = "Table2Snapshot1";
444      admin.snapshot(table2Snapshot1, TABLE_NAME);
445      LOG.debug(table2Snapshot1 + " completed.");
446
447      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
448      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
449    } finally {
450      if (admin != null) {
451        try {
452          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
453        } catch (SnapshotDoesNotExistException ignore) {
454        }
455        admin.close();
456      }
457    }
458  }
459}