Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertThrows;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.HashMap;
027import java.util.List;
028import java.util.Map;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.LocatedFileStatus;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.fs.RemoteIterator;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.MetaTableAccessor;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.ResultScanner;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.client.SnapshotType;
043import org.apache.hadoop.hbase.client.Table;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
046import org.apache.hadoop.hbase.io.HFileLink;
047import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
048import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.mob.MobUtils;
051import org.apache.hadoop.hbase.monitoring.MonitoredTask;
052import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
053import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
054import org.apache.hadoop.hbase.regionserver.HRegion;
055import org.apache.hadoop.hbase.regionserver.HRegionServer;
056import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
057import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.testclassification.RegionServerTests;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.CommonFSUtils;
062import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
063import org.apache.hadoop.hbase.util.FSTableDescriptors;
064import org.apache.hadoop.hbase.wal.WALSplitUtil;
065import org.junit.jupiter.api.AfterAll;
066import org.junit.jupiter.api.AfterEach;
067import org.junit.jupiter.api.BeforeAll;
068import org.junit.jupiter.api.BeforeEach;
069import org.junit.jupiter.api.Tag;
070import org.junit.jupiter.api.Test;
071import org.mockito.Mockito;
072import org.slf4j.Logger;
073import org.slf4j.LoggerFactory;
074
075import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
076
077/**
078 * Test the restore/clone operation from a file-system point of view.
079 */
080@Tag(RegionServerTests.TAG)
081@Tag(MediumTests.TAG)
082public class TestRestoreSnapshotHelper {
083
084  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotHelper.class);
085
086  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
087  protected final static String TEST_HFILE = "abc";
088
089  protected Configuration conf;
090  protected Path archiveDir;
091  protected FileSystem fs;
092  protected Path rootDir;
093
094  protected void setupConf(Configuration conf) {
095  }
096
097  @BeforeAll
098  public static void setupCluster() throws Exception {
099    TEST_UTIL.getConfiguration().setInt(AssignmentManager.ASSIGN_MAX_ATTEMPTS, 3);
100    TEST_UTIL.startMiniCluster();
101  }
102
103  @AfterAll
104  public static void tearDownCluster() throws Exception {
105    TEST_UTIL.shutdownMiniCluster();
106  }
107
108  @BeforeEach
109  public void setup() throws Exception {
110    rootDir = TEST_UTIL.getDataTestDir("testRestore");
111    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
112    fs = TEST_UTIL.getTestFileSystem();
113    conf = TEST_UTIL.getConfiguration();
114    setupConf(conf);
115    CommonFSUtils.setRootDir(conf, rootDir);
116    // Turn off balancer so it doesn't cut in and mess up our placements.
117    TEST_UTIL.getAdmin().balancerSwitch(false, true);
118  }
119
120  @AfterEach
121  public void tearDown() throws Exception {
122    fs.delete(TEST_UTIL.getDataTestDir(), true);
123  }
124
125  protected SnapshotMock createSnapshotMock() throws IOException {
126    return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
127  }
128
129  @Test
130  public void testRestore() throws IOException {
131    restoreAndVerify("snapshot", "testRestore");
132  }
133
134  @Test
135  public void testRestoreWithNamespace() throws IOException {
136    restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
137  }
138
139  @Test
140  public void testNoHFileLinkInRootDir() throws IOException {
141    rootDir = TEST_UTIL.getDefaultRootDirPath();
142    CommonFSUtils.setRootDir(conf, rootDir);
143    fs = rootDir.getFileSystem(conf);
144
145    TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
146    String snapshotName = tableName.getNameAsString() + "-snapshot";
147    createTableAndSnapshot(tableName, snapshotName);
148
149    Path restoreDir = new Path("/hbase/.tmp-restore");
150    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
151    checkNoHFileLinkInTableDir(tableName);
152  }
153
154  @Test
155  public void testSkipReplayAndUpdateSeqId() throws Exception {
156    rootDir = TEST_UTIL.getDefaultRootDirPath();
157    CommonFSUtils.setRootDir(conf, rootDir);
158    TableName tableName = TableName.valueOf("testSkipReplayAndUpdateSeqId");
159    String snapshotName = "testSkipReplayAndUpdateSeqId";
160    createTableAndSnapshot(tableName, snapshotName);
161    // put some data in the table
162    Table table = TEST_UTIL.getConnection().getTable(tableName);
163    TEST_UTIL.loadTable(table, Bytes.toBytes("A"));
164
165    Configuration conf = TEST_UTIL.getConfiguration();
166    Path rootDir = CommonFSUtils.getRootDir(conf);
167    Path restoreDir = new Path("/hbase/.tmp-restore/testScannerWithRestoreScanner2");
168    // restore snapshot.
169    final RestoreSnapshotHelper.RestoreMetaChanges meta =
170      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
171    TableDescriptor htd = meta.getTableDescriptor();
172    final List<RegionInfo> restoredRegions = meta.getRegionsToAdd();
173    for (RegionInfo restoredRegion : restoredRegions) {
174      // open restored region
175      HRegion region = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
176        fs, conf, restoredRegion, htd, null, null);
177      // set restore flag
178      region.setRestoredRegion(true);
179      region.initialize();
180      Path recoveredEdit =
181        CommonFSUtils.getWALRegionDir(conf, tableName, region.getRegionInfo().getEncodedName());
182      long maxSeqId = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
183
184      // open restored region without set restored flag
185      HRegion region2 = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
186        fs, conf, restoredRegion, htd, null, null);
187      region2.initialize();
188      long maxSeqId2 = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
189      assertTrue(maxSeqId2 > maxSeqId);
190    }
191  }
192
193  @Test
194  public void testCopyExpiredSnapshotForScanner() throws IOException, InterruptedException {
195    rootDir = TEST_UTIL.getDefaultRootDirPath();
196    CommonFSUtils.setRootDir(conf, rootDir);
197    TableName tableName = TableName.valueOf("testCopyExpiredSnapshotForScanner");
198    String snapshotName = tableName.getNameAsString() + "-snapshot";
199    Path restoreDir = new Path("/hbase/.tmp-expired-snapshot/copySnapshotDest");
200    // create table and put some data into the table
201    byte[] columnFamily = Bytes.toBytes("A");
202    Table table = TEST_UTIL.createTable(tableName, columnFamily);
203    TEST_UTIL.loadTable(table, columnFamily);
204    // create snapshot with ttl = 10 sec
205    Map<String, Object> properties = new HashMap<>();
206    properties.put("TTL", 10);
207    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDesc =
208      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
209        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
210    TEST_UTIL.getAdmin().snapshot(snapshotDesc);
211    boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
212      .anyMatch(ele -> snapshotName.equals(ele.getName()));
213    assertTrue(isExist);
214    int retry = 6;
215    while (
216      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
217        snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
218    ) {
219      retry--;
220      Thread.sleep(10 * 1000);
221    }
222    boolean isExpiredSnapshot = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
223      snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
224    assertTrue(isExpiredSnapshot);
225    assertThrows(SnapshotTTLExpiredException.class, () -> RestoreSnapshotHelper
226      .copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName));
227  }
228
229  /**
230   * Test scenario for HBASE-29346, which addresses the issue where restoring snapshots after region
231   * merge operations could lead to missing store file references, potentially resulting in data
232   * loss.
233   * <p>
234   * This test performs the following steps:
235   * </p>
236   * <ol>
237   * <li>Creates a table with multiple regions.</li>
238   * <li>Inserts data into each region and flushes to create store files.</li>
239   * <li>Takes snapshot of the table and performs restore.</li>
240   * <li>Disable compactions, merge regions, create a new snapshot, and restore that snapshot on the
241   * same restore path.</li>
242   * <li>Verifies data integrity by scanning all data post region re-open.</li>
243   * </ol>
244   */
245  @Test
246  public void testMultiSnapshotRestoreWithMerge() throws IOException, InterruptedException {
247    rootDir = TEST_UTIL.getDefaultRootDirPath();
248    CommonFSUtils.setRootDir(conf, rootDir);
249    TableName tableName = TableName.valueOf("testMultiSnapshotRestoreWithMerge");
250    Path restoreDir = new Path("/hbase/.tmp-snapshot/restore-snapshot-dest");
251
252    byte[] columnFamily = Bytes.toBytes("A");
253    Table table = TEST_UTIL.createTable(tableName, new byte[][] { columnFamily },
254      new byte[][] { new byte[] { 'b' }, new byte[] { 'd' } });
255    Put put1 = new Put(Bytes.toBytes("a")); // Region 1: [-∞, b)
256    put1.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val1"));
257    table.put(put1);
258    Put put2 = new Put(Bytes.toBytes("b")); // Region 2: [b, d)
259    put2.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val2"));
260    table.put(put2);
261    Put put3 = new Put(Bytes.toBytes("d")); // Region 3: [d, +∞)
262    put3.addColumn(columnFamily, Bytes.toBytes("q"), Bytes.toBytes("val3"));
263    table.put(put3);
264
265    TEST_UTIL.getAdmin().flush(tableName);
266
267    String snapshotOne = tableName.getNameAsString() + "-snapshot-one";
268    createAndAssertSnapshot(tableName, snapshotOne);
269    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotOne);
270    flipCompactions(false);
271    mergeRegions(tableName, 2);
272    String snapshotTwo = tableName.getNameAsString() + "-snapshot-two";
273    createAndAssertSnapshot(tableName, snapshotTwo);
274    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotTwo);
275    flipCompactions(true);
276
277    TEST_UTIL.getAdmin().disableTable(tableName);
278    TEST_UTIL.getAdmin().enableTable(tableName);
279    try (ResultScanner scanner = table.getScanner(new Scan())) {
280      assertEquals(3, scanner.next(4).length);
281    }
282    String snapshotThree = tableName.getNameAsString() + "-snapshot-three";
283    createAndAssertSnapshot(tableName, snapshotThree);
284  }
285
286  private void createAndAssertSnapshot(TableName tableName, String snapshotName)
287    throws SnapshotCreationException, IllegalArgumentException, IOException {
288    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescOne =
289      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
290        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1);
291    TEST_UTIL.getAdmin().snapshot(snapshotDescOne);
292    boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
293      .anyMatch(ele -> snapshotName.equals(ele.getName()));
294    assertTrue(isExist);
295
296  }
297
298  private void flipCompactions(boolean isEnable) {
299    int numLiveRegionServers = TEST_UTIL.getHBaseCluster().getNumLiveRegionServers();
300    for (int serverNumber = 0; serverNumber < numLiveRegionServers; serverNumber++) {
301      HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(serverNumber);
302      regionServer.getCompactSplitThread().setCompactionsEnabled(isEnable);
303    }
304
305  }
306
307  private void mergeRegions(TableName tableName, int mergeCount) throws IOException {
308    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
309    int originalRegionCount = ris.size();
310    assertTrue(originalRegionCount > mergeCount);
311    RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo[] {});
312    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
313    MergeTableRegionsProcedure proc =
314      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
315    long procId = procExec.submitProcedure(proc);
316    ProcedureTestingUtility.waitProcedure(procExec, procId);
317    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
318    MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
319    assertEquals(originalRegionCount - mergeCount + 1,
320      MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size());
321  }
322
323  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
324    return TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
325  }
326
327  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
328    throws IOException {
329    byte[] column = Bytes.toBytes("A");
330    Table table = TEST_UTIL.createTable(tableName, column, 2);
331    TEST_UTIL.loadTable(table, column);
332    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
333  }
334
335  private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
336    Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
337      CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
338      CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
339    for (Path tableDir : tableDirs) {
340      assertFalse(hasHFileLink(tableDir));
341    }
342  }
343
344  private boolean hasHFileLink(Path tableDir) throws IOException {
345    if (fs.exists(tableDir)) {
346      RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
347      while (iterator.hasNext()) {
348        LocatedFileStatus fileStatus = iterator.next();
349        if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
350          return true;
351        }
352      }
353    }
354    return false;
355  }
356
357  private void restoreAndVerify(final String snapshotName, final String tableName)
358    throws IOException {
359    // Test Rolling-Upgrade like Snapshot.
360    // half machines writing using v1 and the others using v2 format.
361    SnapshotMock snapshotMock = createSnapshotMock();
362    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName);
363    builder.addRegionV1();
364    builder.addRegionV2();
365    builder.addRegionV2();
366    builder.addRegionV1();
367    Path snapshotDir = builder.commit();
368    TableDescriptor htd = builder.getTableDescriptor();
369    SnapshotDescription desc = builder.getSnapshotDescription();
370
371    // Test clone a snapshot
372    TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
373    testRestore(snapshotDir, desc, htdClone);
374    verifyRestore(rootDir, htd, htdClone);
375
376    // Test clone a clone ("link to link")
377    SnapshotDescription cloneDesc =
378      SnapshotDescription.newBuilder().setName("cloneSnapshot").setTable("testtb-clone").build();
379    Path cloneDir = CommonFSUtils.getTableDir(rootDir, htdClone.getTableName());
380    TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
381    testRestore(cloneDir, cloneDesc, htdClone2);
382    verifyRestore(rootDir, htd, htdClone2);
383  }
384
385  private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd,
386    final TableDescriptor htdClone) throws IOException {
387    List<String> files = SnapshotTestingUtils.listHFileNames(fs,
388      CommonFSUtils.getTableDir(rootDir, htdClone.getTableName()));
389    assertEquals(12, files.size());
390    for (int i = 0; i < files.size(); i += 2) {
391      String linkFile = files.get(i);
392      String refFile = files.get(i + 1);
393      assertTrue(HFileLink.isHFileLink(linkFile), linkFile + " should be a HFileLink");
394      assertTrue(StoreFileInfo.isReference(refFile), refFile + " should be a Reference");
395      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
396      Path refPath = getReferredToFile(refFile);
397      LOG.debug("get reference name for file " + refFile + " = " + refPath);
398      assertTrue(HFileLink.isHFileLink(refPath.getName()),
399        refPath.getName() + " should be a HFileLink");
400      assertEquals(linkFile, refPath.getName());
401    }
402  }
403
404  /**
405   * Execute the restore operation
406   * @param snapshotDir The snapshot directory to use as "restore source"
407   * @param sd          The snapshot descriptor
408   * @param htdClone    The HTableDescriptor of the table to restore/clone.
409   */
410  private void testRestore(final Path snapshotDir, final SnapshotDescription sd,
411    final TableDescriptor htdClone) throws IOException {
412    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
413    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
414
415    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
416    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
417    helper.restoreHdfsRegions();
418
419    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
420    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
421  }
422
423  /**
424   * Initialize the restore helper, based on the snapshot and table information provided.
425   */
426  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
427    final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException {
428    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
429    MonitoredTask status = Mockito.mock(MonitoredTask.class);
430
431    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
432    return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status);
433  }
434
435  private Path getReferredToFile(final String referenceName) {
436    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
437    return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
438  }
439}