001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.List;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.LocatedFileStatus;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.fs.RemoteIterator;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
037import org.apache.hadoop.hbase.io.HFileLink;
038import org.apache.hadoop.hbase.mob.MobUtils;
039import org.apache.hadoop.hbase.monitoring.MonitoredTask;
040import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
041import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.RegionServerTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.apache.hadoop.hbase.util.FSTableDescriptors;
047import org.apache.hadoop.hbase.util.FSUtils;
048import org.junit.After;
049import org.junit.AfterClass;
050import org.junit.Assert;
051import org.junit.Before;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.mockito.Mockito;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
061
062/**
063 * Test the restore/clone operation from a file-system point of view.
064 */
065@Category({RegionServerTests.class, MediumTests.class})
066public class TestRestoreSnapshotHelper {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070      HBaseClassTestRule.forClass(TestRestoreSnapshotHelper.class);
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotHelper.class);
073
074  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
075  protected final static String TEST_HFILE = "abc";
076
077  protected Configuration conf;
078  protected Path archiveDir;
079  protected FileSystem fs;
080  protected Path rootDir;
081
082  protected void setupConf(Configuration conf) {
083  }
084
085  @BeforeClass
086  public static void setupCluster() throws Exception {
087    TEST_UTIL.startMiniCluster();
088  }
089
090  @AfterClass
091  public static void tearDownCluster() throws Exception {
092    TEST_UTIL.shutdownMiniCluster();
093  }
094
095  @Before
096  public void setup() throws Exception {
097    rootDir = TEST_UTIL.getDataTestDir("testRestore");
098    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
099    fs = TEST_UTIL.getTestFileSystem();
100    conf = TEST_UTIL.getConfiguration();
101    setupConf(conf);
102    FSUtils.setRootDir(conf, rootDir);
103  }
104
105  @After
106  public void tearDown() throws Exception {
107    fs.delete(TEST_UTIL.getDataTestDir(), true);
108  }
109
110  protected SnapshotMock createSnapshotMock() throws IOException {
111    return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
112  }
113
114  @Test
115  public void testRestore() throws IOException {
116    restoreAndVerify("snapshot", "testRestore");
117  }
118
119  @Test
120  public void testRestoreWithNamespace() throws IOException {
121    restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
122  }
123
124  @Test
125  public void testNoHFileLinkInRootDir() throws IOException {
126    rootDir = TEST_UTIL.getDefaultRootDirPath();
127    FSUtils.setRootDir(conf, rootDir);
128    fs = rootDir.getFileSystem(conf);
129
130    TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
131    String snapshotName = tableName.getNameAsString() + "-snapshot";
132    createTableAndSnapshot(tableName, snapshotName);
133
134    Path restoreDir = new Path("/hbase/.tmp-restore");
135    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
136    checkNoHFileLinkInTableDir(tableName);
137  }
138
139  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
140      throws IOException {
141    byte[] column = Bytes.toBytes("A");
142    Table table = TEST_UTIL.createTable(tableName, column, 2);
143    TEST_UTIL.loadTable(table, column);
144    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
145  }
146
147  private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
148    Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
149        CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
150        CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
151    for (Path tableDir : tableDirs) {
152      Assert.assertFalse(hasHFileLink(tableDir));
153    }
154  }
155
156  private boolean hasHFileLink(Path tableDir) throws IOException {
157    if (fs.exists(tableDir)) {
158      RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
159      while (iterator.hasNext()) {
160        LocatedFileStatus fileStatus = iterator.next();
161        if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
162          return true;
163        }
164      }
165    }
166    return false;
167  }
168
169  private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
170    // Test Rolling-Upgrade like Snapshot.
171    // half machines writing using v1 and the others using v2 format.
172    SnapshotMock snapshotMock = createSnapshotMock();
173    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName);
174    builder.addRegionV1();
175    builder.addRegionV2();
176    builder.addRegionV2();
177    builder.addRegionV1();
178    Path snapshotDir = builder.commit();
179    TableDescriptor htd = builder.getTableDescriptor();
180    SnapshotDescription desc = builder.getSnapshotDescription();
181
182    // Test clone a snapshot
183    TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
184    testRestore(snapshotDir, desc, htdClone);
185    verifyRestore(rootDir, htd, htdClone);
186
187    // Test clone a clone ("link to link")
188    SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
189        .setName("cloneSnapshot")
190        .setTable("testtb-clone")
191        .build();
192    Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
193    TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
194    testRestore(cloneDir, cloneDesc, htdClone2);
195    verifyRestore(rootDir, htd, htdClone2);
196  }
197
198  private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd,
199      final TableDescriptor htdClone) throws IOException {
200    List<String> files = SnapshotTestingUtils.listHFileNames(fs,
201      FSUtils.getTableDir(rootDir, htdClone.getTableName()));
202    assertEquals(12, files.size());
203    for (int i = 0; i < files.size(); i += 2) {
204      String linkFile = files.get(i);
205      String refFile = files.get(i+1);
206      assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
207      assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
208      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
209      Path refPath = getReferredToFile(refFile);
210      LOG.debug("get reference name for file " + refFile + " = " + refPath);
211      assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
212      assertEquals(linkFile, refPath.getName());
213    }
214  }
215
216  /**
217   * Execute the restore operation
218   * @param snapshotDir The snapshot directory to use as "restore source"
219   * @param sd The snapshot descriptor
220   * @param htdClone The HTableDescriptor of the table to restore/clone.
221   */
222  private void testRestore(final Path snapshotDir, final SnapshotDescription sd,
223      final TableDescriptor htdClone) throws IOException {
224    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
225    FSUtils.logFileSystemState(fs, rootDir, LOG);
226
227    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
228    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
229    helper.restoreHdfsRegions();
230
231    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
232    FSUtils.logFileSystemState(fs, rootDir, LOG);
233  }
234
235  /**
236   * Initialize the restore helper, based on the snapshot and table information provided.
237   */
238  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
239      final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException {
240    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
241    MonitoredTask status = Mockito.mock(MonitoredTask.class);
242
243    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
244    return new RestoreSnapshotHelper(conf, fs, manifest,
245      htdClone, rootDir, monitor, status);
246  }
247
248  private Path getReferredToFile(final String referenceName) {
249    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
250    return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
251  }
252}