001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
034import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
035import org.apache.hadoop.hbase.regionserver.BloomType;
036import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
037import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
038import org.apache.hadoop.hbase.testclassification.ClientTests;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.junit.After;
043import org.junit.AfterClass;
044import org.junit.Before;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Test class to verify that metadata is consistent before and after a snapshot attempt.
054 */
055@Category({MediumTests.class, ClientTests.class})
056public class TestSnapshotMetadata {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestSnapshotMetadata.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotMetadata.class);
063
064  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
065  private static final int NUM_RS = 2;
066  private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
067
068  private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
069  private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
070
071  private static final String COMPRESSED_FAM_STR = "fam_compressed";
072  private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
073
074  private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
075  private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
076
077  private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
078  private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
079
080  private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
081  private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
082
083  private static final byte[][] families = {
084    MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
085  };
086
087  private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
088  private static final BloomType BLOOM_TYPE = BloomType.ROW;
089  private static final int BLOCK_SIZE = 98;
090  private static final int MAX_VERSIONS = 8;
091
092  private Admin admin;
093  private String originalTableDescription;
094  private TableDescriptor originalTableDescriptor;
095  TableName originalTableName;
096
097  private static FileSystem fs;
098  private static Path rootDir;
099
100  @BeforeClass
101  public static void setupCluster() throws Exception {
102    setupConf(UTIL.getConfiguration());
103    UTIL.startMiniCluster(NUM_RS);
104
105    fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
106    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
107  }
108
109  @AfterClass
110  public static void cleanupTest() throws Exception {
111    try {
112      UTIL.shutdownMiniCluster();
113    } catch (Exception e) {
114      LOG.warn("failure shutting down cluster", e);
115    }
116  }
117
118  private static void setupConf(Configuration conf) {
119    // enable snapshot support
120    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
121    // disable the ui
122    conf.setInt("hbase.regionsever.info.port", -1);
123    // change the flush size to a small amount, regulating number of store files
124    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
125    // so make sure we get a compaction when doing a load, but keep around
126    // some files in the store
127    conf.setInt("hbase.hstore.compaction.min", 10);
128    conf.setInt("hbase.hstore.compactionThreshold", 10);
129    // block writes if we get to 12 store files
130    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
131    conf.setInt("hbase.regionserver.msginterval", 100);
132    conf.setBoolean("hbase.master.enabletable.roundrobin", true);
133    // Avoid potentially aggressive splitting which would cause snapshot to fail
134    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
135      ConstantSizeRegionSplitPolicy.class.getName());
136  }
137
138  @Before
139  public void setup() throws Exception {
140    admin = UTIL.getAdmin();
141    createTableWithNonDefaultProperties();
142  }
143
144  @After
145  public void tearDown() throws Exception {
146    SnapshotTestingUtils.deleteAllSnapshots(admin);
147  }
148
149  /*
150   *  Create a table that has non-default properties so we can see if they hold
151   */
152  private void createTableWithNonDefaultProperties() throws Exception {
153    final long startTime = EnvironmentEdgeManager.currentTime();
154    final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
155    originalTableName = TableName.valueOf(sourceTableNameAsString);
156
157    // enable replication on a column family
158    ColumnFamilyDescriptor maxVersionsColumn = ColumnFamilyDescriptorBuilder
159      .newBuilder(MAX_VERSIONS_FAM).setMaxVersions(MAX_VERSIONS).build();
160    ColumnFamilyDescriptor bloomFilterColumn = ColumnFamilyDescriptorBuilder
161      .newBuilder(BLOOMFILTER_FAM).setBloomFilterType(BLOOM_TYPE).build();
162    ColumnFamilyDescriptor dataBlockColumn = ColumnFamilyDescriptorBuilder
163      .newBuilder(COMPRESSED_FAM).setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE).build();
164    ColumnFamilyDescriptor blockSizeColumn =
165      ColumnFamilyDescriptorBuilder.newBuilder(BLOCKSIZE_FAM).setBlocksize(BLOCK_SIZE).build();
166
167    TableDescriptor tableDescriptor = TableDescriptorBuilder
168      .newBuilder(TableName.valueOf(sourceTableNameAsString)).setColumnFamily(maxVersionsColumn)
169      .setColumnFamily(bloomFilterColumn).setColumnFamily(dataBlockColumn)
170      .setColumnFamily(blockSizeColumn).setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE)
171      .setValue(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE).build();
172    assertTrue(tableDescriptor.getValues().size() > 0);
173
174    admin.createTable(tableDescriptor);
175    Table original = UTIL.getConnection().getTable(originalTableName);
176    originalTableName = TableName.valueOf(sourceTableNameAsString);
177    originalTableDescriptor = admin.getDescriptor(originalTableName);
178    originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
179
180    original.close();
181  }
182
183
184  /**
185   * Verify that the describe for a cloned table matches the describe from the original.
186   */
187  @Test
188  public void testDescribeMatchesAfterClone() throws Exception {
189    // Clone the original table
190    final String clonedTableNameAsString = "clone" + originalTableName;
191    final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
192    final String snapshotNameAsString = "snapshot" + originalTableName
193        + EnvironmentEdgeManager.currentTime();
194    final String snapshotName = snapshotNameAsString;
195
196    // restore the snapshot into a cloned table and examine the output
197    List<byte[]> familiesList = new ArrayList<>();
198    Collections.addAll(familiesList, families);
199
200    // Create a snapshot in which all families are empty
201    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
202      familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
203
204    admin.cloneSnapshot(snapshotName, clonedTableName);
205    Table clonedTable = UTIL.getConnection().getTable(clonedTableName);
206    TableDescriptor cloneHtd = admin.getDescriptor(clonedTableName);
207    assertEquals(
208      originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
209      cloneHtd.toStringCustomizedValues());
210
211    // Verify the custom fields
212    assertEquals(originalTableDescriptor.getValues().size(),
213                        cloneHtd.getValues().size());
214    assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
215    assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getValue(TEST_CONF_CUSTOM_VALUE));
216    assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
217
218    admin.enableTable(originalTableName);
219    clonedTable.close();
220  }
221
222  /**
223   * Verify that the describe for a restored table matches the describe for one the original.
224   */
225  @Test
226  public void testDescribeMatchesAfterRestore() throws Exception {
227    runRestoreWithAdditionalMetadata(false);
228  }
229
230  /**
231   * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
232   * new metadata during a restore
233   */
234  @Test
235  public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
236    runRestoreWithAdditionalMetadata(true);
237  }
238
239  /**
240   * Verify that when the table is empty, making metadata changes after the restore does not affect
241   * the restored table's original metadata
242   * @throws Exception
243   */
244  @Test
245  public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
246    runRestoreWithAdditionalMetadata(true, false);
247  }
248
249  private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
250    runRestoreWithAdditionalMetadata(changeMetadata, true);
251  }
252
253  private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
254      throws Exception {
255
256    if (admin.isTableDisabled(originalTableName)) {
257      admin.enableTable(originalTableName);
258    }
259
260    // populate it with data
261    final byte[] familyForUpdate = BLOCKSIZE_FAM;
262
263    List<byte[]> familiesWithDataList = new ArrayList<>();
264    List<byte[]> emptyFamiliesList = new ArrayList<>();
265    if (addData) {
266      Table original = UTIL.getConnection().getTable(originalTableName);
267      UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
268      original.close();
269
270      for (byte[] family : families) {
271        if (family != familyForUpdate) {
272          emptyFamiliesList.add(family);
273        }
274      }
275      familiesWithDataList.add(familyForUpdate);
276    } else {
277      Collections.addAll(emptyFamiliesList, families);
278    }
279
280    // take a "disabled" snapshot
281    final String snapshotNameAsString = "snapshot" + originalTableName
282        + EnvironmentEdgeManager.currentTime();
283
284    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
285      familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
286      /* onlineSnapshot= */ false);
287
288    admin.enableTable(originalTableName);
289
290    if (changeMetadata) {
291      final String newFamilyNameAsString = "newFamily" + EnvironmentEdgeManager.currentTime();
292      final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
293
294      admin.disableTable(originalTableName);
295      ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.of(newFamilyName);
296      admin.addColumnFamily(originalTableName, familyDescriptor);
297      assertTrue("New column family was not added.",
298        admin.getDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
299    }
300
301    // restore it
302    if (!admin.isTableDisabled(originalTableName)) {
303      admin.disableTable(originalTableName);
304    }
305
306    admin.restoreSnapshot(snapshotNameAsString);
307    admin.enableTable(originalTableName);
308
309    // verify that the descrption is reverted
310    try (Table original = UTIL.getConnection().getTable(originalTableName)) {
311      assertEquals(originalTableDescriptor, admin.getDescriptor(originalTableName));
312      assertEquals(originalTableDescriptor, original.getDescriptor());
313    }
314  }
315}