Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
033import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
034import org.apache.hadoop.hbase.regionserver.BloomType;
035import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
037import org.apache.hadoop.hbase.testclassification.ClientTests;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.jupiter.api.AfterAll;
042import org.junit.jupiter.api.AfterEach;
043import org.junit.jupiter.api.BeforeAll;
044import org.junit.jupiter.api.BeforeEach;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050/**
051 * Test class to verify that metadata is consistent before and after a snapshot attempt.
052 */
053@Tag(MediumTests.TAG)
054@Tag(ClientTests.TAG)
055public class TestSnapshotMetadata {
056
057  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotMetadata.class);
058
059  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
060  private static final int NUM_RS = 2;
061  private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
062
063  private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
064  private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
065
066  private static final String COMPRESSED_FAM_STR = "fam_compressed";
067  private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
068
069  private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
070  private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
071
072  private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
073  private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
074
075  private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
076  private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
077
078  private static final byte[][] families =
079    { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
080
081  private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
082  private static final BloomType BLOOM_TYPE = BloomType.ROW;
083  private static final int BLOCK_SIZE = 98;
084  private static final int MAX_VERSIONS = 8;
085
086  private Admin admin;
087  private String originalTableDescription;
088  private TableDescriptor originalTableDescriptor;
089  TableName originalTableName;
090
091  private static FileSystem fs;
092  private static Path rootDir;
093
094  @BeforeAll
095  public static void setupCluster() throws Exception {
096    setupConf(UTIL.getConfiguration());
097    UTIL.startMiniCluster(NUM_RS);
098
099    fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
100    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
101  }
102
103  @AfterAll
104  public static void cleanupTest() throws Exception {
105    try {
106      UTIL.shutdownMiniCluster();
107    } catch (Exception e) {
108      LOG.warn("failure shutting down cluster", e);
109    }
110  }
111
112  private static void setupConf(Configuration conf) {
113    // enable snapshot support
114    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
115    // disable the ui
116    conf.setInt("hbase.regionsever.info.port", -1);
117    // change the flush size to a small amount, regulating number of store files
118    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
119    // so make sure we get a compaction when doing a load, but keep around
120    // some files in the store
121    conf.setInt("hbase.hstore.compaction.min", 10);
122    conf.setInt("hbase.hstore.compactionThreshold", 10);
123    // block writes if we get to 12 store files
124    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
125    conf.setInt("hbase.regionserver.msginterval", 100);
126    conf.setBoolean("hbase.master.enabletable.roundrobin", true);
127    // Avoid potentially aggressive splitting which would cause snapshot to fail
128    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
129      ConstantSizeRegionSplitPolicy.class.getName());
130  }
131
132  @BeforeEach
133  public void setup() throws Exception {
134    admin = UTIL.getAdmin();
135    createTableWithNonDefaultProperties();
136  }
137
138  @AfterEach
139  public void tearDown() throws Exception {
140    SnapshotTestingUtils.deleteAllSnapshots(admin);
141  }
142
143  /*
144   * Create a table that has non-default properties so we can see if they hold
145   */
146  private void createTableWithNonDefaultProperties() throws Exception {
147    final long startTime = EnvironmentEdgeManager.currentTime();
148    final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
149    originalTableName = TableName.valueOf(sourceTableNameAsString);
150
151    // enable replication on a column family
152    ColumnFamilyDescriptor maxVersionsColumn = ColumnFamilyDescriptorBuilder
153      .newBuilder(MAX_VERSIONS_FAM).setMaxVersions(MAX_VERSIONS).build();
154    ColumnFamilyDescriptor bloomFilterColumn = ColumnFamilyDescriptorBuilder
155      .newBuilder(BLOOMFILTER_FAM).setBloomFilterType(BLOOM_TYPE).build();
156    ColumnFamilyDescriptor dataBlockColumn = ColumnFamilyDescriptorBuilder
157      .newBuilder(COMPRESSED_FAM).setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE).build();
158    ColumnFamilyDescriptor blockSizeColumn =
159      ColumnFamilyDescriptorBuilder.newBuilder(BLOCKSIZE_FAM).setBlocksize(BLOCK_SIZE).build();
160
161    TableDescriptor tableDescriptor = TableDescriptorBuilder
162      .newBuilder(TableName.valueOf(sourceTableNameAsString)).setColumnFamily(maxVersionsColumn)
163      .setColumnFamily(bloomFilterColumn).setColumnFamily(dataBlockColumn)
164      .setColumnFamily(blockSizeColumn).setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE)
165      .setValue(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE).build();
166    assertTrue(tableDescriptor.getValues().size() > 0);
167
168    admin.createTable(tableDescriptor);
169    Table original = UTIL.getConnection().getTable(originalTableName);
170    originalTableName = TableName.valueOf(sourceTableNameAsString);
171    originalTableDescriptor = admin.getDescriptor(originalTableName);
172    originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
173
174    original.close();
175  }
176
177  /**
178   * Verify that the describe for a cloned table matches the describe from the original.
179   */
180  @Test
181  public void testDescribeMatchesAfterClone() throws Exception {
182    // Clone the original table
183    final String clonedTableNameAsString = "clone" + originalTableName;
184    final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
185    final String snapshotNameAsString =
186      "snapshot" + originalTableName + EnvironmentEdgeManager.currentTime();
187    final String snapshotName = snapshotNameAsString;
188
189    // restore the snapshot into a cloned table and examine the output
190    List<byte[]> familiesList = new ArrayList<>();
191    Collections.addAll(familiesList, families);
192
193    // Create a snapshot in which all families are empty
194    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null, familiesList,
195      snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
196
197    admin.cloneSnapshot(snapshotName, clonedTableName);
198    Table clonedTable = UTIL.getConnection().getTable(clonedTableName);
199    TableDescriptor cloneHtd = admin.getDescriptor(clonedTableName);
200    assertEquals(originalTableDescription.replace(originalTableName.getNameAsString(),
201      clonedTableNameAsString), cloneHtd.toStringCustomizedValues());
202
203    // Verify the custom fields
204    assertEquals(originalTableDescriptor.getValues().size(), cloneHtd.getValues().size());
205    assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
206    assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getValue(TEST_CONF_CUSTOM_VALUE));
207    assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
208
209    admin.enableTable(originalTableName);
210    clonedTable.close();
211  }
212
213  /**
214   * Verify that the describe for a restored table matches the describe for one the original.
215   */
216  @Test
217  public void testDescribeMatchesAfterRestore() throws Exception {
218    runRestoreWithAdditionalMetadata(false);
219  }
220
221  /**
222   * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
223   * new metadata during a restore
224   */
225  @Test
226  public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
227    runRestoreWithAdditionalMetadata(true);
228  }
229
230  /**
231   * Verify that when the table is empty, making metadata changes after the restore does not affect
232   * the restored table's original metadata
233   */
234  @Test
235  public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
236    runRestoreWithAdditionalMetadata(true, false);
237  }
238
239  private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
240    runRestoreWithAdditionalMetadata(changeMetadata, true);
241  }
242
243  private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
244    throws Exception {
245
246    if (admin.isTableDisabled(originalTableName)) {
247      admin.enableTable(originalTableName);
248    }
249
250    // populate it with data
251    final byte[] familyForUpdate = BLOCKSIZE_FAM;
252
253    List<byte[]> familiesWithDataList = new ArrayList<>();
254    List<byte[]> emptyFamiliesList = new ArrayList<>();
255    if (addData) {
256      Table original = UTIL.getConnection().getTable(originalTableName);
257      UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
258      original.close();
259
260      for (byte[] family : families) {
261        if (family != familyForUpdate) {
262          emptyFamiliesList.add(family);
263        }
264      }
265      familiesWithDataList.add(familyForUpdate);
266    } else {
267      Collections.addAll(emptyFamiliesList, families);
268    }
269
270    // take a "disabled" snapshot
271    final String snapshotNameAsString =
272      "snapshot" + originalTableName + EnvironmentEdgeManager.currentTime();
273
274    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, familiesWithDataList,
275      emptyFamiliesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
276
277    admin.enableTable(originalTableName);
278
279    if (changeMetadata) {
280      final String newFamilyNameAsString = "newFamily" + EnvironmentEdgeManager.currentTime();
281      final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
282
283      admin.disableTable(originalTableName);
284      ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.of(newFamilyName);
285      admin.addColumnFamily(originalTableName, familyDescriptor);
286      assertTrue(admin.getDescriptor(originalTableName).toString().contains(newFamilyNameAsString),
287        "New column family was not added.");
288    }
289
290    // restore it
291    if (!admin.isTableDisabled(originalTableName)) {
292      admin.disableTable(originalTableName);
293    }
294
295    admin.restoreSnapshot(snapshotNameAsString);
296    admin.enableTable(originalTableName);
297
298    // verify that the descrption is reverted
299    try (Table original = UTIL.getConnection().getTable(originalTableName)) {
300      assertEquals(originalTableDescriptor, admin.getDescriptor(originalTableName));
301      assertEquals(originalTableDescriptor, original.getDescriptor());
302    }
303  }
304}