001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.hamcrest.MatcherAssert.assertThat;
021import static org.hamcrest.Matchers.containsString;
022import static org.hamcrest.Matchers.equalTo;
023import static org.hamcrest.Matchers.greaterThan;
024import static org.hamcrest.Matchers.notNullValue;
025import static org.hamcrest.Matchers.nullValue;
026import static org.junit.jupiter.api.Assertions.assertThrows;
027
028import java.io.IOException;
029import java.util.function.Function;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.LocatedFileStatus;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.fs.RemoteIterator;
034import org.apache.hadoop.hbase.HBaseIOException;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Admin;
038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.io.hfile.HFile;
043import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.apache.hadoop.hbase.util.JVMClusterUtil;
050import org.apache.hadoop.hbase.util.TableDescriptorChecker;
051import org.apache.hadoop.hdfs.DistributedFileSystem;
052import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
053import org.junit.jupiter.api.AfterAll;
054import org.junit.jupiter.api.BeforeAll;
055import org.junit.jupiter.api.Tag;
056import org.junit.jupiter.api.Test;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060@Tag(MasterTests.TAG)
061@Tag(MediumTests.TAG)
062public class TestManageTableErasureCodingPolicy {
063  private static final Logger LOG =
064    LoggerFactory.getLogger(TestManageTableErasureCodingPolicy.class);
065
066  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
067  private static final byte[] FAMILY = Bytes.toBytes("a");
068  private static final TableName NON_EC_TABLE = TableName.valueOf("foo");
069  private static final TableDescriptor NON_EC_TABLE_DESC = TableDescriptorBuilder
070    .newBuilder(NON_EC_TABLE).setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
071  private static final TableName EC_TABLE = TableName.valueOf("bar");
072  private static final TableDescriptor EC_TABLE_DESC =
073    TableDescriptorBuilder.newBuilder(EC_TABLE).setErasureCodingPolicy("XOR-2-1-1024k")
074      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
075
076  @BeforeAll
077  public static void beforeClass() throws Exception {
078    // enable because we are testing the checks below
079    UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, true);
080    UTIL.startMiniDFSCluster(3); // 3 necessary for XOR-2-1-1024k
081    UTIL.startMiniCluster(1);
082    DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration());
083    fs.enableErasureCodingPolicy("XOR-2-1-1024k");
084    fs.enableErasureCodingPolicy("RS-6-3-1024k");
085    Table table = UTIL.createTable(NON_EC_TABLE_DESC, null);
086    UTIL.loadTable(table, FAMILY);
087    UTIL.flush();
088  }
089
090  @AfterAll
091  public static void afterClass() throws Exception {
092    UTIL.shutdownMiniCluster();
093    UTIL.shutdownMiniDFSCluster();
094  }
095
096  @Test
097  public void itValidatesPolicyNameForCreate() {
098    runValidatePolicyNameTest(unused -> EC_TABLE_DESC, Admin::createTable);
099  }
100
101  @Test
102  public void itValidatesPolicyNameForAlter() {
103    runValidatePolicyNameTest(admin -> {
104      try {
105        return admin.getDescriptor(NON_EC_TABLE);
106      } catch (IOException e) {
107        throw new RuntimeException(e);
108      }
109    }, Admin::modifyTable);
110  }
111
112  @FunctionalInterface
113  interface ThrowingTableDescriptorConsumer {
114    void accept(Admin admin, TableDescriptor desc) throws IOException;
115  }
116
117  private void runValidatePolicyNameTest(Function<Admin, TableDescriptor> descriptorSupplier,
118    ThrowingTableDescriptorConsumer consumer) {
119    HBaseIOException thrown = assertThrows(HBaseIOException.class, () -> {
120      try (Admin admin = UTIL.getAdmin()) {
121        TableDescriptor desc = descriptorSupplier.apply(admin);
122        consumer.accept(admin,
123          TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("foo").build());
124      }
125    });
126    assertThat(thrown.getMessage(),
127      containsString("Cannot set Erasure Coding policy: foo. Policy not found"));
128
129    thrown = assertThrows(HBaseIOException.class, () -> {
130      try (Admin admin = UTIL.getAdmin()) {
131        TableDescriptor desc = descriptorSupplier.apply(admin);
132        consumer.accept(admin,
133          TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("RS-10-4-1024k").build());
134      }
135    });
136    assertThat(thrown.getMessage(), containsString(
137      "Cannot set Erasure Coding policy: RS-10-4-1024k. The policy must be enabled"));
138
139    // RS-6-3-1024k requires at least 6 datanodes, so should fail write test
140    thrown = assertThrows(HBaseIOException.class, () -> {
141      try (Admin admin = UTIL.getAdmin()) {
142        TableDescriptor desc = descriptorSupplier.apply(admin);
143        consumer.accept(admin,
144          TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("RS-6-3-1024k").build());
145      }
146    });
147    assertThat(thrown.getMessage(), containsString("Failed write test for EC policy"));
148  }
149
150  @Test
151  public void testCreateTableErasureCodingSync() throws IOException {
152    try (Admin admin = UTIL.getAdmin()) {
153      recreateTable(admin, EC_TABLE_DESC);
154      UTIL.flush(EC_TABLE);
155      Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
156      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration());
157      checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k");
158    }
159  }
160
161  private void recreateTable(Admin admin, TableDescriptor desc) throws IOException {
162    if (admin.tableExists(desc.getTableName())) {
163      admin.disableTable(desc.getTableName());
164      admin.deleteTable(desc.getTableName());
165    }
166    admin.createTable(desc);
167    try (Table table = UTIL.getConnection().getTable(desc.getTableName())) {
168      UTIL.loadTable(table, FAMILY);
169    }
170  }
171
172  @Test
173  public void testModifyTableErasureCodingSync() throws IOException, InterruptedException {
174    try (Admin admin = UTIL.getAdmin()) {
175      Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
176      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration());
177
178      // start off without EC
179      checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, null);
180
181      // add EC
182      TableDescriptor desc = UTIL.getAdmin().getDescriptor(NON_EC_TABLE);
183      TableDescriptor newDesc =
184        TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("XOR-2-1-1024k").build();
185      admin.modifyTable(newDesc);
186
187      // check dirs, but files should not be changed yet
188      checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, "XOR-2-1-1024k", null);
189
190      compactAwayOldFiles(NON_EC_TABLE);
191
192      // expect both dirs and files to be EC now
193      checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k");
194
195      newDesc = TableDescriptorBuilder.newBuilder(newDesc).setErasureCodingPolicy(null).build();
196      // remove EC now
197      admin.modifyTable(newDesc);
198
199      // dirs should no longer be EC, but old EC files remain
200      checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, "XOR-2-1-1024k");
201
202      // compact to rewrite EC files without EC, then run discharger to get rid of the old EC files
203      UTIL.compact(NON_EC_TABLE, true);
204      for (JVMClusterUtil.RegionServerThread regionserver : UTIL.getHBaseCluster()
205        .getLiveRegionServerThreads()) {
206        CompactedHFilesDischarger chore =
207          regionserver.getRegionServer().getCompactedHFilesDischarger();
208        chore.setUseExecutor(false);
209        chore.chore();
210      }
211
212      checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, null);
213    }
214  }
215
216  private void compactAwayOldFiles(TableName tableName) throws IOException {
217    LOG.info("Compacting and discharging files for {}", tableName);
218    // compact to rewrit files, then run discharger to get rid of the old files
219    UTIL.compact(tableName, true);
220    for (JVMClusterUtil.RegionServerThread regionserver : UTIL.getHBaseCluster()
221      .getLiveRegionServerThreads()) {
222      CompactedHFilesDischarger chore =
223        regionserver.getRegionServer().getCompactedHFilesDischarger();
224      chore.setUseExecutor(false);
225      chore.chore();
226    }
227  }
228
229  @Test
230  public void testRestoreSnapshot() throws IOException {
231    String snapshotName = "testRestoreSnapshot_snap";
232    TableName tableName = TableName.valueOf("testRestoreSnapshot_tbl");
233    try (Admin admin = UTIL.getAdmin()) {
234      Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
235      DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration());
236
237      // recreate EC test table and load it
238      recreateTable(admin, EC_TABLE_DESC);
239
240      // Take a snapshot, then clone it into a new table
241      admin.snapshot(snapshotName, EC_TABLE);
242      admin.cloneSnapshot(snapshotName, tableName);
243      compactAwayOldFiles(tableName);
244
245      // Verify the new table has the right EC policy
246      checkRegionDirAndFilePolicies(dfs, rootDir, tableName, "XOR-2-1-1024k", "XOR-2-1-1024k");
247
248      // Remove the EC policy from the EC test table, and verify that worked
249      admin.modifyTable(
250        TableDescriptorBuilder.newBuilder(EC_TABLE_DESC).setErasureCodingPolicy(null).build());
251      compactAwayOldFiles(EC_TABLE);
252      checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, null, null);
253
254      // Restore snapshot, and then verify it has the policy again
255      admin.disableTable(EC_TABLE);
256      admin.restoreSnapshot(snapshotName);
257      admin.enableTable(EC_TABLE);
258      compactAwayOldFiles(EC_TABLE);
259      checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k");
260    }
261  }
262
263  private void checkRegionDirAndFilePolicies(DistributedFileSystem dfs, Path rootDir,
264    TableName testTable, String expectedDirPolicy, String expectedFilePolicy) throws IOException {
265    Path tableDir = CommonFSUtils.getTableDir(rootDir, testTable);
266    checkPolicy(dfs, tableDir, expectedDirPolicy);
267
268    int filesMatched = 0;
269    for (HRegion region : UTIL.getHBaseCluster().getRegions(testTable)) {
270      Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
271      checkPolicy(dfs, regionDir, expectedDirPolicy);
272      RemoteIterator<LocatedFileStatus> itr = dfs.listFiles(regionDir, true);
273      while (itr.hasNext()) {
274        LocatedFileStatus fileStatus = itr.next();
275        Path path = fileStatus.getPath();
276        if (!HFile.isHFileFormat(dfs, path)) {
277          LOG.info("{} is not an hfile", path);
278          continue;
279        }
280        filesMatched++;
281        checkPolicy(dfs, path, expectedFilePolicy);
282      }
283    }
284    assertThat(filesMatched, greaterThan(0));
285  }
286
287  private void checkPolicy(DistributedFileSystem dfs, Path path, String expectedPolicy)
288    throws IOException {
289    ErasureCodingPolicy policy = dfs.getErasureCodingPolicy(path);
290    if (expectedPolicy == null) {
291      assertThat("policy for " + path, policy, nullValue());
292    } else {
293      assertThat("policy for " + path, policy, notNullValue());
294      assertThat("policy for " + path, policy.getName(), equalTo(expectedPolicy));
295    }
296  }
297}