Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.hamcrest.Matchers.containsString;
023import static org.hamcrest.Matchers.everyItem;
024import static org.hamcrest.Matchers.hasItem;
025import static org.hamcrest.Matchers.hasProperty;
026import static org.hamcrest.Matchers.not;
027import static org.junit.jupiter.api.Assertions.assertTrue;
028
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.List;
032import java.util.UUID;
033import java.util.concurrent.TimeUnit;
034import org.apache.hadoop.fs.FileStatus;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.FileUtil;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.TableNameTestExtension;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.RegionInfo;
044import org.apache.hadoop.hbase.client.RegionInfoBuilder;
045import org.apache.hadoop.hbase.client.Table;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
048import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
049import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
050import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
051import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
052import org.apache.hadoop.hbase.testclassification.LargeTests;
053import org.apache.hadoop.hbase.testclassification.RegionServerTests;
054import org.apache.hadoop.hbase.util.Bytes;
055import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
056import org.apache.hadoop.hbase.util.Pair;
057import org.junit.jupiter.api.AfterAll;
058import org.junit.jupiter.api.BeforeAll;
059import org.junit.jupiter.api.BeforeEach;
060import org.junit.jupiter.api.Tag;
061import org.junit.jupiter.api.Test;
062import org.junit.jupiter.api.extension.RegisterExtension;
063
064@Tag(RegionServerTests.TAG)
065@Tag(LargeTests.TAG)
066public class TestMergesSplitsAddToTracker {
067
068  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
069
070  private static final String FAMILY_NAME_STR = "info";
071
072  private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
073
074  @RegisterExtension
075  public TableNameTestExtension name = new TableNameTestExtension();
076
077  @BeforeAll
078  public static void setupClass() throws Exception {
079    TEST_UTIL.startMiniCluster();
080  }
081
082  @AfterAll
083  public static void afterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @BeforeEach
088  public void setup() {
089    StoreFileTrackerForTest.clear();
090  }
091
092  private TableName createTable(byte[] splitKey) throws IOException {
093    TableDescriptor td = TableDescriptorBuilder.newBuilder(name.getTableName())
094      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME))
095      .setValue(TRACKER_IMPL, StoreFileTrackerForTest.class.getName()).build();
096    if (splitKey != null) {
097      TEST_UTIL.getAdmin().createTable(td, new byte[][] { splitKey });
098    } else {
099      TEST_UTIL.getAdmin().createTable(td);
100    }
101    return td.getTableName();
102  }
103
104  @Test
105  public void testCommitDaughterRegion() throws Exception {
106    TableName table = createTable(null);
107    // first put some data in order to have a store file created
108    putThreeRowsAndFlush(table);
109    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
110    HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
111    RegionInfo daughterA =
112      RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey())
113        .setEndKey(Bytes.toBytes("002")).setSplit(false)
114        .setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
115        .build();
116    RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002"))
117      .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false)
118      .setRegionId(region.getRegionInfo().getRegionId()).build();
119    HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
120    List<StoreFileInfo> splitFilesA = new ArrayList<>();
121    HRegionFileSystem regionFs = region.getRegionFileSystem();
122    StoreFileTracker sft = StoreFileTrackerFactory.create(region.getBaseConf(), true,
123      StoreContext.getBuilder()
124        .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), "info"))
125        .withRegionFileSystem(regionFs).build());
126    splitFilesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file,
127      Bytes.toBytes("002"), false, region.getSplitPolicy(), sft));
128    List<StoreFileInfo> splitFilesB = new ArrayList<>();
129    splitFilesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file,
130      Bytes.toBytes("002"), true, region.getSplitPolicy(), sft));
131    MasterProcedureEnv env =
132      TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment();
133    Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env);
134    Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env);
135    FileSystem fs = regionFS.getFileSystem();
136    verifyFilesAreTracked(resultA, fs);
137    verifyFilesAreTracked(resultB, fs);
138  }
139
140  @Test
141  public void testCommitMergedRegion() throws Exception {
142    TableName table = createTable(null);
143    // splitting the table first
144    TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
145    // Add data and flush to create files in the two different regions
146    putThreeRowsAndFlush(table);
147    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
148    HRegion first = regions.get(0);
149    HRegion second = regions.get(1);
150    HRegionFileSystem regionFS = first.getRegionFileSystem();
151
152    RegionInfo mergeResult =
153      RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey())
154        .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false)
155        .setRegionId(first.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
156        .build();
157
158    HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem(
159      TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), regionFS.getFileSystem(),
160      regionFS.getTableDir(), mergeResult);
161
162    List<StoreFileInfo> mergedFiles = new ArrayList<>();
163    // merge file from first region
164    mergedFiles.add(mergeFileFromRegion(first, mergeFS));
165    // merge file from second region
166    mergedFiles.add(mergeFileFromRegion(second, mergeFS));
167    MasterProcedureEnv env =
168      TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment();
169    mergeFS.commitMergedRegion(mergedFiles, env);
170    // validate
171    FileSystem fs = first.getRegionFileSystem().getFileSystem();
172    Path finalMergeDir =
173      new Path(first.getRegionFileSystem().getTableDir(), mergeResult.getEncodedName());
174    verifyFilesAreTracked(finalMergeDir, fs);
175  }
176
177  @Test
178  public void testSplitLoadsFromTracker() throws Exception {
179    TableName table = createTable(null);
180    // Add data and flush to create files in the two different regions
181    putThreeRowsAndFlush(table);
182    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
183    Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(region);
184    StoreFileInfo fileInfo = copyResult.getFirst();
185    String copyName = copyResult.getSecond();
186    // Now splits the region
187    TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
188    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
189    HRegion first = regions.get(0);
190    validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName);
191    HRegion second = regions.get(1);
192    validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName);
193  }
194
195  @Test
196  public void testMergeLoadsFromTracker() throws Exception {
197    TableName table = createTable(Bytes.toBytes("002"));
198    // Add data and flush to create files in the two different regions
199    putThreeRowsAndFlush(table);
200    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
201    HRegion first = regions.get(0);
202    Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(first);
203    StoreFileInfo fileInfo = copyResult.getFirst();
204    String copyName = copyResult.getSecond();
205    // Now merges the first two regions
206    TEST_UTIL.getAdmin()
207      .mergeRegionsAsync(new byte[][] { first.getRegionInfo().getEncodedNameAsBytes(),
208        regions.get(1).getRegionInfo().getEncodedNameAsBytes() }, true)
209      .get(10, TimeUnit.SECONDS);
210    regions = TEST_UTIL.getHBaseCluster().getRegions(table);
211    HRegion merged = regions.get(0);
212    validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName);
213  }
214
215  private Pair<StoreFileInfo, String> copyFileInTheStoreDir(HRegion region) throws IOException {
216    Path storeDir = region.getRegionFileSystem().getStoreDir("info");
217    // gets the single file
218    HRegionFileSystem regionFs = region.getRegionFileSystem();
219    StoreFileTracker sft = StoreFileTrackerFactory.create(region.getBaseConf(), false,
220      StoreContext.getBuilder().withFamilyStoreDirectoryPath(storeDir)
221        .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME))
222        .withRegionFileSystem(regionFs).build());
223    List<StoreFileInfo> infos = sft.load();
224    StoreFileInfo fileInfo = infos.get(0);
225    // make a copy of the valid file staight into the store dir, so that it's not tracked.
226    String copyName = UUID.randomUUID().toString().replaceAll("-", "");
227    Path copy = new Path(storeDir, copyName);
228    FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(), copy,
229      false, false, TEST_UTIL.getConfiguration());
230    return new Pair<>(fileInfo, copyName);
231  }
232
233  private void validateDaughterRegionsFiles(HRegion region, String originalFileName,
234    String untrackedFile) throws IOException {
235    // verify there's no link for the untracked, copied file in first region
236    HRegionFileSystem regionFs = region.getRegionFileSystem();
237    StoreFileTracker sft = StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), false,
238      StoreContext.getBuilder()
239        .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), "info"))
240        .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME))
241        .withRegionFileSystem(regionFs).build());
242    List<StoreFileInfo> infos = sft.load();
243    assertThat(infos, everyItem(hasProperty("activeFileName", not(containsString(untrackedFile)))));
244    assertThat(infos, hasItem(hasProperty("activeFileName", containsString(originalFileName))));
245  }
246
247  private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception {
248    for (FileStatus f : fs.listStatus(new Path(regionDir, FAMILY_NAME_STR))) {
249      assertTrue(
250        StoreFileTrackerForTest.tracked(regionDir.getName(), FAMILY_NAME_STR, f.getPath()));
251    }
252  }
253
254  private StoreFileInfo mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS)
255    throws IOException {
256    HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
257    HRegionFileSystem regionFs = regionToMerge.getRegionFileSystem();
258    StoreFileTracker sft = StoreFileTrackerFactory.create(regionToMerge.getBaseConf(), true,
259      StoreContext.getBuilder()
260        .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), FAMILY_NAME_STR))
261        .withRegionFileSystem(regionFs).build());
262    return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file,
263      sft);
264  }
265
266  private void putThreeRowsAndFlush(TableName table) throws IOException {
267    Table tbl = TEST_UTIL.getConnection().getTable(table);
268    Put put = new Put(Bytes.toBytes("001"));
269    byte[] qualifier = Bytes.toBytes("1");
270    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1));
271    tbl.put(put);
272    put = new Put(Bytes.toBytes("002"));
273    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
274    tbl.put(put);
275    put = new Put(Bytes.toBytes("003"));
276    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
277    tbl.put(put);
278    TEST_UTIL.flush(table);
279  }
280}