Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Put;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure;
035import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
036import org.apache.hadoop.hbase.procedure2.Procedure;
037import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
038import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.testclassification.RegionServerTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.BeforeEach;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.Test;
048import org.junit.jupiter.api.TestInfo;
049
050@Tag(RegionServerTests.TAG)
051@Tag(LargeTests.TAG)
052public class TestDirectStoreSplitsMerges {
053
054  private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
055
056  public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
057  private String name;
058
059  @BeforeEach
060  public void setTestName(TestInfo testInfo) {
061    this.name = testInfo.getTestMethod().get().getName();
062  }
063
064  @BeforeAll
065  public static void setup() throws Exception {
066    TEST_UTIL.startMiniCluster();
067  }
068
069  @AfterAll
070  public static void after() throws Exception {
071    TEST_UTIL.shutdownMiniCluster();
072  }
073
074  @Test
075  public void testSplitStoreDir() throws Exception {
076    TableName table = TableName.valueOf(name);
077    TEST_UTIL.createTable(table, FAMILY_NAME);
078    // first put some data in order to have a store file created
079    putThreeRowsAndFlush(table);
080    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
081    HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
082    RegionInfo daughterA =
083      RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey())
084        .setEndKey(Bytes.toBytes("002")).setSplit(false)
085        .setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
086        .build();
087    HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
088    StoreFileTracker sft =
089      StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
090        true, region.getStores().get(0).getStoreContext());
091    Path result = regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file,
092      Bytes.toBytes("002"), false, region.getSplitPolicy(), sft).getPath();
093    // asserts the reference file naming is correct
094    validateResultingFile(region.getRegionInfo().getEncodedName(), result);
095    // Additionally check if split region dir was created directly under table dir, not on .tmp
096    Path resultGreatGrandParent = result.getParent().getParent().getParent();
097    assertEquals(regionFS.getTableDir().getName(), resultGreatGrandParent.getName());
098  }
099
100  @Test
101  public void testMergeStoreFile() throws Exception {
102    TableName table = TableName.valueOf(name);
103    TEST_UTIL.createTable(table, FAMILY_NAME);
104    // splitting the table first
105    TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
106    waitForSplitProcComplete(1000, 10);
107    // Add data and flush to create files in the two different regions
108    putThreeRowsAndFlush(table);
109    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
110    HRegion first = regions.get(0);
111    HRegion second = regions.get(1);
112    HRegionFileSystem regionFS = first.getRegionFileSystem();
113
114    RegionInfo mergeResult =
115      RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey())
116        .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false)
117        .setRegionId(first.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
118        .build();
119
120    Configuration configuration = TEST_UTIL.getHBaseCluster().getMaster().getConfiguration();
121    HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem(configuration,
122      regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult);
123
124    // merge file from first region
125    HStoreFile file = (HStoreFile) first.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
126    mergeFileFromRegion(mergeRegionFs, first, file, StoreFileTrackerFactory.create(configuration,
127      true, first.getStore(FAMILY_NAME).getStoreContext()));
128    // merge file from second region
129    file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
130    mergeFileFromRegion(mergeRegionFs, second, file, StoreFileTrackerFactory.create(configuration,
131      true, second.getStore(FAMILY_NAME).getStoreContext()));
132  }
133
134  @Test
135  public void testCommitDaughterRegionNoFiles() throws Exception {
136    TableName table = TableName.valueOf(name);
137    TEST_UTIL.createTable(table, FAMILY_NAME);
138    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
139    HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
140    RegionInfo daughterA =
141      RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey())
142        .setEndKey(Bytes.toBytes("002")).setSplit(false)
143        .setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
144        .build();
145    Path splitDir = regionFS.getSplitsDir(daughterA);
146    MasterProcedureEnv env =
147      TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment();
148    Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>(), env);
149    assertEquals(splitDir, result);
150  }
151
152  @Test
153  public void testCommitDaughterRegionWithFiles() throws Exception {
154    TableName table = TableName.valueOf(name);
155    TEST_UTIL.createTable(table, FAMILY_NAME);
156    // first put some data in order to have a store file created
157    putThreeRowsAndFlush(table);
158    HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
159    HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
160    RegionInfo daughterA =
161      RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey())
162        .setEndKey(Bytes.toBytes("002")).setSplit(false)
163        .setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
164        .build();
165    RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002"))
166      .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false)
167      .setRegionId(region.getRegionInfo().getRegionId()).build();
168    Path splitDirA = regionFS.getSplitsDir(daughterA);
169    Path splitDirB = regionFS.getSplitsDir(daughterB);
170    HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
171    List<StoreFileInfo> filesA = new ArrayList<>();
172    StoreFileTracker sft =
173      StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
174        true, region.getStores().get(0).getStoreContext());
175    filesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file,
176      Bytes.toBytes("002"), false, region.getSplitPolicy(), sft));
177    List<StoreFileInfo> filesB = new ArrayList<>();
178    filesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file,
179      Bytes.toBytes("002"), true, region.getSplitPolicy(), sft));
180    MasterProcedureEnv env =
181      TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment();
182    Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env);
183    Path resultB = regionFS.commitDaughterRegion(daughterB, filesB, env);
184    assertEquals(splitDirA, resultA);
185    assertEquals(splitDirB, resultB);
186  }
187
188  @Test
189  public void testCommitMergedRegion() throws Exception {
190    TableName table = TableName.valueOf(name);
191    TEST_UTIL.createTable(table, FAMILY_NAME);
192    // splitting the table first
193    TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
194    waitForSplitProcComplete(1000, 10);
195    // Add data and flush to create files in the two different regions
196    putThreeRowsAndFlush(table);
197    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
198    HRegion first = regions.get(0);
199    HRegion second = regions.get(1);
200    HRegionFileSystem regionFS = first.getRegionFileSystem();
201
202    RegionInfo mergeResult =
203      RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey())
204        .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false)
205        .setRegionId(first.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime())
206        .build();
207
208    Configuration configuration = TEST_UTIL.getHBaseCluster().getMaster().getConfiguration();
209    HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem(configuration,
210      regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult);
211
212    // merge file from first region
213    HStoreFile file = (HStoreFile) first.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
214    mergeFileFromRegion(mergeRegionFs, first, file, StoreFileTrackerFactory.create(configuration,
215      true, first.getStore(FAMILY_NAME).getStoreContext()));
216    // merge file from second region
217    file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
218    List<StoreFileInfo> mergedFiles = new ArrayList<>();
219    mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file, StoreFileTrackerFactory
220      .create(configuration, true, second.getStore(FAMILY_NAME).getStoreContext())));
221    MasterProcedureEnv env =
222      TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment();
223    mergeRegionFs.commitMergedRegion(mergedFiles, env);
224  }
225
226  private void waitForSplitProcComplete(int attempts, int waitTime) throws Exception {
227    List<Procedure<?>> procedures = TEST_UTIL.getHBaseCluster().getMaster().getProcedures();
228    if (procedures.size() > 0) {
229      Procedure splitProc =
230        procedures.stream().filter(p -> p instanceof SplitTableRegionProcedure).findFirst().get();
231      int count = 0;
232      while ((splitProc.isWaiting() || splitProc.isRunnable()) && count < attempts) {
233        synchronized (splitProc) {
234          splitProc.wait(waitTime);
235        }
236        count++;
237      }
238      assertTrue(splitProc.isSuccess());
239    }
240  }
241
242  private StoreFileInfo mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge,
243    HStoreFile file, StoreFileTracker sft) throws IOException {
244    StoreFileInfo mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(),
245      Bytes.toString(FAMILY_NAME), file, sft);
246    validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile.getPath());
247    return mergedFile;
248  }
249
250  private void validateResultingFile(String originalRegion, Path result) {
251    assertEquals(originalRegion, result.getName().split("\\.")[1]);
252    // asserts we are under the cf directory
253    Path resultParent = result.getParent();
254    assertEquals(Bytes.toString(FAMILY_NAME), resultParent.getName());
255  }
256
257  private void putThreeRowsAndFlush(TableName table) throws IOException {
258    Table tbl = TEST_UTIL.getConnection().getTable(table);
259    Put put = new Put(Bytes.toBytes("001"));
260    byte[] qualifier = Bytes.toBytes("1");
261    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1));
262    tbl.put(put);
263    put = new Put(Bytes.toBytes("002"));
264    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
265    tbl.put(put);
266    put = new Put(Bytes.toBytes("003"));
267    put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
268    tbl.put(put);
269    TEST_UTIL.flush(table);
270  }
271}