001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.storefiletracker;
019
020import static org.hamcrest.CoreMatchers.hasItems;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Collectors;
028import java.util.stream.Stream;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.Get;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.regionserver.ChunkCreator;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.wal.WAL;
049import org.junit.jupiter.api.AfterEach;
050import org.junit.jupiter.api.BeforeAll;
051import org.junit.jupiter.api.BeforeEach;
052import org.junit.jupiter.api.Tag;
053import org.junit.jupiter.api.TestInfo;
054import org.junit.jupiter.api.TestTemplate;
055import org.junit.jupiter.params.provider.Arguments;
056
057import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
058import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
059
060@Tag(RegionServerTests.TAG)
061@Tag(MediumTests.TAG)
062@HBaseParameterizedTestTemplate(name = "{index}: src={0}, dst={1}")
063public class TestMigrationStoreFileTracker {
064
065  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
066
067  private static final byte[] CF = Bytes.toBytes("cf");
068
069  private static final byte[] CQ = Bytes.toBytes("cq");
070
071  private static final TableDescriptor TD =
072    TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker"))
073      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
074
075  private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build();
076
077  private String name;
078
079  private final StoreFileTrackerFactory.Trackers srcImpl;
080
081  private final StoreFileTrackerFactory.Trackers dstImpl;
082
083  public TestMigrationStoreFileTracker(StoreFileTrackerFactory.Trackers srcImpl,
084    StoreFileTrackerFactory.Trackers dstImpl) {
085    this.srcImpl = srcImpl;
086    this.dstImpl = dstImpl;
087  }
088
089  private HRegion region;
090
091  private Path rootDir;
092
093  private WAL wal;
094
095  public static Stream<Arguments> parameters() {
096    List<Arguments> params = new ArrayList<>();
097    for (StoreFileTrackerFactory.Trackers src : StoreFileTrackerFactory.Trackers.values()) {
098      for (StoreFileTrackerFactory.Trackers dst : StoreFileTrackerFactory.Trackers.values()) {
099        if (
100          src == StoreFileTrackerFactory.Trackers.MIGRATION
101            || dst == StoreFileTrackerFactory.Trackers.MIGRATION
102        ) {
103          continue;
104        }
105        if (src.equals(dst)) {
106          continue;
107        }
108        params.add(Arguments.of(src, dst));
109      }
110    }
111    return params.stream();
112  }
113
114  @BeforeAll
115  public static void setUpBeforeClass() {
116    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
117      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
118  }
119
120  @BeforeEach
121  public void setUp(TestInfo testInfo) throws IOException {
122    name = testInfo.getTestMethod().get().getName();
123    Configuration conf = UTIL.getConfiguration();
124    conf.set(MigrationStoreFileTracker.SRC_IMPL, srcImpl.name().toLowerCase());
125    conf.set(MigrationStoreFileTracker.DST_IMPL, dstImpl.name().toLowerCase());
126    rootDir = UTIL.getDataTestDir(name.replaceAll("[=:\\[ ]", "_"));
127    wal = HBaseTestingUtil.createWal(conf, rootDir, RI);
128  }
129
130  @AfterEach
131  public void tearDown() throws IOException {
132    if (region != null) {
133      region.close();
134    }
135    Closeables.close(wal, true);
136    UTIL.cleanupTestDir();
137  }
138
139  private List<String> getStoreFiles() {
140    return Iterables.getOnlyElement(region.getStores()).getStorefiles().stream()
141      .map(s -> s.getFileInfo().getPath().getName()).collect(Collectors.toList());
142  }
143
144  private HRegion createRegion(Class<? extends StoreFileTrackerBase> trackerImplClass)
145    throws IOException {
146    Configuration conf = new Configuration(UTIL.getConfiguration());
147    conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class);
148    return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true);
149  }
150
151  private void reopenRegion(Class<? extends StoreFileTrackerBase> trackerImplClass)
152    throws IOException {
153    region.flush(true);
154    List<String> before = getStoreFiles();
155    region.close();
156    Configuration conf = new Configuration(UTIL.getConfiguration());
157    conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class);
158    region = HRegion.openHRegion(rootDir, RI, TD, wal, conf);
159    List<String> after = getStoreFiles();
160    assertEquals(before.size(), after.size());
161    assertThat(after, hasItems(before.toArray(new String[0])));
162  }
163
164  private void putData(int start, int end) throws IOException {
165    for (int i = start; i < end; i++) {
166      region.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
167      if (i % 30 == 0) {
168        region.flush(true);
169      }
170    }
171  }
172
173  private void verifyData(int start, int end) throws IOException {
174    for (int i = start; i < end; i++) {
175      Result result = region.get(new Get(Bytes.toBytes(i)));
176      assertEquals(i, Bytes.toInt(result.getValue(CF, CQ)));
177    }
178  }
179
180  @TestTemplate
181  public void testMigration() throws IOException {
182    region = createRegion(srcImpl.clazz.asSubclass(StoreFileTrackerBase.class));
183    putData(0, 100);
184    verifyData(0, 100);
185    reopenRegion(MigrationStoreFileTracker.class);
186    verifyData(0, 100);
187    region.compact(true);
188    putData(100, 200);
189    reopenRegion(dstImpl.clazz.asSubclass(StoreFileTrackerBase.class));
190    verifyData(0, 200);
191  }
192}