001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.List;
024
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.client.Admin;
027import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
029import org.apache.hadoop.hbase.client.Connection;
030import org.apache.hadoop.hbase.client.ConnectionFactory;
031import org.apache.hadoop.hbase.testclassification.IntegrationTests;
032import org.apache.hadoop.hbase.util.Bytes;
033import org.apache.hadoop.hbase.util.HFileTestUtil;
034import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
035import org.apache.hadoop.hbase.util.LoadTestTool;
036import org.apache.hadoop.util.ToolRunner;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039
040import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
041
042/**
043 * Integration Test for MOB ingest.
044 */
045@Category(IntegrationTests.class)
046public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
047  private static final char COLON = ':';
048
049  private byte[] mobColumnFamily = HFileTestUtil.DEFAULT_COLUMN_FAMILY;
050  public static final String THRESHOLD = "threshold";
051  public static final String MIN_MOB_DATA_SIZE = "minMobDataSize";
052  public static final String MAX_MOB_DATA_SIZE = "maxMobDataSize";
053  private int threshold = 1024; // 1KB
054  private int minMobDataSize = 512; // 512B
055  private int maxMobDataSize = threshold * 5; // 5KB
056  private static final long JUNIT_RUN_TIME = 2 * 60 * 1000; // 2 minutes
057
058  //similar to LOAD_TEST_TOOL_INIT_ARGS except OPT_IN_MEMORY is removed
059  protected String[] LOAD_TEST_TOOL_MOB_INIT_ARGS = {
060      LoadTestTool.OPT_COMPRESSION,
061      HFileTestUtil.OPT_DATA_BLOCK_ENCODING,
062      LoadTestTool.OPT_ENCRYPTION,
063      LoadTestTool.OPT_NUM_REGIONS_PER_SERVER,
064      LoadTestTool.OPT_REGION_REPLICATION,
065  };
066
067  @Override
068  protected String[] getArgsForLoadTestToolInitTable() {
069    List<String> args = new ArrayList<>();
070    args.add("-tn");
071    args.add(getTablename().getNameAsString());
072    // pass all remaining args from conf with keys <test class name>.<load test tool arg>
073    String clazz = this.getClass().getSimpleName();
074    for (String arg : LOAD_TEST_TOOL_MOB_INIT_ARGS) {
075      String val = conf.get(String.format("%s.%s", clazz, arg));
076      if (val != null) {
077        args.add("-" + arg);
078        args.add(val);
079      }
080    }
081    args.add("-init_only");
082    return args.toArray(new String[args.size()]);
083  }
084
085  @Override
086  protected void addOptions() {
087    super.addOptions();
088    super.addOptWithArg(THRESHOLD, "The threshold to classify cells to mob data");
089    super.addOptWithArg(MIN_MOB_DATA_SIZE, "Minimum value size for mob data");
090    super.addOptWithArg(MAX_MOB_DATA_SIZE, "Maximum value size for mob data");
091  }
092
093  @Override
094  protected void processOptions(CommandLine cmd) {
095    super.processOptions(cmd);
096    if (cmd.hasOption(THRESHOLD)) {
097      threshold = Integer.parseInt(cmd.getOptionValue(THRESHOLD));
098    }
099    if (cmd.hasOption(MIN_MOB_DATA_SIZE)) {
100      minMobDataSize = Integer.parseInt(cmd.getOptionValue(MIN_MOB_DATA_SIZE));
101    }
102    if (cmd.hasOption(MAX_MOB_DATA_SIZE)) {
103      maxMobDataSize = Integer.parseInt(cmd.getOptionValue(MAX_MOB_DATA_SIZE));
104    }
105    if (minMobDataSize > maxMobDataSize) {
106      throw new IllegalArgumentException(
107          "The minMobDataSize should not be larger than minMobDataSize");
108    }
109  }
110
111  @Test
112  public void testIngest() throws Exception {
113    runIngestTest(JUNIT_RUN_TIME, 100, 10, 1024, 10, 20);
114  };
115
116  @Override
117  protected void initTable() throws IOException {
118    super.initTable();
119
120    TableName tableName = getTablename();
121    try (Connection connection = ConnectionFactory.createConnection();
122         Admin admin = connection.getAdmin()) {
123      HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
124      LOG.info("Disabling table " + getTablename());
125      admin.disableTable(tableName);
126      ColumnFamilyDescriptor mobColumn = tableDesc.getColumnFamily(mobColumnFamily);
127      ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(mobColumn)
128        .setMobEnabled(true)
129        .setMobThreshold((long) threshold)
130        .build();
131      admin.modifyColumnFamily(tableName, cfd);
132      LOG.info("Enabling table " + getTablename());
133      admin.enableTable(tableName);
134    }
135  }
136
137  @Override
138  protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
139      long numKeys) {
140    String[] args = super.getArgsForLoadTestTool(mode, modeSpecificArg, startKey, numKeys);
141    List<String> tmp = new ArrayList<>(Arrays.asList(args));
142    // LoadTestDataGeneratorMOB:mobColumnFamily:minMobDataSize:maxMobDataSize
143    tmp.add(HIPHEN + LoadTestTool.OPT_GENERATOR);
144    StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithMOB.class.getName());
145    sb.append(COLON);
146    sb.append(Bytes.toString(mobColumnFamily));
147    sb.append(COLON);
148    sb.append(minMobDataSize);
149    sb.append(COLON);
150    sb.append(maxMobDataSize);
151    tmp.add(sb.toString());
152    return tmp.toArray(new String[tmp.size()]);
153  }
154
155  public static void main(String[] args) throws Exception {
156    Configuration conf = HBaseConfiguration.create();
157    IntegrationTestingUtility.setUseDistributedCluster(conf);
158    int ret = ToolRunner.run(conf, new IntegrationTestIngestWithMOB(), args);
159    System.exit(ret);
160  }
161}