001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.Arrays;
022import java.util.Locale;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hbase.MasterNotRunningException;
025import org.apache.hadoop.hbase.TableExistsException;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Admin;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
029import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.ConnectionFactory;
032import org.apache.hadoop.hbase.client.Durability;
033import org.apache.hadoop.hbase.client.TableDescriptor;
034import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
035import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
036import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
037import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042/**
043 * This class was created by moving all load test related code from HFileTestUtil and
044 * HBaseTestingUtil as part of refactoring for hbase-diagnostics module creation in HBASE-28432
045 */
046@InterfaceAudience.Private
047public class LoadTestUtil {
048  private static final Logger LOG = LoggerFactory.getLogger(LoadTestUtil.class);
049
050  public static final String OPT_DATA_BLOCK_ENCODING_USAGE = "Encoding algorithm (e.g. prefix "
051    + "compression) to use for data blocks in the test column family, " + "one of "
052    + Arrays.toString(DataBlockEncoding.values()) + ".";
053  public static final String OPT_DATA_BLOCK_ENCODING =
054    ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT);
055
056  /**
057   * The default number of regions per regionserver when creating a pre-split table.
058   */
059  private static final int DEFAULT_REGIONS_PER_SERVER = 3;
060
061  /**
062   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
063   * continues.
064   * @return the number of regions the table was split into
065   */
066  public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName,
067    byte[] columnFamily, Algorithm compression, DataBlockEncoding dataBlockEncoding)
068    throws IOException {
069    return createPreSplitLoadTestTable(conf, tableName, columnFamily, compression,
070      dataBlockEncoding, DEFAULT_REGIONS_PER_SERVER, 1, Durability.USE_DEFAULT);
071  }
072
073  /**
074   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
075   * continues.
076   * @return the number of regions the table was split into
077   */
078  public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName,
079    byte[] columnFamily, Algorithm compression, DataBlockEncoding dataBlockEncoding,
080    int numRegionsPerServer, int regionReplication, Durability durability) throws IOException {
081    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
082    builder.setDurability(durability);
083    builder.setRegionReplication(regionReplication);
084    ColumnFamilyDescriptorBuilder cfBuilder =
085      ColumnFamilyDescriptorBuilder.newBuilder(columnFamily);
086    cfBuilder.setDataBlockEncoding(dataBlockEncoding);
087    cfBuilder.setCompressionType(compression);
088    return createPreSplitLoadTestTable(conf, builder.build(), cfBuilder.build(),
089      numRegionsPerServer);
090  }
091
092  /**
093   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
094   * continues.
095   * @return the number of regions the table was split into
096   */
097  public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName,
098    byte[][] columnFamilies, Algorithm compression, DataBlockEncoding dataBlockEncoding,
099    int numRegionsPerServer, int regionReplication, Durability durability) throws IOException {
100    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
101    builder.setDurability(durability);
102    builder.setRegionReplication(regionReplication);
103    ColumnFamilyDescriptor[] hcds = new ColumnFamilyDescriptor[columnFamilies.length];
104    for (int i = 0; i < columnFamilies.length; i++) {
105      ColumnFamilyDescriptorBuilder cfBuilder =
106        ColumnFamilyDescriptorBuilder.newBuilder(columnFamilies[i]);
107      cfBuilder.setDataBlockEncoding(dataBlockEncoding);
108      cfBuilder.setCompressionType(compression);
109      hcds[i] = cfBuilder.build();
110    }
111    return createPreSplitLoadTestTable(conf, builder.build(), hcds, numRegionsPerServer);
112  }
113
114  /**
115   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
116   * continues.
117   * @return the number of regions the table was split into
118   */
119  public static int createPreSplitLoadTestTable(Configuration conf, TableDescriptor desc,
120    ColumnFamilyDescriptor hcd) throws IOException {
121    return createPreSplitLoadTestTable(conf, desc, hcd, DEFAULT_REGIONS_PER_SERVER);
122  }
123
124  /**
125   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
126   * continues.
127   * @return the number of regions the table was split into
128   */
129  public static int createPreSplitLoadTestTable(Configuration conf, TableDescriptor desc,
130    ColumnFamilyDescriptor hcd, int numRegionsPerServer) throws IOException {
131    return createPreSplitLoadTestTable(conf, desc, new ColumnFamilyDescriptor[] { hcd },
132      numRegionsPerServer);
133  }
134
135  /**
136   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
137   * continues.
138   * @return the number of regions the table was split into
139   */
140  public static int createPreSplitLoadTestTable(Configuration conf, TableDescriptor desc,
141    ColumnFamilyDescriptor[] hcds, int numRegionsPerServer) throws IOException {
142    return createPreSplitLoadTestTable(conf, desc, hcds, new RegionSplitter.HexStringSplit(),
143      numRegionsPerServer);
144  }
145
146  /**
147   * Creates a pre-split table for load testing. If the table already exists, logs a warning and
148   * continues.
149   * @return the number of regions the table was split into
150   */
151  public static int createPreSplitLoadTestTable(Configuration conf, TableDescriptor td,
152    ColumnFamilyDescriptor[] cds, SplitAlgorithm splitter, int numRegionsPerServer)
153    throws IOException {
154    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(td);
155    for (ColumnFamilyDescriptor cd : cds) {
156      if (!td.hasColumnFamily(cd.getName())) {
157        builder.setColumnFamily(cd);
158      }
159    }
160    td = builder.build();
161    int totalNumberOfRegions = 0;
162    Connection unmanagedConnection = ConnectionFactory.createConnection(conf);
163    Admin admin = unmanagedConnection.getAdmin();
164
165    try {
166      // create a table a pre-splits regions.
167      // The number of splits is set as:
168      // region servers * regions per region server).
169      int numberOfServers = admin.getRegionServers().size();
170      if (numberOfServers == 0) {
171        throw new IllegalStateException("No live regionservers");
172      }
173
174      totalNumberOfRegions = numberOfServers * numRegionsPerServer;
175      LOG.info("Number of live regionservers: " + numberOfServers + ", "
176        + "pre-splitting table into " + totalNumberOfRegions + " regions " + "(regions per server: "
177        + numRegionsPerServer + ")");
178
179      byte[][] splits = splitter.split(totalNumberOfRegions);
180
181      admin.createTable(td, splits);
182    } catch (MasterNotRunningException e) {
183      LOG.error("Master not running", e);
184      throw new IOException(e);
185    } catch (TableExistsException e) {
186      LOG.warn("Table " + td.getTableName() + " already exists, continuing");
187    } finally {
188      admin.close();
189      unmanagedConnection.close();
190    }
191    return totalNumberOfRegions;
192  }
193
194}