001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.master.normalizer;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.List;
024import java.util.Objects;
025import org.apache.hadoop.hbase.RegionMetrics;
026import org.apache.hadoop.hbase.ServerName;
027import org.apache.hadoop.hbase.Size;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.MasterSwitchType;
030import org.apache.hadoop.hbase.client.RegionInfo;
031import org.apache.hadoop.hbase.client.TableDescriptor;
032import org.apache.hadoop.hbase.master.MasterRpcServices;
033import org.apache.hadoop.hbase.master.MasterServices;
034import org.apache.hadoop.hbase.master.RegionState;
035import org.apache.hadoop.hbase.master.assignment.RegionStates;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
040import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
041
042@InterfaceAudience.Private
043public abstract class AbstractRegionNormalizer implements RegionNormalizer {
044  private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class);
045  protected MasterServices masterServices;
046  protected MasterRpcServices masterRpcServices;
047
048  /**
049   * Set the master service.
050   * @param masterServices inject instance of MasterServices
051   */
052  @Override
053  public void setMasterServices(MasterServices masterServices) {
054    this.masterServices = masterServices;
055  }
056
057  @Override
058  public void setMasterRpcServices(MasterRpcServices masterRpcServices) {
059    this.masterRpcServices = masterRpcServices;
060  }
061
062  /**
063   * @param hri regioninfo
064   * @return size of region in MB and if region is not found than -1
065   */
066  protected long getRegionSize(RegionInfo hri) {
067    ServerName sn =
068        masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri);
069    RegionMetrics regionLoad =
070        masterServices.getServerManager().getLoad(sn).getRegionMetrics().get(hri.getRegionName());
071    if (regionLoad == null) {
072      LOG.debug("{} was not found in RegionsLoad", hri.getRegionNameAsString());
073      return -1;
074    }
075    return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE);
076  }
077
078  protected boolean isMergeEnabled() {
079    boolean mergeEnabled = true;
080    try {
081      mergeEnabled = masterRpcServices
082          .isSplitOrMergeEnabled(null,
083            RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE))
084          .getEnabled();
085    } catch (ServiceException e) {
086      LOG.warn("Unable to determine whether merge is enabled", e);
087    }
088    return mergeEnabled;
089  }
090
091  protected boolean isSplitEnabled() {
092    boolean splitEnabled = true;
093    try {
094      splitEnabled = masterRpcServices
095          .isSplitOrMergeEnabled(null,
096            RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT))
097          .getEnabled();
098    } catch (ServiceException se) {
099      LOG.warn("Unable to determine whether split is enabled", se);
100    }
101    return splitEnabled;
102  }
103
104  /**
105   * @param tableRegions regions of table to normalize
106   * @return average region size depending on
107   * @see org.apache.hadoop.hbase.client.TableDescriptor#getNormalizerTargetRegionCount()
108   * Also make sure tableRegions contains regions of the same table
109   */
110  protected double getAverageRegionSize(List<RegionInfo> tableRegions) {
111    long totalSizeMb = 0;
112    int actualRegionCnt = 0;
113    for (RegionInfo hri : tableRegions) {
114      long regionSize = getRegionSize(hri);
115      // don't consider regions that are in bytes for averaging the size.
116      if (regionSize > 0) {
117        actualRegionCnt++;
118        totalSizeMb += regionSize;
119      }
120    }
121    TableName table = tableRegions.get(0).getTable();
122    int targetRegionCount = -1;
123    long targetRegionSize = -1;
124    try {
125      TableDescriptor tableDescriptor = masterServices.getTableDescriptors().get(table);
126      if (tableDescriptor != null) {
127        targetRegionCount = tableDescriptor.getNormalizerTargetRegionCount();
128        targetRegionSize = tableDescriptor.getNormalizerTargetRegionSize();
129        LOG.debug("Table {}:  target region count is {}, target region size is {}", table,
130          targetRegionCount, targetRegionSize);
131      }
132    } catch (IOException e) {
133      LOG.warn(
134        "cannot get the target number and target size of table {}, they will be default value -1.",
135        table, e);
136    }
137
138    double avgRegionSize;
139    if (targetRegionSize > 0) {
140      avgRegionSize = targetRegionSize;
141    } else if (targetRegionCount > 0) {
142      avgRegionSize = totalSizeMb / (double) targetRegionCount;
143    } else {
144      avgRegionSize = actualRegionCnt == 0 ? 0 : totalSizeMb / (double) actualRegionCnt;
145    }
146
147    LOG.debug("Table {}, total aggregated regions size: {} and average region size {}", table,
148      totalSizeMb, avgRegionSize);
149    return avgRegionSize;
150  }
151
152  /**
153   * Determine if a region in {@link RegionState} should be considered for a merge operation.
154   */
155  private static boolean skipForMerge(final RegionState state) {
156    return state == null || !Objects.equals(state.getState(), RegionState.State.OPEN);
157  }
158
159  /**
160   * Computes the merge plans that should be executed for this table to converge average region
161   * towards target average or target region count
162   * @param table table to normalize
163   * @return list of merge normalization plans
164   */
165  protected List<NormalizationPlan> getMergeNormalizationPlan(TableName table) {
166    final RegionStates regionStates = masterServices.getAssignmentManager().getRegionStates();
167    final List<RegionInfo> tableRegions = regionStates.getRegionsOfTable(table);
168    final double avgRegionSize = getAverageRegionSize(tableRegions);
169    LOG.debug("Table {}, average region size: {}. Computing normalization plan for table: {}, "
170        + "number of regions: {}",
171      table, avgRegionSize, table, tableRegions.size());
172
173    final List<NormalizationPlan> plans = new ArrayList<>();
174    for (int candidateIdx = 0; candidateIdx < tableRegions.size() - 1; candidateIdx++) {
175      final RegionInfo hri = tableRegions.get(candidateIdx);
176      final RegionInfo hri2 = tableRegions.get(candidateIdx + 1);
177      if (skipForMerge(regionStates.getRegionState(hri))) {
178        continue;
179      }
180      if (skipForMerge(regionStates.getRegionState(hri2))) {
181        continue;
182      }
183      final long regionSize = getRegionSize(hri);
184      final long regionSize2 = getRegionSize(hri2);
185
186      if (regionSize >= 0 && regionSize2 >= 0 && regionSize + regionSize2 < avgRegionSize) {
187        // at least one of the two regions should be older than MIN_REGION_DURATION days
188        plans.add(new MergeNormalizationPlan(hri, hri2));
189        candidateIdx++;
190      } else {
191        LOG.debug("Skipping region {} of table {} with size {}", hri.getRegionNameAsString(), table,
192          regionSize);
193      }
194    }
195    return plans;
196  }
197
198  /**
199   * Determine if a region in {@link RegionState} should be considered for a split operation.
200   */
201  private static boolean skipForSplit(final RegionState state) {
202    return state == null || !Objects.equals(state.getState(), RegionState.State.OPEN);
203  }
204
205  /**
206   * Computes the split plans that should be executed for this table to converge average region size
207   * towards target average or target region count
208   * @param table table to normalize
209   * @return list of split normalization plans
210   */
211  protected List<NormalizationPlan> getSplitNormalizationPlan(TableName table) {
212    final RegionStates regionStates = masterServices.getAssignmentManager().getRegionStates();
213    final List<RegionInfo> tableRegions = regionStates.getRegionsOfTable(table);
214    final double avgRegionSize = getAverageRegionSize(tableRegions);
215    LOG.debug("Table {}, average region size: {}", table, avgRegionSize);
216
217    final List<NormalizationPlan> plans = new ArrayList<>();
218    for (final RegionInfo hri : tableRegions) {
219      if (skipForSplit(regionStates.getRegionState(hri))) {
220        continue;
221      }
222      long regionSize = getRegionSize(hri);
223      // if the region is > 2 times larger than average, we split it, split
224      // is more high priority normalization action than merge.
225      if (regionSize > 2 * avgRegionSize) {
226        LOG.info("Table {}, large region {} has size {}, more than twice avg size {}, splitting",
227          table, hri.getRegionNameAsString(), regionSize, avgRegionSize);
228        plans.add(new SplitNormalizationPlan(hri, null));
229      }
230    }
231    return plans;
232  }
233}