View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master.normalizer;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.HBaseIOException;
24  import org.apache.hadoop.hbase.HRegionInfo;
25  import org.apache.hadoop.hbase.RegionLoad;
26  import org.apache.hadoop.hbase.ServerName;
27  import org.apache.hadoop.hbase.TableName;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.master.MasterServices;
30  import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
31  
32  import java.util.ArrayList;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.List;
36  
37  /**
38   * Simple implementation of region normalizer.
39   *
40   * Logic in use:
41   *
42   *  <ol>
43   *  <li> get all regions of a given table
44   *  <li> get avg size S of each region (by total size of store files reported in RegionLoad)
45   *  <li> If biggest region is bigger than S * 2, it is kindly requested to split,
46   *    and normalization stops
47   *  <li> Otherwise, two smallest region R1 and its smallest neighbor R2 are kindly requested
48   *    to merge, if R1 + R1 &lt;  S, and normalization stops
49   *  <li> Otherwise, no action is performed
50   * </ol>
51   * <p>
52   * Region sizes are coarse and approximate on the order of megabytes. Additionally,
53   * "empty" regions (less than 1MB, with the previous note) are not merged away. This
54   * is by design to prevent normalization from undoing the pre-splitting of a table.
55   */
56  @InterfaceAudience.Private
57  public class SimpleRegionNormalizer implements RegionNormalizer {
58  
59    private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
60    private static final int MIN_REGION_COUNT = 3;
61    private MasterServices masterServices;
62  
63    /**
64     * Set the master service.
65     * @param masterServices inject instance of MasterServices
66     */
67    @Override
68    public void setMasterServices(MasterServices masterServices) {
69      this.masterServices = masterServices;
70    }
71  
72    // Comparator that gives higher priority to region Split plan
73    private Comparator<NormalizationPlan> planComparator =
74        new Comparator<NormalizationPlan>() {
75      @Override
76      public int compare(NormalizationPlan plan, NormalizationPlan plan2) {
77        if (plan instanceof SplitNormalizationPlan) {
78          return -1;
79        }
80        if (plan2 instanceof SplitNormalizationPlan) {
81          return 1;
82        }
83        return 0;
84      }
85    };
86  
87    /**
88     * Computes next most "urgent" normalization action on the table.
89     * Action may be either a split, or a merge, or no action.
90     *
91     * @param table table to normalize
92     * @return normalization plan to execute
93     */
94    @Override
95    public List<NormalizationPlan> computePlanForTable(TableName table) throws HBaseIOException {
96      if (table == null || table.isSystemTable()) {
97        LOG.debug("Normalization of system table " + table + " isn't allowed");
98        return null;
99      }
100 
101     List<NormalizationPlan> plans = new ArrayList<NormalizationPlan>();
102     List<HRegionInfo> tableRegions = masterServices.getAssignmentManager().getRegionStates().
103       getRegionsOfTable(table);
104 
105     //TODO: should we make min number of regions a config param?
106     if (tableRegions == null || tableRegions.size() < MIN_REGION_COUNT) {
107       int nrRegions = tableRegions == null ? 0 : tableRegions.size();
108       LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
109         + " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
110       return null;
111     }
112 
113     LOG.debug("Computing normalization plan for table: " + table +
114       ", number of regions: " + tableRegions.size());
115 
116     long totalSizeMb = 0;
117 
118     for (int i = 0; i < tableRegions.size(); i++) {
119       HRegionInfo hri = tableRegions.get(i);
120       long regionSize = getRegionSize(hri);
121       totalSizeMb += regionSize;
122     }
123 
124     double avgRegionSize = totalSizeMb / (double) tableRegions.size();
125 
126     LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb);
127     LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
128 
129     int candidateIdx = 0;
130     while (candidateIdx < tableRegions.size()) {
131       HRegionInfo hri = tableRegions.get(candidateIdx);
132       long regionSize = getRegionSize(hri);
133       // if the region is > 2 times larger than average, we split it, split
134       // is more high priority normalization action than merge.
135       if (regionSize > 2 * avgRegionSize) {
136         LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size "
137             + regionSize + ", more than twice avg size, splitting");
138         plans.add(new SplitNormalizationPlan(hri, null));
139       } else {
140         if (candidateIdx == tableRegions.size()-1) {
141           break;
142         }
143         HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
144         long regionSize2 = getRegionSize(hri2);
145         if (regionSize + regionSize2 < avgRegionSize) {
146           LOG.info("Table " + table + ", small region size: " + regionSize
147             + " plus its neighbor size: " + regionSize2
148             + ", less than the avg size " + avgRegionSize + ", merging them");
149           plans.add(new MergeNormalizationPlan(hri, hri2));
150           candidateIdx++;
151         }
152       }
153       candidateIdx++;
154     }
155     if (plans.isEmpty()) {
156       LOG.debug("No normalization needed, regions look good for table: " + table);
157       return null;
158     }
159     Collections.sort(plans, planComparator);
160     return plans;
161   }
162 
163   private long getRegionSize(HRegionInfo hri) {
164     ServerName sn = masterServices.getAssignmentManager().getRegionStates().
165       getRegionServerOfRegion(hri);
166     RegionLoad regionLoad = masterServices.getServerManager().getLoad(sn).
167       getRegionsLoad().get(hri.getRegionName());
168     return regionLoad.getStorefileSizeMB();
169   }
170 }