View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.util;
21  
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.ArrayList;
25  import java.util.Collection;
26  import java.util.List;
27  import java.util.concurrent.Callable;
28  import java.util.concurrent.CompletionService;
29  import java.util.concurrent.ExecutionException;
30  import java.util.concurrent.ExecutorCompletionService;
31  import java.util.concurrent.ThreadFactory;
32  import java.util.concurrent.ThreadPoolExecutor;
33  import java.util.concurrent.TimeUnit;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.master.AssignmentManager;
45  
46  /**
47   * Utility methods for interacting with the regions.
48   */
49  @InterfaceAudience.Private
50  public abstract class ModifyRegionUtils {
51    private static final Log LOG = LogFactory.getLog(ModifyRegionUtils.class);
52  
53    private ModifyRegionUtils() {
54    }
55  
56    public interface RegionFillTask {
57      void fillRegion(final HRegion region) throws IOException;
58    }
59  
60    public interface RegionEditTask {
61      void editRegion(final HRegionInfo region) throws IOException;
62    }
63  
64    /**
65     * Create new set of regions on the specified file-system.
66     * NOTE: that you should add the regions to hbase:meta after this operation.
67     *
68     * @param conf {@link Configuration}
69     * @param rootDir Root directory for HBase instance
70     * @param hTableDescriptor description of the table
71     * @param newRegions {@link HRegionInfo} that describes the regions to create
72     * @throws IOException
73     */
74    public static List<HRegionInfo> createRegions(final Configuration conf, final Path rootDir,
75        final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions) throws IOException {
76      return createRegions(conf, rootDir, hTableDescriptor, newRegions, null);
77    }
78  
79    /**
80     * Create new set of regions on the specified file-system.
81     * NOTE: that you should add the regions to hbase:meta after this operation.
82     *
83     * @param conf {@link Configuration}
84     * @param rootDir Root directory for HBase instance
85     * @param hTableDescriptor description of the table
86     * @param newRegions {@link HRegionInfo} that describes the regions to create
87     * @param task {@link RegionFillTask} custom code to populate region after creation
88     * @throws IOException
89     */
90    public static List<HRegionInfo> createRegions(final Configuration conf, final Path rootDir,
91        final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
92        final RegionFillTask task) throws IOException {
93  
94        Path tableDir = FSUtils.getTableDir(rootDir, hTableDescriptor.getTableName());
95        return createRegions(conf, rootDir, tableDir, hTableDescriptor, newRegions, task);
96    }
97  
98    /**
99     * Create new set of regions on the specified file-system.
100    * NOTE: that you should add the regions to hbase:meta after this operation.
101    *
102    * @param conf {@link Configuration}
103    * @param rootDir Root directory for HBase instance
104    * @param tableDir table directory
105    * @param hTableDescriptor description of the table
106    * @param newRegions {@link HRegionInfo} that describes the regions to create
107    * @param task {@link RegionFillTask} custom code to populate region after creation
108    * @throws IOException
109    */
110   public static List<HRegionInfo> createRegions(final Configuration conf, final Path rootDir,
111       final Path tableDir, final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
112       final RegionFillTask task) throws IOException {
113     if (newRegions == null) return null;
114     int regionNumber = newRegions.length;
115     ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf,
116         "RegionOpenAndInitThread-" + hTableDescriptor.getTableName(), regionNumber);
117     try {
118       return createRegions(exec, conf, rootDir, tableDir, hTableDescriptor, newRegions, task);
119     } finally {
120       exec.shutdownNow();
121     }
122   }
123 
124   /**
125    * Create new set of regions on the specified file-system.
126    * NOTE: that you should add the regions to hbase:meta after this operation.
127    *
128    * @param exec Thread Pool Executor
129    * @param conf {@link Configuration}
130    * @param rootDir Root directory for HBase instance
131    * @param tableDir table directory
132    * @param hTableDescriptor description of the table
133    * @param newRegions {@link HRegionInfo} that describes the regions to create
134    * @param task {@link RegionFillTask} custom code to populate region after creation
135    * @throws IOException
136    */
137   public static List<HRegionInfo> createRegions(final ThreadPoolExecutor exec,
138       final Configuration conf, final Path rootDir, final Path tableDir,
139       final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
140       final RegionFillTask task) throws IOException {
141     if (newRegions == null) return null;
142     int regionNumber = newRegions.length;
143     CompletionService<HRegionInfo> completionService =
144       new ExecutorCompletionService<HRegionInfo>(exec);
145     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
146     for (final HRegionInfo newRegion : newRegions) {
147       completionService.submit(new Callable<HRegionInfo>() {
148         @Override
149         public HRegionInfo call() throws IOException {
150           return createRegion(conf, rootDir, tableDir, hTableDescriptor, newRegion, task);
151         }
152       });
153     }
154     try {
155       // wait for all regions to finish creation
156       for (int i = 0; i < regionNumber; i++) {
157         regionInfos.add(completionService.take().get());
158       }
159     } catch (InterruptedException e) {
160       LOG.error("Caught " + e + " during region creation");
161       throw new InterruptedIOException(e.getMessage());
162     } catch (ExecutionException e) {
163       throw new IOException(e);
164     }
165     return regionInfos;
166   }
167 
168   /**
169    * Create new set of regions on the specified file-system.
170    * @param conf {@link Configuration}
171    * @param rootDir Root directory for HBase instance
172    * @param tableDir table directory
173    * @param hTableDescriptor description of the table
174    * @param newRegion {@link HRegionInfo} that describes the region to create
175    * @param task {@link RegionFillTask} custom code to populate region after creation
176    * @throws IOException
177    */
178   public static HRegionInfo createRegion(final Configuration conf, final Path rootDir,
179       final Path tableDir, final HTableDescriptor hTableDescriptor, final HRegionInfo newRegion,
180       final RegionFillTask task) throws IOException {
181     // 1. Create HRegion
182     HRegion region = HRegion.createHRegion(newRegion,
183       rootDir, tableDir, conf, hTableDescriptor, null,
184       false, true);
185     try {
186       // 2. Custom user code to interact with the created region
187       if (task != null) {
188         task.fillRegion(region);
189       }
190     } finally {
191       // 3. Close the new region to flush to disk. Close log file too.
192       region.close();
193     }
194     return region.getRegionInfo();
195   }
196 
197   /**
198    * Execute the task on the specified set of regions.
199    *
200    * @param exec Thread Pool Executor
201    * @param regions {@link HRegionInfo} that describes the regions to edit
202    * @param task {@link RegionFillTask} custom code to edit the region
203    * @throws IOException
204    */
205   public static void editRegions(final ThreadPoolExecutor exec,
206       final Collection<HRegionInfo> regions, final RegionEditTask task) throws IOException {
207     final ExecutorCompletionService<Void> completionService =
208       new ExecutorCompletionService<Void>(exec);
209     for (final HRegionInfo hri: regions) {
210       completionService.submit(new Callable<Void>() {
211         @Override
212         public Void call() throws IOException {
213           task.editRegion(hri);
214           return null;
215         }
216       });
217     }
218 
219     try {
220       for (HRegionInfo hri: regions) {
221         completionService.take().get();
222       }
223     } catch (InterruptedException e) {
224       throw new InterruptedIOException(e.getMessage());
225     } catch (ExecutionException e) {
226       IOException ex = new IOException();
227       ex.initCause(e.getCause());
228       throw ex;
229     }
230   }
231 
232   /*
233    * used by createRegions() to get the thread pool executor based on the
234    * "hbase.hregion.open.and.init.threads.max" property.
235    */
236   static ThreadPoolExecutor getRegionOpenAndInitThreadPool(final Configuration conf,
237       final String threadNamePrefix, int regionNumber) {
238     int maxThreads = Math.min(regionNumber, conf.getInt(
239         "hbase.hregion.open.and.init.threads.max", 10));
240     ThreadPoolExecutor regionOpenAndInitThreadPool = Threads
241     .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
242         new ThreadFactory() {
243           private int count = 1;
244 
245           @Override
246           public Thread newThread(Runnable r) {
247             Thread t = new Thread(r, threadNamePrefix + "-" + count++);
248             return t;
249           }
250         });
251     return regionOpenAndInitThreadPool;
252   }
253 
254   /**
255    * Triggers a bulk assignment of the specified regions
256    *
257    * @param assignmentManager the Assignment Manger
258    * @param regionInfos the list of regions to assign
259    * @throws IOException if an error occurred during the assignment
260    */
261   public static void assignRegions(final AssignmentManager assignmentManager,
262       final List<HRegionInfo> regionInfos) throws IOException {
263     try {
264       assignmentManager.getRegionStates().createRegionStates(regionInfos);
265       assignmentManager.assign(regionInfos);
266     } catch (InterruptedException e) {
267       LOG.error("Caught " + e + " during round-robin assignment");
268       InterruptedIOException ie = new InterruptedIOException(e.getMessage());
269       ie.initCause(e);
270       throw ie;
271     }
272   }
273 }