View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HRegionLocation;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.RegionLocations;
37  import org.apache.hadoop.hbase.ServerName;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.TableNotFoundException;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ExceptionUtil;
43  
44  /**
45   * Scanner class that contains the <code>hbase:meta</code> table scanning logic.
46   * Provided visitors will be called for each row.
47   *
48   * Although public visibility, this is not a public-facing API and may evolve in
49   * minor releases.
50   *
51   * <p> Note that during concurrent region splits, the scanner might not see
52   * hbase:meta changes across rows (for parent and daughter entries) consistently.
53   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
54   */
55  @InterfaceAudience.Private
56  //TODO: merge this to MetaTableAccessor, get rid of it.
57  public class MetaScanner {
58    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
59    /**
60     * Scans the meta table and calls a visitor on each RowResult and uses a empty
61     * start row value as table name.
62     *
63     * @param configuration conf
64     * @param visitor A custom visitor
65     * @throws IOException e
66     */
67    public static void metaScan(Configuration configuration,
68        MetaScannerVisitor visitor)
69    throws IOException {
70      metaScan(configuration, visitor, null, null, Integer.MAX_VALUE);
71    }
72  
73    /**
74     * Scans the meta table and calls a visitor on each RowResult. Uses a table
75     * name to locate meta regions.
76     *
77     * @param configuration config
78     * @param connection connection to use internally (null to use a new instance)
79     * @param visitor visitor object
80     * @param userTableName User table name in meta table to start scan at.  Pass
81     * null if not interested in a particular table.
82     * @throws IOException e
83     */
84    public static void metaScan(Configuration configuration, Connection connection,
85        MetaScannerVisitor visitor, TableName userTableName) throws IOException {
86      metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
87          TableName.META_TABLE_NAME);
88    }
89  
90    /**
91     * Scans the meta table and calls a visitor on each RowResult. Uses a table
92     * name and a row name to locate meta regions. And it only scans at most
93     * <code>rowLimit</code> of rows.
94     *
95     * @param configuration HBase configuration.
96     * @param visitor Visitor object.
97     * @param userTableName User table name in meta table to start scan at.  Pass
98     * null if not interested in a particular table.
99     * @param row Name of the row at the user table. The scan will start from
100    * the region row where the row resides.
101    * @param rowLimit Max of processed rows. If it is less than 0, it
102    * will be set to default value <code>Integer.MAX_VALUE</code>.
103    * @throws IOException e
104    */
105   public static void metaScan(Configuration configuration,
106       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
107       int rowLimit)
108   throws IOException {
109     metaScan(configuration, null, visitor, userTableName, row, rowLimit,
110       TableName.META_TABLE_NAME);
111   }
112 
113   /**
114    * Scans the meta table and calls a visitor on each RowResult. Uses a table
115    * name and a row name to locate meta regions. And it only scans at most
116    * <code>rowLimit</code> of rows.
117    *
118    * @param configuration HBase configuration.
119    * @param connection connection to use internally (null to use a new instance)
120    * @param visitor Visitor object. Closes the visitor before returning.
121    * @param tableName User table name in meta table to start scan at.  Pass
122    * null if not interested in a particular table.
123    * @param row Name of the row at the user table. The scan will start from
124    * the region row where the row resides.
125    * @param rowLimit Max of processed rows. If it is less than 0, it
126    * will be set to default value <code>Integer.MAX_VALUE</code>.
127    * @param metaTableName Meta table to scan, root or meta.
128    * @throws IOException e
129    */
130   static void metaScan(Configuration configuration, Connection connection,
131       final MetaScannerVisitor visitor, final TableName tableName,
132       final byte[] row, final int rowLimit, final TableName metaTableName)
133     throws IOException {
134 
135     boolean closeConnection = false;
136     if (connection == null){
137       connection = ConnectionFactory.createConnection(configuration);
138       closeConnection = true;
139     }
140 
141     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
142     // Calculate startrow for scan.
143     byte[] startRow;
144     ResultScanner scanner = null;
145     HTable metaTable = null;
146     try {
147       metaTable = new HTable(TableName.META_TABLE_NAME, connection, null);
148       if (row != null) {
149         // Scan starting at a particular row in a particular table
150         byte[] searchRow = HRegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
151 
152         Result startRowResult = metaTable.getRowOrBefore(searchRow, HConstants.CATALOG_FAMILY);
153 
154         if (startRowResult == null) {
155           throw new TableNotFoundException("Cannot find row in "+ TableName
156               .META_TABLE_NAME.getNameAsString()+" for table: "
157               + tableName + ", row=" + Bytes.toStringBinary(searchRow));
158         }
159         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
160         if (regionInfo == null) {
161           throw new IOException("HRegionInfo was null or empty in Meta for " +
162             tableName + ", row=" + Bytes.toStringBinary(searchRow));
163         }
164         byte[] rowBefore = regionInfo.getStartKey();
165         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
166       } else if (tableName == null || tableName.getName().length == 0) {
167         // Full hbase:meta scan
168         startRow = HConstants.EMPTY_START_ROW;
169       } else {
170         // Scan hbase:meta for an entire table
171         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
172           HConstants.ZEROES, false);
173       }
174       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
175       int scannerCaching = configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
176         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
177       if (rowUpperLimit <= scannerCaching) {
178           scan.setSmall(true);
179       }
180       int rows = Math.min(rowLimit, scannerCaching);
181       scan.setCaching(rows);
182       if (LOG.isTraceEnabled()) {
183         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
184           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
185       }
186       // Run the scan
187       scanner = metaTable.getScanner(scan);
188       Result result;
189       int processedRows = 0;
190       while ((result = scanner.next()) != null) {
191         if (visitor != null) {
192           if (!visitor.processRow(result)) break;
193         }
194         processedRows++;
195         if (processedRows >= rowUpperLimit) break;
196       }
197     } finally {
198       if (scanner != null) {
199         try {
200           scanner.close();
201         } catch (Throwable t) {
202           ExceptionUtil.rethrowIfInterrupt(t);
203           LOG.debug("Got exception in closing the result scanner", t);
204         }
205       }
206       if (visitor != null) {
207         try {
208           visitor.close();
209         } catch (Throwable t) {
210           ExceptionUtil.rethrowIfInterrupt(t);
211           LOG.debug("Got exception in closing the meta scanner visitor", t);
212         }
213       }
214       if (metaTable != null) {
215         try {
216           metaTable.close();
217         } catch (Throwable t) {
218           ExceptionUtil.rethrowIfInterrupt(t);
219           LOG.debug("Got exception in closing meta table", t);
220         }
221       }
222       if (closeConnection) {
223         connection.close();
224       }
225     }
226   }
227 
228   /**
229    * Returns HRegionInfo object from the column
230    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
231    * table Result.
232    * @param data a Result object from the catalog table scan
233    * @return HRegionInfo or null
234    * @deprecated Use {@link org.apache.hadoop.hbase.MetaTableAccessor#getRegionLocations(Result)}
235    */
236   @Deprecated
237   public static HRegionInfo getHRegionInfo(Result data) {
238     return HRegionInfo.getHRegionInfo(data);
239   }
240 
241   /**
242    * Lists all of the regions currently in META.
243    * @param conf
244    * @param offlined True if we are to include offlined regions, false and we'll
245    * leave out offlined regions from returned list.
246    * @return List of all user-space regions.
247    * @throws IOException
248    */
249   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
250   throws IOException {
251     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
252     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
253         @Override
254         public boolean processRow(Result result) throws IOException {
255           if (result == null || result.isEmpty()) {
256             return true;
257           }
258 
259           RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
260           if (locations == null) return true;
261           for (HRegionLocation loc : locations.getRegionLocations()) {
262             if (loc != null) {
263               HRegionInfo regionInfo = loc.getRegionInfo();
264               // If region offline AND we are not to include offlined regions, return.
265               if (regionInfo.isOffline() && !offlined) continue;
266               regions.add(regionInfo);
267             }
268           }
269           return true;
270         }
271     };
272     metaScan(conf, visitor);
273     return regions;
274   }
275 
276   /**
277    * Lists all of the table regions currently in META.
278    * @param conf
279    * @param offlined True if we are to include offlined regions, false and we'll
280    * leave out offlined regions from returned list.
281    * @return Map of all user-space regions to servers
282    * @throws IOException
283    * @deprecated Use {@link #allTableRegions(Configuration, Connection, TableName)} instead
284    */
285   @Deprecated
286   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
287       Connection connection, final TableName tableName, boolean offlined) throws IOException {
288     return allTableRegions(conf, connection, tableName);
289   }
290 
291   /**
292    * Lists all of the table regions currently in META.
293    * @param conf
294    * @param connection
295    * @param tableName
296    * @return Map of all user-space regions to servers
297    * @throws IOException
298    */
299   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
300       Connection connection, final TableName tableName) throws IOException {
301     final NavigableMap<HRegionInfo, ServerName> regions =
302       new TreeMap<HRegionInfo, ServerName>();
303     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
304       @Override
305       public boolean processRowInternal(Result result) throws IOException {
306         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
307         if (locations == null) return true;
308         for (HRegionLocation loc : locations.getRegionLocations()) {
309           if (loc != null) {
310             HRegionInfo regionInfo = loc.getRegionInfo();
311             regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName());
312           }
313         }
314         return true;
315       }
316     };
317     metaScan(conf, connection, visitor, tableName);
318     return regions;
319   }
320 
321   /**
322    * Lists table regions and locations grouped by region range from META.
323    */
324   public static List<RegionLocations> listTableRegionLocations(Configuration conf,
325       Connection connection, final TableName tableName) throws IOException {
326     final List<RegionLocations> regions = new ArrayList<RegionLocations>();
327     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
328       @Override
329       public boolean processRowInternal(Result result) throws IOException {
330         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
331         if (locations == null) return true;
332         regions.add(locations);
333         return true;
334       }
335     };
336     metaScan(conf, connection, visitor, tableName);
337     return regions;
338   }
339 
340   /**
341    * Visitor class called to process each row of the hbase:meta table
342    */
343   public interface MetaScannerVisitor extends Closeable {
344     /**
345      * Visitor method that accepts a RowResult and the meta region location.
346      * Implementations can return false to stop the region's loop if it becomes
347      * unnecessary for some reason.
348      *
349      * @param rowResult result
350      * @return A boolean to know if it should continue to loop in the region
351      * @throws IOException e
352      */
353     boolean processRow(Result rowResult) throws IOException;
354   }
355 
356   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
357     @Override
358     public void close() throws IOException {
359     }
360   }
361 
362   /**
363    * A MetaScannerVisitor that skips offline regions and split parents
364    */
365   public static abstract class DefaultMetaScannerVisitor
366     extends MetaScannerVisitorBase {
367 
368     public DefaultMetaScannerVisitor() {
369       super();
370     }
371 
372     public abstract boolean processRowInternal(Result rowResult) throws IOException;
373 
374     @Override
375     public boolean processRow(Result rowResult) throws IOException {
376       HRegionInfo info = getHRegionInfo(rowResult);
377       if (info == null) {
378         return true;
379       }
380 
381       //skip over offline and split regions
382       if (!(info.isOffline() || info.isSplit())) {
383         return processRowInternal(rowResult);
384       }
385       return true;
386     }
387   }
388 
389   /**
390    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
391    * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class
392    * does not guarantee ordered traversal of meta entries, and can block until the
393    * hbase:meta entries for daughters are available during splits.
394    */
395   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
396     private TableName tableName;
397 
398     public TableMetaScannerVisitor(TableName tableName) {
399       super();
400       this.tableName = tableName;
401     }
402 
403     @Override
404     public final boolean processRow(Result rowResult) throws IOException {
405       HRegionInfo info = getHRegionInfo(rowResult);
406       if (info == null) {
407         return true;
408       }
409       if (!(info.getTable().equals(tableName))) {
410         return false;
411       }
412       return super.processRow(rowResult);
413     }
414   }
415 }