View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.ServerName;
37  import org.apache.hadoop.hbase.TableNotFoundException;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.ExceptionUtil;
40  
41  /**
42   * Scanner class that contains the <code>hbase:meta</code> table scanning logic.
43   * Provided visitors will be called for each row.
44   *
45   * Although public visibility, this is not a public-facing API and may evolve in
46   * minor releases.
47   *
48   * <p> Note that during concurrent region splits, the scanner might not see
49   * hbase:meta changes across rows (for parent and daughter entries) consistently.
50   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
51   */
52  @InterfaceAudience.Private
53  public class MetaScanner {
54    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
55    /**
56     * Scans the meta table and calls a visitor on each RowResult and uses a empty
57     * start row value as table name.
58     *
59     * @param configuration conf
60     * @param visitor A custom visitor
61     * @throws IOException e
62     */
63    public static void metaScan(Configuration configuration,
64        MetaScannerVisitor visitor)
65    throws IOException {
66      metaScan(configuration, visitor, null, null, Integer.MAX_VALUE);
67    }
68  
69    /**
70     * Scans the meta table and calls a visitor on each RowResult. Uses a table
71     * name to locate meta regions.
72     *
73     * @param configuration config
74     * @param connection connection to use internally (null to use a new instance)
75     * @param visitor visitor object
76     * @param userTableName User table name in meta table to start scan at.  Pass
77     * null if not interested in a particular table.
78     * @throws IOException e
79     */
80    public static void metaScan(Configuration configuration, ClusterConnection connection,
81        MetaScannerVisitor visitor, TableName userTableName) throws IOException {
82      metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
83          TableName.META_TABLE_NAME);
84    }
85  
86    /**
87     * Scans the meta table and calls a visitor on each RowResult. Uses a table
88     * name and a row name to locate meta regions. And it only scans at most
89     * <code>rowLimit</code> of rows.
90     *
91     * @param configuration HBase configuration.
92     * @param visitor Visitor object.
93     * @param userTableName User table name in meta table to start scan at.  Pass
94     * null if not interested in a particular table.
95     * @param row Name of the row at the user table. The scan will start from
96     * the region row where the row resides.
97     * @param rowLimit Max of processed rows. If it is less than 0, it
98     * will be set to default value <code>Integer.MAX_VALUE</code>.
99     * @throws IOException e
100    */
101   public static void metaScan(Configuration configuration,
102       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
103       int rowLimit)
104   throws IOException {
105     metaScan(configuration, null, visitor, userTableName, row, rowLimit,
106       TableName.META_TABLE_NAME);
107   }
108 
109   /**
110    * Scans the meta table and calls a visitor on each RowResult. Uses a table
111    * name and a row name to locate meta regions. And it only scans at most
112    * <code>rowLimit</code> of rows.
113    *
114    * @param configuration HBase configuration.
115    * @param connection connection to use internally (null to use a new instance)
116    * @param visitor Visitor object. Closes the visitor before returning.
117    * @param tableName User table name in meta table to start scan at.  Pass
118    * null if not interested in a particular table.
119    * @param row Name of the row at the user table. The scan will start from
120    * the region row where the row resides.
121    * @param rowLimit Max of processed rows. If it is less than 0, it
122    * will be set to default value <code>Integer.MAX_VALUE</code>.
123    * @param metaTableName Meta table to scan, root or meta.
124    * @throws IOException e
125    */
126   static void metaScan(Configuration configuration, ClusterConnection connection,
127       final MetaScannerVisitor visitor, final TableName tableName,
128       final byte[] row, final int rowLimit, final TableName metaTableName)
129     throws IOException {
130 
131     if (connection == null){
132       connection = ConnectionManager.getConnectionInternal(configuration);
133     }
134 
135     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
136     // Calculate startrow for scan.
137     byte[] startRow;
138     ResultScanner scanner = null;
139 
140     try {
141       if (row != null) {
142         // Scan starting at a particular row in a particular table
143         byte[] searchRow = HRegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
144         HTable metaTable;
145         metaTable = new HTable(TableName.META_TABLE_NAME, connection, null);
146         Result startRowResult = metaTable.getRowOrBefore(searchRow, HConstants.CATALOG_FAMILY);
147         metaTable.close();
148         if (startRowResult == null) {
149           throw new TableNotFoundException("Cannot find row in "+ TableName
150               .META_TABLE_NAME.getNameAsString()+" for table: "
151               + tableName + ", row=" + Bytes.toStringBinary(searchRow));
152         }
153         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
154         if (regionInfo == null) {
155           throw new IOException("HRegionInfo was null or empty in Meta for " +
156             tableName + ", row=" + Bytes.toStringBinary(searchRow));
157         }
158         byte[] rowBefore = regionInfo.getStartKey();
159         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
160       } else if (tableName == null || tableName.getName().length == 0) {
161         // Full hbase:meta scan
162         startRow = HConstants.EMPTY_START_ROW;
163       } else {
164         // Scan hbase:meta for an entire table
165         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
166           HConstants.ZEROES, false);
167       }
168       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
169       int scannerCaching = configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
170         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
171       if (rowUpperLimit <= scannerCaching) {
172           scan.setSmall(true);
173       }
174       int rows = Math.min(rowLimit, scannerCaching);
175       scan.setCaching(rows);
176       if (LOG.isTraceEnabled()) {
177         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
178           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
179       }
180       // Run the scan
181       scanner = (scan.isSmall() ?
182           new ClientSmallScanner(configuration, scan, TableName.META_TABLE_NAME, connection) :
183           new ClientScanner(configuration, scan, TableName.META_TABLE_NAME, connection));
184       Result result;
185       int processedRows = 0;
186       while ((result = scanner.next()) != null) {
187         if (visitor != null) {
188           if (!visitor.processRow(result)) break;
189         }
190         processedRows++;
191         if (processedRows >= rowUpperLimit) break;
192       }
193     } finally {
194       if (scanner != null) {
195         try {
196           scanner.close();
197         } catch (Throwable t) {
198           ExceptionUtil.rethrowIfInterrupt(t);
199           LOG.debug("Got exception in closing the result scanner", t);
200         }
201       }
202       if (visitor != null) {
203         try {
204           visitor.close();
205         } catch (Throwable t) {
206           ExceptionUtil.rethrowIfInterrupt(t);
207           LOG.debug("Got exception in closing the meta scanner visitor", t);
208         }
209       }
210     }
211   }
212 
213   /**
214    * Returns HRegionInfo object from the column
215    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
216    * table Result.
217    * @param data a Result object from the catalog table scan
218    * @return HRegionInfo or null
219    */
220   public static HRegionInfo getHRegionInfo(Result data) {
221     return HRegionInfo.getHRegionInfo(data);
222   }
223 
224   /**
225    * Used in tests.
226    *
227    * Lists all of the regions currently in META.
228    * @param conf
229    * @param offlined True if we are to include offlined regions, false and we'll
230    * leave out offlined regions from returned list.
231    * @return List of all user-space regions.
232    * @throws IOException
233    */
234   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
235   throws IOException {
236     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
237     MetaScannerVisitor visitor = new DefaultMetaScannerVisitor() {
238         @Override
239         public boolean processRowInternal(Result result) throws IOException {
240           if (result == null || result.isEmpty()) {
241             return true;
242           }
243 
244           HRegionInfo regionInfo = getHRegionInfo(result);
245           if (regionInfo == null) {
246             LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
247             return true;
248           }
249 
250           // If region offline AND we are not to include offlined regions, return.
251           if (regionInfo.isOffline() && !offlined) return true;
252           regions.add(regionInfo);
253           return true;
254         }
255     };
256     metaScan(conf, visitor);
257     return regions;
258   }
259 
260   /**
261    * Lists all of the table regions currently in META.
262    * @param conf
263    * @param offlined True if we are to include offlined regions, false and we'll
264    * leave out offlined regions from returned list.
265    * @return Map of all user-space regions to servers
266    * @throws IOException
267    */
268   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
269       ClusterConnection connection, final TableName tableName,
270       final boolean offlined) throws IOException {
271     final NavigableMap<HRegionInfo, ServerName> regions =
272       new TreeMap<HRegionInfo, ServerName>();
273     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
274       @Override
275       public boolean processRowInternal(Result rowResult) throws IOException {
276         HRegionInfo info = getHRegionInfo(rowResult);
277         ServerName serverName = HRegionInfo.getServerName(rowResult);
278         regions.put(new UnmodifyableHRegionInfo(info), serverName);
279         return true;
280       }
281     };
282     metaScan(conf, connection, visitor, tableName);
283     return regions;
284   }
285 
286   /**
287    * Visitor class called to process each row of the hbase:meta table
288    */
289   public interface MetaScannerVisitor extends Closeable {
290     /**
291      * Visitor method that accepts a RowResult and the meta region location.
292      * Implementations can return false to stop the region's loop if it becomes
293      * unnecessary for some reason.
294      *
295      * @param rowResult result
296      * @return A boolean to know if it should continue to loop in the region
297      * @throws IOException e
298      */
299     boolean processRow(Result rowResult) throws IOException;
300   }
301 
302   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
303     @Override
304     public void close() throws IOException {
305     }
306   }
307 
308   /**
309    * A MetaScannerVisitor that skips offline regions and split parents
310    */
311   public static abstract class DefaultMetaScannerVisitor
312     extends MetaScannerVisitorBase {
313 
314     public DefaultMetaScannerVisitor() {
315       super();
316     }
317 
318     public abstract boolean processRowInternal(Result rowResult) throws IOException;
319 
320     @Override
321     public boolean processRow(Result rowResult) throws IOException {
322       HRegionInfo info = getHRegionInfo(rowResult);
323       if (info == null) {
324         return true;
325       }
326 
327       //skip over offline and split regions
328       if (!(info.isOffline() || info.isSplit())) {
329         return processRowInternal(rowResult);
330       }
331       return true;
332     }
333   }
334 
335   /**
336    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
337    * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class
338    * does not guarantee ordered traversal of meta entries, and can block until the
339    * hbase:meta entries for daughters are available during splits.
340    */
341   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
342     private TableName tableName;
343 
344     public TableMetaScannerVisitor(TableName tableName) {
345       super();
346       this.tableName = tableName;
347     }
348 
349     @Override
350     public final boolean processRow(Result rowResult) throws IOException {
351       HRegionInfo info = getHRegionInfo(rowResult);
352       if (info == null) {
353         return true;
354       }
355       if (!(info.getTable().equals(tableName))) {
356         return false;
357       }
358       return super.processRow(rowResult);
359     }
360   }
361 }