View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HRegionLocation;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.RegionLocations;
37  import org.apache.hadoop.hbase.ServerName;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.TableNotFoundException;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ExceptionUtil;
43  
44  import com.google.common.annotations.VisibleForTesting;
45  
46  /**
47   * Scanner class that contains the <code>hbase:meta</code> table scanning logic.
48   * Provided visitors will be called for each row.
49   *
50   * Although public visibility, this is not a public-facing API and may evolve in
51   * minor releases.
52   *
53   * <p> Note that during concurrent region splits, the scanner might not see
54   * hbase:meta changes across rows (for parent and daughter entries) consistently.
55   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
56   */
57  @InterfaceAudience.Private
58  //TODO: merge this to MetaTableAccessor, get rid of it.
59  public class MetaScanner {
60    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
61    /**
62     * Scans the meta table and calls a visitor on each RowResult and uses a empty
63     * start row value as table name.
64     * 
65     * <p>Visible for testing. Use {@link
66     * #metaScan(Configuration, Connection, MetaScannerVisitor, TableName)} instead.
67     *
68     * @param configuration conf
69     * @param visitor A custom visitor
70     * @throws IOException e
71     */
72    @VisibleForTesting // Do not use. Used by tests only and hbck.
73    public static void metaScan(Configuration configuration, MetaScannerVisitor visitor)
74    throws IOException {
75      metaScan(configuration, visitor, null, null, Integer.MAX_VALUE);
76    }
77  
78    /**
79     * Scans the meta table and calls a visitor on each RowResult. Uses a table
80     * name to locate meta regions.
81     *
82     * @param configuration config
83     * @param connection connection to use internally (null to use a new instance)
84     * @param visitor visitor object
85     * @param userTableName User table name in meta table to start scan at.  Pass
86     * null if not interested in a particular table.
87     * @throws IOException e
88     */
89    public static void metaScan(Configuration configuration, Connection connection,
90        MetaScannerVisitor visitor, TableName userTableName) throws IOException {
91      metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
92          TableName.META_TABLE_NAME);
93    }
94  
95    /**
96     * Scans the meta table and calls a visitor on each RowResult. Uses a table
97     * name and a row name to locate meta regions. And it only scans at most
98     * <code>rowLimit</code> of rows.
99     * 
100    * <p>Visible for testing. Use {@link
101    * #metaScan(Configuration, Connection, MetaScannerVisitor, TableName)} instead.
102    *
103    * @param configuration HBase configuration.
104    * @param visitor Visitor object.
105    * @param userTableName User table name in meta table to start scan at.  Pass
106    * null if not interested in a particular table.
107    * @param row Name of the row at the user table. The scan will start from
108    * the region row where the row resides.
109    * @param rowLimit Max of processed rows. If it is less than 0, it
110    * will be set to default value <code>Integer.MAX_VALUE</code>.
111    * @throws IOException e
112    */
113   @VisibleForTesting // Do not use. Used by Master but by a method that is used testing.
114   public static void metaScan(Configuration configuration,
115       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
116       int rowLimit)
117   throws IOException {
118     metaScan(configuration, null, visitor, userTableName, row, rowLimit, TableName.META_TABLE_NAME);
119   }
120 
121   /**
122    * Scans the meta table and calls a visitor on each RowResult. Uses a table
123    * name and a row name to locate meta regions. And it only scans at most
124    * <code>rowLimit</code> of rows.
125    *
126    * @param configuration HBase configuration.
127    * @param connection connection to use internally (null to use a new instance)
128    * @param visitor Visitor object. Closes the visitor before returning.
129    * @param tableName User table name in meta table to start scan at.  Pass
130    * null if not interested in a particular table.
131    * @param row Name of the row at the user table. The scan will start from
132    * the region row where the row resides.
133    * @param rowLimit Max of processed rows. If it is less than 0, it
134    * will be set to default value <code>Integer.MAX_VALUE</code>.
135    * @param metaTableName Meta table to scan, root or meta.
136    * @throws IOException e
137    */
138   static void metaScan(Configuration configuration, Connection connection,
139       final MetaScannerVisitor visitor, final TableName tableName,
140       final byte[] row, final int rowLimit, final TableName metaTableName)
141     throws IOException {
142 
143     boolean closeConnection = false;
144     if (connection == null) {
145       connection = ConnectionFactory.createConnection(configuration);
146       closeConnection = true;
147     }
148 
149     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
150     // Calculate startrow for scan.
151     byte[] startRow;
152     // If the passed in 'connection' is 'managed' -- i.e. every second test uses
153     // an HTable or an HBaseAdmin with managed connections -- then doing
154     // connection.getTable will throw an exception saying you are NOT to use
155     // managed connections getting tables.  Leaving this as it is for now. Will
156     // revisit when inclined to change all tests.  User code probaby makes use of
157     // managed connections too so don't change it till post hbase 1.0.
158     try (Table metaTable = new HTable(TableName.META_TABLE_NAME, connection, null)) {
159       if (row != null) {
160         // Scan starting at a particular row in a particular table
161         Result startRowResult = getClosestRowOrBefore(metaTable, tableName, row);
162         if (startRowResult == null) {
163           throw new TableNotFoundException("Cannot find row in " + metaTable.getName() +
164             " for table: " + tableName + ", row=" + Bytes.toStringBinary(row));
165         }
166         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
167         if (regionInfo == null) {
168           throw new IOException("HRegionInfo was null or empty in Meta for " +
169             tableName + ", row=" + Bytes.toStringBinary(row));
170         }
171         byte[] rowBefore = regionInfo.getStartKey();
172         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
173       } else if (tableName == null || tableName.getName().length == 0) {
174         // Full hbase:meta scan
175         startRow = HConstants.EMPTY_START_ROW;
176       } else {
177         // Scan hbase:meta for an entire table
178         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
179           HConstants.ZEROES, false);
180       }
181       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
182       int scannerCaching = configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
183         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
184       if (rowUpperLimit <= scannerCaching) {
185           scan.setSmall(true);
186       }
187       int rows = Math.min(rowLimit, scannerCaching);
188       scan.setCaching(rows);
189       if (LOG.isTraceEnabled()) {
190         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
191           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
192       }
193       // Run the scan
194       try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
195         Result result;
196         int processedRows = 0;
197         while ((result = resultScanner.next()) != null) {
198           if (visitor != null) {
199             if (!visitor.processRow(result)) break;
200           }
201           processedRows++;
202           if (processedRows >= rowUpperLimit) break;
203         }
204       }
205     } finally {
206       if (visitor != null) {
207         try {
208           visitor.close();
209         } catch (Throwable t) {
210           ExceptionUtil.rethrowIfInterrupt(t);
211           LOG.debug("Got exception in closing the meta scanner visitor", t);
212         }
213       }
214       if (closeConnection) {
215         if (connection != null) connection.close();
216       }
217     }
218   }
219 
220   /**
221    * @return Get closest metatable region row to passed <code>row</code>
222    * @throws IOException
223    */
224   private static Result getClosestRowOrBefore(final Table metaTable, final TableName userTableName,
225       final byte [] row)
226   throws IOException {
227     byte[] searchRow = HRegionInfo.createRegionName(userTableName, row, HConstants.NINES, false);
228     Scan scan = Scan.createGetClosestRowOrBeforeReverseScan(searchRow);
229     try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
230       return resultScanner.next();
231     }
232   }
233 
234   /**
235    * Returns HRegionInfo object from the column
236    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
237    * table Result.
238    * @param data a Result object from the catalog table scan
239    * @return HRegionInfo or null
240    * @deprecated Use {@link org.apache.hadoop.hbase.MetaTableAccessor#getRegionLocations(Result)}
241    */
242   @Deprecated
243   public static HRegionInfo getHRegionInfo(Result data) {
244     return HRegionInfo.getHRegionInfo(data);
245   }
246 
247   /**
248    * Lists all of the regions currently in META.
249    * @param conf
250    * @param offlined True if we are to include offlined regions, false and we'll
251    * leave out offlined regions from returned list.
252    * @return List of all user-space regions.
253    * @throws IOException
254    */
255   @VisibleForTesting // And for hbck.
256   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
257   throws IOException {
258     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
259     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
260         @Override
261         public boolean processRow(Result result) throws IOException {
262           if (result == null || result.isEmpty()) {
263             return true;
264           }
265 
266           RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
267           if (locations == null) return true;
268           for (HRegionLocation loc : locations.getRegionLocations()) {
269             if (loc != null) {
270               HRegionInfo regionInfo = loc.getRegionInfo();
271               // If region offline AND we are not to include offlined regions, return.
272               if (regionInfo.isOffline() && !offlined) continue;
273               regions.add(regionInfo);
274             }
275           }
276           return true;
277         }
278     };
279     metaScan(conf, visitor);
280     return regions;
281   }
282 
283   /**
284    * Lists all of the table regions currently in META.
285    * @param conf
286    * @param offlined True if we are to include offlined regions, false and we'll
287    * leave out offlined regions from returned list.
288    * @return Map of all user-space regions to servers
289    * @throws IOException
290    * @deprecated Use {@link #allTableRegions(Configuration, Connection, TableName)} instead
291    */
292   @Deprecated
293   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
294       Connection connection, final TableName tableName, boolean offlined) throws IOException {
295     return allTableRegions(conf, connection, tableName);
296   }
297 
298   /**
299    * Lists all of the table regions currently in META.
300    * @param conf
301    * @param connection
302    * @param tableName
303    * @return Map of all user-space regions to servers
304    * @throws IOException
305    */
306   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
307       Connection connection, final TableName tableName) throws IOException {
308     final NavigableMap<HRegionInfo, ServerName> regions =
309       new TreeMap<HRegionInfo, ServerName>();
310     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
311       @Override
312       public boolean processRowInternal(Result result) throws IOException {
313         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
314         if (locations == null) return true;
315         for (HRegionLocation loc : locations.getRegionLocations()) {
316           if (loc != null) {
317             HRegionInfo regionInfo = loc.getRegionInfo();
318             regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName());
319           }
320         }
321         return true;
322       }
323     };
324     metaScan(conf, connection, visitor, tableName);
325     return regions;
326   }
327 
328   /**
329    * Lists table regions and locations grouped by region range from META.
330    */
331   public static List<RegionLocations> listTableRegionLocations(Configuration conf,
332       Connection connection, final TableName tableName) throws IOException {
333     final List<RegionLocations> regions = new ArrayList<RegionLocations>();
334     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
335       @Override
336       public boolean processRowInternal(Result result) throws IOException {
337         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
338         if (locations == null) return true;
339         regions.add(locations);
340         return true;
341       }
342     };
343     metaScan(conf, connection, visitor, tableName);
344     return regions;
345   }
346 
347   /**
348    * Visitor class called to process each row of the hbase:meta table
349    */
350   public interface MetaScannerVisitor extends Closeable {
351     /**
352      * Visitor method that accepts a RowResult and the meta region location.
353      * Implementations can return false to stop the region's loop if it becomes
354      * unnecessary for some reason.
355      *
356      * @param rowResult result
357      * @return A boolean to know if it should continue to loop in the region
358      * @throws IOException e
359      */
360     boolean processRow(Result rowResult) throws IOException;
361   }
362 
363   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
364     @Override
365     public void close() throws IOException {
366     }
367   }
368 
369   /**
370    * A MetaScannerVisitor that skips offline regions and split parents
371    */
372   public static abstract class DefaultMetaScannerVisitor
373     extends MetaScannerVisitorBase {
374 
375     public DefaultMetaScannerVisitor() {
376       super();
377     }
378 
379     public abstract boolean processRowInternal(Result rowResult) throws IOException;
380 
381     @Override
382     public boolean processRow(Result rowResult) throws IOException {
383       HRegionInfo info = getHRegionInfo(rowResult);
384       if (info == null) {
385         return true;
386       }
387 
388       //skip over offline and split regions
389       if (!(info.isOffline() || info.isSplit())) {
390         return processRowInternal(rowResult);
391       }
392       return true;
393     }
394   }
395 
396   /**
397    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
398    * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class
399    * does not guarantee ordered traversal of meta entries, and can block until the
400    * hbase:meta entries for daughters are available during splits.
401    */
402   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
403     private TableName tableName;
404 
405     public TableMetaScannerVisitor(TableName tableName) {
406       super();
407       this.tableName = tableName;
408     }
409 
410     @Override
411     public final boolean processRow(Result rowResult) throws IOException {
412       HRegionInfo info = getHRegionInfo(rowResult);
413       if (info == null) {
414         return true;
415       }
416       if (!(info.getTable().equals(tableName))) {
417         return false;
418       }
419       return super.processRow(rowResult);
420     }
421   }
422 }