View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HRegionLocation;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.RegionLocations;
37  import org.apache.hadoop.hbase.ServerName;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.TableNotFoundException;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ExceptionUtil;
43  
44  import com.google.common.annotations.VisibleForTesting;
45  
46  /**
47   * Scanner class that contains the <code>hbase:meta</code> table scanning logic.
48   * Provided visitors will be called for each row.
49   *
50   * Although public visibility, this is not a public-facing API and may evolve in
51   * minor releases.
52   *
53   * <p> Note that during concurrent region splits, the scanner might not see
54   * hbase:meta changes across rows (for parent and daughter entries) consistently.
55   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
56   */
57  @InterfaceAudience.Private
58  //TODO: merge this to MetaTableAccessor, get rid of it.
59  public final class MetaScanner {
60    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
61  
62    private MetaScanner() {}
63  
64    /**
65     * Scans the meta table and calls a visitor on each RowResult and uses a empty
66     * start row value as table name.
67     * 
68     * <p>Visible for testing. Use {@link
69     * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
70     *
71     * @param visitor A custom visitor
72     * @throws IOException e
73     */
74    @VisibleForTesting // Do not use. Used by tests only and hbck.
75    public static void metaScan(Connection connection,
76        MetaScannerVisitor visitor) throws IOException {
77      metaScan(connection, visitor, null, null, Integer.MAX_VALUE);
78    }
79  
80    /**
81     * Scans the meta table and calls a visitor on each RowResult. Uses a table
82     * name to locate meta regions.
83     *
84     * @param connection connection to use internally (null to use a new instance)
85     * @param visitor visitor object
86     * @param userTableName User table name in meta table to start scan at.  Pass
87     * null if not interested in a particular table.
88     * @throws IOException e
89     */
90    public static void metaScan(Connection connection,
91        MetaScannerVisitor visitor, TableName userTableName) throws IOException {
92      metaScan(connection, visitor, userTableName, null, Integer.MAX_VALUE,
93          TableName.META_TABLE_NAME);
94    }
95  
96    /**
97     * Scans the meta table and calls a visitor on each RowResult. Uses a table
98     * name and a row name to locate meta regions. And it only scans at most
99     * <code>rowLimit</code> of rows.
100    * 
101    * <p>Visible for testing. Use {@link
102    * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
103    *
104    * @param connection to scan on
105    * @param visitor Visitor object.
106    * @param userTableName User table name in meta table to start scan at.  Pass
107    * null if not interested in a particular table.
108    * @param row Name of the row at the user table. The scan will start from
109    * the region row where the row resides.
110    * @param rowLimit Max of processed rows. If it is less than 0, it
111    * will be set to default value <code>Integer.MAX_VALUE</code>.
112    * @throws IOException e
113    */
114   @VisibleForTesting // Do not use. Used by Master but by a method that is used testing.
115   public static void metaScan(Connection connection,
116       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
117       int rowLimit)
118   throws IOException {
119     metaScan(connection, visitor, userTableName, row, rowLimit, TableName
120         .META_TABLE_NAME);
121   }
122 
123   /**
124    * Scans the meta table and calls a visitor on each RowResult. Uses a table
125    * name and a row name to locate meta regions. And it only scans at most
126    * <code>rowLimit</code> of rows.
127    *
128    * @param connection connection to use internally (null to use a new instance)
129    * @param visitor Visitor object. Closes the visitor before returning.
130    * @param tableName User table name in meta table to start scan at.  Pass
131    * null if not interested in a particular table.
132    * @param row Name of the row at the user table. The scan will start from
133    * the region row where the row resides.
134    * @param rowLimit Max of processed rows. If it is less than 0, it
135    * will be set to default value <code>Integer.MAX_VALUE</code>.
136    * @param metaTableName Meta table to scan, root or meta.
137    * @throws IOException e
138    */
139   static void metaScan(Connection connection,
140       final MetaScannerVisitor visitor, final TableName tableName,
141       final byte[] row, final int rowLimit, final TableName metaTableName)
142     throws IOException {
143 
144     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
145     // Calculate startrow for scan.
146     byte[] startRow;
147     // If the passed in 'connection' is 'managed' -- i.e. every second test uses
148     // an HTable or an HBaseAdmin with managed connections -- then doing
149     // connection.getTable will throw an exception saying you are NOT to use
150     // managed connections getting tables.  Leaving this as it is for now. Will
151     // revisit when inclined to change all tests.  User code probaby makes use of
152     // managed connections too so don't change it till post hbase 1.0.
153     try (Table metaTable = new HTable(TableName.META_TABLE_NAME, connection, null)) {
154       if (row != null) {
155         // Scan starting at a particular row in a particular table
156         Result startRowResult = getClosestRowOrBefore(metaTable, tableName, row,
157             connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS,
158                 HConstants.DEFAULT_USE_META_REPLICAS));
159         if (startRowResult == null) {
160           throw new TableNotFoundException("Cannot find row in " + metaTable.getName() +
161             " for table: " + tableName + ", row=" + Bytes.toStringBinary(row));
162         }
163         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
164         if (regionInfo == null) {
165           throw new IOException("HRegionInfo was null or empty in Meta for " +
166             tableName + ", row=" + Bytes.toStringBinary(row));
167         }
168         byte[] rowBefore = regionInfo.getStartKey();
169         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
170       } else if (tableName == null || tableName.getName().length == 0) {
171         // Full hbase:meta scan
172         startRow = HConstants.EMPTY_START_ROW;
173       } else {
174         // Scan hbase:meta for an entire table
175         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
176           HConstants.ZEROES, false);
177       }
178       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
179       int scannerCaching = connection.getConfiguration()
180           .getInt(HConstants.HBASE_META_SCANNER_CACHING,
181               HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
182       if (connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS,
183                 HConstants.DEFAULT_USE_META_REPLICAS)) {
184         scan.setConsistency(Consistency.TIMELINE);
185       }
186       if (rowUpperLimit <= scannerCaching) {
187           scan.setSmall(true);
188       }
189       int rows = Math.min(rowLimit, scannerCaching);
190       scan.setCaching(rows);
191       if (LOG.isTraceEnabled()) {
192         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
193           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
194       }
195       // Run the scan
196       try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
197         Result result;
198         int processedRows = 0;
199         while ((result = resultScanner.next()) != null) {
200           if (visitor != null) {
201             if (!visitor.processRow(result)) break;
202           }
203           processedRows++;
204           if (processedRows >= rowUpperLimit) break;
205         }
206       }
207     } finally {
208       if (visitor != null) {
209         try {
210           visitor.close();
211         } catch (Throwable t) {
212           ExceptionUtil.rethrowIfInterrupt(t);
213           LOG.debug("Got exception in closing the meta scanner visitor", t);
214         }
215       }
216     }
217   }
218 
219   /**
220    * @return Get closest metatable region row to passed <code>row</code>
221    * @throws IOException
222    */
223   private static Result getClosestRowOrBefore(final Table metaTable, final TableName userTableName,
224       final byte [] row, boolean useMetaReplicas)
225   throws IOException {
226     byte[] searchRow = HRegionInfo.createRegionName(userTableName, row, HConstants.NINES, false);
227     Scan scan = Scan.createGetClosestRowOrBeforeReverseScan(searchRow);
228     if (useMetaReplicas) {
229       scan.setConsistency(Consistency.TIMELINE);
230     }
231     try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
232       return resultScanner.next();
233     }
234   }
235 
236   /**
237    * Returns HRegionInfo object from the column
238    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
239    * table Result.
240    * @param data a Result object from the catalog table scan
241    * @return HRegionInfo or null
242    * @deprecated Use {@link org.apache.hadoop.hbase.MetaTableAccessor#getRegionLocations(Result)}
243    */
244   @Deprecated
245   public static HRegionInfo getHRegionInfo(Result data) {
246     return HRegionInfo.getHRegionInfo(data);
247   }
248 
249   /**
250    * Lists all of the regions currently in META.
251    * @param conf configuration
252    * @param connection to connect with
253    * @param offlined True if we are to include offlined regions, false and we'll
254    * leave out offlined regions from returned list.
255    * @return List of all user-space regions.
256    * @throws IOException
257    */
258   @VisibleForTesting // And for hbck.
259   public static List<HRegionInfo> listAllRegions(Configuration conf, Connection connection,
260       final boolean offlined)
261   throws IOException {
262     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
263     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
264         @Override
265         public boolean processRow(Result result) throws IOException {
266           if (result == null || result.isEmpty()) {
267             return true;
268           }
269 
270           RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
271           if (locations == null) return true;
272           for (HRegionLocation loc : locations.getRegionLocations()) {
273             if (loc != null) {
274               HRegionInfo regionInfo = loc.getRegionInfo();
275               // If region offline AND we are not to include offlined regions, return.
276               if (regionInfo.isOffline() && !offlined) continue;
277               regions.add(regionInfo);
278             }
279           }
280           return true;
281         }
282     };
283     metaScan(connection, visitor);
284     return regions;
285   }
286 
287   /**
288    * Lists all of the table regions currently in META.
289    * @param conf
290    * @param offlined True if we are to include offlined regions, false and we'll
291    * leave out offlined regions from returned list.
292    * @return Map of all user-space regions to servers
293    * @throws IOException
294    * @deprecated Use {@link #allTableRegions(Connection, TableName)} instead
295    */
296   @Deprecated
297   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
298       Connection connection, final TableName tableName, boolean offlined) throws IOException {
299     return allTableRegions(connection, tableName);
300   }
301 
302   /**
303    * Lists all of the table regions currently in META.
304    * @param connection
305    * @param tableName
306    * @return Map of all user-space regions to servers
307    * @throws IOException
308    */
309   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(
310       Connection connection, final TableName tableName) throws IOException {
311     final NavigableMap<HRegionInfo, ServerName> regions =
312       new TreeMap<HRegionInfo, ServerName>();
313     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
314       @Override
315       public boolean processRowInternal(Result result) throws IOException {
316         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
317         if (locations == null) return true;
318         for (HRegionLocation loc : locations.getRegionLocations()) {
319           if (loc != null) {
320             HRegionInfo regionInfo = loc.getRegionInfo();
321             regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName());
322           }
323         }
324         return true;
325       }
326     };
327     metaScan(connection, visitor, tableName);
328     return regions;
329   }
330 
331   /**
332    * Lists table regions and locations grouped by region range from META.
333    */
334   public static List<RegionLocations> listTableRegionLocations(Configuration conf,
335       Connection connection, final TableName tableName) throws IOException {
336     final List<RegionLocations> regions = new ArrayList<RegionLocations>();
337     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
338       @Override
339       public boolean processRowInternal(Result result) throws IOException {
340         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
341         if (locations == null) return true;
342         regions.add(locations);
343         return true;
344       }
345     };
346     metaScan(connection, visitor, tableName);
347     return regions;
348   }
349 
350   /**
351    * Visitor class called to process each row of the hbase:meta table
352    */
353   public interface MetaScannerVisitor extends Closeable {
354     /**
355      * Visitor method that accepts a RowResult and the meta region location.
356      * Implementations can return false to stop the region's loop if it becomes
357      * unnecessary for some reason.
358      *
359      * @param rowResult result
360      * @return A boolean to know if it should continue to loop in the region
361      * @throws IOException e
362      */
363     boolean processRow(Result rowResult) throws IOException;
364   }
365 
366   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
367     @Override
368     public void close() throws IOException {
369     }
370   }
371 
372   /**
373    * A MetaScannerVisitor that skips offline regions and split parents
374    */
375   public static abstract class DefaultMetaScannerVisitor
376     extends MetaScannerVisitorBase {
377 
378     public DefaultMetaScannerVisitor() {
379       super();
380     }
381 
382     public abstract boolean processRowInternal(Result rowResult) throws IOException;
383 
384     @Override
385     public boolean processRow(Result rowResult) throws IOException {
386       HRegionInfo info = getHRegionInfo(rowResult);
387       if (info == null) {
388         return true;
389       }
390 
391       //skip over offline and split regions
392       if (!(info.isOffline() || info.isSplit())) {
393         return processRowInternal(rowResult);
394       }
395       return true;
396     }
397   }
398 
399   /**
400    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
401    * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class
402    * does not guarantee ordered traversal of meta entries, and can block until the
403    * hbase:meta entries for daughters are available during splits.
404    */
405   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
406     private TableName tableName;
407 
408     public TableMetaScannerVisitor(TableName tableName) {
409       super();
410       this.tableName = tableName;
411     }
412 
413     @Override
414     public final boolean processRow(Result rowResult) throws IOException {
415       HRegionInfo info = getHRegionInfo(rowResult);
416       if (info == null) {
417         return true;
418       }
419       if (!(info.getTable().equals(tableName))) {
420         return false;
421       }
422       return super.processRow(rowResult);
423     }
424   }
425 }