View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.classification.InterfaceAudience;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HTableDescriptor;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.client.Get;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Result;
32  import org.apache.hadoop.hbase.client.ResultScanner;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.Pair;
36  
37  import java.io.IOException;
38  import java.io.InterruptedIOException;
39  import java.util.ArrayList;
40  import java.util.List;
41  import java.util.Map;
42  import java.util.NavigableMap;
43  import java.util.Set;
44  import java.util.TreeMap;
45  
46  /**
47   * Reads region and assignment information from <code>hbase:meta</code>.
48   */
49  @InterfaceAudience.Private
50  public class MetaReader {
51    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
52    // its only used to get its Configuration so we can get associated
53    // Connection.
54    private static final Log LOG = LogFactory.getLog(MetaReader.class);
55  
56    static final byte [] META_REGION_PREFIX;
57    static {
58      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
59      // FIRST_META_REGIONINFO == 'hbase:meta,,1'.  META_REGION_PREFIX == 'hbase:meta,'
60      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
61      META_REGION_PREFIX = new byte [len];
62      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
63        META_REGION_PREFIX, 0, len);
64    }
65  
66    /**
67     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
68     * tables in the specified set of disabled tables.
69     * @param catalogTracker
70     * @param disabledTables set of disabled tables that will not be returned
71     * @return Returns a map of every region to it's currently assigned server,
72     * according to META.  If the region does not have an assignment it will have
73     * a null value in the map.
74     * @throws IOException
75     */
76    public static Map<HRegionInfo, ServerName> fullScan(
77        CatalogTracker catalogTracker, final Set<TableName> disabledTables)
78    throws IOException {
79      return fullScan(catalogTracker, disabledTables, false);
80    }
81  
82    /**
83     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
84     * tables in the specified set of disabled tables.
85     * @param catalogTracker
86     * @param disabledTables set of disabled tables that will not be returned
87     * @param excludeOfflinedSplitParents If true, do not include offlined split
88     * parents in the return.
89     * @return Returns a map of every region to it's currently assigned server,
90     * according to META.  If the region does not have an assignment it will have
91     * a null value in the map.
92     * @throws IOException
93     */
94    public static Map<HRegionInfo, ServerName> fullScan(
95        CatalogTracker catalogTracker, final Set<TableName> disabledTables,
96        final boolean excludeOfflinedSplitParents)
97    throws IOException {
98      final Map<HRegionInfo, ServerName> regions =
99        new TreeMap<HRegionInfo, ServerName>();
100     Visitor v = new Visitor() {
101       @Override
102       public boolean visit(Result r) throws IOException {
103         if (r ==  null || r.isEmpty()) return true;
104         Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
105         HRegionInfo hri = region.getFirst();
106         if (hri  == null) return true;
107         if (hri.getTable() == null) return true;
108         if (disabledTables.contains(
109             hri.getTable())) return true;
110         // Are we to include split parents in the list?
111         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
112         regions.put(hri, region.getSecond());
113         return true;
114       }
115     };
116     fullScan(catalogTracker, v);
117     return regions;
118   }
119 
120   /**
121    * Performs a full scan of <code>hbase:meta</code>.
122    * @return List of {@link Result}
123    * @throws IOException
124    */
125   public static List<Result> fullScan(CatalogTracker catalogTracker)
126   throws IOException {
127     CollectAllVisitor v = new CollectAllVisitor();
128     fullScan(catalogTracker, v, null);
129     return v.getResults();
130   }
131 
132   /**
133    * Performs a full scan of a <code>hbase:meta</code> table.
134    * @return List of {@link Result}
135    * @throws IOException
136    */
137   public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
138   throws IOException {
139     CollectAllVisitor v = new CollectAllVisitor();
140     fullScan(catalogTracker, v, null);
141     return v.getResults();
142   }
143 
144   /**
145    * Performs a full scan of <code>hbase:meta</code>.
146    * @param catalogTracker
147    * @param visitor Visitor invoked against each row.
148    * @throws IOException
149    */
150   public static void fullScan(CatalogTracker catalogTracker,
151       final Visitor visitor)
152   throws IOException {
153     fullScan(catalogTracker, visitor, null);
154   }
155 
156   /**
157    * Callers should call close on the returned {@link HTable} instance.
158    * @param catalogTracker We'll use this catalogtracker's connection
159    * @param tableName Table to get an {@link HTable} against.
160    * @return An {@link HTable} for <code>tableName</code>
161    * @throws IOException
162    */
163   private static HTable getHTable(final CatalogTracker catalogTracker,
164       final TableName tableName)
165   throws IOException {
166     // Passing the CatalogTracker's connection ensures this
167     // HTable instance uses the CatalogTracker's connection.
168     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
169     if (c == null) throw new NullPointerException("No connection");
170     return new HTable(tableName, c);
171   }
172 
173   /**
174    * Callers should call close on the returned {@link HTable} instance.
175    * @param catalogTracker
176    * @return An {@link HTable} for <code>hbase:meta</code>
177    * @throws IOException
178    */
179   static HTable getCatalogHTable(final CatalogTracker catalogTracker)
180   throws IOException {
181     return getMetaHTable(catalogTracker);
182   }
183 
184   /**
185    * Callers should call close on the returned {@link HTable} instance.
186    * @param ct
187    * @return An {@link HTable} for <code>hbase:meta</code>
188    * @throws IOException
189    */
190   static HTable getMetaHTable(final CatalogTracker ct)
191   throws IOException {
192     return getHTable(ct, TableName.META_TABLE_NAME);
193   }
194 
195   /**
196    * @param t Table to use (will be closed when done).
197    * @param g Get to run
198    * @throws IOException
199    */
200   private static Result get(final HTable t, final Get g) throws IOException {
201     try {
202       return t.get(g);
203     } finally {
204       t.close();
205     }
206   }
207 
208   /**
209    * Reads the location of the specified region
210    * @param catalogTracker
211    * @param regionName region whose location we are after
212    * @return location of region as a {@link ServerName} or null if not found
213    * @throws IOException
214    */
215   static ServerName readRegionLocation(CatalogTracker catalogTracker,
216       byte [] regionName)
217   throws IOException {
218     Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
219     return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
220   }
221 
222   /**
223    * Gets the region info and assignment for the specified region.
224    * @param catalogTracker
225    * @param regionName Region to lookup.
226    * @return Location and HRegionInfo for <code>regionName</code>
227    * @throws IOException
228    */
229   public static Pair<HRegionInfo, ServerName> getRegion(
230       CatalogTracker catalogTracker, byte [] regionName)
231   throws IOException {
232     Get get = new Get(regionName);
233     get.addFamily(HConstants.CATALOG_FAMILY);
234     Result r = get(getCatalogHTable(catalogTracker), get);
235     return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
236   }
237 
238   /**
239    * Gets the result in hbase:meta for the specified region.
240    * @param catalogTracker
241    * @param regionName
242    * @return result of the specified region
243    * @throws IOException
244    */
245   public static Result getRegionResult(CatalogTracker catalogTracker,
246       byte[] regionName) throws IOException {
247     Get get = new Get(regionName);
248     get.addFamily(HConstants.CATALOG_FAMILY);
249     return get(getCatalogHTable(catalogTracker), get);
250   }
251 
252   /**
253    * Get regions from the merge qualifier of the specified merged region
254    * @return null if it doesn't contain merge qualifier, else two merge regions
255    * @throws IOException
256    */
257   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
258       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
259     Result result = getRegionResult(catalogTracker, regionName);
260     HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
261         HConstants.MERGEA_QUALIFIER);
262     HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
263         HConstants.MERGEB_QUALIFIER);
264     if (mergeA == null && mergeB == null) {
265       return null;
266     }
267     return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
268  }
269 
270   /**
271    * Checks if the specified table exists.  Looks at the hbase:meta table hosted on
272    * the specified server.
273    * @param catalogTracker
274    * @param tableName table to check
275    * @return true if the table exists in meta, false if not
276    * @throws IOException
277    */
278   public static boolean tableExists(CatalogTracker catalogTracker,
279       final TableName tableName)
280   throws IOException {
281     if (tableName.equals(HTableDescriptor.META_TABLEDESC.getTableName())) {
282       // Catalog tables always exist.
283       return true;
284     }
285     // Make a version of ResultCollectingVisitor that only collects the first
286     CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
287       private HRegionInfo current = null;
288 
289       @Override
290       public boolean visit(Result r) throws IOException {
291         this.current =
292           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
293         if (this.current == null) {
294           LOG.warn("No serialized HRegionInfo in " + r);
295           return true;
296         }
297         if (!isInsideTable(this.current, tableName)) return false;
298         // Else call super and add this Result to the collection.
299         super.visit(r);
300         // Stop collecting regions from table after we get one.
301         return false;
302       }
303 
304       @Override
305       void add(Result r) {
306         // Add the current HRI.
307         this.results.add(this.current);
308       }
309     };
310     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
311     // If visitor has results >= 1 then table exists.
312     return visitor.getResults().size() >= 1;
313   }
314 
315   /**
316    * Gets all of the regions of the specified table.
317    * @param catalogTracker
318    * @param tableName
319    * @return Ordered list of {@link HRegionInfo}.
320    * @throws IOException
321    */
322   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
323       TableName tableName)
324   throws IOException {
325     return getTableRegions(catalogTracker, tableName, false);
326   }
327 
328   /**
329    * Gets all of the regions of the specified table.
330    * @param catalogTracker
331    * @param tableName
332    * @param excludeOfflinedSplitParents If true, do not include offlined split
333    * parents in the return.
334    * @return Ordered list of {@link HRegionInfo}.
335    * @throws IOException
336    */
337   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
338       TableName tableName, final boolean excludeOfflinedSplitParents)
339   throws IOException {
340     List<Pair<HRegionInfo, ServerName>> result = null;
341     try {
342       result = getTableRegionsAndLocations(catalogTracker, tableName,
343         excludeOfflinedSplitParents);
344     } catch (InterruptedException e) {
345       throw (InterruptedIOException)new InterruptedIOException().initCause(e);
346     }
347     return getListOfHRegionInfos(result);
348   }
349 
350   static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
351     if (pairs == null || pairs.isEmpty()) return null;
352     List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
353     for (Pair<HRegionInfo, ServerName> pair: pairs) {
354       result.add(pair.getFirst());
355     }
356     return result;
357   }
358 
359   /**
360    * @param current
361    * @param tableName
362    * @return True if <code>current</code> tablename is equal to
363    * <code>tableName</code>
364    */
365   static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
366     return tableName.equals(current.getTable());
367   }
368 
369   /**
370    * @param tableName
371    * @return Place to start Scan in <code>hbase:meta</code> when passed a
372    * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
373    */
374   static byte [] getTableStartRowForMeta(TableName tableName) {
375     byte [] startRow = new byte[tableName.getName().length + 2];
376     System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
377     startRow[startRow.length - 2] = HConstants.DELIMITER;
378     startRow[startRow.length - 1] = HConstants.DELIMITER;
379     return startRow;
380   }
381 
382   /**
383    * This method creates a Scan object that will only scan catalog rows that
384    * belong to the specified table. It doesn't specify any columns.
385    * This is a better alternative to just using a start row and scan until
386    * it hits a new table since that requires parsing the HRI to get the table
387    * name.
388    * @param tableName bytes of table's name
389    * @return configured Scan object
390    */
391   public static Scan getScanForTableName(TableName tableName) {
392     String strName = tableName.getNameAsString();
393     // Start key is just the table name with delimiters
394     byte[] startKey = Bytes.toBytes(strName + ",,");
395     // Stop key appends the smallest possible char to the table name
396     byte[] stopKey = Bytes.toBytes(strName + " ,,");
397 
398     Scan scan = new Scan(startKey);
399     scan.setStopRow(stopKey);
400     return scan;
401   }
402 
403   /**
404    * @param catalogTracker
405    * @param tableName
406    * @return Return list of regioninfos and server.
407    * @throws IOException
408    * @throws InterruptedException
409    */
410   public static List<Pair<HRegionInfo, ServerName>>
411   getTableRegionsAndLocations(CatalogTracker catalogTracker, TableName tableName)
412   throws IOException, InterruptedException {
413     return getTableRegionsAndLocations(catalogTracker, tableName,
414       true);
415   }
416 
417   /**
418    * @param catalogTracker
419    * @param tableName
420    * @return Return list of regioninfos and server addresses.
421    * @throws IOException
422    * @throws InterruptedException
423    */
424   public static List<Pair<HRegionInfo, ServerName>>
425   getTableRegionsAndLocations(final CatalogTracker catalogTracker,
426       final TableName tableName, final boolean excludeOfflinedSplitParents)
427   throws IOException, InterruptedException {
428     if (tableName.equals(TableName.META_TABLE_NAME)) {
429       // If meta, do a bit of special handling.
430       ServerName serverName = catalogTracker.getMetaLocation();
431       List<Pair<HRegionInfo, ServerName>> list =
432           new ArrayList<Pair<HRegionInfo, ServerName>>();
433       list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
434           serverName));
435       return list;
436     }
437     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
438     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
439         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
440       private Pair<HRegionInfo, ServerName> current = null;
441 
442       @Override
443       public boolean visit(Result r) throws IOException {
444         HRegionInfo hri =
445           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
446         if (hri == null) {
447           LOG.warn("No serialized HRegionInfo in " + r);
448           return true;
449         }
450         if (!isInsideTable(hri, tableName)) return false;
451         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
452         ServerName sn = HRegionInfo.getServerName(r);
453         // Populate this.current so available when we call #add
454         this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
455         // Else call super and add this Result to the collection.
456         return super.visit(r);
457       }
458 
459       @Override
460       void add(Result r) {
461         this.results.add(this.current);
462       }
463     };
464     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
465     return visitor.getResults();
466   }
467 
468   /**
469    * @param catalogTracker
470    * @param serverName
471    * @return List of user regions installed on this server (does not include
472    * catalog regions).
473    * @throws IOException
474    */
475   public static NavigableMap<HRegionInfo, Result>
476   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
477   throws IOException {
478     final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
479     // Fill the above hris map with entries from hbase:meta that have the passed
480     // servername.
481     CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
482       @Override
483       void add(Result r) {
484         if (r == null || r.isEmpty()) return;
485         ServerName sn = HRegionInfo.getServerName(r);
486         if (sn != null && sn.equals(serverName)) this.results.add(r);
487       }
488     };
489     fullScan(catalogTracker, v);
490     List<Result> results = v.getResults();
491     if (results != null && !results.isEmpty()) {
492       // Convert results to Map keyed by HRI
493       for (Result r: results) {
494         Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
495         if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
496       }
497     }
498     return hris;
499   }
500 
501   public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
502   throws IOException {
503     Visitor v = new Visitor() {
504       @Override
505       public boolean visit(Result r) throws IOException {
506         if (r ==  null || r.isEmpty()) return true;
507         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
508         HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
509         LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
510         return true;
511       }
512     };
513     fullScan(catalogTracker, v);
514   }
515 
516   /**
517    * Performs a full scan of a catalog table.
518    * @param catalogTracker
519    * @param visitor Visitor invoked against each row.
520    * @param startrow Where to start the scan. Pass null if want to begin scan
521    * at first row.
522    * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
523    * @throws IOException
524    */
525   public static void fullScan(CatalogTracker catalogTracker,
526     final Visitor visitor, final byte [] startrow)
527   throws IOException {
528     Scan scan = new Scan();
529     if (startrow != null) scan.setStartRow(startrow);
530     if (startrow == null) {
531       int caching = catalogTracker.getConnection().getConfiguration()
532           .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
533       scan.setCaching(caching);
534     }
535     scan.addFamily(HConstants.CATALOG_FAMILY);
536     HTable metaTable = getMetaHTable(catalogTracker);
537     ResultScanner scanner = null;
538     try {
539       scanner = metaTable.getScanner(scan);
540       Result data;
541       while((data = scanner.next()) != null) {
542         if (data.isEmpty()) continue;
543         // Break if visit returns false.
544         if (!visitor.visit(data)) break;
545       }
546     } finally {
547       if (scanner != null) scanner.close();
548       metaTable.close();
549     }
550     return;
551   }
552 
553   /**
554    * Implementations 'visit' a catalog table row.
555    */
556   public interface Visitor {
557     /**
558      * Visit the catalog table row.
559      * @param r A row from catalog table
560      * @return True if we are to proceed scanning the table, else false if
561      * we are to stop now.
562      */
563     boolean visit(final Result r) throws IOException;
564   }
565 
566   /**
567    * A {@link Visitor} that collects content out of passed {@link Result}.
568    */
569   static abstract class CollectingVisitor<T> implements Visitor {
570     final List<T> results = new ArrayList<T>();
571     @Override
572     public boolean visit(Result r) throws IOException {
573       if (r ==  null || r.isEmpty()) return true;
574       add(r);
575       return true;
576     }
577 
578     abstract void add(Result r);
579 
580     /**
581      * @return Collected results; wait till visits complete to collect all
582      * possible results
583      */
584     List<T> getResults() {
585       return this.results;
586     }
587   }
588 
589   /**
590    * Collects all returned.
591    */
592   static class CollectAllVisitor extends CollectingVisitor<Result> {
593     @Override
594     void add(Result r) {
595       this.results.add(r);
596     }
597   }
598 
599   /**
600    * Count regions in <code>hbase:meta</code> for passed table.
601    * @param c
602    * @param tableName
603    * @return Count or regions in table <code>tableName</code>
604    * @throws IOException
605    */
606   public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
607     HTable t = new HTable(c, tableName);
608     try {
609       return t.getRegionLocations().size();
610     } finally {
611       t.close();
612     }
613   }
614 }