View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.classification.InterfaceAudience;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HTableDescriptor;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.client.Get;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Result;
32  import org.apache.hadoop.hbase.client.ResultScanner;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.Pair;
36  
37  import java.io.IOException;
38  import java.io.InterruptedIOException;
39  import java.util.ArrayList;
40  import java.util.List;
41  import java.util.Map;
42  import java.util.NavigableMap;
43  import java.util.Set;
44  import java.util.TreeMap;
45  
46  /**
47   * Reads region and assignment information from <code>hbase:meta</code>.
48   */
49  @InterfaceAudience.Private
50  public class MetaReader {
51    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
52    // its only used to get its Configuration so we can get associated
53    // Connection.
54    private static final Log LOG = LogFactory.getLog(MetaReader.class);
55  
56    static final byte [] META_REGION_PREFIX;
57    static {
58      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
59      // FIRST_META_REGIONINFO == 'hbase:meta,,1'.  META_REGION_PREFIX == 'hbase:meta,'
60      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
61      META_REGION_PREFIX = new byte [len];
62      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
63        META_REGION_PREFIX, 0, len);
64    }
65  
66    /**
67     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
68     * tables in the specified set of disabled tables.
69     * @param catalogTracker
70     * @param disabledTables set of disabled tables that will not be returned
71     * @return Returns a map of every region to it's currently assigned server,
72     * according to META.  If the region does not have an assignment it will have
73     * a null value in the map.
74     * @throws IOException
75     */
76    public static Map<HRegionInfo, ServerName> fullScan(
77        CatalogTracker catalogTracker, final Set<TableName> disabledTables)
78    throws IOException {
79      return fullScan(catalogTracker, disabledTables, false);
80    }
81  
82    /**
83     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
84     * tables in the specified set of disabled tables.
85     * @param catalogTracker
86     * @param disabledTables set of disabled tables that will not be returned
87     * @param excludeOfflinedSplitParents If true, do not include offlined split
88     * parents in the return.
89     * @return Returns a map of every region to it's currently assigned server,
90     * according to META.  If the region does not have an assignment it will have
91     * a null value in the map.
92     * @throws IOException
93     */
94    public static Map<HRegionInfo, ServerName> fullScan(
95        CatalogTracker catalogTracker, final Set<TableName> disabledTables,
96        final boolean excludeOfflinedSplitParents)
97    throws IOException {
98      final Map<HRegionInfo, ServerName> regions =
99        new TreeMap<HRegionInfo, ServerName>();
100     Visitor v = new Visitor() {
101       @Override
102       public boolean visit(Result r) throws IOException {
103         if (r ==  null || r.isEmpty()) return true;
104         Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
105         HRegionInfo hri = region.getFirst();
106         if (hri  == null) return true;
107         if (hri.getTable() == null) return true;
108         if (disabledTables.contains(
109             hri.getTable())) return true;
110         // Are we to include split parents in the list?
111         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
112         regions.put(hri, region.getSecond());
113         return true;
114       }
115     };
116     fullScan(catalogTracker, v);
117     return regions;
118   }
119 
120   /**
121    * Performs a full scan of <code>hbase:meta</code>.
122    * @return List of {@link Result}
123    * @throws IOException
124    */
125   public static List<Result> fullScan(CatalogTracker catalogTracker)
126   throws IOException {
127     CollectAllVisitor v = new CollectAllVisitor();
128     fullScan(catalogTracker, v, null);
129     return v.getResults();
130   }
131 
132   /**
133    * Performs a full scan of a <code>hbase:meta</code> table.
134    * @return List of {@link Result}
135    * @throws IOException
136    */
137   public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
138   throws IOException {
139     CollectAllVisitor v = new CollectAllVisitor();
140     fullScan(catalogTracker, v, null);
141     return v.getResults();
142   }
143 
144   /**
145    * Performs a full scan of <code>hbase:meta</code>.
146    * @param catalogTracker
147    * @param visitor Visitor invoked against each row.
148    * @throws IOException
149    */
150   public static void fullScan(CatalogTracker catalogTracker,
151       final Visitor visitor)
152   throws IOException {
153     fullScan(catalogTracker, visitor, null);
154   }
155 
156   /**
157    * Callers should call close on the returned {@link HTable} instance.
158    * @param catalogTracker We'll use this catalogtracker's connection
159    * @param tableName Table to get an {@link HTable} against.
160    * @return An {@link HTable} for <code>tableName</code>
161    * @throws IOException
162    */
163   @SuppressWarnings("deprecation")
164   private static HTable getHTable(final CatalogTracker catalogTracker,
165       final TableName tableName)
166   throws IOException {
167     // Passing the CatalogTracker's connection ensures this
168     // HTable instance uses the CatalogTracker's connection.
169     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
170     if (c == null) throw new NullPointerException("No connection");
171     return new HTable(tableName, c);
172   }
173 
174   /**
175    * Callers should call close on the returned {@link HTable} instance.
176    * @param catalogTracker
177    * @return An {@link HTable} for <code>hbase:meta</code>
178    * @throws IOException
179    */
180   static HTable getCatalogHTable(final CatalogTracker catalogTracker)
181   throws IOException {
182     return getMetaHTable(catalogTracker);
183   }
184 
185   /**
186    * Callers should call close on the returned {@link HTable} instance.
187    * @param ct
188    * @return An {@link HTable} for <code>hbase:meta</code>
189    * @throws IOException
190    */
191   static HTable getMetaHTable(final CatalogTracker ct)
192   throws IOException {
193     return getHTable(ct, TableName.META_TABLE_NAME);
194   }
195 
196   /**
197    * @param t Table to use (will be closed when done).
198    * @param g Get to run
199    * @throws IOException
200    */
201   private static Result get(final HTable t, final Get g) throws IOException {
202     try {
203       return t.get(g);
204     } finally {
205       t.close();
206     }
207   }
208 
209   /**
210    * Reads the location of the specified region
211    * @param catalogTracker
212    * @param regionName region whose location we are after
213    * @return location of region as a {@link ServerName} or null if not found
214    * @throws IOException
215    */
216   static ServerName readRegionLocation(CatalogTracker catalogTracker,
217       byte [] regionName)
218   throws IOException {
219     Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
220     return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
221   }
222 
223   /**
224    * Gets the region info and assignment for the specified region.
225    * @param catalogTracker
226    * @param regionName Region to lookup.
227    * @return Location and HRegionInfo for <code>regionName</code>
228    * @throws IOException
229    */
230   public static Pair<HRegionInfo, ServerName> getRegion(
231       CatalogTracker catalogTracker, byte [] regionName)
232   throws IOException {
233     Get get = new Get(regionName);
234     get.addFamily(HConstants.CATALOG_FAMILY);
235     Result r = get(getCatalogHTable(catalogTracker), get);
236     return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
237   }
238 
239   /**
240    * Gets the result in hbase:meta for the specified region.
241    * @param catalogTracker
242    * @param regionName
243    * @return result of the specified region
244    * @throws IOException
245    */
246   public static Result getRegionResult(CatalogTracker catalogTracker,
247       byte[] regionName) throws IOException {
248     Get get = new Get(regionName);
249     get.addFamily(HConstants.CATALOG_FAMILY);
250     return get(getCatalogHTable(catalogTracker), get);
251   }
252 
253   /**
254    * Get regions from the merge qualifier of the specified merged region
255    * @return null if it doesn't contain merge qualifier, else two merge regions
256    * @throws IOException
257    */
258   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
259       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
260     Result result = getRegionResult(catalogTracker, regionName);
261     HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
262         HConstants.MERGEA_QUALIFIER);
263     HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
264         HConstants.MERGEB_QUALIFIER);
265     if (mergeA == null && mergeB == null) {
266       return null;
267     }
268     return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
269  }
270 
271   /**
272    * Checks if the specified table exists.  Looks at the hbase:meta table hosted on
273    * the specified server.
274    * @param catalogTracker
275    * @param tableName table to check
276    * @return true if the table exists in meta, false if not
277    * @throws IOException
278    */
279   public static boolean tableExists(CatalogTracker catalogTracker,
280       final TableName tableName)
281   throws IOException {
282     if (tableName.equals(HTableDescriptor.META_TABLEDESC.getTableName())) {
283       // Catalog tables always exist.
284       return true;
285     }
286     // Make a version of ResultCollectingVisitor that only collects the first
287     CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
288       private HRegionInfo current = null;
289 
290       @Override
291       public boolean visit(Result r) throws IOException {
292         this.current =
293           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
294         if (this.current == null) {
295           LOG.warn("No serialized HRegionInfo in " + r);
296           return true;
297         }
298         if (!isInsideTable(this.current, tableName)) return false;
299         // Else call super and add this Result to the collection.
300         super.visit(r);
301         // Stop collecting regions from table after we get one.
302         return false;
303       }
304 
305       @Override
306       void add(Result r) {
307         // Add the current HRI.
308         this.results.add(this.current);
309       }
310     };
311     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
312     // If visitor has results >= 1 then table exists.
313     return visitor.getResults().size() >= 1;
314   }
315 
316   /**
317    * Gets all of the regions of the specified table.
318    * @param catalogTracker
319    * @param tableName
320    * @return Ordered list of {@link HRegionInfo}.
321    * @throws IOException
322    */
323   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
324       TableName tableName)
325   throws IOException {
326     return getTableRegions(catalogTracker, tableName, false);
327   }
328 
329   /**
330    * Gets all of the regions of the specified table.
331    * @param catalogTracker
332    * @param tableName
333    * @param excludeOfflinedSplitParents If true, do not include offlined split
334    * parents in the return.
335    * @return Ordered list of {@link HRegionInfo}.
336    * @throws IOException
337    */
338   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
339       TableName tableName, final boolean excludeOfflinedSplitParents)
340   throws IOException {
341     List<Pair<HRegionInfo, ServerName>> result = null;
342     try {
343       result = getTableRegionsAndLocations(catalogTracker, tableName,
344         excludeOfflinedSplitParents);
345     } catch (InterruptedException e) {
346       throw (InterruptedIOException)new InterruptedIOException().initCause(e);
347     }
348     return getListOfHRegionInfos(result);
349   }
350 
351   static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
352     if (pairs == null || pairs.isEmpty()) return null;
353     List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
354     for (Pair<HRegionInfo, ServerName> pair: pairs) {
355       result.add(pair.getFirst());
356     }
357     return result;
358   }
359 
360   /**
361    * @param current
362    * @param tableName
363    * @return True if <code>current</code> tablename is equal to
364    * <code>tableName</code>
365    */
366   static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
367     return tableName.equals(current.getTable());
368   }
369 
370   /**
371    * @param tableName
372    * @return Place to start Scan in <code>hbase:meta</code> when passed a
373    * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
374    */
375   static byte [] getTableStartRowForMeta(TableName tableName) {
376     byte [] startRow = new byte[tableName.getName().length + 2];
377     System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
378     startRow[startRow.length - 2] = HConstants.DELIMITER;
379     startRow[startRow.length - 1] = HConstants.DELIMITER;
380     return startRow;
381   }
382 
383   /**
384    * This method creates a Scan object that will only scan catalog rows that
385    * belong to the specified table. It doesn't specify any columns.
386    * This is a better alternative to just using a start row and scan until
387    * it hits a new table since that requires parsing the HRI to get the table
388    * name.
389    * @param tableName bytes of table's name
390    * @return configured Scan object
391    */
392   public static Scan getScanForTableName(TableName tableName) {
393     String strName = tableName.getNameAsString();
394     // Start key is just the table name with delimiters
395     byte[] startKey = Bytes.toBytes(strName + ",,");
396     // Stop key appends the smallest possible char to the table name
397     byte[] stopKey = Bytes.toBytes(strName + " ,,");
398 
399     Scan scan = new Scan(startKey);
400     scan.setStopRow(stopKey);
401     return scan;
402   }
403 
404   /**
405    * @param catalogTracker
406    * @param tableName
407    * @return Return list of regioninfos and server.
408    * @throws IOException
409    * @throws InterruptedException
410    */
411   public static List<Pair<HRegionInfo, ServerName>>
412   getTableRegionsAndLocations(CatalogTracker catalogTracker, TableName tableName)
413   throws IOException, InterruptedException {
414     return getTableRegionsAndLocations(catalogTracker, tableName,
415       true);
416   }
417 
418   /**
419    * @param catalogTracker
420    * @param tableName
421    * @return Return list of regioninfos and server addresses.
422    * @throws IOException
423    * @throws InterruptedException
424    */
425   public static List<Pair<HRegionInfo, ServerName>>
426   getTableRegionsAndLocations(final CatalogTracker catalogTracker,
427       final TableName tableName, final boolean excludeOfflinedSplitParents)
428   throws IOException, InterruptedException {
429     if (tableName.equals(TableName.META_TABLE_NAME)) {
430       // If meta, do a bit of special handling.
431       ServerName serverName = catalogTracker.getMetaLocation();
432       List<Pair<HRegionInfo, ServerName>> list =
433           new ArrayList<Pair<HRegionInfo, ServerName>>();
434       list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
435           serverName));
436       return list;
437     }
438     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
439     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
440         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
441       private Pair<HRegionInfo, ServerName> current = null;
442 
443       @Override
444       public boolean visit(Result r) throws IOException {
445         HRegionInfo hri =
446           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
447         if (hri == null) {
448           LOG.warn("No serialized HRegionInfo in " + r);
449           return true;
450         }
451         if (!isInsideTable(hri, tableName)) return false;
452         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
453         ServerName sn = HRegionInfo.getServerName(r);
454         // Populate this.current so available when we call #add
455         this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
456         // Else call super and add this Result to the collection.
457         return super.visit(r);
458       }
459 
460       @Override
461       void add(Result r) {
462         this.results.add(this.current);
463       }
464     };
465     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
466     return visitor.getResults();
467   }
468 
469   /**
470    * @param catalogTracker
471    * @param serverName
472    * @return List of user regions installed on this server (does not include
473    * catalog regions).
474    * @throws IOException
475    */
476   public static NavigableMap<HRegionInfo, Result>
477   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
478   throws IOException {
479     final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
480     // Fill the above hris map with entries from hbase:meta that have the passed
481     // servername.
482     CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
483       @Override
484       void add(Result r) {
485         if (r == null || r.isEmpty()) return;
486         if (HRegionInfo.getHRegionInfo(r) == null) return;
487         ServerName sn = HRegionInfo.getServerName(r);
488         if (sn != null && sn.equals(serverName)) {
489           this.results.add(r);
490         }
491       }
492     };
493     fullScan(catalogTracker, v);
494     List<Result> results = v.getResults();
495     if (results != null && !results.isEmpty()) {
496       // Convert results to Map keyed by HRI
497       for (Result r: results) {
498         HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
499         if (hri != null) hris.put(hri, r);
500       }
501     }
502     return hris;
503   }
504 
505   public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
506   throws IOException {
507     Visitor v = new Visitor() {
508       @Override
509       public boolean visit(Result r) throws IOException {
510         if (r ==  null || r.isEmpty()) return true;
511         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
512         HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
513         LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
514         return true;
515       }
516     };
517     fullScan(catalogTracker, v);
518   }
519 
520   /**
521    * Performs a full scan of a catalog table.
522    * @param catalogTracker
523    * @param visitor Visitor invoked against each row.
524    * @param startrow Where to start the scan. Pass null if want to begin scan
525    * at first row.
526    * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
527    * @throws IOException
528    */
529   public static void fullScan(CatalogTracker catalogTracker,
530     final Visitor visitor, final byte [] startrow)
531   throws IOException {
532     Scan scan = new Scan();
533     if (startrow != null) scan.setStartRow(startrow);
534     if (startrow == null) {
535       int caching = catalogTracker.getConnection().getConfiguration()
536           .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
537       scan.setCaching(caching);
538     }
539     scan.addFamily(HConstants.CATALOG_FAMILY);
540     HTable metaTable = getMetaHTable(catalogTracker);
541     ResultScanner scanner = null;
542     try {
543       scanner = metaTable.getScanner(scan);
544       Result data;
545       while((data = scanner.next()) != null) {
546         if (data.isEmpty()) continue;
547         // Break if visit returns false.
548         if (!visitor.visit(data)) break;
549       }
550     } finally {
551       if (scanner != null) scanner.close();
552       metaTable.close();
553     }
554     return;
555   }
556 
557   /**
558    * Implementations 'visit' a catalog table row.
559    */
560   public interface Visitor {
561     /**
562      * Visit the catalog table row.
563      * @param r A row from catalog table
564      * @return True if we are to proceed scanning the table, else false if
565      * we are to stop now.
566      */
567     boolean visit(final Result r) throws IOException;
568   }
569 
570   /**
571    * A {@link Visitor} that collects content out of passed {@link Result}.
572    */
573   static abstract class CollectingVisitor<T> implements Visitor {
574     final List<T> results = new ArrayList<T>();
575     @Override
576     public boolean visit(Result r) throws IOException {
577       if (r ==  null || r.isEmpty()) return true;
578       add(r);
579       return true;
580     }
581 
582     abstract void add(Result r);
583 
584     /**
585      * @return Collected results; wait till visits complete to collect all
586      * possible results
587      */
588     List<T> getResults() {
589       return this.results;
590     }
591   }
592 
593   /**
594    * Collects all returned.
595    */
596   static class CollectAllVisitor extends CollectingVisitor<Result> {
597     @Override
598     void add(Result r) {
599       this.results.add(r);
600     }
601   }
602 
603   /**
604    * Count regions in <code>hbase:meta</code> for passed table.
605    * @param c
606    * @param tableName
607    * @return Count or regions in table <code>tableName</code>
608    * @throws IOException
609    */
610   public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
611     HTable t = new HTable(c, tableName);
612     try {
613       return t.getRegionLocations().size();
614     } finally {
615       t.close();
616     }
617   }
618 }