View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.TreeMap;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseConfiguration;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.client.Delete;
41  import org.apache.hadoop.hbase.client.Get;
42  import org.apache.hadoop.hbase.client.HTable;
43  import org.apache.hadoop.hbase.client.Put;
44  import org.apache.hadoop.hbase.client.Result;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.regionserver.HRegion;
47  import org.apache.hadoop.hbase.regionserver.InternalScanner;
48  import org.apache.hadoop.hbase.regionserver.wal.HLog;
49  
50  /**
51   * Contains utility methods for manipulating HBase meta tables.
52   * Be sure to call {@link #shutdown()} when done with this class so it closes
53   * resources opened during meta processing (ROOT, META, etc.).  Be careful
54   * how you use this class.  If used during migrations, be careful when using
55   * this class to check whether migration is needed.
56   */
57  public class MetaUtils {
58    private static final Log LOG = LogFactory.getLog(MetaUtils.class);
59    private final Configuration conf;
60    private FileSystem fs;
61    private HLog log;
62    private HRegion rootRegion;
63    private Map<byte [], HRegion> metaRegions = Collections.synchronizedSortedMap(
64      new TreeMap<byte [], HRegion>(Bytes.BYTES_COMPARATOR));
65  
66    /** Default constructor
67     * @throws IOException e
68     */
69    public MetaUtils() throws IOException {
70      this(HBaseConfiguration.create());
71    }
72  
73    /**
74     * @param conf Configuration
75     * @throws IOException e
76     */
77    public MetaUtils(Configuration conf) throws IOException {
78      this.conf = conf;
79      conf.setInt("hbase.client.retries.number", 1);
80      this.rootRegion = null;
81      initialize();
82    }
83  
84    /**
85     * Verifies that DFS is available and that HBase is off-line.
86     * @throws IOException e
87     */
88    private void initialize() throws IOException {
89      this.fs = FileSystem.get(this.conf);
90    }
91  
92    /**
93     * @return the HLog
94     * @throws IOException e
95     */
96    public synchronized HLog getLog() throws IOException {
97      if (this.log == null) {
98        Path logdir = new Path(this.fs.getHomeDirectory(),
99            HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis());
100       Path oldLogDir = new Path(this.fs.getHomeDirectory(),
101           HConstants.HREGION_OLDLOGDIR_NAME);
102       this.log = new HLog(this.fs, logdir, oldLogDir, this.conf);
103     }
104     return this.log;
105   }
106 
107   /**
108    * @return HRegion for root region
109    * @throws IOException e
110    */
111   public HRegion getRootRegion() throws IOException {
112     if (this.rootRegion == null) {
113       openRootRegion();
114     }
115     return this.rootRegion;
116   }
117 
118   /**
119    * Open or return cached opened meta region
120    *
121    * @param metaInfo HRegionInfo for meta region
122    * @return meta HRegion
123    * @throws IOException e
124    */
125   public HRegion getMetaRegion(HRegionInfo metaInfo) throws IOException {
126     HRegion meta = metaRegions.get(metaInfo.getRegionName());
127     if (meta == null) {
128       meta = openMetaRegion(metaInfo);
129       LOG.info("OPENING META " + meta.toString());
130       this.metaRegions.put(metaInfo.getRegionName(), meta);
131     }
132     return meta;
133   }
134 
135   /**
136    * Closes catalog regions if open. Also closes and deletes the HLog. You
137    * must call this method if you want to persist changes made during a
138    * MetaUtils edit session.
139    */
140   public void shutdown() {
141     if (this.rootRegion != null) {
142       try {
143         this.rootRegion.close();
144       } catch (IOException e) {
145         LOG.error("closing root region", e);
146       } finally {
147         this.rootRegion = null;
148       }
149     }
150     try {
151       for (HRegion r: metaRegions.values()) {
152         LOG.info("CLOSING META " + r.toString());
153         r.close();
154       }
155     } catch (IOException e) {
156       LOG.error("closing meta region", e);
157     } finally {
158       metaRegions.clear();
159     }
160     try {
161       if (this.log != null) {
162         this.log.rollWriter();
163         this.log.closeAndDelete();
164       }
165     } catch (IOException e) {
166       LOG.error("closing HLog", e);
167     } finally {
168       this.log = null;
169     }
170   }
171 
172   /**
173    * Used by scanRootRegion and scanMetaRegion to call back the caller so it
174    * can process the data for a row.
175    */
176   public interface ScannerListener {
177     /**
178      * Callback so client of scanner can process row contents
179      *
180      * @param info HRegionInfo for row
181      * @return false to terminate the scan
182      * @throws IOException e
183      */
184     public boolean processRow(HRegionInfo info) throws IOException;
185   }
186 
187   /**
188    * Scans the root region. For every meta region found, calls the listener with
189    * the HRegionInfo of the meta region.
190    *
191    * @param listener method to be called for each meta region found
192    * @throws IOException e
193    */
194   public void scanRootRegion(ScannerListener listener) throws IOException {
195     // Open root region so we can scan it
196     if (this.rootRegion == null) {
197       openRootRegion();
198     }
199     scanMetaRegion(this.rootRegion, listener);
200   }
201 
202   /**
203    * Scan the passed in metaregion <code>m</code> invoking the passed
204    * <code>listener</code> per row found.
205    * @param r region
206    * @param listener scanner listener
207    * @throws IOException e
208    */
209   public void scanMetaRegion(final HRegion r, final ScannerListener listener)
210   throws IOException {
211     Scan scan = new Scan();
212     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
213     InternalScanner s = r.getScanner(scan);
214     try {
215       List<KeyValue> results = new ArrayList<KeyValue>();
216       boolean hasNext = true;
217       do {
218         hasNext = s.next(results);
219         HRegionInfo info = null;
220         for (KeyValue kv: results) {
221           info = Writables.getHRegionInfoOrNull(kv.getValue());
222           if (info == null) {
223             LOG.warn("Region info is null for row " +
224               Bytes.toStringBinary(kv.getRow()) + " in table " +
225               r.getTableDesc().getNameAsString());
226           }
227           continue;
228         }
229         if (!listener.processRow(info)) {
230           break;
231         }
232         results.clear();
233       } while (hasNext);
234     } finally {
235       s.close();
236     }
237   }
238 
239   /**
240    * Scans a meta region. For every region found, calls the listener with
241    * the HRegionInfo of the region.
242    * TODO: Use Visitor rather than Listener pattern.  Allow multiple Visitors.
243    * Use this everywhere we scan meta regions: e.g. in metascanners, in close
244    * handling, etc.  Have it pass in the whole row, not just HRegionInfo.
245    * <p>Use for reading meta only.  Does not close region when done.
246    * Use {@link #getMetaRegion(HRegionInfo)} instead if writing.  Adds
247    * meta region to list that will get a close on {@link #shutdown()}.
248    *
249    * @param metaRegionInfo HRegionInfo for meta region
250    * @param listener method to be called for each meta region found
251    * @throws IOException e
252    */
253   public void scanMetaRegion(HRegionInfo metaRegionInfo,
254     ScannerListener listener)
255   throws IOException {
256     // Open meta region so we can scan it
257     HRegion metaRegion = openMetaRegion(metaRegionInfo);
258     scanMetaRegion(metaRegion, listener);
259   }
260 
261   private synchronized HRegion openRootRegion() throws IOException {
262     if (this.rootRegion != null) {
263       return this.rootRegion;
264     }
265     this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO,
266       HTableDescriptor.ROOT_TABLEDESC, getLog(),
267       this.conf);
268     this.rootRegion.compactStores();
269     return this.rootRegion;
270   }
271 
272   private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException {
273     HRegion meta = HRegion.openHRegion(metaInfo, HTableDescriptor.META_TABLEDESC,
274       getLog(), this.conf);
275     meta.compactStores();
276     return meta;
277   }
278 
279   /**
280    * Set a single region on/offline.
281    * This is a tool to repair tables that have offlined tables in their midst.
282    * Can happen on occasion.  Use at your own risk.  Call from a bit of java
283    * or jython script.  This method is 'expensive' in that it creates a
284    * {@link HTable} instance per invocation to go against <code>.META.</code>
285    * @param c A configuration that has its <code>hbase.master</code>
286    * properly set.
287    * @param row Row in the catalog .META. table whose HRegionInfo's offline
288    * status we want to change.
289    * @param onlineOffline Pass <code>true</code> to OFFLINE the region.
290    * @throws IOException e
291    */
292   public static void changeOnlineStatus (final Configuration c,
293       final byte [] row, final boolean onlineOffline)
294   throws IOException {
295     HTable t = new HTable(c, HConstants.META_TABLE_NAME);
296     Get get = new Get(row);
297     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
298     Result res = t.get(get);
299     KeyValue [] kvs = res.raw();
300     if(kvs.length <= 0) {
301       throw new IOException("no information for row " + Bytes.toString(row));
302     }
303     byte [] value = kvs[0].getValue();
304     if (value == null) {
305       throw new IOException("no information for row " + Bytes.toString(row));
306     }
307     HRegionInfo info = Writables.getHRegionInfo(value);
308     Put put = new Put(row);
309     info.setOffline(onlineOffline);
310     put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
311         Writables.getBytes(info));
312     t.put(put);
313 
314     Delete delete = new Delete(row);
315     delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
316     delete.deleteColumns(HConstants.CATALOG_FAMILY,
317         HConstants.STARTCODE_QUALIFIER);
318 
319     t.delete(delete);
320   }
321 
322   /**
323    * Update COL_REGIONINFO in meta region r with HRegionInfo hri
324    *
325    * @param r region
326    * @param hri region info
327    * @throws IOException e
328    */
329   public void updateMETARegionInfo(HRegion r, final HRegionInfo hri)
330   throws IOException {
331     if (LOG.isDebugEnabled()) {
332       Get get = new Get(hri.getRegionName());
333       get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
334       Result res = r.get(get, null);
335       KeyValue [] kvs = res.raw();
336       if(kvs.length <= 0) {
337         return;
338       }
339       byte [] value = kvs[0].getValue();
340       if (value == null) {
341         return;
342       }
343       HRegionInfo h = Writables.getHRegionInfoOrNull(value);
344 
345       LOG.debug("Old " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
346           Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " +
347           hri.toString() + " in " + r.toString() + " is: " + h.toString());
348     }
349 
350     Put put = new Put(hri.getRegionName());
351     put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
352         Writables.getBytes(hri));
353     r.put(put);
354 
355     if (LOG.isDebugEnabled()) {
356       Get get = new Get(hri.getRegionName());
357       get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
358       Result res = r.get(get, null);
359       KeyValue [] kvs = res.raw();
360       if(kvs.length <= 0) {
361         return;
362       }
363       byte [] value = kvs[0].getValue();
364       if (value == null) {
365         return;
366       }
367       HRegionInfo h = Writables.getHRegionInfoOrNull(value);
368         LOG.debug("New " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
369             Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " +
370             hri.toString() + " in " + r.toString() + " is: " +  h.toString());
371     }
372   }
373 
374   /**
375    * @return List of {@link HRegionInfo} rows found in the ROOT or META
376    * catalog table.
377    * @param tableName Name of table to go looking for.
378    * @throws IOException e
379    * @see #getMetaRegion(HRegionInfo)
380    */
381   public List<HRegionInfo> getMETARows(final byte [] tableName)
382   throws IOException {
383     final List<HRegionInfo> result = new ArrayList<HRegionInfo>();
384     // If passed table name is META, then  return the root region.
385     if (Bytes.equals(HConstants.META_TABLE_NAME, tableName)) {
386       result.add(openRootRegion().getRegionInfo());
387       return result;
388     }
389     // Return all meta regions that contain the passed tablename.
390     scanRootRegion(new ScannerListener() {
391       private final Log SL_LOG = LogFactory.getLog(this.getClass());
392 
393       public boolean processRow(HRegionInfo info) throws IOException {
394         SL_LOG.debug("Testing " + info);
395         if (Bytes.equals(info.getTableName(),
396             HConstants.META_TABLE_NAME)) {
397           result.add(info);
398           return false;
399         }
400         return true;
401       }});
402     return result;
403   }
404 }