View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import java.io.IOException;
23  import java.util.List;
24  import java.util.Map;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.HConnection;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.ipc.HRegionInterface;
40  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
41  import org.apache.hadoop.hbase.regionserver.HRegion;
42  import org.apache.hadoop.hbase.regionserver.wal.HLog;
43  import org.apache.zookeeper.KeeperException;
44  
45  /**
46   * This class contains helper methods that repair parts of hbase's filesystem
47   * contents.
48   */
49  public class HBaseFsckRepair {
50    public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
51  
52    /**
53     * Fix multiple assignment by doing silent closes on each RS hosting the region
54     * and then force ZK unassigned node to OFFLINE to trigger assignment by
55     * master.
56     *
57     * @param admin HBase admin used to undeploy
58     * @param region Region to undeploy
59     * @param servers list of Servers to undeploy from
60     */
61    public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region,
62        List<ServerName> servers)
63    throws IOException, KeeperException, InterruptedException {
64      HRegionInfo actualRegion = new HRegionInfo(region);
65  
66      // Close region on the servers silently
67      for(ServerName server : servers) {
68        closeRegionSilentlyAndWait(admin, server, actualRegion);
69      }
70  
71      // Force ZK node to OFFLINE so master assigns
72      forceOfflineInZK(admin, actualRegion);
73    }
74  
75    /**
76     * Fix unassigned by creating/transition the unassigned ZK node for this
77     * region to OFFLINE state with a special flag to tell the master that this is
78     * a forced operation by HBCK.
79     *
80     * This assumes that info is in META.
81     *
82     * @param conf
83     * @param region
84     * @throws IOException
85     * @throws KeeperException
86     */
87    public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region)
88        throws IOException, KeeperException {
89      HRegionInfo actualRegion = new HRegionInfo(region);
90  
91      // Force ZK node to OFFLINE so master assigns
92      forceOfflineInZK(admin, actualRegion);
93    }
94  
95    /**
96     * In 0.90, this forces an HRI offline by setting the RegionTransitionData
97     * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
98     * the AssignmentManager that attempts an assign call by the master.
99     *
100    * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK
101    *
102    * This doesn't seem to work properly in the updated version of 0.92+'s hbck
103    * so we use assign to force the region into transition.  This has the
104    * side-effect of requiring a HRegionInfo that considers regionId (timestamp)
105    * in comparators that is addressed by HBASE-5563.
106    */
107   private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
108   throws ZooKeeperConnectionException, KeeperException, IOException {
109     admin.assign(region.getRegionName());
110   }
111 
112   /*
113    * Should we check all assignments or just not in RIT?
114    */
115   public static void waitUntilAssigned(HBaseAdmin admin,
116       HRegionInfo region) throws IOException, InterruptedException {
117     long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
118     long expiration = timeout + System.currentTimeMillis();
119     while (System.currentTimeMillis() < expiration) {
120       try {
121         Map<String, RegionState> rits=
122             admin.getClusterStatus().getRegionsInTransition();
123 
124         if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
125           // yay! no longer RIT
126           return;
127         }
128         // still in rit
129         LOG.info("Region still in transition, waiting for "
130             + "it to become assigned: " + region);
131       } catch (IOException e) {
132         LOG.warn("Exception when waiting for region to become assigned,"
133             + " retrying", e);
134       }
135       Thread.sleep(1000);
136     }
137     throw new IOException("Region " + region + " failed to move out of " +
138         "transition within timeout " + timeout + "ms");
139   }
140 
141   /**
142    * Contacts a region server and waits up to hbase.hbck.close.timeout ms
143    * (default 120s) to close the region.  This bypasses the active hmaster.
144    */
145   public static void closeRegionSilentlyAndWait(HBaseAdmin admin,
146       ServerName server, HRegionInfo region) throws IOException, InterruptedException {
147     HConnection connection = admin.getConnection();
148     HRegionInterface rs = connection.getHRegionConnection(server.getHostname(),
149         server.getPort());
150     try {
151       rs.closeRegion(region, false);
152     } catch (IOException ioe) {
153       LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), ioe);
154     }
155     long timeout = admin.getConfiguration()
156       .getLong("hbase.hbck.close.timeout", 120000);
157     long expiration = timeout + System.currentTimeMillis();
158     while (System.currentTimeMillis() < expiration) {
159       try {
160         HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName());
161         if (rsRegion == null)
162           return;
163       } catch (IOException ioe) {
164         return;
165       }
166       Thread.sleep(1000);
167     }
168     throw new IOException("Region " + region + " failed to close within"
169         + " timeout " + timeout);
170   }
171 
172   /**
173    * Puts the specified HRegionInfo into META.
174    */
175   public static void fixMetaHoleOnline(Configuration conf,
176       HRegionInfo hri) throws IOException {
177     Put p = new Put(hri.getRegionName());
178     p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
179         Writables.getBytes(hri));
180     HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
181     meta.put(p);
182     meta.close();
183   }
184 
185   /**
186    * Creates, flushes, and closes a new region.
187    */
188   public static HRegion createHDFSRegionDir(Configuration conf,
189       HRegionInfo hri, HTableDescriptor htd) throws IOException {
190     // Create HRegion
191     Path root = FSUtils.getRootDir(conf);
192     HRegion region = HRegion.createHRegion(hri, root, conf, htd);
193     HLog hlog = region.getLog();
194 
195     // Close the new region to flush to disk. Close log file too.
196     region.close();
197     hlog.closeAndDelete();
198     return region;
199   }
200 }