View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.ServerName;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
35  import org.apache.hadoop.hbase.catalog.MetaEditor;
36  import org.apache.hadoop.hbase.client.HBaseAdmin;
37  import org.apache.hadoop.hbase.client.HConnection;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.master.RegionState;
40  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.regionserver.wal.HLog;
44  import org.apache.zookeeper.KeeperException;
45  
46  /**
47   * This class contains helper methods that repair parts of hbase's filesystem
48   * contents.
49   */
50  @InterfaceAudience.Private
51  public class HBaseFsckRepair {
52    public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
53  
54    /**
55     * Fix multiple assignment by doing silent closes on each RS hosting the region
56     * and then force ZK unassigned node to OFFLINE to trigger assignment by
57     * master.
58     *
59     * @param admin HBase admin used to undeploy
60     * @param region Region to undeploy
61     * @param servers list of Servers to undeploy from
62     */
63    public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region,
64        List<ServerName> servers)
65    throws IOException, KeeperException, InterruptedException {
66      HRegionInfo actualRegion = new HRegionInfo(region);
67  
68      // Close region on the servers silently
69      for(ServerName server : servers) {
70        closeRegionSilentlyAndWait(admin, server, actualRegion);
71      }
72  
73      // Force ZK node to OFFLINE so master assigns
74      forceOfflineInZK(admin, actualRegion);
75    }
76  
77    /**
78     * Fix unassigned by creating/transition the unassigned ZK node for this
79     * region to OFFLINE state with a special flag to tell the master that this is
80     * a forced operation by HBCK.
81     *
82     * This assumes that info is in META.
83     *
84     * @param admin
85     * @param region
86     * @throws IOException
87     * @throws KeeperException
88     */
89    public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region)
90        throws IOException, KeeperException {
91      HRegionInfo actualRegion = new HRegionInfo(region);
92  
93      // Force ZK node to OFFLINE so master assigns
94      forceOfflineInZK(admin, actualRegion);
95    }
96  
97    /**
98     * In 0.90, this forces an HRI offline by setting the RegionTransitionData
99     * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
100    * the AssignmentManager that attempts an assign call by the master.
101    *
102    * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK
103    *
104    * This doesn't seem to work properly in the updated version of 0.92+'s hbck
105    * so we use assign to force the region into transition.  This has the
106    * side-effect of requiring a HRegionInfo that considers regionId (timestamp)
107    * in comparators that is addressed by HBASE-5563.
108    */
109   private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
110   throws ZooKeeperConnectionException, KeeperException, IOException {
111     admin.assign(region.getRegionName());
112   }
113 
114   /*
115    * Should we check all assignments or just not in RIT?
116    */
117   public static void waitUntilAssigned(HBaseAdmin admin,
118       HRegionInfo region) throws IOException, InterruptedException {
119     long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
120     long expiration = timeout + System.currentTimeMillis();
121     while (System.currentTimeMillis() < expiration) {
122       try {
123         Map<String, RegionState> rits=
124             admin.getClusterStatus().getRegionsInTransition();
125 
126         if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
127           // yay! no longer RIT
128           return;
129         }
130         // still in rit
131         LOG.info("Region still in transition, waiting for "
132             + "it to become assigned: " + region);
133       } catch (IOException e) {
134         LOG.warn("Exception when waiting for region to become assigned,"
135             + " retrying", e);
136       }
137       Thread.sleep(1000);
138     }
139     throw new IOException("Region " + region + " failed to move out of " +
140         "transition within timeout " + timeout + "ms");
141   }
142 
143   /**
144    * Contacts a region server and waits up to hbase.hbck.close.timeout ms
145    * (default 120s) to close the region.  This bypasses the active hmaster.
146    */
147   public static void closeRegionSilentlyAndWait(HBaseAdmin admin,
148       ServerName server, HRegionInfo region) throws IOException, InterruptedException {
149     HConnection connection = admin.getConnection();
150     AdminService.BlockingInterface rs = connection.getAdmin(server);
151     try {
152       ProtobufUtil.closeRegion(rs, server, region.getRegionName(), false);
153     } catch (IOException e) {
154       LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
155     }
156     long timeout = admin.getConfiguration()
157       .getLong("hbase.hbck.close.timeout", 120000);
158     long expiration = timeout + System.currentTimeMillis();
159     while (System.currentTimeMillis() < expiration) {
160       try {
161         HRegionInfo rsRegion =
162           ProtobufUtil.getRegionInfo(rs, region.getRegionName());
163         if (rsRegion == null) return;
164       } catch (IOException ioe) {
165         return;
166       }
167       Thread.sleep(1000);
168     }
169     throw new IOException("Region " + region + " failed to close within"
170         + " timeout " + timeout);
171   }
172 
173   /**
174    * Puts the specified HRegionInfo into META.
175    */
176   public static void fixMetaHoleOnline(Configuration conf,
177       HRegionInfo hri) throws IOException {
178     HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
179     MetaEditor.addRegionToMeta(meta, hri);
180     meta.close();
181   }
182 
183   /**
184    * Creates, flushes, and closes a new region.
185    */
186   public static HRegion createHDFSRegionDir(Configuration conf,
187       HRegionInfo hri, HTableDescriptor htd) throws IOException {
188     // Create HRegion
189     Path root = FSUtils.getRootDir(conf);
190     HRegion region = HRegion.createHRegion(hri, root, conf, htd);
191     HLog hlog = region.getLog();
192 
193     // Close the new region to flush to disk. Close log file too.
194     region.close();
195     hlog.closeAndDelete();
196     return region;
197   }
198 }