View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.util.Collection;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Random;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.ServerName;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
37  import org.apache.hadoop.hbase.MetaTableAccessor;
38  import org.apache.hadoop.hbase.client.Admin;
39  import org.apache.hadoop.hbase.client.HBaseAdmin;
40  import org.apache.hadoop.hbase.client.HConnection;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.master.RegionState;
44  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
45  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
46  import org.apache.hadoop.hbase.regionserver.HRegion;
47  import org.apache.zookeeper.KeeperException;
48  
49  /**
50   * This class contains helper methods that repair parts of hbase's filesystem
51   * contents.
52   */
53  @InterfaceAudience.Private
54  public class HBaseFsckRepair {
55    public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
56  
57    /**
58     * Fix multiple assignment by doing silent closes on each RS hosting the region
59     * and then force ZK unassigned node to OFFLINE to trigger assignment by
60     * master.
61     *
62     * @param admin HBase admin used to undeploy
63     * @param region Region to undeploy
64     * @param servers list of Servers to undeploy from
65     */
66    public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region,
67        List<ServerName> servers)
68    throws IOException, KeeperException, InterruptedException {
69      HRegionInfo actualRegion = new HRegionInfo(region);
70  
71      // Close region on the servers silently
72      for(ServerName server : servers) {
73        closeRegionSilentlyAndWait(admin, server, actualRegion);
74      }
75  
76      // Force ZK node to OFFLINE so master assigns
77      forceOfflineInZK(admin, actualRegion);
78    }
79  
80    /**
81     * Fix unassigned by creating/transition the unassigned ZK node for this
82     * region to OFFLINE state with a special flag to tell the master that this is
83     * a forced operation by HBCK.
84     *
85     * This assumes that info is in META.
86     *
87     * @param admin
88     * @param region
89     * @throws IOException
90     * @throws KeeperException
91     */
92    public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region)
93        throws IOException, KeeperException {
94      HRegionInfo actualRegion = new HRegionInfo(region);
95  
96      // Force ZK node to OFFLINE so master assigns
97      forceOfflineInZK(admin, actualRegion);
98    }
99  
100   /**
101    * In 0.90, this forces an HRI offline by setting the RegionTransitionData
102    * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
103    * the AssignmentManager that attempts an assign call by the master.
104    *
105    * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK
106    *
107    * This doesn't seem to work properly in the updated version of 0.92+'s hbck
108    * so we use assign to force the region into transition.  This has the
109    * side-effect of requiring a HRegionInfo that considers regionId (timestamp)
110    * in comparators that is addressed by HBASE-5563.
111    */
112   private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
113   throws ZooKeeperConnectionException, KeeperException, IOException {
114     admin.assign(region.getRegionName());
115   }
116 
117   /*
118    * Should we check all assignments or just not in RIT?
119    */
120   public static void waitUntilAssigned(HBaseAdmin admin,
121       HRegionInfo region) throws IOException, InterruptedException {
122     long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
123     long expiration = timeout + System.currentTimeMillis();
124     while (System.currentTimeMillis() < expiration) {
125       try {
126         Map<String, RegionState> rits=
127             admin.getClusterStatus().getRegionsInTransition();
128 
129         if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
130           // yay! no longer RIT
131           return;
132         }
133         // still in rit
134         LOG.info("Region still in transition, waiting for "
135             + "it to become assigned: " + region);
136       } catch (IOException e) {
137         LOG.warn("Exception when waiting for region to become assigned,"
138             + " retrying", e);
139       }
140       Thread.sleep(1000);
141     }
142     throw new IOException("Region " + region + " failed to move out of " +
143         "transition within timeout " + timeout + "ms");
144   }
145 
146   /**
147    * Contacts a region server and waits up to hbase.hbck.close.timeout ms
148    * (default 120s) to close the region.  This bypasses the active hmaster.
149    */
150   @SuppressWarnings("deprecation")
151   public static void closeRegionSilentlyAndWait(Admin admin,
152       ServerName server, HRegionInfo region) throws IOException, InterruptedException {
153     HConnection connection = admin.getConnection();
154     AdminService.BlockingInterface rs = connection.getAdmin(server);
155     try {
156       ProtobufUtil.closeRegion(rs, server, region.getRegionName());
157     } catch (IOException e) {
158       LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
159     }
160     long timeout = admin.getConfiguration()
161       .getLong("hbase.hbck.close.timeout", 120000);
162     long expiration = timeout + System.currentTimeMillis();
163     while (System.currentTimeMillis() < expiration) {
164       try {
165         HRegionInfo rsRegion =
166           ProtobufUtil.getRegionInfo(rs, region.getRegionName());
167         if (rsRegion == null) return;
168       } catch (IOException ioe) {
169         return;
170       }
171       Thread.sleep(1000);
172     }
173     throw new IOException("Region " + region + " failed to close within"
174         + " timeout " + timeout);
175   }
176 
177   /**
178    * Puts the specified HRegionInfo into META with replica related columns
179    */
180   public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf,
181       HRegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
182     HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
183     Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
184     if (numReplicas > 1) {
185       Random r = new Random();
186       ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
187       for (int i = 1; i < numReplicas; i++) {
188         ServerName sn = serversArr[r.nextInt(serversArr.length)];
189         // the column added here is just to make sure the master is able to
190         // see the additional replicas when it is asked to assign. The
191         // final value of these columns will be different and will be updated
192         // by the actual regionservers that start hosting the respective replicas
193         MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
194       }
195     }
196     meta.put(put);
197     meta.close();
198   }
199 
200   /**
201    * Creates, flushes, and closes a new region.
202    */
203   public static HRegion createHDFSRegionDir(Configuration conf,
204       HRegionInfo hri, HTableDescriptor htd) throws IOException {
205     // Create HRegion
206     Path root = FSUtils.getRootDir(conf);
207     HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
208 
209     // Close the new region to flush to disk. Close log file too.
210     region.close();
211     return region;
212   }
213 }