View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.fs.Path;
25  import org.apache.hadoop.hbase.HRegionInfo;
26  import org.apache.hadoop.hbase.HTableDescriptor;
27  import org.apache.hadoop.hbase.MetaTableAccessor;
28  import org.apache.hadoop.hbase.NotServingRegionException;
29  import org.apache.hadoop.hbase.ServerName;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.client.Admin;
34  import org.apache.hadoop.hbase.client.Connection;
35  import org.apache.hadoop.hbase.client.ConnectionFactory;
36  import org.apache.hadoop.hbase.client.HConnection;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Table;
40  import org.apache.hadoop.hbase.master.RegionState;
41  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.zookeeper.KeeperException;
45  
46  import java.io.IOException;
47  import java.util.Collection;
48  import java.util.List;
49  import java.util.Map;
50  import java.util.Random;
51  
52  /**
53   * This class contains helper methods that repair parts of hbase's filesystem
54   * contents.
55   */
56  @InterfaceAudience.Private
57  public class HBaseFsckRepair {
58    public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
59  
60    /**
61     * Fix multiple assignment by doing silent closes on each RS hosting the region
62     * and then force ZK unassigned node to OFFLINE to trigger assignment by
63     * master.
64     *
65     * @param connection HBase connection to the cluster
66     * @param region Region to undeploy
67     * @param servers list of Servers to undeploy from
68     */
69    public static void fixMultiAssignment(HConnection connection, HRegionInfo region,
70        List<ServerName> servers)
71    throws IOException, KeeperException, InterruptedException {
72      HRegionInfo actualRegion = new HRegionInfo(region);
73  
74      // Close region on the servers silently
75      for(ServerName server : servers) {
76        closeRegionSilentlyAndWait(connection, server, actualRegion);
77      }
78  
79      // Force ZK node to OFFLINE so master assigns
80      forceOfflineInZK(connection.getAdmin(), actualRegion);
81    }
82  
83    /**
84     * Fix unassigned by creating/transition the unassigned ZK node for this
85     * region to OFFLINE state with a special flag to tell the master that this is
86     * a forced operation by HBCK.
87     *
88     * This assumes that info is in META.
89     *
90     * @param admin
91     * @param region
92     * @throws IOException
93     * @throws KeeperException
94     */
95    public static void fixUnassigned(Admin admin, HRegionInfo region)
96        throws IOException, KeeperException, InterruptedException {
97      HRegionInfo actualRegion = new HRegionInfo(region);
98  
99      // Force ZK node to OFFLINE so master assigns
100     forceOfflineInZK(admin, actualRegion);
101   }
102 
103   /**
104    * In 0.90, this forces an HRI offline by setting the RegionTransitionData
105    * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
106    * the AssignmentManager that attempts an assign call by the master.
107    *
108    * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK
109    *
110    * This doesn't seem to work properly in the updated version of 0.92+'s hbck
111    * so we use assign to force the region into transition.  This has the
112    * side-effect of requiring a HRegionInfo that considers regionId (timestamp)
113    * in comparators that is addressed by HBASE-5563.
114    */
115   private static void forceOfflineInZK(Admin admin, final HRegionInfo region)
116   throws ZooKeeperConnectionException, KeeperException, IOException, InterruptedException {
117     admin.assign(region.getRegionName());
118   }
119 
120   /*
121    * Should we check all assignments or just not in RIT?
122    */
123   public static void waitUntilAssigned(Admin admin,
124       HRegionInfo region) throws IOException, InterruptedException {
125     long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
126     long expiration = timeout + System.currentTimeMillis();
127     while (System.currentTimeMillis() < expiration) {
128       try {
129         Map<String, RegionState> rits=
130             admin.getClusterStatus().getRegionsInTransition();
131 
132         if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
133           // yay! no longer RIT
134           return;
135         }
136         // still in rit
137         LOG.info("Region still in transition, waiting for "
138             + "it to become assigned: " + region);
139       } catch (IOException e) {
140         LOG.warn("Exception when waiting for region to become assigned,"
141             + " retrying", e);
142       }
143       Thread.sleep(1000);
144     }
145     throw new IOException("Region " + region + " failed to move out of " +
146         "transition within timeout " + timeout + "ms");
147   }
148 
149   /**
150    * Contacts a region server and waits up to hbase.hbck.close.timeout ms
151    * (default 120s) to close the region.  This bypasses the active hmaster.
152    */
153   @SuppressWarnings("deprecation")
154   public static void closeRegionSilentlyAndWait(HConnection connection, 
155       ServerName server, HRegionInfo region) throws IOException, InterruptedException {
156     AdminService.BlockingInterface rs = connection.getAdmin(server);
157     try {
158       ProtobufUtil.closeRegion(rs, server, region.getRegionName());
159     } catch (IOException e) {
160       LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
161     }
162     long timeout = connection.getConfiguration()
163       .getLong("hbase.hbck.close.timeout", 120000);
164     long expiration = timeout + System.currentTimeMillis();
165     while (System.currentTimeMillis() < expiration) {
166       try {
167         HRegionInfo rsRegion =
168           ProtobufUtil.getRegionInfo(rs, region.getRegionName());
169         if (rsRegion == null) return;
170       } catch (IOException ioe) {
171         if (ioe instanceof NotServingRegionException) // no need to retry again
172           return;
173         LOG.warn("Exception when retrieving regioninfo from: " + region.getRegionNameAsString(), ioe);
174       }
175       Thread.sleep(1000);
176     }
177     throw new IOException("Region " + region + " failed to close within"
178         + " timeout " + timeout);
179   }
180 
181   /**
182    * Puts the specified HRegionInfo into META with replica related columns
183    */
184   public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf,
185       HRegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
186     Connection conn = ConnectionFactory.createConnection(conf);
187     Table meta = conn.getTable(TableName.META_TABLE_NAME);
188     Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
189     if (numReplicas > 1) {
190       Random r = new Random();
191       ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
192       for (int i = 1; i < numReplicas; i++) {
193         ServerName sn = serversArr[r.nextInt(serversArr.length)];
194         // the column added here is just to make sure the master is able to
195         // see the additional replicas when it is asked to assign. The
196         // final value of these columns will be different and will be updated
197         // by the actual regionservers that start hosting the respective replicas
198         MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
199       }
200     }
201     meta.put(put);
202     meta.close();
203     conn.close();
204   }
205 
206   /**
207    * Creates, flushes, and closes a new region.
208    */
209   public static HRegion createHDFSRegionDir(Configuration conf,
210       HRegionInfo hri, HTableDescriptor htd) throws IOException {
211     // Create HRegion
212     Path root = FSUtils.getRootDir(conf);
213     HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
214 
215     // Close the new region to flush to disk. Close log file too.
216     region.close();
217     return region;
218   }
219 }