001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.util;
020
021import java.io.IOException;
022import java.util.Collection;
023import java.util.EnumSet;
024import java.util.List;
025import java.util.Random;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.ClusterMetrics.Option;
029import org.apache.hadoop.hbase.MetaTableAccessor;
030import org.apache.hadoop.hbase.ServerName;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.ZooKeeperConnectionException;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.AsyncClusterConnection;
035import org.apache.hadoop.hbase.client.ClusterConnectionFactory;
036import org.apache.hadoop.hbase.client.Connection;
037import org.apache.hadoop.hbase.client.ConnectionFactory;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.master.RegionState;
043import org.apache.hadoop.hbase.master.ServerManager;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.security.User;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.apache.zookeeper.KeeperException;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051/**
052 * This class contains helper methods that repair parts of hbase's filesystem
053 * contents.
054 */
055@InterfaceAudience.Private
056public class HBaseFsckRepair {
057  private static final Logger LOG = LoggerFactory.getLogger(HBaseFsckRepair.class);
058
059  /**
060   * Fix multiple assignment by doing silent closes on each RS hosting the region
061   * and then force ZK unassigned node to OFFLINE to trigger assignment by
062   * master.
063   *
064   * @param connection HBase connection to the cluster
065   * @param region Region to undeploy
066   * @param servers list of Servers to undeploy from
067   */
068  public static void fixMultiAssignment(Connection connection, RegionInfo region,
069      List<ServerName> servers)
070  throws IOException, KeeperException, InterruptedException {
071    // Close region on the servers silently
072    for(ServerName server : servers) {
073      closeRegionSilentlyAndWait(connection, server, region);
074    }
075
076    // Force ZK node to OFFLINE so master assigns
077    forceOfflineInZK(connection.getAdmin(), region);
078  }
079
080  /**
081   * Fix unassigned by creating/transition the unassigned ZK node for this
082   * region to OFFLINE state with a special flag to tell the master that this is
083   * a forced operation by HBCK.
084   *
085   * This assumes that info is in META.
086   *
087   * @param admin
088   * @param region
089   * @throws IOException
090   * @throws KeeperException
091   */
092  public static void fixUnassigned(Admin admin, RegionInfo region)
093      throws IOException, KeeperException, InterruptedException {
094    // Force ZK node to OFFLINE so master assigns
095    forceOfflineInZK(admin, region);
096  }
097
098  /**
099   * In 0.90, this forces an HRI offline by setting the RegionTransitionData
100   * in ZK to have HBCK_CODE_NAME as the server.  This is a special case in
101   * the AssignmentManager that attempts an assign call by the master.
102   *
103   * This doesn't seem to work properly in the updated version of 0.92+'s hbck
104   * so we use assign to force the region into transition.  This has the
105   * side-effect of requiring a RegionInfo that considers regionId (timestamp)
106   * in comparators that is addressed by HBASE-5563.
107   */
108  private static void forceOfflineInZK(Admin admin, final RegionInfo region)
109  throws ZooKeeperConnectionException, KeeperException, IOException, InterruptedException {
110    admin.assign(region.getRegionName());
111  }
112
113  /*
114   * Should we check all assignments or just not in RIT?
115   */
116  public static void waitUntilAssigned(Admin admin,
117      RegionInfo region) throws IOException, InterruptedException {
118    long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
119    long expiration = timeout + EnvironmentEdgeManager.currentTime();
120    while (EnvironmentEdgeManager.currentTime() < expiration) {
121      try {
122        boolean inTransition = false;
123        for (RegionState rs : admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
124                                   .getRegionStatesInTransition()) {
125          if (RegionInfo.COMPARATOR.compare(rs.getRegion(), region) == 0) {
126            inTransition = true;
127            break;
128          }
129        }
130        if (!inTransition) {
131          // yay! no longer RIT
132          return;
133        }
134        // still in rit
135        LOG.info("Region still in transition, waiting for "
136            + "it to become assigned: " + region);
137      } catch (IOException e) {
138        LOG.warn("Exception when waiting for region to become assigned,"
139            + " retrying", e);
140      }
141      Thread.sleep(1000);
142    }
143    throw new IOException("Region " + region + " failed to move out of " +
144        "transition within timeout " + timeout + "ms");
145  }
146
147  /**
148   * Contacts a region server and waits up to hbase.hbck.close.timeout ms (default 120s) to close
149   * the region. This bypasses the active hmaster.
150   */
151  public static void closeRegionSilentlyAndWait(Connection connection, ServerName server,
152      RegionInfo region) throws IOException, InterruptedException {
153    long timeout = connection.getConfiguration().getLong("hbase.hbck.close.timeout", 120000);
154    // this is a bit ugly but it is only used in the old hbck and tests, so I think it is fine.
155    try (AsyncClusterConnection asyncConn = ClusterConnectionFactory
156      .createAsyncClusterConnection(connection.getConfiguration(), null, User.getCurrent())) {
157      ServerManager.closeRegionSilentlyAndWait(asyncConn, server, region, timeout);
158    }
159  }
160
161  /**
162   * Puts the specified RegionInfo into META with replica related columns
163   */
164  public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf,
165      RegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
166    Connection conn = ConnectionFactory.createConnection(conf);
167    Table meta = conn.getTable(TableName.META_TABLE_NAME);
168    Put put = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
169    if (numReplicas > 1) {
170      Random r = new Random();
171      ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
172      for (int i = 1; i < numReplicas; i++) {
173        ServerName sn = serversArr[r.nextInt(serversArr.length)];
174        // the column added here is just to make sure the master is able to
175        // see the additional replicas when it is asked to assign. The
176        // final value of these columns will be different and will be updated
177        // by the actual regionservers that start hosting the respective replicas
178        MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
179      }
180    }
181    meta.put(put);
182    meta.close();
183    conn.close();
184  }
185
186  /**
187   * Creates, flushes, and closes a new region.
188   */
189  public static HRegion createHDFSRegionDir(Configuration conf,
190      RegionInfo hri, TableDescriptor htd) throws IOException {
191    // Create HRegion
192    Path root = FSUtils.getRootDir(conf);
193    HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
194
195    // Close the new region to flush to disk. Close log file too.
196    region.close();
197    return region;
198  }
199
200  /*
201   * Remove parent
202   */
203  public static void removeParentInMeta(Configuration conf, RegionInfo hri) throws IOException {
204    Connection conn = ConnectionFactory.createConnection(conf);
205    MetaTableAccessor.deleteRegionInfo(conn, hri);
206  }
207}