001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.EnumSet;
023import java.util.List;
024import java.util.Random;
025import java.util.concurrent.ThreadLocalRandom;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.ClusterMetrics.Option;
029import org.apache.hadoop.hbase.MetaTableAccessor;
030import org.apache.hadoop.hbase.ServerName;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.ZooKeeperConnectionException;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.AsyncClusterConnection;
035import org.apache.hadoop.hbase.client.ClusterConnectionFactory;
036import org.apache.hadoop.hbase.client.Connection;
037import org.apache.hadoop.hbase.client.ConnectionFactory;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.master.RegionState;
043import org.apache.hadoop.hbase.master.ServerManager;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.security.User;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.apache.zookeeper.KeeperException;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051/**
052 * This class contains helper methods that repair parts of hbase's filesystem contents.
053 */
054@InterfaceAudience.Private
055public class HBaseFsckRepair {
056  private static final Logger LOG = LoggerFactory.getLogger(HBaseFsckRepair.class);
057
058  /**
059   * Fix multiple assignment by doing silent closes on each RS hosting the region and then force ZK
060   * unassigned node to OFFLINE to trigger assignment by master.
061   * @param connection HBase connection to the cluster
062   * @param region     Region to undeploy
063   * @param servers    list of Servers to undeploy from
064   */
065  public static void fixMultiAssignment(Connection connection, RegionInfo region,
066    List<ServerName> servers) throws IOException, KeeperException, InterruptedException {
067    // Close region on the servers silently
068    for (ServerName server : servers) {
069      closeRegionSilentlyAndWait(connection, server, region);
070    }
071
072    // Force ZK node to OFFLINE so master assigns
073    forceOfflineInZK(connection.getAdmin(), region);
074  }
075
076  /**
077   * Fix unassigned by creating/transition the unassigned ZK node for this region to OFFLINE state
078   * with a special flag to tell the master that this is a forced operation by HBCK. This assumes
079   * that info is in META. nnnn
080   */
081  public static void fixUnassigned(Admin admin, RegionInfo region)
082    throws IOException, KeeperException, InterruptedException {
083    // Force ZK node to OFFLINE so master assigns
084    forceOfflineInZK(admin, region);
085  }
086
087  /**
088   * In 0.90, this forces an HRI offline by setting the RegionTransitionData in ZK to have
089   * HBCK_CODE_NAME as the server. This is a special case in the AssignmentManager that attempts an
090   * assign call by the master. This doesn't seem to work properly in the updated version of 0.92+'s
091   * hbck so we use assign to force the region into transition. This has the side-effect of
092   * requiring a RegionInfo that considers regionId (timestamp) in comparators that is addressed by
093   * HBASE-5563.
094   */
095  private static void forceOfflineInZK(Admin admin, final RegionInfo region)
096    throws ZooKeeperConnectionException, KeeperException, IOException, InterruptedException {
097    admin.assign(region.getRegionName());
098  }
099
100  /*
101   * Should we check all assignments or just not in RIT?
102   */
103  public static void waitUntilAssigned(Admin admin, RegionInfo region)
104    throws IOException, InterruptedException {
105    long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
106    long expiration = timeout + EnvironmentEdgeManager.currentTime();
107    while (EnvironmentEdgeManager.currentTime() < expiration) {
108      try {
109        boolean inTransition = false;
110        for (RegionState rs : admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
111          .getRegionStatesInTransition()) {
112          if (RegionInfo.COMPARATOR.compare(rs.getRegion(), region) == 0) {
113            inTransition = true;
114            break;
115          }
116        }
117        if (!inTransition) {
118          // yay! no longer RIT
119          return;
120        }
121        // still in rit
122        LOG.info("Region still in transition, waiting for " + "it to become assigned: " + region);
123      } catch (IOException e) {
124        LOG.warn("Exception when waiting for region to become assigned," + " retrying", e);
125      }
126      Thread.sleep(1000);
127    }
128    throw new IOException("Region " + region + " failed to move out of "
129      + "transition within timeout " + timeout + "ms");
130  }
131
132  /**
133   * Contacts a region server and waits up to hbase.hbck.close.timeout ms (default 120s) to close
134   * the region. This bypasses the active hmaster.
135   */
136  public static void closeRegionSilentlyAndWait(Connection connection, ServerName server,
137    RegionInfo region) throws IOException, InterruptedException {
138    long timeout = connection.getConfiguration().getLong("hbase.hbck.close.timeout", 120000);
139    // this is a bit ugly but it is only used in the old hbck and tests, so I think it is fine.
140    try (AsyncClusterConnection asyncConn = ClusterConnectionFactory
141      .createAsyncClusterConnection(connection.getConfiguration(), null, User.getCurrent())) {
142      ServerManager.closeRegionSilentlyAndWait(asyncConn, server, region, timeout);
143    }
144  }
145
146  /**
147   * Puts the specified RegionInfo into META with replica related columns
148   */
149  public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf, RegionInfo hri,
150    Collection<ServerName> servers, int numReplicas) throws IOException {
151    Connection conn = ConnectionFactory.createConnection(conf);
152    Table meta = conn.getTable(TableName.META_TABLE_NAME);
153    Put put = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
154    if (numReplicas > 1) {
155      Random rand = ThreadLocalRandom.current();
156      ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
157      for (int i = 1; i < numReplicas; i++) {
158        ServerName sn = serversArr[rand.nextInt(serversArr.length)];
159        // the column added here is just to make sure the master is able to
160        // see the additional replicas when it is asked to assign. The
161        // final value of these columns will be different and will be updated
162        // by the actual regionservers that start hosting the respective replicas
163        MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
164      }
165    }
166    meta.put(put);
167    meta.close();
168    conn.close();
169  }
170
171  /**
172   * Creates, flushes, and closes a new region.
173   */
174  public static HRegion createHDFSRegionDir(Configuration conf, RegionInfo hri, TableDescriptor htd)
175    throws IOException {
176    // Create HRegion
177    Path root = CommonFSUtils.getRootDir(conf);
178    HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
179
180    // Close the new region to flush to disk. Close log file too.
181    region.close();
182    return region;
183  }
184
185  /*
186   * Remove parent
187   */
188  public static void removeParentInMeta(Configuration conf, RegionInfo hri) throws IOException {
189    throw new UnsupportedOperationException("HBCK1 is read-only now, use HBCK2 instead");
190  }
191}