Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.io.IOException;
021import java.util.Collections;
022import java.util.List;
023import java.util.SortedMap;
024import java.util.TreeMap;
025
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellBuilderFactory;
028import org.apache.hadoop.hbase.CellBuilderType;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.HRegionLocation;
031import org.apache.hadoop.hbase.MetaTableAccessor;
032import org.apache.hadoop.hbase.RegionLocations;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.master.MasterFileSystem;
041import org.apache.hadoop.hbase.master.MasterServices;
042import org.apache.hadoop.hbase.master.RegionState.State;
043import org.apache.hadoop.hbase.procedure2.Procedure;
044import org.apache.hadoop.hbase.procedure2.util.StringUtils;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.wal.WALSplitter;
048import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
049import org.apache.yetus.audience.InterfaceAudience;
050import org.apache.zookeeper.KeeperException;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
055import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
056
057/**
058 * Store Region State to hbase:meta table.
059 */
060@InterfaceAudience.Private
061public class RegionStateStore {
062  private static final Logger LOG = LoggerFactory.getLogger(RegionStateStore.class);
063
064  /** The delimiter for meta columns for replicaIds &gt; 0 */
065  protected static final char META_REPLICA_ID_DELIMITER = '_';
066
067  private final MasterServices master;
068
069  public RegionStateStore(final MasterServices master) {
070    this.master = master;
071  }
072
073  public interface RegionStateVisitor {
074    void visitRegionState(Result result, RegionInfo regionInfo, State state,
075      ServerName regionLocation, ServerName lastHost, long openSeqNum);
076  }
077
078  public void visitMeta(final RegionStateVisitor visitor) throws IOException {
079    MetaTableAccessor.fullScanRegions(master.getConnection(), new MetaTableAccessor.Visitor() {
080      final boolean isDebugEnabled = LOG.isDebugEnabled();
081
082      @Override
083      public boolean visit(final Result r) throws IOException {
084        if (r !=  null && !r.isEmpty()) {
085          long st = 0;
086          if (LOG.isTraceEnabled()) {
087            st = System.currentTimeMillis();
088          }
089          visitMetaEntry(visitor, r);
090          if (LOG.isTraceEnabled()) {
091            long et = System.currentTimeMillis();
092            LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st));
093          }
094        } else if (isDebugEnabled) {
095          LOG.debug("NULL result from meta - ignoring but this is strange.");
096        }
097        return true;
098      }
099    });
100  }
101
102  private void visitMetaEntry(final RegionStateVisitor visitor, final Result result)
103      throws IOException {
104    final RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
105    if (rl == null) return;
106
107    final HRegionLocation[] locations = rl.getRegionLocations();
108    if (locations == null) return;
109
110    for (int i = 0; i < locations.length; ++i) {
111      final HRegionLocation hrl = locations[i];
112      if (hrl == null) continue;
113
114      final RegionInfo regionInfo = hrl.getRegion();
115      if (regionInfo == null) continue;
116
117      final int replicaId = regionInfo.getReplicaId();
118      final State state = getRegionState(result, replicaId);
119
120      final ServerName lastHost = hrl.getServerName();
121      final ServerName regionLocation = getRegionServer(result, replicaId);
122      final long openSeqNum = hrl.getSeqNum();
123
124      // TODO: move under trace, now is visible for debugging
125      LOG.info(
126        "Load hbase:meta entry region={}, regionState={}, lastHost={}, " +
127          "regionLocation={}, openSeqNum={}",
128        regionInfo.getEncodedName(), state, lastHost, regionLocation, openSeqNum);
129      visitor.visitRegionState(result, regionInfo, state, regionLocation, lastHost, openSeqNum);
130    }
131  }
132
133  public void updateRegionLocation(RegionStates.RegionStateNode regionStateNode)
134      throws IOException {
135    if (regionStateNode.getRegionInfo().isMetaRegion()) {
136      updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(),
137        regionStateNode.getState());
138    } else {
139      long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum()
140        : HConstants.NO_SEQNUM;
141      updateUserRegionLocation(regionStateNode.getRegionInfo(), regionStateNode.getState(),
142        regionStateNode.getRegionLocation(), openSeqNum,
143        // The regionStateNode may have no procedure in a test scenario; allow for this.
144        regionStateNode.getProcedure() != null ? regionStateNode.getProcedure().getProcId()
145          : Procedure.NO_PROC_ID);
146    }
147  }
148
149  private void updateMetaLocation(RegionInfo regionInfo, ServerName serverName, State state)
150      throws IOException {
151    try {
152      MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(),
153        state);
154    } catch (KeeperException e) {
155      throw new IOException(e);
156    }
157  }
158
159  private void updateUserRegionLocation(RegionInfo regionInfo, State state,
160      ServerName regionLocation, long openSeqNum,
161       long pid) throws IOException {
162    long time = EnvironmentEdgeManager.currentTime();
163    final int replicaId = regionInfo.getReplicaId();
164    final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo), time);
165    MetaTableAccessor.addRegionInfo(put, regionInfo);
166    final StringBuilder info =
167      new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=")
168        .append(regionInfo.getEncodedName()).append(", regionState=").append(state);
169    if (openSeqNum >= 0) {
170      Preconditions.checkArgument(state == State.OPEN && regionLocation != null,
171          "Open region should be on a server");
172      MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, replicaId);
173      // only update replication barrier for default replica
174      if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID &&
175        hasGlobalReplicationScope(regionInfo.getTable())) {
176        MetaTableAccessor.addReplicationBarrier(put, openSeqNum);
177        info.append(", repBarrier=").append(openSeqNum);
178      }
179      info.append(", openSeqNum=").append(openSeqNum);
180      info.append(", regionLocation=").append(regionLocation);
181    } else if (regionLocation != null) {
182      // Ideally, if no regionLocation, write null to the hbase:meta but this will confuse clients
183      // currently; they want a server to hit. TODO: Make clients wait if no location.
184      put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
185          .setRow(put.getRow())
186          .setFamily(HConstants.CATALOG_FAMILY)
187          .setQualifier(getServerNameColumn(replicaId))
188          .setTimestamp(put.getTimestamp())
189          .setType(Cell.Type.Put)
190          .setValue(Bytes.toBytes(regionLocation.getServerName()))
191          .build());
192      info.append(", regionLocation=").append(regionLocation);
193    }
194    put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
195        .setRow(put.getRow())
196        .setFamily(HConstants.CATALOG_FAMILY)
197        .setQualifier(getStateColumn(replicaId))
198        .setTimestamp(put.getTimestamp())
199        .setType(Cell.Type.Put)
200        .setValue(Bytes.toBytes(state.name()))
201        .build());
202    LOG.info(info.toString());
203    updateRegionLocation(regionInfo, state, put);
204  }
205
206  private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
207      throws IOException {
208    try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
209      table.put(put);
210    } catch (IOException e) {
211      // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host!
212      // In tests we abort the Master!
213      String msg = String.format("FAILED persisting region=%s state=%s",
214        regionInfo.getShortNameToLog(), state);
215      LOG.error(msg, e);
216      master.abort(msg, e);
217      throw e;
218    }
219  }
220
221  private long getOpenSeqNumForParentRegion(RegionInfo region) throws IOException {
222    MasterFileSystem fs = master.getMasterFileSystem();
223    long maxSeqId = WALSplitter.getMaxRegionSequenceId(master.getConfiguration(), region,
224      fs::getFileSystem, fs::getWALFileSystem);
225    return maxSeqId > 0 ? maxSeqId + 1 : HConstants.NO_SEQNUM;
226  }
227
228  // ============================================================================================
229  //  Update Region Splitting State helpers
230  // ============================================================================================
231  public void splitRegion(RegionInfo parent, RegionInfo hriA, RegionInfo hriB,
232      ServerName serverName) throws IOException {
233    TableDescriptor htd = getTableDescriptor(parent.getTable());
234    long parentOpenSeqNum = HConstants.NO_SEQNUM;
235    if (htd.hasGlobalReplicationScope()) {
236      parentOpenSeqNum = getOpenSeqNumForParentRegion(parent);
237    }
238    MetaTableAccessor.splitRegion(master.getConnection(), parent, parentOpenSeqNum, hriA, hriB,
239      serverName, getRegionReplication(htd));
240  }
241
242  // ============================================================================================
243  //  Update Region Merging State helpers
244  // ============================================================================================
245  public void mergeRegions(RegionInfo child, RegionInfo [] parents, ServerName serverName)
246      throws IOException {
247    TableDescriptor htd = getTableDescriptor(child.getTable());
248    boolean globalScope = htd.hasGlobalReplicationScope();
249    SortedMap<RegionInfo, Long> parentSeqNums = new TreeMap<>();
250    for (RegionInfo ri: parents) {
251      parentSeqNums.put(ri, globalScope? getOpenSeqNumForParentRegion(ri): -1);
252    }
253    MetaTableAccessor.mergeRegions(master.getConnection(), child, parentSeqNums,
254        serverName, getRegionReplication(htd));
255  }
256
257  // ============================================================================================
258  //  Delete Region State helpers
259  // ============================================================================================
260  public void deleteRegion(final RegionInfo regionInfo) throws IOException {
261    deleteRegions(Collections.singletonList(regionInfo));
262  }
263
264  public void deleteRegions(final List<RegionInfo> regions) throws IOException {
265    MetaTableAccessor.deleteRegionInfos(master.getConnection(), regions);
266  }
267
268  // ==========================================================================
269  //  Table Descriptors helpers
270  // ==========================================================================
271  private boolean hasGlobalReplicationScope(TableName tableName) throws IOException {
272    return hasGlobalReplicationScope(getTableDescriptor(tableName));
273  }
274
275  private boolean hasGlobalReplicationScope(TableDescriptor htd) {
276    return htd != null ? htd.hasGlobalReplicationScope() : false;
277  }
278
279  private int getRegionReplication(TableDescriptor htd) {
280    return htd != null ? htd.getRegionReplication() : 1;
281  }
282
283  private TableDescriptor getTableDescriptor(TableName tableName) throws IOException {
284    return master.getTableDescriptors().get(tableName);
285  }
286
287  // ==========================================================================
288  //  Server Name
289  // ==========================================================================
290
291  /**
292   * Returns the {@link ServerName} from catalog table {@link Result}
293   * where the region is transitioning. It should be the same as
294   * {@link MetaTableAccessor#getServerName(Result,int)} if the server is at OPEN state.
295   * @param r Result to pull the transitioning server name from
296   * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)}
297   * if necessary fields not found or empty.
298   */
299  static ServerName getRegionServer(final Result r, int replicaId) {
300    final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY,
301        getServerNameColumn(replicaId));
302    if (cell == null || cell.getValueLength() == 0) {
303      RegionLocations locations = MetaTableAccessor.getRegionLocations(r);
304      if (locations != null) {
305        HRegionLocation location = locations.getRegionLocation(replicaId);
306        if (location != null) {
307          return location.getServerName();
308        }
309      }
310      return null;
311    }
312    return ServerName.parseServerName(Bytes.toString(cell.getValueArray(),
313      cell.getValueOffset(), cell.getValueLength()));
314  }
315
316  private static byte[] getServerNameColumn(int replicaId) {
317    return replicaId == 0
318        ? HConstants.SERVERNAME_QUALIFIER
319        : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
320          + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
321  }
322
323  // ==========================================================================
324  //  Region State
325  // ==========================================================================
326
327  /**
328   * Pull the region state from a catalog table {@link Result}.
329   * @param r Result to pull the region state from
330   * @return the region state, or null if unknown.
331   */
332  @VisibleForTesting
333  public static State getRegionState(final Result r, int replicaId) {
334    Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getStateColumn(replicaId));
335    if (cell == null || cell.getValueLength() == 0) {
336      return null;
337    }
338    return State.valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(),
339        cell.getValueLength()));
340  }
341
342  private static byte[] getStateColumn(int replicaId) {
343    return replicaId == 0
344        ? HConstants.STATE_QUALIFIER
345        : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
346          + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
347  }
348}