Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.io.IOException;
021import java.util.Collections;
022import java.util.List;
023import java.util.SortedMap;
024import java.util.TreeMap;
025
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellBuilderFactory;
028import org.apache.hadoop.hbase.CellBuilderType;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.HRegionLocation;
031import org.apache.hadoop.hbase.MetaTableAccessor;
032import org.apache.hadoop.hbase.RegionLocations;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.Result;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.master.MasterFileSystem;
041import org.apache.hadoop.hbase.master.MasterServices;
042import org.apache.hadoop.hbase.master.RegionState.State;
043import org.apache.hadoop.hbase.procedure2.Procedure;
044import org.apache.hadoop.hbase.procedure2.util.StringUtils;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.wal.WALSplitUtil;
048import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
049import org.apache.yetus.audience.InterfaceAudience;
050import org.apache.zookeeper.KeeperException;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
055
056/**
057 * Store Region State to hbase:meta table.
058 */
059@InterfaceAudience.Private
060public class RegionStateStore {
061  private static final Logger LOG = LoggerFactory.getLogger(RegionStateStore.class);
062
063  /** The delimiter for meta columns for replicaIds &gt; 0 */
064  protected static final char META_REPLICA_ID_DELIMITER = '_';
065
066  private final MasterServices master;
067
068  public RegionStateStore(final MasterServices master) {
069    this.master = master;
070  }
071
072  public interface RegionStateVisitor {
073    void visitRegionState(Result result, RegionInfo regionInfo, State state,
074      ServerName regionLocation, ServerName lastHost, long openSeqNum);
075  }
076
077  public void visitMeta(final RegionStateVisitor visitor) throws IOException {
078    MetaTableAccessor.fullScanRegions(master.getConnection(), new MetaTableAccessor.Visitor() {
079      final boolean isDebugEnabled = LOG.isDebugEnabled();
080
081      @Override
082      public boolean visit(final Result r) throws IOException {
083        if (r !=  null && !r.isEmpty()) {
084          long st = 0;
085          if (LOG.isTraceEnabled()) {
086            st = System.currentTimeMillis();
087          }
088          visitMetaEntry(visitor, r);
089          if (LOG.isTraceEnabled()) {
090            long et = System.currentTimeMillis();
091            LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st));
092          }
093        } else if (isDebugEnabled) {
094          LOG.debug("NULL result from meta - ignoring but this is strange.");
095        }
096        return true;
097      }
098    });
099  }
100
101  /**
102   * Queries META table for the passed region encoded name,
103   * delegating action upon results to the <code>RegionStateVisitor</code>
104   * passed as second parameter.
105   * @param regionEncodedName encoded name for the Region we want to query META for.
106   * @param visitor The <code>RegionStateVisitor</code> instance to react over the query results.
107   * @throws IOException If some error occurs while querying META or parsing results.
108   */
109  public void visitMetaForRegion(final String regionEncodedName, final RegionStateVisitor visitor)
110      throws IOException {
111    Result result = MetaTableAccessor.
112      scanByRegionEncodedName(master.getConnection(), regionEncodedName);
113    if (result != null) {
114      visitMetaEntry(visitor, result);
115    }
116  }
117
118  private void visitMetaEntry(final RegionStateVisitor visitor, final Result result)
119      throws IOException {
120    final RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
121    if (rl == null) return;
122
123    final HRegionLocation[] locations = rl.getRegionLocations();
124    if (locations == null) return;
125
126    for (int i = 0; i < locations.length; ++i) {
127      final HRegionLocation hrl = locations[i];
128      if (hrl == null) continue;
129
130      final RegionInfo regionInfo = hrl.getRegion();
131      if (regionInfo == null) continue;
132
133      final int replicaId = regionInfo.getReplicaId();
134      final State state = getRegionState(result, regionInfo);
135
136      final ServerName lastHost = hrl.getServerName();
137      final ServerName regionLocation = getRegionServer(result, replicaId);
138      final long openSeqNum = hrl.getSeqNum();
139
140      // TODO: move under trace, now is visible for debugging
141      LOG.info(
142        "Load hbase:meta entry region={}, regionState={}, lastHost={}, " +
143          "regionLocation={}, openSeqNum={}",
144        regionInfo.getEncodedName(), state, lastHost, regionLocation, openSeqNum);
145      visitor.visitRegionState(result, regionInfo, state, regionLocation, lastHost, openSeqNum);
146    }
147  }
148
149  void updateRegionLocation(RegionStateNode regionStateNode) throws IOException {
150    if (regionStateNode.getRegionInfo().isMetaRegion()) {
151      updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(),
152        regionStateNode.getState());
153    } else {
154      long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum()
155        : HConstants.NO_SEQNUM;
156      updateUserRegionLocation(regionStateNode.getRegionInfo(), regionStateNode.getState(),
157        regionStateNode.getRegionLocation(), openSeqNum,
158        // The regionStateNode may have no procedure in a test scenario; allow for this.
159        regionStateNode.getProcedure() != null ? regionStateNode.getProcedure().getProcId()
160          : Procedure.NO_PROC_ID);
161    }
162  }
163
164  private void updateMetaLocation(RegionInfo regionInfo, ServerName serverName, State state)
165      throws IOException {
166    try {
167      MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(),
168        state);
169    } catch (KeeperException e) {
170      throw new IOException(e);
171    }
172  }
173
174  private void updateUserRegionLocation(RegionInfo regionInfo, State state,
175      ServerName regionLocation, long openSeqNum,
176       long pid) throws IOException {
177    long time = EnvironmentEdgeManager.currentTime();
178    final int replicaId = regionInfo.getReplicaId();
179    final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo), time);
180    MetaTableAccessor.addRegionInfo(put, regionInfo);
181    final StringBuilder info =
182      new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=")
183        .append(regionInfo.getEncodedName()).append(", regionState=").append(state);
184    if (openSeqNum >= 0) {
185      Preconditions.checkArgument(state == State.OPEN && regionLocation != null,
186          "Open region should be on a server");
187      MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, replicaId);
188      // only update replication barrier for default replica
189      if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID &&
190        hasGlobalReplicationScope(regionInfo.getTable())) {
191        MetaTableAccessor.addReplicationBarrier(put, openSeqNum);
192        info.append(", repBarrier=").append(openSeqNum);
193      }
194      info.append(", openSeqNum=").append(openSeqNum);
195      info.append(", regionLocation=").append(regionLocation);
196    } else if (regionLocation != null) {
197      // Ideally, if no regionLocation, write null to the hbase:meta but this will confuse clients
198      // currently; they want a server to hit. TODO: Make clients wait if no location.
199      put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
200          .setRow(put.getRow())
201          .setFamily(HConstants.CATALOG_FAMILY)
202          .setQualifier(getServerNameColumn(replicaId))
203          .setTimestamp(put.getTimestamp())
204          .setType(Cell.Type.Put)
205          .setValue(Bytes.toBytes(regionLocation.getServerName()))
206          .build());
207      info.append(", regionLocation=").append(regionLocation);
208    }
209    put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
210        .setRow(put.getRow())
211        .setFamily(HConstants.CATALOG_FAMILY)
212        .setQualifier(getStateColumn(replicaId))
213        .setTimestamp(put.getTimestamp())
214        .setType(Cell.Type.Put)
215        .setValue(Bytes.toBytes(state.name()))
216        .build());
217    LOG.info(info.toString());
218    updateRegionLocation(regionInfo, state, put);
219  }
220
221  private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
222      throws IOException {
223    try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
224      table.put(put);
225    } catch (IOException e) {
226      // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host!
227      // In tests we abort the Master!
228      String msg = String.format("FAILED persisting region=%s state=%s",
229        regionInfo.getShortNameToLog(), state);
230      LOG.error(msg, e);
231      master.abort(msg, e);
232      throw e;
233    }
234  }
235
236  private long getOpenSeqNumForParentRegion(RegionInfo region) throws IOException {
237    MasterFileSystem fs = master.getMasterFileSystem();
238    long maxSeqId = WALSplitUtil.getMaxRegionSequenceId(master.getConfiguration(), region,
239      fs::getFileSystem, fs::getWALFileSystem);
240    return maxSeqId > 0 ? maxSeqId + 1 : HConstants.NO_SEQNUM;
241  }
242
243  // ============================================================================================
244  //  Update Region Splitting State helpers
245  // ============================================================================================
246  public void splitRegion(RegionInfo parent, RegionInfo hriA, RegionInfo hriB,
247      ServerName serverName) throws IOException {
248    TableDescriptor htd = getDescriptor(parent.getTable());
249    long parentOpenSeqNum = HConstants.NO_SEQNUM;
250    if (htd.hasGlobalReplicationScope()) {
251      parentOpenSeqNum = getOpenSeqNumForParentRegion(parent);
252    }
253    MetaTableAccessor.splitRegion(master.getConnection(), parent, parentOpenSeqNum, hriA, hriB,
254      serverName, getRegionReplication(htd));
255  }
256
257  // ============================================================================================
258  //  Update Region Merging State helpers
259  // ============================================================================================
260  public void mergeRegions(RegionInfo child, RegionInfo [] parents, ServerName serverName)
261      throws IOException {
262    TableDescriptor htd = getDescriptor(child.getTable());
263    boolean globalScope = htd.hasGlobalReplicationScope();
264    SortedMap<RegionInfo, Long> parentSeqNums = new TreeMap<>();
265    for (RegionInfo ri: parents) {
266      parentSeqNums.put(ri, globalScope? getOpenSeqNumForParentRegion(ri): -1);
267    }
268    MetaTableAccessor.mergeRegions(master.getConnection(), child, parentSeqNums,
269        serverName, getRegionReplication(htd));
270  }
271
272  // ============================================================================================
273  //  Delete Region State helpers
274  // ============================================================================================
275  public void deleteRegion(final RegionInfo regionInfo) throws IOException {
276    deleteRegions(Collections.singletonList(regionInfo));
277  }
278
279  public void deleteRegions(final List<RegionInfo> regions) throws IOException {
280    MetaTableAccessor.deleteRegionInfos(master.getConnection(), regions);
281  }
282
283  // ==========================================================================
284  //  Table Descriptors helpers
285  // ==========================================================================
286  private boolean hasGlobalReplicationScope(TableName tableName) throws IOException {
287    return hasGlobalReplicationScope(getDescriptor(tableName));
288  }
289
290  private boolean hasGlobalReplicationScope(TableDescriptor htd) {
291    return htd != null ? htd.hasGlobalReplicationScope() : false;
292  }
293
294  private int getRegionReplication(TableDescriptor htd) {
295    return htd != null ? htd.getRegionReplication() : 1;
296  }
297
298  private TableDescriptor getDescriptor(TableName tableName) throws IOException {
299    return master.getTableDescriptors().get(tableName);
300  }
301
302  // ==========================================================================
303  //  Server Name
304  // ==========================================================================
305
306  /**
307   * Returns the {@link ServerName} from catalog table {@link Result}
308   * where the region is transitioning. It should be the same as
309   * {@link MetaTableAccessor#getServerName(Result,int)} if the server is at OPEN state.
310   * @param r Result to pull the transitioning server name from
311   * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)}
312   * if necessary fields not found or empty.
313   */
314  static ServerName getRegionServer(final Result r, int replicaId) {
315    final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY,
316        getServerNameColumn(replicaId));
317    if (cell == null || cell.getValueLength() == 0) {
318      RegionLocations locations = MetaTableAccessor.getRegionLocations(r);
319      if (locations != null) {
320        HRegionLocation location = locations.getRegionLocation(replicaId);
321        if (location != null) {
322          return location.getServerName();
323        }
324      }
325      return null;
326    }
327    return ServerName.parseServerName(Bytes.toString(cell.getValueArray(),
328      cell.getValueOffset(), cell.getValueLength()));
329  }
330
331  private static byte[] getServerNameColumn(int replicaId) {
332    return replicaId == 0
333        ? HConstants.SERVERNAME_QUALIFIER
334        : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
335          + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
336  }
337
338  // ==========================================================================
339  //  Region State
340  // ==========================================================================
341
342  /**
343   * Pull the region state from a catalog table {@link Result}.
344   * @return the region state, or null if unknown.
345   */
346  public static State getRegionState(final Result r, RegionInfo regionInfo) {
347    Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY,
348        getStateColumn(regionInfo.getReplicaId()));
349    if (cell == null || cell.getValueLength() == 0) {
350      return null;
351    }
352
353    String state = Bytes.toString(cell.getValueArray(), cell.getValueOffset(),
354        cell.getValueLength());
355    try {
356      return State.valueOf(state);
357    } catch (IllegalArgumentException e) {
358      LOG.warn("BAD value {} in hbase:meta info:state column for region {} , " +
359              "Consider using HBCK2 setRegionState ENCODED_REGION_NAME STATE",
360          state, regionInfo.getEncodedName());
361      return null;
362    }
363  }
364
365  private static byte[] getStateColumn(int replicaId) {
366    return replicaId == 0
367        ? HConstants.STATE_QUALIFIER
368        : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
369          + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
370  }
371}