001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import edu.umd.cs.findbugs.annotations.Nullable;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.LinkedHashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.NavigableMap;
028import java.util.SortedMap;
029import java.util.regex.Matcher;
030import java.util.regex.Pattern;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.client.RegionReplicaUtil;
034import org.apache.hadoop.hbase.client.Result;
035import org.apache.hadoop.hbase.client.TableState;
036import org.apache.hadoop.hbase.exceptions.DeserializationException;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042/**
043 * Helper class for generating/parsing
044 * {@value org.apache.hadoop.hbase.HConstants#CATALOG_FAMILY_STR} family cells in meta table.
045 * <p/>
046 * The cells in catalog family are:
047 *
048 * <pre>
049 * For each table range ('Region'), there is a single row, formatted as:
050 * &lt;tableName&gt;,&lt;startKey&gt;,&lt;regionId&gt;,&lt;encodedRegionName&gt;.
051 * This row is the serialized regionName of the default region replica.
052 * Columns are:
053 * info:regioninfo         => contains serialized HRI for the default region replica
054 * info:server             => contains hostname:port (in string form) for the server hosting
055 *                            the default regionInfo replica
056 * info:server_&lt;replicaId&gt => contains hostname:port (in string form) for the server hosting
057 *                                 the regionInfo replica with replicaId
058 * info:serverstartcode    => contains server start code (in binary long form) for the server
059 *                            hosting the default regionInfo replica
060 * info:serverstartcode_&lt;replicaId&gt => contains server start code (in binary long form) for
061 *                                          the server hosting the regionInfo replica with
062 *                                          replicaId
063 * info:seqnumDuringOpen   => contains seqNum (in binary long form) for the region at the time
064 *                            the server opened the region with default replicaId
065 * info:seqnumDuringOpen_&lt;replicaId&gt => contains seqNum (in binary long form) for the region
066 *                                           at the time the server opened the region with
067 *                                           replicaId
068 * info:splitA             => contains a serialized HRI for the first daughter region if the
069 *                            region is split
070 * info:splitB             => contains a serialized HRI for the second daughter region if the
071 *                            region is split
072 * info:merge*             => contains a serialized HRI for a merge parent region. There will be two
073 *                            or more of these columns in a row. A row that has these columns is
074 *                            undergoing a merge and is the result of the merge. Columns listed
075 *                            in marge* columns are the parents of this merged region. Example
076 *                            columns: info:merge0001, info:merge0002. You make also see 'mergeA',
077 *                            and 'mergeB'. This is old form replaced by the new format that allows
078 *                            for more than two parents to be merged at a time.
079 * </pre>
080 */
081@InterfaceAudience.Private
082public class CatalogFamilyFormat {
083
084  private static final Logger LOG = LoggerFactory.getLogger(CatalogFamilyFormat.class);
085
086  /** A regex for parsing server columns from meta. See above javadoc for meta layout */
087  private static final Pattern SERVER_COLUMN_PATTERN =
088    Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
089
090  /**
091   * Returns an HRI parsed from this regionName. Not all the fields of the HRI is stored in the
092   * name, so the returned object should only be used for the fields in the regionName.
093   * <p/>
094   * Since the returned object does not contain all the fields, we do not expose this method in
095   * public API, such as {@link RegionInfo} or {@link RegionInfoBuilder}.
096   */
097  public static RegionInfo parseRegionInfoFromRegionName(byte[] regionName) throws IOException {
098    byte[][] fields = RegionInfo.parseRegionName(regionName);
099    long regionId = Long.parseLong(Bytes.toString(fields[2]));
100    int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
101    return RegionInfoBuilder.newBuilder(TableName.valueOf(fields[0])).setStartKey(fields[1])
102      .setRegionId(regionId).setReplicaId(replicaId).build();
103  }
104
105  /**
106   * Returns the RegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and
107   * <code>qualifier</code> of the catalog table result.
108   * @param r         a Result object from the catalog table scan
109   * @param qualifier Column family qualifier
110   * @return An RegionInfo instance or null.
111   */
112  @Nullable
113  public static RegionInfo getRegionInfo(final Result r, byte[] qualifier) {
114    Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, qualifier);
115    if (cell == null) {
116      return null;
117    }
118    return RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(),
119      cell.getValueLength());
120  }
121
122  /**
123   * Returns RegionInfo object from the column
124   * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog table Result.
125   * @param data a Result object from the catalog table scan
126   * @return RegionInfo or null
127   */
128  public static RegionInfo getRegionInfo(Result data) {
129    return getRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
130  }
131
132  /**
133   * Returns the HRegionLocation parsed from the given meta row Result for the given regionInfo and
134   * replicaId. The regionInfo can be the default region info for the replica.
135   * @param r          the meta row result
136   * @param regionInfo RegionInfo for default replica
137   * @param replicaId  the replicaId for the HRegionLocation
138   * @return HRegionLocation parsed from the given meta row Result for the given replicaId
139   */
140  public static HRegionLocation getRegionLocation(final Result r, final RegionInfo regionInfo,
141    final int replicaId) {
142    ServerName serverName = getServerName(r, replicaId);
143    long seqNum = getSeqNumDuringOpen(r, replicaId);
144    RegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
145    return new HRegionLocation(replicaInfo, serverName, seqNum);
146  }
147
148  /**
149   * Returns an HRegionLocationList extracted from the result.
150   * @return an HRegionLocationList containing all locations for the region range or null if we
151   *         can't deserialize the result.
152   */
153  @Nullable
154  public static RegionLocations getRegionLocations(final Result r) {
155    if (r == null) {
156      return null;
157    }
158    RegionInfo regionInfo = getRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
159    if (regionInfo == null) {
160      return null;
161    }
162
163    List<HRegionLocation> locations = new ArrayList<>(1);
164    NavigableMap<byte[], NavigableMap<byte[], byte[]>> familyMap = r.getNoVersionMap();
165
166    locations.add(getRegionLocation(r, regionInfo, 0));
167
168    NavigableMap<byte[], byte[]> infoMap = familyMap.get(HConstants.CATALOG_FAMILY);
169    if (infoMap == null) {
170      return new RegionLocations(locations);
171    }
172
173    // iterate until all serverName columns are seen
174    int replicaId = 0;
175    byte[] serverColumn = getServerColumn(replicaId);
176    SortedMap<byte[], byte[]> serverMap;
177    serverMap = infoMap.tailMap(serverColumn, false);
178
179    if (serverMap.isEmpty()) {
180      return new RegionLocations(locations);
181    }
182
183    for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) {
184      replicaId = parseReplicaIdFromServerColumn(entry.getKey());
185      if (replicaId < 0) {
186        break;
187      }
188      HRegionLocation location = getRegionLocation(r, regionInfo, replicaId);
189      // In case the region replica is newly created, it's location might be null. We usually do not
190      // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs.
191      if (location.getServerName() == null) {
192        locations.add(null);
193      } else {
194        locations.add(location);
195      }
196    }
197
198    return new RegionLocations(locations);
199  }
200
201  /**
202   * Returns a {@link ServerName} from catalog table {@link Result}.
203   * @param r Result to pull from
204   * @return A ServerName instance or null if necessary fields not found or empty.
205   */
206  @Nullable
207  public static ServerName getServerName(Result r, int replicaId) {
208    byte[] serverColumn = getServerColumn(replicaId);
209    Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, serverColumn);
210    if (cell == null || cell.getValueLength() == 0) {
211      return null;
212    }
213    String hostAndPort =
214      Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
215    byte[] startcodeColumn = getStartCodeColumn(replicaId);
216    cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, startcodeColumn);
217    if (cell == null || cell.getValueLength() == 0) {
218      return null;
219    }
220    try {
221      return ServerName.valueOf(hostAndPort,
222        Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
223    } catch (IllegalArgumentException e) {
224      LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e);
225      return null;
226    }
227  }
228
229  /**
230   * Returns the column qualifier for server column for replicaId
231   * @param replicaId the replicaId of the region
232   * @return a byte[] for server column qualifier
233   */
234  public static byte[] getServerColumn(int replicaId) {
235    return replicaId == 0
236      ? HConstants.SERVER_QUALIFIER
237      : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
238        + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
239  }
240
241  /**
242   * Returns the column qualifier for server start code column for replicaId
243   * @param replicaId the replicaId of the region
244   * @return a byte[] for server start code column qualifier
245   */
246  public static byte[] getStartCodeColumn(int replicaId) {
247    return replicaId == 0
248      ? HConstants.STARTCODE_QUALIFIER
249      : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
250        + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
251  }
252
253  /**
254   * The latest seqnum that the server writing to meta observed when opening the region. E.g. the
255   * seqNum when the result of {@link getServerName} was written.
256   * @param r Result to pull the seqNum from
257   * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
258   */
259  private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
260    Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId));
261    if (cell == null || cell.getValueLength() == 0) {
262      return HConstants.NO_SEQNUM;
263    }
264    return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
265  }
266
267  /**
268   * Returns the column qualifier for seqNum column for replicaId
269   * @param replicaId the replicaId of the region
270   * @return a byte[] for seqNum column qualifier
271   */
272  public static byte[] getSeqNumColumn(int replicaId) {
273    return replicaId == 0
274      ? HConstants.SEQNUM_QUALIFIER
275      : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
276        + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
277  }
278
279  /** The delimiter for meta columns for replicaIds &gt; 0 */
280  static final char META_REPLICA_ID_DELIMITER = '_';
281
282  /**
283   * Parses the replicaId from the server column qualifier. See top of the class javadoc for the
284   * actual meta layout
285   * @param serverColumn the column qualifier
286   * @return an int for the replicaId
287   */
288  static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
289    String serverStr = Bytes.toString(serverColumn);
290
291    Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
292    if (matcher.matches() && matcher.groupCount() > 0) {
293      String group = matcher.group(1);
294      if (group != null && group.length() > 0) {
295        return Integer.parseInt(group.substring(1), 16);
296      } else {
297        return 0;
298      }
299    }
300    return -1;
301  }
302
303  /** Returns the row key to use for this regionInfo */
304  public static byte[] getMetaKeyForRegion(RegionInfo regionInfo) {
305    return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
306  }
307
308  /**
309   * Returns the column qualifier for serialized region state
310   * @param replicaId the replicaId of the region
311   * @return a byte[] for state qualifier
312   */
313  public static byte[] getRegionStateColumn(int replicaId) {
314    return replicaId == 0
315      ? HConstants.STATE_QUALIFIER
316      : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
317        + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
318  }
319
320  /**
321   * Returns the column qualifier for serialized region state
322   * @param replicaId the replicaId of the region
323   * @return a byte[] for sn column qualifier
324   */
325  public static byte[] getServerNameColumn(int replicaId) {
326    return replicaId == 0
327      ? HConstants.SERVERNAME_QUALIFIER
328      : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
329        + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
330  }
331
332  /**
333   * Decode table state from META Result. Should contain cell from HConstants.TABLE_FAMILY
334   * @return null if not found
335   */
336  @Nullable
337  public static TableState getTableState(Result r) throws IOException {
338    Cell cell = r.getColumnLatestCell(HConstants.TABLE_FAMILY, HConstants.TABLE_STATE_QUALIFIER);
339    if (cell == null) {
340      return null;
341    }
342    try {
343      return TableState.parseFrom(TableName.valueOf(r.getRow()),
344        Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(),
345          cell.getValueOffset() + cell.getValueLength()));
346    } catch (DeserializationException e) {
347      throw new IOException(e);
348    }
349  }
350
351  /**
352   * Returns Deserialized values of &lt;qualifier,regioninfo&gt; pairs taken from column values that
353   * match the regex 'info:merge.*' in array of <code>cells</code>.
354   */
355  @Nullable
356  public static Map<String, RegionInfo> getMergeRegionsWithName(Cell[] cells) {
357    if (cells == null) {
358      return null;
359    }
360    Map<String, RegionInfo> regionsToMerge = null;
361    for (Cell cell : cells) {
362      if (!isMergeQualifierPrefix(cell)) {
363        continue;
364      }
365      // Ok. This cell is that of a info:merge* column.
366      RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(),
367        cell.getValueLength());
368      if (ri != null) {
369        if (regionsToMerge == null) {
370          regionsToMerge = new LinkedHashMap<>();
371        }
372        regionsToMerge.put(Bytes.toString(CellUtil.cloneQualifier(cell)), ri);
373      }
374    }
375    return regionsToMerge;
376  }
377
378  /**
379   * Returns Deserialized regioninfo values taken from column values that match the regex
380   * 'info:merge.*' in array of <code>cells</code>.
381   */
382  @Nullable
383  public static List<RegionInfo> getMergeRegions(Cell[] cells) {
384    Map<String, RegionInfo> mergeRegionsWithName = getMergeRegionsWithName(cells);
385    return (mergeRegionsWithName == null) ? null : new ArrayList<>(mergeRegionsWithName.values());
386  }
387
388  /**
389   * Returns True if any merge regions present in <code>cells</code>; i.e. the column in
390   * <code>cell</code> matches the regex 'info:merge.*'.
391   */
392  public static boolean hasMergeRegions(Cell[] cells) {
393    for (Cell cell : cells) {
394      if (isMergeQualifierPrefix(cell)) {
395        return true;
396      }
397    }
398    return false;
399  }
400
401  /** Returns True if the column in <code>cell</code> matches the regex 'info:merge.*'. */
402  public static boolean isMergeQualifierPrefix(Cell cell) {
403    // Check to see if has family and that qualifier starts with the merge qualifier 'merge'
404    return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY)
405      && PrivateCellUtil.qualifierStartsWith(cell, HConstants.MERGE_QUALIFIER_PREFIX);
406  }
407}