001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import edu.umd.cs.findbugs.annotations.Nullable; 021import java.io.IOException; 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.LinkedHashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.NavigableMap; 028import java.util.SortedMap; 029import java.util.regex.Matcher; 030import java.util.regex.Pattern; 031import org.apache.hadoop.hbase.client.RegionInfo; 032import org.apache.hadoop.hbase.client.RegionInfoBuilder; 033import org.apache.hadoop.hbase.client.RegionReplicaUtil; 034import org.apache.hadoop.hbase.client.Result; 035import org.apache.hadoop.hbase.client.TableState; 036import org.apache.hadoop.hbase.exceptions.DeserializationException; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.apache.yetus.audience.InterfaceAudience; 039import org.slf4j.Logger; 040import org.slf4j.LoggerFactory; 041 042/** 043 * Helper class for generating/parsing 044 * {@value org.apache.hadoop.hbase.HConstants#CATALOG_FAMILY_STR} family cells in meta table. 045 * <p/> 046 * The cells in catalog family are: 047 * 048 * <pre> 049 * For each table range ('Region'), there is a single row, formatted as: 050 * <tableName>,<startKey>,<regionId>,<encodedRegionName>. 051 * This row is the serialized regionName of the default region replica. 052 * Columns are: 053 * info:regioninfo => contains serialized HRI for the default region replica 054 * info:server => contains hostname:port (in string form) for the server hosting 055 * the default regionInfo replica 056 * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting 057 * the regionInfo replica with replicaId 058 * info:serverstartcode => contains server start code (in binary long form) for the server 059 * hosting the default regionInfo replica 060 * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for 061 * the server hosting the regionInfo replica with 062 * replicaId 063 * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time 064 * the server opened the region with default replicaId 065 * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region 066 * at the time the server opened the region with 067 * replicaId 068 * info:splitA => contains a serialized HRI for the first daughter region if the 069 * region is split 070 * info:splitB => contains a serialized HRI for the second daughter region if the 071 * region is split 072 * info:merge* => contains a serialized HRI for a merge parent region. There will be two 073 * or more of these columns in a row. A row that has these columns is 074 * undergoing a merge and is the result of the merge. Columns listed 075 * in marge* columns are the parents of this merged region. Example 076 * columns: info:merge0001, info:merge0002. You make also see 'mergeA', 077 * and 'mergeB'. This is old form replaced by the new format that allows 078 * for more than two parents to be merged at a time. 079 * </pre> 080 */ 081@InterfaceAudience.Private 082public class CatalogFamilyFormat { 083 084 private static final Logger LOG = LoggerFactory.getLogger(CatalogFamilyFormat.class); 085 086 /** A regex for parsing server columns from meta. See above javadoc for meta layout */ 087 private static final Pattern SERVER_COLUMN_PATTERN = 088 Pattern.compile("^server(_[0-9a-fA-F]{4})?$"); 089 090 /** 091 * Returns an HRI parsed from this regionName. Not all the fields of the HRI is stored in the 092 * name, so the returned object should only be used for the fields in the regionName. 093 * <p/> 094 * Since the returned object does not contain all the fields, we do not expose this method in 095 * public API, such as {@link RegionInfo} or {@link RegionInfoBuilder}. 096 */ 097 public static RegionInfo parseRegionInfoFromRegionName(byte[] regionName) throws IOException { 098 byte[][] fields = RegionInfo.parseRegionName(regionName); 099 long regionId = Long.parseLong(Bytes.toString(fields[2])); 100 int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0; 101 return RegionInfoBuilder.newBuilder(TableName.valueOf(fields[0])).setStartKey(fields[1]) 102 .setRegionId(regionId).setReplicaId(replicaId).build(); 103 } 104 105 /** 106 * Returns the RegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and 107 * <code>qualifier</code> of the catalog table result. 108 * @param r a Result object from the catalog table scan 109 * @param qualifier Column family qualifier 110 * @return An RegionInfo instance or null. 111 */ 112 @Nullable 113 public static RegionInfo getRegionInfo(final Result r, byte[] qualifier) { 114 Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, qualifier); 115 if (cell == null) { 116 return null; 117 } 118 return RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), 119 cell.getValueLength()); 120 } 121 122 /** 123 * Returns RegionInfo object from the column 124 * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog table Result. 125 * @param data a Result object from the catalog table scan 126 * @return RegionInfo or null 127 */ 128 public static RegionInfo getRegionInfo(Result data) { 129 return getRegionInfo(data, HConstants.REGIONINFO_QUALIFIER); 130 } 131 132 /** 133 * Returns the HRegionLocation parsed from the given meta row Result for the given regionInfo and 134 * replicaId. The regionInfo can be the default region info for the replica. 135 * @param r the meta row result 136 * @param regionInfo RegionInfo for default replica 137 * @param replicaId the replicaId for the HRegionLocation 138 * @return HRegionLocation parsed from the given meta row Result for the given replicaId 139 */ 140 public static HRegionLocation getRegionLocation(final Result r, final RegionInfo regionInfo, 141 final int replicaId) { 142 ServerName serverName = getServerName(r, replicaId); 143 long seqNum = getSeqNumDuringOpen(r, replicaId); 144 RegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId); 145 return new HRegionLocation(replicaInfo, serverName, seqNum); 146 } 147 148 /** 149 * Returns an HRegionLocationList extracted from the result. 150 * @return an HRegionLocationList containing all locations for the region range or null if we 151 * can't deserialize the result. 152 */ 153 @Nullable 154 public static RegionLocations getRegionLocations(final Result r) { 155 if (r == null) { 156 return null; 157 } 158 RegionInfo regionInfo = getRegionInfo(r, HConstants.REGIONINFO_QUALIFIER); 159 if (regionInfo == null) { 160 return null; 161 } 162 163 List<HRegionLocation> locations = new ArrayList<>(1); 164 NavigableMap<byte[], NavigableMap<byte[], byte[]>> familyMap = r.getNoVersionMap(); 165 166 locations.add(getRegionLocation(r, regionInfo, 0)); 167 168 NavigableMap<byte[], byte[]> infoMap = familyMap.get(HConstants.CATALOG_FAMILY); 169 if (infoMap == null) { 170 return new RegionLocations(locations); 171 } 172 173 // iterate until all serverName columns are seen 174 int replicaId = 0; 175 byte[] serverColumn = getServerColumn(replicaId); 176 SortedMap<byte[], byte[]> serverMap; 177 serverMap = infoMap.tailMap(serverColumn, false); 178 179 if (serverMap.isEmpty()) { 180 return new RegionLocations(locations); 181 } 182 183 for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) { 184 replicaId = parseReplicaIdFromServerColumn(entry.getKey()); 185 if (replicaId < 0) { 186 break; 187 } 188 HRegionLocation location = getRegionLocation(r, regionInfo, replicaId); 189 // In case the region replica is newly created, it's location might be null. We usually do not 190 // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. 191 if (location.getServerName() == null) { 192 locations.add(null); 193 } else { 194 locations.add(location); 195 } 196 } 197 198 return new RegionLocations(locations); 199 } 200 201 /** 202 * Returns a {@link ServerName} from catalog table {@link Result}. 203 * @param r Result to pull from 204 * @return A ServerName instance or null if necessary fields not found or empty. 205 */ 206 @Nullable 207 public static ServerName getServerName(Result r, int replicaId) { 208 byte[] serverColumn = getServerColumn(replicaId); 209 Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, serverColumn); 210 if (cell == null || cell.getValueLength() == 0) { 211 return null; 212 } 213 String hostAndPort = 214 Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); 215 byte[] startcodeColumn = getStartCodeColumn(replicaId); 216 cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, startcodeColumn); 217 if (cell == null || cell.getValueLength() == 0) { 218 return null; 219 } 220 try { 221 return ServerName.valueOf(hostAndPort, 222 Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); 223 } catch (IllegalArgumentException e) { 224 LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e); 225 return null; 226 } 227 } 228 229 /** 230 * Returns the column qualifier for server column for replicaId 231 * @param replicaId the replicaId of the region 232 * @return a byte[] for server column qualifier 233 */ 234 public static byte[] getServerColumn(int replicaId) { 235 return replicaId == 0 236 ? HConstants.SERVER_QUALIFIER 237 : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER 238 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); 239 } 240 241 /** 242 * Returns the column qualifier for server start code column for replicaId 243 * @param replicaId the replicaId of the region 244 * @return a byte[] for server start code column qualifier 245 */ 246 public static byte[] getStartCodeColumn(int replicaId) { 247 return replicaId == 0 248 ? HConstants.STARTCODE_QUALIFIER 249 : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER 250 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); 251 } 252 253 /** 254 * The latest seqnum that the server writing to meta observed when opening the region. E.g. the 255 * seqNum when the result of {@link getServerName} was written. 256 * @param r Result to pull the seqNum from 257 * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written. 258 */ 259 private static long getSeqNumDuringOpen(final Result r, final int replicaId) { 260 Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId)); 261 if (cell == null || cell.getValueLength() == 0) { 262 return HConstants.NO_SEQNUM; 263 } 264 return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); 265 } 266 267 /** 268 * Returns the column qualifier for seqNum column for replicaId 269 * @param replicaId the replicaId of the region 270 * @return a byte[] for seqNum column qualifier 271 */ 272 public static byte[] getSeqNumColumn(int replicaId) { 273 return replicaId == 0 274 ? HConstants.SEQNUM_QUALIFIER 275 : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER 276 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); 277 } 278 279 /** The delimiter for meta columns for replicaIds > 0 */ 280 static final char META_REPLICA_ID_DELIMITER = '_'; 281 282 /** 283 * Parses the replicaId from the server column qualifier. See top of the class javadoc for the 284 * actual meta layout 285 * @param serverColumn the column qualifier 286 * @return an int for the replicaId 287 */ 288 static int parseReplicaIdFromServerColumn(byte[] serverColumn) { 289 String serverStr = Bytes.toString(serverColumn); 290 291 Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr); 292 if (matcher.matches() && matcher.groupCount() > 0) { 293 String group = matcher.group(1); 294 if (group != null && group.length() > 0) { 295 return Integer.parseInt(group.substring(1), 16); 296 } else { 297 return 0; 298 } 299 } 300 return -1; 301 } 302 303 /** Returns the row key to use for this regionInfo */ 304 public static byte[] getMetaKeyForRegion(RegionInfo regionInfo) { 305 return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName(); 306 } 307 308 /** 309 * Returns the column qualifier for serialized region state 310 * @param replicaId the replicaId of the region 311 * @return a byte[] for state qualifier 312 */ 313 public static byte[] getRegionStateColumn(int replicaId) { 314 return replicaId == 0 315 ? HConstants.STATE_QUALIFIER 316 : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER 317 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); 318 } 319 320 /** 321 * Returns the column qualifier for serialized region state 322 * @param replicaId the replicaId of the region 323 * @return a byte[] for sn column qualifier 324 */ 325 public static byte[] getServerNameColumn(int replicaId) { 326 return replicaId == 0 327 ? HConstants.SERVERNAME_QUALIFIER 328 : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER 329 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); 330 } 331 332 /** 333 * Decode table state from META Result. Should contain cell from HConstants.TABLE_FAMILY 334 * @return null if not found 335 */ 336 @Nullable 337 public static TableState getTableState(Result r) throws IOException { 338 Cell cell = r.getColumnLatestCell(HConstants.TABLE_FAMILY, HConstants.TABLE_STATE_QUALIFIER); 339 if (cell == null) { 340 return null; 341 } 342 try { 343 return TableState.parseFrom(TableName.valueOf(r.getRow()), 344 Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), 345 cell.getValueOffset() + cell.getValueLength())); 346 } catch (DeserializationException e) { 347 throw new IOException(e); 348 } 349 } 350 351 /** 352 * Returns Deserialized values of <qualifier,regioninfo> pairs taken from column values that 353 * match the regex 'info:merge.*' in array of <code>cells</code>. 354 */ 355 @Nullable 356 public static Map<String, RegionInfo> getMergeRegionsWithName(Cell[] cells) { 357 if (cells == null) { 358 return null; 359 } 360 Map<String, RegionInfo> regionsToMerge = null; 361 for (Cell cell : cells) { 362 if (!isMergeQualifierPrefix(cell)) { 363 continue; 364 } 365 // Ok. This cell is that of a info:merge* column. 366 RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), 367 cell.getValueLength()); 368 if (ri != null) { 369 if (regionsToMerge == null) { 370 regionsToMerge = new LinkedHashMap<>(); 371 } 372 regionsToMerge.put(Bytes.toString(CellUtil.cloneQualifier(cell)), ri); 373 } 374 } 375 return regionsToMerge; 376 } 377 378 /** 379 * Returns Deserialized regioninfo values taken from column values that match the regex 380 * 'info:merge.*' in array of <code>cells</code>. 381 */ 382 @Nullable 383 public static List<RegionInfo> getMergeRegions(Cell[] cells) { 384 Map<String, RegionInfo> mergeRegionsWithName = getMergeRegionsWithName(cells); 385 return (mergeRegionsWithName == null) ? null : new ArrayList<>(mergeRegionsWithName.values()); 386 } 387 388 /** 389 * Returns True if any merge regions present in <code>cells</code>; i.e. the column in 390 * <code>cell</code> matches the regex 'info:merge.*'. 391 */ 392 public static boolean hasMergeRegions(Cell[] cells) { 393 for (Cell cell : cells) { 394 if (isMergeQualifierPrefix(cell)) { 395 return true; 396 } 397 } 398 return false; 399 } 400 401 /** Returns True if the column in <code>cell</code> matches the regex 'info:merge.*'. */ 402 public static boolean isMergeQualifierPrefix(Cell cell) { 403 // Check to see if has family and that qualifier starts with the merge qualifier 'merge' 404 return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) 405 && PrivateCellUtil.qualifierStartsWith(cell, HConstants.MERGE_QUALIFIER_PREFIX); 406 } 407}