001/* 002 * Copyright The Apache Software Foundation 003 * 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021package org.apache.hadoop.hbase.io.hfile.bucket; 022 023import java.io.IOException; 024import java.util.Map; 025import java.util.concurrent.ConcurrentHashMap; 026 027import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 028import org.apache.hadoop.hbase.io.hfile.BlockPriority; 029import org.apache.hadoop.hbase.io.hfile.BlockType; 030import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager; 031import org.apache.hadoop.hbase.io.hfile.HFileBlock; 032import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 033import org.apache.yetus.audience.InterfaceAudience; 034 035import org.apache.hadoop.hbase.shaded.protobuf.generated.BucketCacheProtos; 036 037@InterfaceAudience.Private 038final class BucketProtoUtils { 039 private BucketProtoUtils() { 040 041 } 042 043 static BucketCacheProtos.BucketCacheEntry toPB(BucketCache cache) { 044 return BucketCacheProtos.BucketCacheEntry.newBuilder() 045 .setCacheCapacity(cache.getMaxSize()) 046 .setIoClass(cache.ioEngine.getClass().getName()) 047 .setMapClass(cache.backingMap.getClass().getName()) 048 .putAllDeserializers(CacheableDeserializerIdManager.save()) 049 .setBackingMap(BucketProtoUtils.toPB(cache.backingMap)) 050 .setChecksum(ByteString.copyFrom(((PersistentIOEngine) cache.ioEngine). 051 calculateChecksum(cache.getAlgorithm()))).build(); 052 } 053 054 private static BucketCacheProtos.BackingMap toPB( 055 Map<BlockCacheKey, BucketEntry> backingMap) { 056 BucketCacheProtos.BackingMap.Builder builder = BucketCacheProtos.BackingMap.newBuilder(); 057 for (Map.Entry<BlockCacheKey, BucketEntry> entry : backingMap.entrySet()) { 058 builder.addEntry(BucketCacheProtos.BackingMapEntry.newBuilder() 059 .setKey(toPB(entry.getKey())) 060 .setValue(toPB(entry.getValue())) 061 .build()); 062 } 063 return builder.build(); 064 } 065 066 private static BucketCacheProtos.BlockCacheKey toPB(BlockCacheKey key) { 067 return BucketCacheProtos.BlockCacheKey.newBuilder() 068 .setHfilename(key.getHfileName()) 069 .setOffset(key.getOffset()) 070 .setPrimaryReplicaBlock(key.isPrimary()) 071 .setBlockType(toPB(key.getBlockType())) 072 .build(); 073 } 074 075 private static BucketCacheProtos.BlockType toPB(BlockType blockType) { 076 switch(blockType) { 077 case DATA: 078 return BucketCacheProtos.BlockType.data; 079 case META: 080 return BucketCacheProtos.BlockType.meta; 081 case TRAILER: 082 return BucketCacheProtos.BlockType.trailer; 083 case INDEX_V1: 084 return BucketCacheProtos.BlockType.index_v1; 085 case FILE_INFO: 086 return BucketCacheProtos.BlockType.file_info; 087 case LEAF_INDEX: 088 return BucketCacheProtos.BlockType.leaf_index; 089 case ROOT_INDEX: 090 return BucketCacheProtos.BlockType.root_index; 091 case BLOOM_CHUNK: 092 return BucketCacheProtos.BlockType.bloom_chunk; 093 case ENCODED_DATA: 094 return BucketCacheProtos.BlockType.encoded_data; 095 case GENERAL_BLOOM_META: 096 return BucketCacheProtos.BlockType.general_bloom_meta; 097 case INTERMEDIATE_INDEX: 098 return BucketCacheProtos.BlockType.intermediate_index; 099 case DELETE_FAMILY_BLOOM_META: 100 return BucketCacheProtos.BlockType.delete_family_bloom_meta; 101 default: 102 throw new Error("Unrecognized BlockType."); 103 } 104 } 105 106 private static BucketCacheProtos.BucketEntry toPB(BucketEntry entry) { 107 return BucketCacheProtos.BucketEntry.newBuilder() 108 .setOffset(entry.offset()) 109 .setLength(entry.getLength()) 110 .setDeserialiserIndex(entry.deserializerIndex) 111 .setAccessCounter(entry.getAccessCounter()) 112 .setPriority(toPB(entry.getPriority())) 113 .build(); 114 } 115 116 private static BucketCacheProtos.BlockPriority toPB(BlockPriority p) { 117 switch (p) { 118 case MULTI: 119 return BucketCacheProtos.BlockPriority.multi; 120 case MEMORY: 121 return BucketCacheProtos.BlockPriority.memory; 122 case SINGLE: 123 return BucketCacheProtos.BlockPriority.single; 124 default: 125 throw new Error("Unrecognized BlockPriority."); 126 } 127 } 128 129 static ConcurrentHashMap<BlockCacheKey, BucketEntry> fromPB( 130 Map<Integer, String> deserializers, BucketCacheProtos.BackingMap backingMap) 131 throws IOException { 132 ConcurrentHashMap<BlockCacheKey, BucketEntry> result = new ConcurrentHashMap<>(); 133 for (BucketCacheProtos.BackingMapEntry entry : backingMap.getEntryList()) { 134 BucketCacheProtos.BlockCacheKey protoKey = entry.getKey(); 135 BlockCacheKey key = new BlockCacheKey(protoKey.getHfilename(), protoKey.getOffset(), 136 protoKey.getPrimaryReplicaBlock(), fromPb(protoKey.getBlockType())); 137 BucketCacheProtos.BucketEntry protoValue = entry.getValue(); 138 BucketEntry value = new BucketEntry( 139 protoValue.getOffset(), 140 protoValue.getLength(), 141 protoValue.getAccessCounter(), 142 protoValue.getPriority() == BucketCacheProtos.BlockPriority.memory); 143 // This is the deserializer that we stored 144 int oldIndex = protoValue.getDeserialiserIndex(); 145 String deserializerClass = deserializers.get(oldIndex); 146 if (deserializerClass == null) { 147 throw new IOException("Found deserializer index without matching entry."); 148 } 149 // Convert it to the identifier for the deserializer that we have in this runtime 150 if (deserializerClass.equals(HFileBlock.BlockDeserializer.class.getName())) { 151 int actualIndex = HFileBlock.BLOCK_DESERIALIZER.getDeserializerIdentifier(); 152 value.deserializerIndex = (byte) actualIndex; 153 } else { 154 // We could make this more plugable, but right now HFileBlock is the only implementation 155 // of Cacheable outside of tests, so this might not ever matter. 156 throw new IOException("Unknown deserializer class found: " + deserializerClass); 157 } 158 result.put(key, value); 159 } 160 return result; 161 } 162 163 private static BlockType fromPb(BucketCacheProtos.BlockType blockType) { 164 switch (blockType) { 165 case data: 166 return BlockType.DATA; 167 case meta: 168 return BlockType.META; 169 case trailer: 170 return BlockType.TRAILER; 171 case index_v1: 172 return BlockType.INDEX_V1; 173 case file_info: 174 return BlockType.FILE_INFO; 175 case leaf_index: 176 return BlockType.LEAF_INDEX; 177 case root_index: 178 return BlockType.ROOT_INDEX; 179 case bloom_chunk: 180 return BlockType.BLOOM_CHUNK; 181 case encoded_data: 182 return BlockType.ENCODED_DATA; 183 case general_bloom_meta: 184 return BlockType.GENERAL_BLOOM_META; 185 case intermediate_index: 186 return BlockType.INTERMEDIATE_INDEX; 187 case delete_family_bloom_meta: 188 return BlockType.DELETE_FAMILY_BLOOM_META; 189 default: 190 throw new Error("Unrecognized BlockType."); 191 } 192 } 193}