1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.zip.Checksum;
24
25 import org.apache.hadoop.fs.ChecksumException;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.util.ByteBufferUtils;
29 import org.apache.hadoop.hbase.util.Bytes;
30 import org.apache.hadoop.hbase.util.ChecksumType;
31 import org.apache.hadoop.util.DataChecksum;
32
33
34
35
36 @InterfaceAudience.Private
37 public class ChecksumUtil {
38
39
40 private static byte[] DUMMY_VALUE = new byte[128 * HFileBlock.CHECKSUM_SIZE];
41
42
43
44
45
46
47
48
49 private static boolean generateExceptions = false;
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65 static void generateChecksums(byte[] indata,
66 int startOffset, int endOffset,
67 byte[] outdata, int outOffset,
68 ChecksumType checksumType,
69 int bytesPerChecksum) throws IOException {
70
71 if (checksumType == ChecksumType.NULL) {
72 return;
73 }
74
75 Checksum checksum = checksumType.getChecksumObject();
76 int bytesLeft = endOffset - startOffset;
77 int chunkNum = 0;
78
79 while (bytesLeft > 0) {
80
81 checksum.reset();
82 int count = Math.min(bytesLeft, bytesPerChecksum);
83 checksum.update(indata, startOffset, count);
84
85
86 int cksumValue = (int)checksum.getValue();
87 outOffset = Bytes.putInt(outdata, outOffset, cksumValue);
88 chunkNum++;
89 startOffset += count;
90 bytesLeft -= count;
91 }
92 }
93
94
95
96
97
98
99
100
101
102
103 static boolean validateChecksum(ByteBuffer buffer, Path path, long offset, int hdrSize)
104 throws IOException {
105
106
107 ChecksumType cktype =
108 ChecksumType.codeToType(buffer.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
109 if (cktype == ChecksumType.NULL) {
110 return true;
111 }
112
113 int bytesPerChecksum = buffer.getInt(HFileBlock.Header.BYTES_PER_CHECKSUM_INDEX);
114 int onDiskDataSizeWithHeader =
115 buffer.getInt(HFileBlock.Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
116
117 if (HFile.LOG.isTraceEnabled()) {
118 HFile.LOG.info("dataLength=" + buffer.capacity()
119 + ", sizeWithHeader=" + onDiskDataSizeWithHeader
120 + ", checksumType=" + cktype.getName()
121 + ", file=" + path.toString()
122 + ", offset=" + offset
123 + ", headerSize=" + hdrSize
124 + ", bytesPerChecksum=" + bytesPerChecksum);
125 }
126
127 if (bytesPerChecksum < hdrSize) {
128 String msg = "Unsupported value of bytesPerChecksum. " +
129 " Minimum is " + hdrSize +
130 " but the configured value is " + bytesPerChecksum;
131 HFile.LOG.warn(msg);
132 return false;
133 }
134 byte[] data;
135 if (buffer.hasArray()) {
136 data = buffer.array();
137 } else {
138 data = ByteBufferUtils.toBytes(buffer, 0);
139 }
140
141 Checksum checksumObject = cktype.getChecksumObject();
142 checksumObject.reset();
143
144 checksumObject.update(data, 0, hdrSize);
145
146 int off = hdrSize;
147 int consumed = hdrSize;
148 int cksumOffset = onDiskDataSizeWithHeader;
149 int bytesLeft = cksumOffset - off;
150
151
152 while (bytesLeft > 0) {
153 int thisChunkSize = bytesPerChecksum - consumed;
154 int count = Math.min(bytesLeft, thisChunkSize);
155 checksumObject.update(data, off, count);
156
157 int storedChecksum = Bytes.toInt(data, cksumOffset);
158 if (storedChecksum != (int)checksumObject.getValue()) {
159 String msg = "File " + path +
160 " Stored checksum value of " + storedChecksum +
161 " at offset " + cksumOffset +
162 " does not match computed checksum " +
163 checksumObject.getValue() +
164 ", total data size " + data.length +
165 " Checksum data range offset " + off + " len " + count +
166 HFileBlock.toStringHeader(buffer);
167 HFile.LOG.warn(msg);
168 if (generateExceptions) {
169 throw new IOException(msg);
170 } else {
171 return false;
172 }
173 }
174 cksumOffset += HFileBlock.CHECKSUM_SIZE;
175 bytesLeft -= count;
176 off += count;
177 consumed = 0;
178 checksumObject.reset();
179 }
180 return true;
181 }
182
183
184
185
186
187
188
189
190 static long numBytes(long datasize, int bytesPerChecksum) {
191 return numChunks(datasize, bytesPerChecksum) *
192 HFileBlock.CHECKSUM_SIZE;
193 }
194
195
196
197
198
199
200
201
202 static long numChunks(long datasize, int bytesPerChecksum) {
203 long numChunks = datasize/bytesPerChecksum;
204 if (datasize % bytesPerChecksum != 0) {
205 numChunks++;
206 }
207 return numChunks;
208 }
209
210
211
212
213
214
215
216
217
218 static void reserveSpaceForChecksums(ByteArrayOutputStream baos,
219 int numBytes, int bytesPerChecksum) throws IOException {
220 long numChunks = numChunks(numBytes, bytesPerChecksum);
221 long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE;
222 while (bytesLeft > 0) {
223 long count = Math.min(bytesLeft, DUMMY_VALUE.length);
224 baos.write(DUMMY_VALUE, 0, (int)count);
225 bytesLeft -= count;
226 }
227 }
228
229
230
231
232
233
234
235 public static void generateExceptionForChecksumFailureForTest(boolean value) {
236 generateExceptions = value;
237 }
238 }
239