1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.GZ;
23 import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
24 import static org.junit.Assert.assertEquals;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.DataInputStream;
28 import java.io.DataOutputStream;
29 import java.io.IOException;
30 import java.nio.ByteBuffer;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FSDataOutputStream;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.fs.HFileSystem;
41 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
42 import org.apache.hadoop.hbase.io.compress.Compression;
43 import org.apache.hadoop.hbase.testclassification.SmallTests;
44 import org.apache.hadoop.hbase.util.ChecksumType;
45 import org.junit.Before;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49 @Category(SmallTests.class)
50 public class TestChecksum {
51
52 private static final boolean detailedLogging = true;
53 private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
54
55 private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
56
57 static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
58 NONE, GZ };
59
60 static final int[] BYTES_PER_CHECKSUM = {
61 50, 500, 688, 16*1024, (16*1024+980), 64 * 1024};
62
63 private static final HBaseTestingUtility TEST_UTIL =
64 new HBaseTestingUtility();
65 private FileSystem fs;
66 private HFileSystem hfs;
67
68 @Before
69 public void setUp() throws Exception {
70 fs = HFileSystem.get(TEST_UTIL.getConfiguration());
71 hfs = (HFileSystem)fs;
72 }
73
74
75
76
77
78 @Test
79 public void testChecksumCorruption() throws IOException {
80 testChecksumCorruptionInternals(false);
81 testChecksumCorruptionInternals(true);
82 }
83
84 protected void testChecksumCorruptionInternals(boolean useTags) throws IOException {
85 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
86 for (boolean pread : new boolean[] { false, true }) {
87 LOG.info("testChecksumCorruption: Compression algorithm: " + algo +
88 ", pread=" + pread);
89 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
90 + algo);
91 FSDataOutputStream os = fs.create(path);
92 HFileContext meta = new HFileContextBuilder()
93 .withCompression(algo)
94 .withIncludesMvcc(true)
95 .withIncludesTags(useTags)
96 .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
97 .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
98 .build();
99 HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
100 long totalSize = 0;
101 for (int blockId = 0; blockId < 2; ++blockId) {
102 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
103 for (int i = 0; i < 1234; ++i)
104 dos.writeInt(i);
105 hbw.writeHeaderAndData(os);
106 totalSize += hbw.getOnDiskSizeWithHeader();
107 }
108 os.close();
109
110
111 assertEquals(true, hfs.useHBaseChecksum());
112
113
114 FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
115 meta = new HFileContextBuilder()
116 .withCompression(algo)
117 .withIncludesMvcc(true)
118 .withIncludesTags(useTags)
119 .withHBaseCheckSum(true)
120 .build();
121 HFileBlock.FSReader hbr = new FSReaderV2Test(is, totalSize, fs, path, meta);
122 HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
123 b.sanityCheck();
124 assertEquals(4936, b.getUncompressedSizeWithoutHeader());
125 assertEquals(algo == GZ ? 2173 : 4936,
126 b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
127
128 ByteBuffer bb = b.unpack(meta, hbr).getBufferWithoutHeader();
129 DataInputStream in = new DataInputStream(
130 new ByteArrayInputStream(
131 bb.array(), bb.arrayOffset(), bb.limit()));
132
133
134
135 assertEquals(1, HFile.getChecksumFailuresCount());
136 validateData(in);
137
138
139
140
141 for (int i = 0; i <
142 HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) {
143 b = hbr.readBlockData(0, -1, -1, pread);
144 assertEquals(0, HFile.getChecksumFailuresCount());
145 }
146
147
148 b = hbr.readBlockData(0, -1, -1, pread);
149 assertEquals(1, HFile.getChecksumFailuresCount());
150
151
152
153 b = hbr.readBlockData(0, -1, -1, pread);
154 assertEquals(0, HFile.getChecksumFailuresCount());
155 is.close();
156
157
158
159
160 HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
161 assertEquals(false, newfs.useHBaseChecksum());
162 is = new FSDataInputStreamWrapper(newfs, path);
163 hbr = new FSReaderV2Test(is, totalSize, newfs, path, meta);
164 b = hbr.readBlockData(0, -1, -1, pread);
165 is.close();
166 b.sanityCheck();
167 b = b.unpack(meta, hbr);
168 assertEquals(4936, b.getUncompressedSizeWithoutHeader());
169 assertEquals(algo == GZ ? 2173 : 4936,
170 b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
171
172 bb = b.getBufferWithoutHeader();
173 in = new DataInputStream(new ByteArrayInputStream(
174 bb.array(), bb.arrayOffset(), bb.limit()));
175
176
177
178 assertEquals(0, HFile.getChecksumFailuresCount());
179 validateData(in);
180 }
181 }
182 }
183
184
185
186
187 @Test
188 public void testChecksumChunks() throws IOException {
189 testChecksumInternals(false);
190 testChecksumInternals(true);
191 }
192
193 protected void testChecksumInternals(boolean useTags) throws IOException {
194 Compression.Algorithm algo = NONE;
195 for (boolean pread : new boolean[] { false, true }) {
196 for (int bytesPerChecksum : BYTES_PER_CHECKSUM) {
197 Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" +
198 algo + bytesPerChecksum);
199 FSDataOutputStream os = fs.create(path);
200 HFileContext meta = new HFileContextBuilder()
201 .withCompression(algo)
202 .withIncludesMvcc(true)
203 .withIncludesTags(useTags)
204 .withHBaseCheckSum(true)
205 .withBytesPerCheckSum(bytesPerChecksum)
206 .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
207 .build();
208 HFileBlock.Writer hbw = new HFileBlock.Writer(null,
209 meta);
210
211
212
213 long dataSize = 0;
214 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
215 for (; dataSize < 6 * bytesPerChecksum;) {
216 for (int i = 0; i < 1234; ++i) {
217 dos.writeInt(i);
218 dataSize += 4;
219 }
220 }
221 hbw.writeHeaderAndData(os);
222 long totalSize = hbw.getOnDiskSizeWithHeader();
223 os.close();
224
225 long expectedChunks = ChecksumUtil.numChunks(
226 dataSize + HConstants.HFILEBLOCK_HEADER_SIZE,
227 bytesPerChecksum);
228 LOG.info("testChecksumChunks: pread=" + pread +
229 ", bytesPerChecksum=" + bytesPerChecksum +
230 ", fileSize=" + totalSize +
231 ", dataSize=" + dataSize +
232 ", expectedChunks=" + expectedChunks);
233
234
235 assertEquals(true, hfs.useHBaseChecksum());
236
237
238 FSDataInputStream is = fs.open(path);
239 FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);
240 meta = new HFileContextBuilder()
241 .withCompression(algo)
242 .withIncludesMvcc(true)
243 .withIncludesTags(useTags)
244 .withHBaseCheckSum(true)
245 .withBytesPerCheckSum(bytesPerChecksum)
246 .build();
247 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(
248 is, nochecksum), totalSize, hfs, path, meta);
249 HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
250 is.close();
251 b.sanityCheck();
252 assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
253
254
255 assertEquals(totalSize, HConstants.HFILEBLOCK_HEADER_SIZE + dataSize +
256 expectedChunks * HFileBlock.CHECKSUM_SIZE);
257
258
259 assertEquals(0, HFile.getChecksumFailuresCount());
260 }
261 }
262 }
263
264
265
266
267 @Test
268 public void testChecksumAlgorithm() throws IOException {
269 ChecksumType type = ChecksumType.CRC32;
270 assertEquals(ChecksumType.nameToType(type.getName()), type);
271 assertEquals(ChecksumType.valueOf(type.toString()), type);
272 }
273
274 private void validateData(DataInputStream in) throws IOException {
275
276 for (int i = 0; i < 1234; i++) {
277 int val = in.readInt();
278 assertEquals("testChecksumCorruption: data mismatch at index " + i, i, val);
279 }
280 }
281
282
283
284
285
286
287 static private class FSReaderV2Test extends HFileBlock.FSReaderV2 {
288 public FSReaderV2Test(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs,
289 Path path, HFileContext meta) throws IOException {
290 super(istream, fileSize, (HFileSystem) fs, path, meta);
291 }
292
293 @Override
294 protected boolean validateBlockChecksum(HFileBlock block,
295 byte[] data, int hdrSize) throws IOException {
296 return false;
297 }
298 }
299 }
300