1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.hfile;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataOutputStream;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.List;
29
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.testclassification.SmallTests;
33 import org.apache.hadoop.hbase.io.HeapSize;
34 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
35 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
36 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
37 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
38 import org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer.BufferGrabbingByteArrayOutputStream;
39 import org.apache.hadoop.hbase.util.ChecksumType;
40 import org.apache.hadoop.hbase.util.RedundantKVGenerator;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43 import org.junit.runner.RunWith;
44 import org.junit.runners.Parameterized;
45 import org.junit.runners.Parameterized.Parameters;
46
47 @RunWith(Parameterized.class)
48 @Category(SmallTests.class)
49 public class TestHFileDataBlockEncoder {
50 private HFileDataBlockEncoder blockEncoder;
51 private RedundantKVGenerator generator = new RedundantKVGenerator();
52 private boolean includesMemstoreTS;
53
54
55
56
57
58 public TestHFileDataBlockEncoder(HFileDataBlockEncoder blockEncoder,
59 boolean includesMemstoreTS) {
60 this.blockEncoder = blockEncoder;
61 this.includesMemstoreTS = includesMemstoreTS;
62 System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding()
63 + ", includesMemstoreTS: " + includesMemstoreTS);
64 }
65
66
67
68
69
70 @Test
71 public void testEncodingWithCache() throws IOException {
72 testEncodingWithCacheInternals(false);
73 testEncodingWithCacheInternals(true);
74 }
75
76 private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
77 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
78 HFileBlock block = getSampleHFileBlock(kvs, useTag);
79 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag);
80
81 LruBlockCache blockCache =
82 new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
83 BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
84 blockCache.cacheBlock(cacheKey, cacheBlock);
85
86 HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
87 assertTrue(heapSize instanceof HFileBlock);
88
89 HFileBlock returnedBlock = (HFileBlock) heapSize;;
90
91 if (blockEncoder.getDataBlockEncoding() ==
92 DataBlockEncoding.NONE) {
93 assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly());
94 } else {
95 if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
96 System.out.println(blockEncoder);
97 }
98 assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
99 }
100 }
101
102
103 @Test
104 public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
105 testHeaderSizeInCacheWithoutChecksumInternals(false);
106 testHeaderSizeInCacheWithoutChecksumInternals(true);
107 }
108
109 private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException {
110 int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
111
112 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags);
113 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
114 int size = keyValues.limit();
115 ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
116 buf.position(headerSize);
117 keyValues.rewind();
118 buf.put(keyValues);
119 HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false)
120 .withIncludesMvcc(includesMemstoreTS)
121 .withIncludesTags(useTags)
122 .withBlockSize(0)
123 .withChecksumType(ChecksumType.NULL)
124 .build();
125 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
126 HFileBlock.FILL_HEADER, 0,
127 0, -1, hfileContext);
128 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags);
129 assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
130 }
131
132
133
134
135
136 @Test
137 public void testEncoding() throws IOException {
138 testEncodingInternals(false);
139 testEncodingInternals(true);
140 }
141
142 private void testEncodingInternals(boolean useTag) throws IOException {
143
144 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
145 HFileBlock block = getSampleHFileBlock(kvs, useTag);
146 HFileBlock blockOnDisk = createBlockOnDisk(kvs, block, useTag);
147
148 if (blockEncoder.getDataBlockEncoding() !=
149 DataBlockEncoding.NONE) {
150 assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
151 assertEquals(blockEncoder.getDataBlockEncoding().getId(),
152 blockOnDisk.getDataBlockEncodingId());
153 } else {
154 assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
155 }
156 }
157
158 private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) {
159 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
160 int size = keyValues.limit();
161 ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
162 buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
163 keyValues.rewind();
164 buf.put(keyValues);
165 HFileContext meta = new HFileContextBuilder()
166 .withIncludesMvcc(includesMemstoreTS)
167 .withIncludesTags(useTag)
168 .withHBaseCheckSum(true)
169 .withCompression(Algorithm.NONE)
170 .withBlockSize(0)
171 .withChecksumType(ChecksumType.NULL)
172 .build();
173 HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
174 HFileBlock.FILL_HEADER, 0,
175 0, -1, meta);
176 return b;
177 }
178
179 private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
180 throws IOException {
181 int size;
182 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
183 blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
184 block.getHFileContext());
185
186 ByteArrayOutputStream baos = new ByteArrayOutputStream();
187 baos.write(block.getDummyHeaderForVersion());
188 DataOutputStream dos = new DataOutputStream(baos);
189 blockEncoder.startBlockEncoding(context, dos);
190 for (KeyValue kv : kvs) {
191 blockEncoder.encode(kv, context, dos);
192 }
193 BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
194 baos.writeTo(stream);
195 blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
196 byte[] encodedBytes = baos.toByteArray();
197 size = encodedBytes.length - block.getDummyHeaderForVersion().length;
198 return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
199 HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1,
200 block.getHFileContext());
201 }
202
203
204
205
206 @Parameters
207 public static Collection<Object[]> getAllConfigurations() {
208 List<Object[]> configurations =
209 new ArrayList<Object[]>();
210
211 for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
212 for (boolean includesMemstoreTS : new boolean[] { false, true }) {
213 HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ?
214 NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo);
215 configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) });
216 }
217 }
218
219 return configurations;
220 }
221 }