1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.filter;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.List;
25
26 import static org.junit.Assert.assertEquals;
27 import static org.junit.Assert.assertFalse;
28 import static org.junit.Assert.assertNotNull;
29 import static org.junit.Assert.assertTrue;
30 import static org.junit.Assert.assertNull;
31
32 import org.apache.hadoop.hbase.Cell;
33 import org.apache.hadoop.hbase.KeyValue;
34 import org.apache.hadoop.hbase.KeyValueUtil;
35 import org.apache.hadoop.hbase.testclassification.SmallTests;
36 import org.apache.hadoop.hbase.exceptions.DeserializationException;
37 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
38 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
39 import org.apache.hadoop.hbase.filter.FilterList.Operator;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45 import com.google.common.collect.Lists;
46
47
48
49
50
51 @Category(SmallTests.class)
52 public class TestFilterList {
53 static final int MAX_PAGES = 2;
54 static final char FIRST_CHAR = 'a';
55 static final char LAST_CHAR = 'e';
56 static byte[] GOOD_BYTES = Bytes.toBytes("abc");
57 static byte[] BAD_BYTES = Bytes.toBytes("def");
58
59
60 @Test
61 public void testAddFilter() throws Exception {
62 Filter filter1 = new FirstKeyOnlyFilter();
63 Filter filter2 = new FirstKeyOnlyFilter();
64
65 FilterList filterList = new FilterList(filter1, filter2);
66 filterList.addFilter(new FirstKeyOnlyFilter());
67
68 filterList = new FilterList(Arrays.asList(filter1, filter2));
69 filterList.addFilter(new FirstKeyOnlyFilter());
70
71 filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
72 filterList.addFilter(new FirstKeyOnlyFilter());
73
74 filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
75 filterList.addFilter(new FirstKeyOnlyFilter());
76
77 }
78
79
80
81
82
83
84 @Test
85 public void testMPONE() throws Exception {
86 mpOneTest(getFilterMPONE());
87 }
88
89 private Filter getFilterMPONE() {
90 List<Filter> filters = new ArrayList<Filter>();
91 filters.add(new PageFilter(MAX_PAGES));
92 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
93 Filter filterMPONE =
94 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
95 return filterMPONE;
96 }
97
98 private void mpOneTest(Filter filterMPONE) throws Exception {
99
100
101
102
103
104
105
106
107
108
109
110
111 filterMPONE.reset();
112 assertFalse(filterMPONE.filterAllRemaining());
113
114
115 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
116 for (int i = 0; i < MAX_PAGES - 1; i++) {
117 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
118 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
119 Bytes.toBytes(i));
120 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
121 assertFalse(filterMPONE.filterRow());
122 }
123
124
125 rowkey = Bytes.toBytes("z");
126 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
127 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
128 Bytes.toBytes(0));
129 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
130 assertFalse(filterMPONE.filterRow());
131
132
133 rowkey = Bytes.toBytes("yyy");
134 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
135 kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
136 Bytes.toBytes(0));
137 assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
138 assertFalse(filterMPONE.filterRow());
139
140
141 rowkey = Bytes.toBytes("z");
142 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
143 assertTrue(filterMPONE.filterAllRemaining());
144 }
145
146
147
148
149
150 @Test
151 public void testMPALL() throws Exception {
152 mpAllTest(getMPALLFilter());
153 }
154
155 private Filter getMPALLFilter() {
156 List<Filter> filters = new ArrayList<Filter>();
157 filters.add(new PageFilter(MAX_PAGES));
158 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
159 Filter filterMPALL =
160 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
161 return filterMPALL;
162 }
163
164 private void mpAllTest(Filter filterMPALL) throws Exception {
165
166
167
168
169
170
171
172
173
174
175
176
177 filterMPALL.reset();
178 assertFalse(filterMPALL.filterAllRemaining());
179 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
180 for (int i = 0; i < MAX_PAGES - 1; i++) {
181 assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
182 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
183 Bytes.toBytes(i));
184 assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv));
185 }
186 filterMPALL.reset();
187 rowkey = Bytes.toBytes("z");
188 assertTrue(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
189
190 KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey);
191 assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv));
192 }
193
194
195
196
197
198 @Test
199 public void testOrdering() throws Exception {
200 orderingTest(getOrderingFilter());
201 }
202
203 public Filter getOrderingFilter() {
204 List<Filter> filters = new ArrayList<Filter>();
205 filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
206 filters.add(new PageFilter(MAX_PAGES));
207 Filter filterMPONE =
208 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
209 return filterMPONE;
210 }
211
212 public void orderingTest(Filter filterMPONE) throws Exception {
213
214
215
216
217
218
219
220
221
222
223
224
225 filterMPONE.reset();
226 assertFalse(filterMPONE.filterAllRemaining());
227
228
229 byte [] rowkey = Bytes.toBytes("yyyyyyyy");
230 for (int i = 0; i < MAX_PAGES; i++) {
231 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
232 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
233 Bytes.toBytes(i));
234 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
235 assertFalse(filterMPONE.filterRow());
236 }
237
238
239 rowkey = Bytes.toBytes("xxxxxxx");
240 for (int i = 0; i < MAX_PAGES; i++) {
241 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
242 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
243 Bytes.toBytes(i));
244 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
245 assertFalse(filterMPONE.filterRow());
246 }
247
248
249 rowkey = Bytes.toBytes("yyy");
250 for (int i = 0; i < MAX_PAGES; i++) {
251 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
252 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
253 Bytes.toBytes(i));
254 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
255 assertFalse(filterMPONE.filterRow());
256 }
257 }
258
259
260
261
262
263
264 @Test
265 public void testFilterListTwoFiltersMustPassOne() throws Exception {
266 byte[] r1 = Bytes.toBytes("Row1");
267 byte[] r11 = Bytes.toBytes("Row11");
268 byte[] r2 = Bytes.toBytes("Row2");
269
270 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
271 flist.addFilter(new PrefixFilter(r1));
272 flist.filterRowKey(r1, 0, r1.length);
273 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
274 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
275
276 flist.reset();
277 flist.filterRowKey(r2, 0, r2.length);
278 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
279
280 flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
281 flist.addFilter(new AlwaysNextColFilter());
282 flist.addFilter(new PrefixFilter(r1));
283 flist.filterRowKey(r1, 0, r1.length);
284 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
285 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
286
287 flist.reset();
288 flist.filterRowKey(r2, 0, r2.length);
289 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
290 }
291
292
293
294
295
296
297 @Test
298 public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
299 byte[] r1 = Bytes.toBytes("Row1");
300 byte[] r11 = Bytes.toBytes("Row11");
301 byte[] r2 = Bytes.toBytes("Row2");
302
303 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
304 flist.addFilter(new AlwaysNextColFilter());
305 flist.addFilter(new InclusiveStopFilter(r1));
306 flist.filterRowKey(r1, 0, r1.length);
307 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
308 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
309
310 flist.reset();
311 flist.filterRowKey(r2, 0, r2.length);
312 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
313 }
314
315 public static class AlwaysNextColFilter extends FilterBase {
316 public AlwaysNextColFilter() {
317 super();
318 }
319 @Override
320 public ReturnCode filterKeyValue(Cell v) {
321 return ReturnCode.NEXT_COL;
322 }
323 public static AlwaysNextColFilter parseFrom(final byte [] pbBytes)
324 throws DeserializationException {
325 return new AlwaysNextColFilter();
326 }
327 }
328
329
330
331
332
333 @Test
334 public void testSerialization() throws Exception {
335 List<Filter> filters = new ArrayList<Filter>();
336 filters.add(new PageFilter(MAX_PAGES));
337 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
338 Filter filterMPALL =
339 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
340
341
342 byte[] buffer = filterMPALL.toByteArray();
343
344
345 FilterList newFilter = FilterList.parseFrom(buffer);
346
347
348 mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
349 mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
350 orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
351 }
352
353
354
355
356
357 @Test
358 public void testFilterKeyValue() throws Exception {
359 Filter includeFilter = new FilterBase() {
360 @Override
361 public Filter.ReturnCode filterKeyValue(Cell v) {
362 return Filter.ReturnCode.INCLUDE;
363 }
364 };
365
366 Filter alternateFilter = new FilterBase() {
367 boolean returnInclude = true;
368
369 @Override
370 public Filter.ReturnCode filterKeyValue(Cell v) {
371 Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE :
372 Filter.ReturnCode.SKIP;
373 returnInclude = !returnInclude;
374 return returnCode;
375 }
376 };
377
378 Filter alternateIncludeFilter = new FilterBase() {
379 boolean returnIncludeOnly = false;
380
381 @Override
382 public Filter.ReturnCode filterKeyValue(Cell v) {
383 Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE :
384 Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
385 returnIncludeOnly = !returnIncludeOnly;
386 return returnCode;
387 }
388 };
389
390
391 FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
392 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
393
394 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
395
396 assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
397
398
399 FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL,
400 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
401
402 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
403
404 assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
405 }
406
407
408
409
410 @Test
411 public void testHintPassThru() throws Exception {
412
413 final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null);
414 final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE),
415 null, null);
416
417 Filter filterNoHint = new FilterBase() {
418 @Override
419 public byte [] toByteArray() {
420 return null;
421 }
422
423 @Override
424 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
425 return ReturnCode.INCLUDE;
426 }
427 };
428
429 Filter filterMinHint = new FilterBase() {
430 @Override
431 public ReturnCode filterKeyValue(Cell ignored) {
432 return ReturnCode.SEEK_NEXT_USING_HINT;
433 }
434
435 @Override
436 public Cell getNextCellHint(Cell currentKV) {
437 return minKeyValue;
438 }
439
440 @Override
441 public byte [] toByteArray() {return null;}
442 };
443
444 Filter filterMaxHint = new FilterBase() {
445 @Override
446 public ReturnCode filterKeyValue(Cell ignored) {
447 return ReturnCode.SEEK_NEXT_USING_HINT;
448 }
449
450 @Override
451 public Cell getNextCellHint(Cell currentKV) {
452 return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
453 }
454
455 @Override
456 public byte [] toByteArray() {return null;}
457 };
458
459
460
461
462 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE,
463 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
464 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
465 minKeyValue));
466
467
468 filterList = new FilterList(Operator.MUST_PASS_ONE,
469 Arrays.asList(
470 new Filter [] { filterMinHint, filterMaxHint, filterNoHint } ));
471 assertNull(filterList.getNextKeyHint(null));
472 filterList = new FilterList(Operator.MUST_PASS_ONE,
473 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
474 assertNull(filterList.getNextKeyHint(null));
475
476
477 filterList = new FilterList(Operator.MUST_PASS_ONE,
478 Arrays.asList(new Filter [] { filterMaxHint, filterMaxHint } ));
479 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
480 maxKeyValue));
481
482
483
484
485 filterList = new FilterList(Operator.MUST_PASS_ALL,
486 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
487 filterList.filterKeyValue(null);
488 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
489 minKeyValue));
490
491 filterList = new FilterList(Operator.MUST_PASS_ALL,
492 Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } ));
493 filterList.filterKeyValue(null);
494 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
495 maxKeyValue));
496
497
498 filterList = new FilterList(Operator.MUST_PASS_ALL,
499 Arrays.asList(
500 new Filter [] { filterNoHint, filterMinHint, filterMaxHint } ));
501 filterList.filterKeyValue(null);
502 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
503 minKeyValue));
504 filterList = new FilterList(Operator.MUST_PASS_ALL,
505 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
506 filterList.filterKeyValue(null);
507 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
508 maxKeyValue));
509 filterList = new FilterList(Operator.MUST_PASS_ALL,
510 Arrays.asList(new Filter [] { filterNoHint, filterMinHint } ));
511 filterList.filterKeyValue(null);
512 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
513 minKeyValue));
514 }
515
516
517
518
519
520
521
522 @Test
523 public void testTransformMPO() throws Exception {
524
525
526
527 final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
528 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
529 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
530 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
531 new KeyOnlyFilter())),
532 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
533 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
534 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
535
536 final KeyValue kvQual1 = new KeyValue(
537 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
538 final KeyValue kvQual2 = new KeyValue(
539 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
540 final KeyValue kvQual3 = new KeyValue(
541 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
542
543
544 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1));
545 final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual1));
546 assertEquals(0, transformedQual1.getValue().length);
547
548
549 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual2));
550 final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual2));
551 assertEquals("value", Bytes.toString(transformedQual2.getValue()));
552
553
554 assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
555 }
556
557 }
558