1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.filter;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import com.google.common.collect.Lists;
28
29 import java.io.IOException;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.List;
33
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValueUtil;
37 import org.apache.hadoop.hbase.exceptions.DeserializationException;
38 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
39 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
40 import org.apache.hadoop.hbase.filter.FilterList.Operator;
41 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42 import org.apache.hadoop.hbase.testclassification.FilterTests;
43 import org.apache.hadoop.hbase.testclassification.SmallTests;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.junit.Assert;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49 @Category({FilterTests.class, SmallTests.class})
50 public class TestFilterList {
51 static final int MAX_PAGES = 2;
52
53 @Test
54 public void testAddFilter() throws Exception {
55 Filter filter1 = new FirstKeyOnlyFilter();
56 Filter filter2 = new FirstKeyOnlyFilter();
57
58 FilterList filterList = new FilterList(filter1, filter2);
59 filterList.addFilter(new FirstKeyOnlyFilter());
60
61 filterList = new FilterList(Arrays.asList(filter1, filter2));
62 filterList.addFilter(new FirstKeyOnlyFilter());
63
64 filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
65 filterList.addFilter(new FirstKeyOnlyFilter());
66
67 filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
68 filterList.addFilter(new FirstKeyOnlyFilter());
69
70 filterList.setReversed(false);
71 FirstKeyOnlyFilter f = new FirstKeyOnlyFilter();
72 f.setReversed(true);
73 try {
74 filterList.addFilter(f);
75 fail("The IllegalArgumentException should be thrown because the added filter is reversed");
76 } catch (IllegalArgumentException e) {
77 }
78
79 }
80
81 @Test
82 public void testConstruction() {
83 FirstKeyOnlyFilter f1 = new FirstKeyOnlyFilter();
84 FirstKeyOnlyFilter f2 = new FirstKeyOnlyFilter();
85 f1.setReversed(true);
86 f2.setReversed(false);
87
88 try {
89 FilterList ff = new FilterList(f1, f2);
90 fail("The IllegalArgumentException should be thrown");
91 } catch (IllegalArgumentException e) {
92 }
93
94 try {
95 FilterList ff = new FilterList(Arrays.asList((Filter) f1, (Filter) f2));
96 fail("The IllegalArgumentException should be thrown because the added filter is reversed");
97 } catch (IllegalArgumentException e) {
98 }
99
100 try {
101 FilterList ff = new FilterList(FilterList.Operator.MUST_PASS_ALL,
102 Arrays.asList((Filter) f1, (Filter) f2));
103 fail("The IllegalArgumentException should be thrown because the added filter is reversed");
104 } catch (IllegalArgumentException e) {
105 }
106
107 try {
108 FilterList ff = new FilterList(FilterList.Operator.MUST_PASS_ALL, f1, f2);
109 fail("The IllegalArgumentException should be thrown because the added filter is reversed");
110 } catch (IllegalArgumentException e) {
111 }
112 }
113
114
115
116
117 @Test
118 public void testMPONE() throws Exception {
119 mpOneTest(getFilterMPONE());
120 }
121
122 private Filter getFilterMPONE() {
123 List<Filter> filters = new ArrayList<Filter>();
124 filters.add(new PageFilter(MAX_PAGES));
125 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
126 return new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
127 }
128
129 private void mpOneTest(Filter filterMPONE) throws Exception {
130
131
132
133
134
135
136
137
138
139
140
141
142 filterMPONE.reset();
143 assertFalse(filterMPONE.filterAllRemaining());
144
145
146 byte[] rowkey = Bytes.toBytes("yyyyyyyyy");
147 for (int i = 0; i < MAX_PAGES - 1; i++) {
148 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
149 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i));
150 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
151 assertFalse(filterMPONE.filterRow());
152 }
153
154
155 rowkey = Bytes.toBytes("z");
156 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
157 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0));
158 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
159 assertFalse(filterMPONE.filterRow());
160
161
162 rowkey = Bytes.toBytes("yyy");
163 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
164 kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0));
165 assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
166 assertFalse(filterMPONE.filterRow());
167
168
169 rowkey = Bytes.toBytes("z");
170 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
171 assertTrue(filterMPONE.filterAllRemaining());
172 }
173
174
175
176
177
178 @Test
179 public void testMPALL() throws Exception {
180 mpAllTest(getMPALLFilter());
181 }
182
183 private Filter getMPALLFilter() {
184 List<Filter> filters = new ArrayList<Filter>();
185 filters.add(new PageFilter(MAX_PAGES));
186 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
187 Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
188 return filterMPALL;
189 }
190
191 private void mpAllTest(Filter filterMPALL) throws Exception {
192
193
194
195
196
197
198
199
200
201
202
203
204 filterMPALL.reset();
205 assertFalse(filterMPALL.filterAllRemaining());
206 byte[] rowkey = Bytes.toBytes("yyyyyyyyy");
207 for (int i = 0; i < MAX_PAGES - 1; i++) {
208 assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
209 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i));
210 assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv));
211 }
212 filterMPALL.reset();
213 rowkey = Bytes.toBytes("z");
214 assertTrue(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
215
216 KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey);
217 assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv));
218 }
219
220
221
222
223
224 @Test
225 public void testOrdering() throws Exception {
226 orderingTest(getOrderingFilter());
227 }
228
229 public Filter getOrderingFilter() {
230 List<Filter> filters = new ArrayList<Filter>();
231 filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
232 filters.add(new PageFilter(MAX_PAGES));
233 Filter filterMPONE = new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
234 return filterMPONE;
235 }
236
237 public void orderingTest(Filter filterMPONE) throws Exception {
238
239
240
241
242
243
244
245
246
247
248
249
250 filterMPONE.reset();
251 assertFalse(filterMPONE.filterAllRemaining());
252
253
254 byte[] rowkey = Bytes.toBytes("yyyyyyyy");
255 for (int i = 0; i < MAX_PAGES; i++) {
256 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
257 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i));
258 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
259 assertFalse(filterMPONE.filterRow());
260 }
261
262
263 rowkey = Bytes.toBytes("xxxxxxx");
264 for (int i = 0; i < MAX_PAGES; i++) {
265 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
266 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i));
267 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
268 assertFalse(filterMPONE.filterRow());
269 }
270
271
272 rowkey = Bytes.toBytes("yyy");
273 for (int i = 0; i < MAX_PAGES; i++) {
274 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
275 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i));
276 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
277 assertFalse(filterMPONE.filterRow());
278 }
279 }
280
281
282
283
284
285
286 @Test
287 public void testFilterListTwoFiltersMustPassOne() throws Exception {
288 byte[] r1 = Bytes.toBytes("Row1");
289 byte[] r11 = Bytes.toBytes("Row11");
290 byte[] r2 = Bytes.toBytes("Row2");
291
292 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
293 flist.addFilter(new PrefixFilter(r1));
294 flist.filterRowKey(r1, 0, r1.length);
295 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
296 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
297
298 flist.reset();
299 flist.filterRowKey(r2, 0, r2.length);
300 assertEquals(ReturnCode.SKIP, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
301
302 flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
303 flist.addFilter(new AlwaysNextColFilter());
304 flist.addFilter(new PrefixFilter(r1));
305 flist.filterRowKey(r1, 0, r1.length);
306 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
307 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
308
309 flist.reset();
310 flist.filterRowKey(r2, 0, r2.length);
311 assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
312 }
313
314
315
316
317
318
319 @Test
320 public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception {
321 byte[] r1 = Bytes.toBytes("Row1");
322 byte[] r11 = Bytes.toBytes("Row11");
323 byte[] r2 = Bytes.toBytes("Row2");
324
325 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
326 flist.addFilter(new AlwaysNextColFilter());
327 flist.addFilter(new InclusiveStopFilter(r1));
328 flist.filterRowKey(r1, 0, r1.length);
329 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
330 assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
331
332 flist.reset();
333 flist.filterRowKey(r2, 0, r2.length);
334 assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
335 }
336
337 static class AlwaysNextColFilter extends FilterBase {
338 public AlwaysNextColFilter() {
339 super();
340 }
341
342 @Override
343 public ReturnCode filterKeyValue(Cell v) {
344 return ReturnCode.NEXT_COL;
345 }
346
347 public static AlwaysNextColFilter parseFrom(final byte[] pbBytes)
348 throws DeserializationException {
349 return new AlwaysNextColFilter();
350 }
351 }
352
353
354
355
356
357 @Test
358 public void testSerialization() throws Exception {
359 List<Filter> filters = new ArrayList<Filter>();
360 filters.add(new PageFilter(MAX_PAGES));
361 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
362 Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
363
364
365 byte[] buffer = filterMPALL.toByteArray();
366
367
368 FilterList.parseFrom(buffer);
369
370
371 mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
372 mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
373 orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
374 }
375
376
377
378
379
380 @Test
381 public void testFilterKeyValue() throws Exception {
382 Filter includeFilter = new FilterBase() {
383 @Override
384 public Filter.ReturnCode filterKeyValue(Cell v) {
385 return Filter.ReturnCode.INCLUDE;
386 }
387 };
388
389 Filter alternateFilter = new FilterBase() {
390 boolean returnInclude = true;
391
392 @Override
393 public Filter.ReturnCode filterKeyValue(Cell v) {
394 Filter.ReturnCode returnCode =
395 returnInclude ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.SKIP;
396 returnInclude = !returnInclude;
397 return returnCode;
398 }
399 };
400
401 Filter alternateIncludeFilter = new FilterBase() {
402 boolean returnIncludeOnly = false;
403
404 @Override
405 public Filter.ReturnCode filterKeyValue(Cell v) {
406 Filter.ReturnCode returnCode =
407 returnIncludeOnly ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
408 returnIncludeOnly = !returnIncludeOnly;
409 return returnCode;
410 }
411 };
412
413
414 FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
415 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
416
417 assertEquals(ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
418
419 assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
420
421
422 FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL,
423 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
424
425 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
426
427 assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
428 }
429
430
431
432
433 @Test
434 public void testHintPassThru() throws Exception {
435
436 final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null);
437 final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
438
439 Filter filterNoHint = new FilterBase() {
440 @Override
441 public byte[] toByteArray() {
442 return null;
443 }
444
445 @Override
446 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
447 return ReturnCode.INCLUDE;
448 }
449 };
450
451 Filter filterMinHint = new FilterBase() {
452 @Override
453 public ReturnCode filterKeyValue(Cell ignored) {
454 return ReturnCode.SEEK_NEXT_USING_HINT;
455 }
456
457 @Override
458 public Cell getNextCellHint(Cell currentKV) {
459 return minKeyValue;
460 }
461
462 @Override
463 public byte[] toByteArray() {
464 return null;
465 }
466 };
467
468 Filter filterMaxHint = new FilterBase() {
469 @Override
470 public ReturnCode filterKeyValue(Cell ignored) {
471 return ReturnCode.SEEK_NEXT_USING_HINT;
472 }
473
474 @Override
475 public Cell getNextCellHint(Cell cell) {
476 return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
477 }
478
479 @Override
480 public byte[] toByteArray() {
481 return null;
482 }
483 };
484
485
486
487
488 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE,
489 Arrays.asList(new Filter[] { filterMinHint, filterMaxHint }));
490 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), minKeyValue));
491
492
493 filterList = new FilterList(Operator.MUST_PASS_ONE,
494 Arrays.asList(new Filter[] { filterMinHint, filterMaxHint, filterNoHint }));
495 assertNull(filterList.getNextCellHint(null));
496 filterList = new FilterList(Operator.MUST_PASS_ONE,
497 Arrays.asList(new Filter[] { filterNoHint, filterMaxHint }));
498 assertNull(filterList.getNextCellHint(null));
499
500
501 filterList = new FilterList(Operator.MUST_PASS_ONE,
502 Arrays.asList(new Filter[] { filterMaxHint, filterMaxHint }));
503 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), maxKeyValue));
504
505
506
507
508 filterList = new FilterList(Operator.MUST_PASS_ALL,
509 Arrays.asList(new Filter[] { filterMinHint, filterMaxHint }));
510 filterList.filterKeyValue(null);
511 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), minKeyValue));
512
513 filterList = new FilterList(Operator.MUST_PASS_ALL,
514 Arrays.asList(new Filter[] { filterMaxHint, filterMinHint }));
515 filterList.filterKeyValue(null);
516 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), maxKeyValue));
517
518
519 filterList = new FilterList(Operator.MUST_PASS_ALL,
520 Arrays.asList(new Filter[] { filterNoHint, filterMinHint, filterMaxHint }));
521 filterList.filterKeyValue(null);
522 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), minKeyValue));
523 filterList = new FilterList(Operator.MUST_PASS_ALL,
524 Arrays.asList(new Filter[] { filterNoHint, filterMaxHint }));
525 filterList.filterKeyValue(null);
526 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), maxKeyValue));
527 filterList = new FilterList(Operator.MUST_PASS_ALL,
528 Arrays.asList(new Filter[] { filterNoHint, filterMinHint }));
529 filterList.filterKeyValue(null);
530 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextCellHint(null), minKeyValue));
531 }
532
533
534
535
536
537 @Test
538 public void testTransformMPO() throws Exception {
539
540
541
542 final FilterList flist =
543 new FilterList(Operator.MUST_PASS_ONE,
544 Lists.<Filter> newArrayList(
545 new FilterList(Operator.MUST_PASS_ALL,
546 Lists.<Filter> newArrayList(
547 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
548 new QualifierFilter(CompareOp.EQUAL,
549 new BinaryComparator(Bytes.toBytes("qual1"))),
550 new KeyOnlyFilter())),
551 new FilterList(Operator.MUST_PASS_ALL,
552 Lists.<Filter> newArrayList(
553 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
554 new QualifierFilter(CompareOp.EQUAL,
555 new BinaryComparator(Bytes.toBytes("qual2")))))));
556
557 final KeyValue kvQual1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"),
558 Bytes.toBytes("qual1"), Bytes.toBytes("value"));
559 final KeyValue kvQual2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"),
560 Bytes.toBytes("qual2"), Bytes.toBytes("value"));
561 final KeyValue kvQual3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"),
562 Bytes.toBytes("qual3"), Bytes.toBytes("value"));
563
564
565 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1));
566 final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
567 assertEquals(0, transformedQual1.getValueLength());
568
569
570 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual2));
571 final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
572 assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
573 transformedQual2.getValueOffset(), transformedQual2.getValueLength()));
574
575
576 assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
577 }
578
579 @Test
580 public void testWithMultiVersionsInSameRow() throws Exception {
581 FilterList filterList01 =
582 new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0));
583
584 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
585 1, Bytes.toBytes("value"));
586 KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
587 2, Bytes.toBytes("value"));
588 KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
589 3, Bytes.toBytes("value"));
590
591 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterKeyValue(kv1));
592 assertEquals(ReturnCode.SKIP, filterList01.filterKeyValue(kv2));
593 assertEquals(ReturnCode.SKIP, filterList01.filterKeyValue(kv3));
594
595 FilterList filterList11 =
596 new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));
597
598 assertEquals(ReturnCode.NEXT_COL, filterList11.filterKeyValue(kv1));
599 assertEquals(ReturnCode.SKIP, filterList11.filterKeyValue(kv2));
600 assertEquals(ReturnCode.SKIP, filterList11.filterKeyValue(kv3));
601 }
602
603 @Test
604 public void testMPONEWithSeekNextUsingHint() throws Exception {
605 byte[] col = Bytes.toBytes("c");
606 FilterList filterList =
607 new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col));
608
609 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
610 Bytes.toBytes("value"));
611 KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2,
612 Bytes.toBytes("value"));
613 KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3,
614 Bytes.toBytes("value"));
615 KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
616 Bytes.toBytes("value"));
617
618 assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
619 assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv2));
620 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv3));
621 assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv4));
622 }
623
624 private static class MockFilter extends FilterBase {
625 private ReturnCode targetRetCode;
626 public boolean didCellPassToTheFilter = false;
627
628 public MockFilter(ReturnCode targetRetCode) {
629 this.targetRetCode = targetRetCode;
630 }
631
632 @Override
633 public ReturnCode filterKeyValue(Cell v) throws IOException {
634 this.didCellPassToTheFilter = true;
635 return targetRetCode;
636 }
637 }
638
639 @Test
640 public void testShouldPassCurrentCellToFilter() throws IOException {
641 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
642 Bytes.toBytes("value"));
643 KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 2,
644 Bytes.toBytes("value"));
645 KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 3,
646 Bytes.toBytes("value"));
647 KeyValue kv4 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
648 Bytes.toBytes("value"));
649
650 MockFilter mockFilter = new MockFilter(ReturnCode.NEXT_COL);
651 FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
652
653 filter.filterKeyValue(kv1);
654 assertTrue(mockFilter.didCellPassToTheFilter);
655
656 mockFilter.didCellPassToTheFilter = false;
657 filter.filterKeyValue(kv2);
658 assertFalse(mockFilter.didCellPassToTheFilter);
659
660 mockFilter.didCellPassToTheFilter = false;
661 filter.filterKeyValue(kv3);
662 assertTrue(mockFilter.didCellPassToTheFilter);
663
664 mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
665 filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
666
667 filter.filterKeyValue(kv1);
668 assertTrue(mockFilter.didCellPassToTheFilter);
669
670 mockFilter.didCellPassToTheFilter = false;
671 filter.filterKeyValue(kv2);
672 assertFalse(mockFilter.didCellPassToTheFilter);
673
674 mockFilter.didCellPassToTheFilter = false;
675 filter.filterKeyValue(kv3);
676 assertTrue(mockFilter.didCellPassToTheFilter);
677
678 mockFilter = new MockFilter(ReturnCode.NEXT_ROW);
679 filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
680 filter.filterKeyValue(kv1);
681 assertTrue(mockFilter.didCellPassToTheFilter);
682
683 mockFilter.didCellPassToTheFilter = false;
684 filter.filterKeyValue(kv2);
685 assertFalse(mockFilter.didCellPassToTheFilter);
686
687 mockFilter.didCellPassToTheFilter = false;
688 filter.filterKeyValue(kv3);
689 assertFalse(mockFilter.didCellPassToTheFilter);
690
691 filter.reset();
692 mockFilter.didCellPassToTheFilter = false;
693 filter.filterKeyValue(kv4);
694 assertTrue(mockFilter.didCellPassToTheFilter);
695
696 mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
697 filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
698 filter.filterKeyValue(kv1);
699 assertTrue(mockFilter.didCellPassToTheFilter);
700
701 mockFilter.didCellPassToTheFilter = false;
702 filter.filterKeyValue(kv2);
703 assertFalse(mockFilter.didCellPassToTheFilter);
704
705 mockFilter.didCellPassToTheFilter = false;
706 filter.filterKeyValue(kv3);
707 assertFalse(mockFilter.didCellPassToTheFilter);
708
709 filter.reset();
710 mockFilter.didCellPassToTheFilter = false;
711 filter.filterKeyValue(kv4);
712 assertTrue(mockFilter.didCellPassToTheFilter);
713 }
714
715 @Test
716 public void testTheMaximalRule() throws IOException {
717 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
718 Bytes.toBytes("value"));
719 MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
720 MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
721 MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
722 MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
723 MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
724 MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
725 MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW);
726
727 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
728 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
729
730 filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3);
731 assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1));
732
733 filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6);
734 assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(kv1));
735
736 filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6);
737 assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(kv1));
738
739 filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1);
740 assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1));
741
742 filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5);
743 assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
744
745 filterList = new FilterList(Operator.MUST_PASS_ALL, filter2,
746 new FilterList(Operator.MUST_PASS_ALL, filter3, filter4));
747 assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
748
749 filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7);
750 assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
751 }
752
753 @Test
754 public void testTheMinimalRule() throws IOException {
755 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
756 Bytes.toBytes("value"));
757 MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
758 MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
759 MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
760 MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
761 MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
762 MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
763 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
764 assertEquals(filterList.filterKeyValue(kv1), ReturnCode.INCLUDE);
765
766 filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
767 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
768
769 filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6);
770 assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1));
771
772 filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6);
773 assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1));
774
775 filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1);
776 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1));
777
778 filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5);
779 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1));
780
781 filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
782 new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
783 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
784
785 filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
786 new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
787 assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
788
789 filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
790 assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
791 }
792
793 static class MockSeekHintFilter extends FilterBase {
794 private Cell returnCell;
795
796 public MockSeekHintFilter(Cell returnCell) {
797 this.returnCell = returnCell;
798 }
799
800 @Override
801 public ReturnCode filterKeyValue(Cell v) throws IOException {
802 return ReturnCode.SEEK_NEXT_USING_HINT;
803 }
804
805 @Override
806 public Cell getNextCellHint(Cell currentCell) throws IOException {
807 return this.returnCell;
808 }
809 }
810
811 @Test
812 public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
813 KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
814 Bytes.toBytes("value"));
815 KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
816 Bytes.toBytes("value"));
817 KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
818 Bytes.toBytes("value"));
819 Filter filter1 = new MockSeekHintFilter(kv1);
820 filter1.setReversed(true);
821 Filter filter2 = new MockSeekHintFilter(kv2);
822 filter2.setReversed(true);
823 Filter filter3 = new MockSeekHintFilter(kv3);
824 filter3.setReversed(true);
825
826 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
827 filterList.setReversed(true);
828 filterList.addFilter(filter1);
829 filterList.addFilter(filter2);
830 filterList.addFilter(filter3);
831
832 Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
833 Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));
834
835 filterList = new FilterList(Operator.MUST_PASS_ALL);
836 filterList.setReversed(true);
837 filterList.addFilter(filter1);
838 filterList.addFilter(filter2);
839 filterList.addFilter(filter3);
840
841 Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
842 Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
843 }
844
845 @Test
846 public void testReversedFilterListWithOR() throws IOException {
847 byte[] r22 = Bytes.toBytes("Row22");
848 byte[] r2 = Bytes.toBytes("Row2");
849 byte[] r1 = Bytes.toBytes("Row1");
850
851 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
852 filterList.setReversed(true);
853 PrefixFilter prefixFilter = new PrefixFilter(r2);
854 prefixFilter.setReversed(true);
855 filterList.addFilter(prefixFilter);
856 filterList.filterRowKey(r22, 0, r22.length);
857 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22)));
858 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2)));
859
860 filterList.reset();
861 filterList.filterRowKey(r1, 0, r1.length);
862 assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(new KeyValue(r1, r1, r1)));
863
864 filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
865 filterList.setReversed(true);
866 AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter();
867 alwaysNextColFilter.setReversed(true);
868 prefixFilter = new PrefixFilter(r2);
869 prefixFilter.setReversed(true);
870 filterList.addFilter(alwaysNextColFilter);
871 filterList.addFilter(prefixFilter);
872 filterList.filterRowKey(r22, 0, r22.length);
873 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22)));
874 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2)));
875
876 filterList.reset();
877 filterList.filterRowKey(r1, 0, r1.length);
878 assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(new KeyValue(r1, r1, r1)));
879 }
880
881 @Test
882 public void testKeyOnlyFilterTransformCell() throws IOException {
883 Cell c;
884 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
885 1, Bytes.toBytes("value1"));
886 KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
887 2, Bytes.toBytes("value2"));
888
889 Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
890 CompareOp.EQUAL, Bytes.toBytes("value1"));
891 Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
892 CompareOp.EQUAL, Bytes.toBytes("value2"));
893 FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
894
895 FilterList keyOnlyFilterFirst =
896 new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
897
898 assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv1));
899 c = keyOnlyFilterFirst.transformCell(kv1);
900 assertEquals(0, c.getValueLength());
901 assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv2));
902 c = keyOnlyFilterFirst.transformCell(kv2);
903 assertEquals(0, c.getValueLength());
904
905 internalFilterList.reset();
906 FilterList keyOnlyFilterLast =
907 new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
908 assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv1));
909 c = keyOnlyFilterLast.transformCell(kv1);
910 assertEquals(0, c.getValueLength());
911 assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv2));
912 c = keyOnlyFilterLast.transformCell(kv2);
913 assertEquals(0, c.getValueLength());
914 }
915
916 @Test
917 public void testEmptyFilterListTransformCell() throws IOException {
918 KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
919 1, Bytes.toBytes("value"));
920 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
921 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
922 assertEquals(kv, filterList.transformCell(kv));
923
924 filterList = new FilterList(Operator.MUST_PASS_ONE);
925 assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
926 assertEquals(kv, filterList.transformCell(kv));
927 }
928
929 private static class MockNextRowFilter extends FilterBase {
930 private int hitCount = 0;
931
932 public ReturnCode filterKeyValue(Cell v) throws IOException {
933 hitCount++;
934 return ReturnCode.NEXT_ROW;
935 }
936
937 public int getHitCount() {
938 return hitCount;
939 }
940 }
941
942 @Test
943 public void testRowCountFilter() throws IOException {
944 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1,
945 Bytes.toBytes("value"));
946 KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2,
947 Bytes.toBytes("value"));
948 MockNextRowFilter mockNextRowFilter = new MockNextRowFilter();
949 FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter);
950 filter.filterKeyValue(kv1);
951 filter.filterKeyValue(kv2);
952 assertEquals(2, mockNextRowFilter.getHitCount());
953 }
954
955 private static class TransformFilter extends FilterBase {
956 private ReturnCode targetRetCode;
957 private boolean transformed = false;
958
959 public TransformFilter(ReturnCode targetRetCode) {
960 this.targetRetCode = targetRetCode;
961 }
962
963 @Override
964 public ReturnCode filterKeyValue(final Cell v) throws IOException {
965 return targetRetCode;
966 }
967
968 @Override
969 public Cell transformCell(Cell c) throws IOException {
970 transformed = true;
971 return super.transformCell(c);
972 }
973
974 public boolean getTransformed() {
975 return this.transformed;
976 }
977 }
978
979 @Test
980 public void testTransformCell() throws IOException {
981 KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
982 1, Bytes.toBytes("value"));
983
984
985 TransformFilter filter1 = new TransformFilter(ReturnCode.INCLUDE);
986 TransformFilter filter2 = new TransformFilter(ReturnCode.NEXT_ROW);
987 TransformFilter filter3 = new TransformFilter(ReturnCode.SEEK_NEXT_USING_HINT);
988 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2, filter3);
989 Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
990 Assert.assertEquals(kv, filterList.transformCell(kv));
991 Assert.assertEquals(true, filter1.getTransformed());
992 Assert.assertEquals(false, filter2.getTransformed());
993 Assert.assertEquals(false, filter3.getTransformed());
994
995
996 filter1 = new TransformFilter(ReturnCode.INCLUDE);
997 filter2 = new TransformFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
998 filter3 = new TransformFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
999 filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2, filter3);
1000
1001 Assert.assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv));
1002 Assert.assertEquals(kv, filterList.transformCell(kv));
1003 Assert.assertEquals(true, filter1.getTransformed());
1004 Assert.assertEquals(true, filter2.getTransformed());
1005 Assert.assertEquals(true, filter3.getTransformed());
1006 }
1007 }