1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.NavigableSet;
24
25 import org.apache.hadoop.hbase.KeyValue.Type;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.CellUtil;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.KeepDeletedCells;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.KeyValueUtil;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.filter.Filter;
35 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
36 import org.apache.hadoop.hbase.io.TimeRange;
37 import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40
41 import com.google.common.base.Preconditions;
42
43
44
45
46 @InterfaceAudience.Private
47 public class ScanQueryMatcher {
48
49
50 private boolean stickyNextRow;
51 private final byte[] stopRow;
52
53 private final TimeRange tr;
54
55 private final Filter filter;
56
57
58 private final DeleteTracker deletes;
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 private boolean retainDeletesInOutput;
75
76
77 private final KeepDeletedCells keepDeletedCells;
78
79 private final boolean seePastDeleteMarkers;
80
81
82
83 private final ColumnTracker columns;
84
85
86 private final Cell startKey;
87
88
89 private final KeyValue.KVComparator rowComparator;
90
91
92
93 byte [] row;
94 int rowOffset;
95 short rowLength;
96
97
98
99
100
101
102
103 private final long earliestPutTs;
104 private final long ttl;
105
106
107 private final long oldestUnexpiredTS;
108 private final long now;
109
110
111 protected long maxReadPointToTrackVersions;
112
113 private byte[] dropDeletesFromRow = null, dropDeletesToRow = null;
114
115
116
117
118
119
120
121 private boolean hasNullColumn = true;
122
123 private RegionCoprocessorHost regionCoprocessorHost= null;
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142 private final long timeToPurgeDeletes;
143
144 private final boolean isUserScan;
145
146 private final boolean isReversed;
147
148
149
150
151 private final boolean get;
152
153
154
155
156
157
158
159
160 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
161 ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS,
162 long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
163 TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily());
164 if (timeRange == null) {
165 this.tr = scan.getTimeRange();
166 } else {
167 this.tr = timeRange;
168 }
169 this.get = scan.isGetScan();
170 this.rowComparator = scanInfo.getComparator();
171 this.regionCoprocessorHost = regionCoprocessorHost;
172 this.deletes = instantiateDeleteTracker();
173 this.stopRow = scan.getStopRow();
174 this.startKey = KeyValueUtil.createFirstDeleteFamilyOnRow(scan.getStartRow(),
175 scanInfo.getFamily());
176 this.filter = scan.getFilter();
177 this.earliestPutTs = earliestPutTs;
178 this.oldestUnexpiredTS = oldestUnexpiredTS;
179 this.now = now;
180
181 this.maxReadPointToTrackVersions = readPointToUse;
182 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes();
183 this.ttl = oldestUnexpiredTS;
184
185
186 this.isUserScan = scanType == ScanType.USER_SCAN;
187
188 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE :
189 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells();
190
191 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw();
192
193 this.seePastDeleteMarkers =
194 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan;
195
196 int maxVersions =
197 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(),
198 scanInfo.getMaxVersions());
199
200
201 if (columns == null || columns.size() == 0) {
202
203 hasNullColumn = true;
204
205
206 this.columns = new ScanWildcardColumnTracker(
207 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
208 } else {
209
210 hasNullColumn = (columns.first().length == 0);
211
212
213
214 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
215 oldestUnexpiredTS);
216 }
217 this.isReversed = scan.isReversed();
218 }
219
220 private DeleteTracker instantiateDeleteTracker() throws IOException {
221 DeleteTracker tracker = new ScanDeleteTracker();
222 if (regionCoprocessorHost != null) {
223 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker);
224 }
225 return tracker;
226 }
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
242 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now,
243 byte[] dropDeletesFromRow, byte[] dropDeletesToRow,
244 RegionCoprocessorHost regionCoprocessorHost) throws IOException {
245 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs,
246 oldestUnexpiredTS, now, regionCoprocessorHost);
247 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null));
248 this.dropDeletesFromRow = dropDeletesFromRow;
249 this.dropDeletesToRow = dropDeletesToRow;
250 }
251
252
253
254
255 ScanQueryMatcher(Scan scan, ScanInfo scanInfo,
256 NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException {
257 this(scan, scanInfo, columns, ScanType.USER_SCAN,
258 Long.MAX_VALUE,
259 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null);
260 }
261
262
263
264
265
266 public boolean hasNullColumnInQuery() {
267 return hasNullColumn;
268 }
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283 public MatchCode match(Cell cell) throws IOException {
284 if (filter != null && filter.filterAllRemaining()) {
285 return MatchCode.DONE_SCAN;
286 }
287 if (row != null) {
288 int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
289 cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
290 if (!this.isReversed) {
291 if (ret <= -1) {
292 return MatchCode.DONE;
293 } else if (ret >= 1) {
294
295
296
297 return MatchCode.SEEK_NEXT_ROW;
298 }
299 } else {
300 if (ret <= -1) {
301 return MatchCode.SEEK_NEXT_ROW;
302 } else if (ret >= 1) {
303 return MatchCode.DONE;
304 }
305 }
306 } else {
307 return MatchCode.DONE;
308 }
309
310
311 if (this.stickyNextRow)
312 return MatchCode.SEEK_NEXT_ROW;
313
314 if (this.columns.done()) {
315 stickyNextRow = true;
316 return MatchCode.SEEK_NEXT_ROW;
317 }
318
319 int qualifierOffset = cell.getQualifierOffset();
320 int qualifierLength = cell.getQualifierLength();
321
322 long timestamp = cell.getTimestamp();
323
324 if (columns.isDone(timestamp)) {
325 return columns.getNextRowOrNextColumn(cell.getQualifierArray(), qualifierOffset,
326 qualifierLength);
327 }
328
329 if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) {
330 return MatchCode.SKIP;
331 }
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346 byte typeByte = cell.getTypeByte();
347 long mvccVersion = cell.getMvccVersion();
348 if (CellUtil.isDelete(cell)) {
349 if (keepDeletedCells == KeepDeletedCells.FALSE
350 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) {
351
352
353
354
355
356
357 boolean includeDeleteMarker = seePastDeleteMarkers ?
358 tr.withinTimeRange(timestamp) :
359 tr.withinOrAfterTimeRange(timestamp);
360 if (includeDeleteMarker
361 && mvccVersion <= maxReadPointToTrackVersions) {
362 this.deletes.add(cell);
363 }
364
365 }
366
367 if ((!isUserScan)
368 && timeToPurgeDeletes > 0
369 && (EnvironmentEdgeManager.currentTime() - timestamp)
370 <= timeToPurgeDeletes) {
371 return MatchCode.INCLUDE;
372 } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) {
373
374
375 if (!isUserScan) {
376
377
378 return MatchCode.INCLUDE;
379 }
380 } else if (keepDeletedCells == KeepDeletedCells.TRUE
381 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) {
382 if (timestamp < earliestPutTs) {
383
384
385 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
386 qualifierOffset, qualifierLength);
387 }
388
389
390 } else {
391 return MatchCode.SKIP;
392 }
393
394
395 } else if (!this.deletes.isEmpty()) {
396 DeleteResult deleteResult = deletes.isDeleted(cell);
397 switch (deleteResult) {
398 case FAMILY_DELETED:
399 case COLUMN_DELETED:
400 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
401 qualifierOffset, qualifierLength);
402 case VERSION_DELETED:
403 case FAMILY_VERSION_DELETED:
404 return MatchCode.SKIP;
405 case NOT_DELETED:
406 break;
407 default:
408 throw new RuntimeException("UNEXPECTED");
409 }
410 }
411
412
413
414
415
416
417
418
419
420
421 int timestampComparison = timestamp == HConstants.OLDEST_TIMESTAMP? -1: tr.compare(timestamp);
422 if (timestampComparison >= 1) {
423 return MatchCode.SKIP;
424 } else if (timestampComparison <= -1) {
425 return columns.getNextRowOrNextColumn(cell.getQualifierArray(), qualifierOffset,
426 qualifierLength);
427 }
428
429
430 MatchCode colChecker = columns.checkColumn(cell.getQualifierArray(),
431 qualifierOffset, qualifierLength, typeByte);
432 if (colChecker == MatchCode.INCLUDE) {
433 ReturnCode filterResponse = ReturnCode.SKIP;
434
435 if (filter != null) {
436
437 filterResponse = filter.filterKeyValue(cell);
438 switch (filterResponse) {
439 case SKIP:
440 return MatchCode.SKIP;
441 case NEXT_COL:
442 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
443 qualifierOffset, qualifierLength);
444 case NEXT_ROW:
445 stickyNextRow = true;
446 return MatchCode.SEEK_NEXT_ROW;
447 case SEEK_NEXT_USING_HINT:
448 return MatchCode.SEEK_NEXT_USING_HINT;
449 default:
450
451 break;
452 }
453 }
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473 colChecker =
474 columns.checkVersions(cell.getQualifierArray(), qualifierOffset,
475 qualifierLength, timestamp, typeByte,
476 mvccVersion > maxReadPointToTrackVersions);
477
478 stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
479 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
480 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
481 : colChecker;
482 }
483 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
484 : stickyNextRow;
485 return colChecker;
486 }
487
488
489
490
491 private void checkPartialDropDeleteRange(byte [] row, int offset, short length) {
492
493
494
495
496 if ((dropDeletesFromRow != null)
497 && ((dropDeletesFromRow == HConstants.EMPTY_START_ROW)
498 || (Bytes.compareTo(row, offset, length,
499 dropDeletesFromRow, 0, dropDeletesFromRow.length) >= 0))) {
500 retainDeletesInOutput = false;
501 dropDeletesFromRow = null;
502 }
503
504
505
506 if ((dropDeletesFromRow == null)
507 && (dropDeletesToRow != null) && (dropDeletesToRow != HConstants.EMPTY_END_ROW)
508 && (Bytes.compareTo(row, offset, length,
509 dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) {
510 retainDeletesInOutput = true;
511 dropDeletesToRow = null;
512 }
513 }
514
515
516
517
518
519 public boolean moreRowsMayExistAfter(Cell kv) {
520
521
522
523 if (this.get) {
524 return false;
525 }
526
527
528
529 if (this.stopRow == null || this.stopRow == HConstants.EMPTY_BYTE_ARRAY) {
530 return true;
531 }
532 return this.isReversed?
533 rowComparator.compareRows(kv, stopRow, 0, stopRow.length) > 0:
534 Bytes.equals(stopRow, HConstants.EMPTY_END_ROW) ||
535 rowComparator.compareRows(kv, stopRow, 0, stopRow.length) < 0;
536 }
537
538
539
540
541
542 public void setRow(byte [] row, int offset, short length) {
543 checkPartialDropDeleteRange(row, offset, length);
544 this.row = row;
545 this.rowOffset = offset;
546 this.rowLength = length;
547 reset();
548 }
549
550 public void reset() {
551 this.deletes.reset();
552 this.columns.reset();
553
554 stickyNextRow = false;
555 }
556
557
558
559
560
561 public Cell getStartKey() {
562 return this.startKey;
563 }
564
565
566
567
568
569 Filter getFilter() {
570 return this.filter;
571 }
572
573 public Cell getNextKeyHint(Cell kv) throws IOException {
574 if (filter == null) {
575 return null;
576 } else {
577 return filter.getNextCellHint(kv);
578 }
579 }
580
581 public Cell getKeyForNextColumn(Cell kv) {
582
583
584
585
586
587 if (kv.getQualifierLength() == 0) {
588 Cell nextKey = createNextOnRowCol(kv);
589 if (nextKey != kv) {
590 return nextKey;
591 }
592
593
594 }
595 ColumnCount nextColumn = columns.getColumnHint();
596 if (nextColumn == null) {
597 return KeyValueUtil.createLastOnRow(
598 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
599 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
600 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
601 } else {
602 return KeyValueUtil.createFirstOnRow(
603 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
604 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
605 nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
606 }
607 }
608
609
610
611
612
613
614 public int compareKeyForNextRow(Cell nextIndexed, Cell kv) {
615 return rowComparator.compareKey(nextIndexed,
616 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
617 null, 0, 0,
618 null, 0, 0,
619 HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode());
620 }
621
622
623
624
625
626
627 public int compareKeyForNextColumn(Cell nextIndexed, Cell kv) {
628 ColumnCount nextColumn = columns.getColumnHint();
629 if (nextColumn == null) {
630 return rowComparator.compareKey(nextIndexed,
631 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
632 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
633 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
634 HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode());
635 } else {
636 return rowComparator.compareKey(nextIndexed,
637 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
638 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
639 nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength(),
640 HConstants.LATEST_TIMESTAMP, Type.Maximum.getCode());
641 }
642 }
643
644
645 static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset,
646 int length, long ttl, byte type, boolean ignoreCount) throws IOException {
647 MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
648 if (matchCode == MatchCode.INCLUDE) {
649 return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
650 }
651 return matchCode;
652 }
653
654
655
656
657
658
659
660
661 public static enum MatchCode {
662
663
664
665 INCLUDE,
666
667
668
669
670 SKIP,
671
672
673
674
675 NEXT,
676
677
678
679
680 DONE,
681
682
683
684
685
686
687
688
689 SEEK_NEXT_ROW,
690
691
692
693 SEEK_NEXT_COL,
694
695
696
697
698 DONE_SCAN,
699
700
701
702
703 SEEK_NEXT_USING_HINT,
704
705
706
707
708 INCLUDE_AND_SEEK_NEXT_COL,
709
710
711
712
713 INCLUDE_AND_SEEK_NEXT_ROW,
714 }
715
716
717
718
719
720 private static Cell createNextOnRowCol(Cell cell) {
721 long ts = cell.getTimestamp();
722 byte type = cell.getTypeByte();
723 if (type != Type.Minimum.getCode()) {
724 type = KeyValue.Type.values()[KeyValue.Type.codeToType(type).ordinal() - 1].getCode();
725 } else if (ts != HConstants.OLDEST_TIMESTAMP) {
726 ts = ts - 1;
727 type = Type.Maximum.getCode();
728 } else {
729 return cell;
730 }
731 return createNextOnRowCol(cell, ts, type);
732 }
733
734 private static Cell createNextOnRowCol(final Cell cell, final long ts, final byte type) {
735 return new Cell() {
736 @Override
737 public byte[] getRowArray() { return cell.getRowArray(); }
738
739 @Override
740 public int getRowOffset() { return cell.getRowOffset(); }
741
742 @Override
743 public short getRowLength() { return cell.getRowLength(); }
744
745 @Override
746 public byte[] getFamilyArray() { return cell.getFamilyArray(); }
747
748 @Override
749 public int getFamilyOffset() { return cell.getFamilyOffset(); }
750
751 @Override
752 public byte getFamilyLength() { return cell.getFamilyLength(); }
753
754 @Override
755 public byte[] getQualifierArray() { return cell.getQualifierArray(); }
756
757 @Override
758 public int getQualifierOffset() { return cell.getQualifierOffset(); }
759
760 @Override
761 public int getQualifierLength() { return cell.getQualifierLength(); }
762
763 @Override
764 public long getTimestamp() { return ts; }
765
766 @Override
767 public byte getTypeByte() {return type; }
768
769 @Override
770 public long getMvccVersion() { return cell.getMvccVersion(); }
771
772 @Override
773 public long getSequenceId() { return cell.getSequenceId(); }
774
775 @Override
776 public byte[] getValueArray() { return cell.getValueArray(); }
777
778 @Override
779 public int getValueOffset() { return cell.getValueOffset(); }
780
781 @Override
782 public int getValueLength() { return cell.getValueLength(); }
783
784 @Override
785 public byte[] getTagsArray() { return cell.getTagsArray(); }
786
787 @Override
788 public int getTagsOffset() { return cell.getTagsOffset(); }
789
790 @Override
791 public int getTagsLength() { return cell.getTagsLength(); }
792
793 @Override
794 public byte[] getValue() { return cell.getValue(); }
795
796 @Override
797 public byte[] getFamily() { return cell.getFamily(); }
798
799 @Override
800 public byte[] getQualifier() { return cell.getQualifier(); }
801
802 @Override
803 public byte[] getRow() {return cell.getRow(); }
804 };
805 }
806 }