View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.filter;
21  
22  import org.apache.hadoop.hbase.Cell;
23  import org.apache.hadoop.hbase.CellUtil;
24  import org.apache.hadoop.hbase.KeyValueUtil;
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.List;
31  
32  /**
33   * FilterListWithOR represents an ordered list of filters which will be evaluated with an OR
34   * operator.
35   */
36  @InterfaceAudience.Private
37  public class FilterListWithOR extends FilterListBase {
38  
39    /**
40     * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE,
41     * we use the previous return code to decide whether we should pass current cell encountered to
42     * the filter. For MUST_PASS_ALL, the two list are meaningless.
43     */
44    private List<ReturnCode> prevFilterRCList = null;
45    private List<Cell> prevCellList = null;
46  
47    public FilterListWithOR(List<Filter> filters) {
48      super(filters);
49      prevFilterRCList =
50          new ArrayList<ReturnCode>(Collections.nCopies(filters.size(), (ReturnCode) null));
51      prevCellList = new ArrayList<Cell>(Collections.nCopies(filters.size(), (Cell) null));
52      subFiltersIncludedCell = new ArrayList<Boolean>(Collections.nCopies(filters.size(), false));
53    }
54  
55    @Override
56    public void addFilterLists(List<Filter> filters) {
57      if (checkAndGetReversed(filters, isReversed()) != isReversed()) {
58        throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
59      }
60      this.filters.addAll(filters);
61      this.subFiltersIncludedCell.addAll(Collections.nCopies(filters.size(), false));
62      this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), (ReturnCode) null));
63      this.prevCellList.addAll(Collections.nCopies(filters.size(), (Cell) null));
64    }
65  
66    @Override
67    protected String formatLogFilters(List<Filter> logFilters) {
68      return String.format("FilterList OR (%d/%d): %s", logFilters.size(), this.size(),
69        logFilters.toString());
70    }
71  
72    /**
73     * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then
74     * the next cell passing to filterList will be the first cell in next column, because if filter-B
75     * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass
76     * the cell following the previous cell, and it's possible that the next cell has the same column
77     * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should
78     * save the previous cell and the return code list when checking previous cell for every filter in
79     * filter list, and verify if currentCell fit the previous return code, if fit then pass the
80     * currentCell to the corresponding filter. (HBASE-17678) <br>
81     * Note that: In StoreScanner level, NEXT_ROW will skip to the next row in current family, and in
82     * RegionScanner level, NEXT_ROW will skip to the next row in current family and switch to the
83     * next family for RegionScanner, INCLUDE_AND_NEXT_ROW is the same. so we should pass current cell
84     * to the filter, if row mismatch or row match but column family mismatch. (HBASE-18368)
85     * @see org.apache.hadoop.hbase.filter.Filter.ReturnCode
86     */
87    private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell,
88        ReturnCode prevCode) throws IOException {
89      if (prevCell == null || prevCode == null) {
90        return true;
91      }
92      switch (prevCode) {
93      case INCLUDE:
94      case SKIP:
95        return true;
96      case SEEK_NEXT_USING_HINT:
97        Cell nextHintCell = getNextCellHint(prevCell);
98        return nextHintCell == null || this.compareCell(currentCell, nextHintCell) >= 0;
99      case NEXT_COL:
100     case INCLUDE_AND_NEXT_COL:
101       // Once row changed, reset() will clear prevCells, so we need not to compare their rows
102       // because rows are the same here.
103       return !CellUtil.matchingColumn(prevCell, currentCell);
104     case NEXT_ROW:
105     case INCLUDE_AND_SEEK_NEXT_ROW:
106       // As described above, rows are definitely the same, so we only compare the family.
107       return !CellUtil.matchingFamily(prevCell, currentCell);
108     default:
109       throw new IllegalStateException("Received code is not valid.");
110     }
111   }
112 
113   /**
114    * FilterList with MUST_PASS_ONE choose the minimal forward step among sub-filter in filter list.
115    * Let's call it: The Minimal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
116    * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return INCLUDE. For
117    * SEEK_NEXT_USING_HINT, it's more special, because we do not know how far it will forward, so we
118    * use SKIP by default.<br/>
119    * <br/>
120    * The jump step will be:
121    *
122    * <pre>
123    * INCLUDE &lt; SKIP &lt; INCLUDE_AND_NEXT_COL &lt; NEXT_COL &lt; INCLUDE_AND_SEEK_NEXT_ROW &lt; NEXT_ROW &lt; SEEK_NEXT_USING_HINT
124    * </pre>
125    *
126    * Here, we have the following map to describe The Minimal Step Rule. if current return code (for
127    * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
128    * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
129    * according to The Minimal Step Rule.<br/>
130    *
131    * <pre>
132    * LocalCode\ReturnCode       INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  SKIP      NEXT_COL              NEXT_ROW                  SEEK_NEXT_USING_HINT
133    * INCLUDE                    INCLUDE INCLUDE                  INCLUDE                    INCLUDE   INCLUDE               INCLUDE                   INCLUDE
134    * INCLUDE_AND_NEXT_COL       INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_NEXT_COL       INCLUDE   INCLUDE_AND_NEXT_COL  INCLUDE_AND_NEXT_COL      INCLUDE
135    * INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  INCLUDE   INCLUDE_AND_NEXT_COL  INCLUDE_AND_SEEK_NEXT_ROW INCLUDE
136    * SKIP                       INCLUDE INCLUDE                  INCLUDE                    SKIP      SKIP                  SKIP                      SKIP
137    * NEXT_COL                   INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_NEXT_COL       SKIP      NEXT_COL              NEXT_COL                  SKIP
138    * NEXT_ROW                   INCLUDE INCLUDE_AND_NEXT_COL     INCLUDE_AND_SEEK_NEXT_ROW  SKIP      NEXT_COL              NEXT_ROW                  SKIP
139    * SEEK_NEXT_USING_HINT       INCLUDE INCLUDE                  INCLUDE                    SKIP      SKIP                  SKIP                      SEEK_NEXT_USING_HINT
140    * </pre>
141    *
142    * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
143    * @param localRC Return code of the current sub-filter in filter list.
144    * @return Return code which is merged by the return code of previous sub-filter(s) and the return
145    *         code of current sub-filter.
146    */
147   private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) {
148     if (rc == null) return localRC;
149     switch (localRC) {
150     case INCLUDE:
151       return ReturnCode.INCLUDE;
152     case INCLUDE_AND_NEXT_COL:
153       if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
154         ReturnCode.SEEK_NEXT_USING_HINT)) {
155         return ReturnCode.INCLUDE;
156       }
157       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW,
158         ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
159         return ReturnCode.INCLUDE_AND_NEXT_COL;
160       }
161       break;
162     case INCLUDE_AND_SEEK_NEXT_ROW:
163       if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
164         ReturnCode.SEEK_NEXT_USING_HINT)) {
165         return ReturnCode.INCLUDE;
166       }
167       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
168         return ReturnCode.INCLUDE_AND_NEXT_COL;
169       }
170       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
171         return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
172       }
173       break;
174     case SKIP:
175       if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
176         ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
177         return ReturnCode.INCLUDE;
178       }
179       if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW,
180         ReturnCode.SEEK_NEXT_USING_HINT)) {
181         return ReturnCode.SKIP;
182       }
183       break;
184     case NEXT_COL:
185       if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
186         return ReturnCode.INCLUDE;
187       }
188       if (isInReturnCodes(rc, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
189         return ReturnCode.NEXT_COL;
190       }
191       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL,
192         ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
193         return ReturnCode.INCLUDE_AND_NEXT_COL;
194       }
195       if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
196         return ReturnCode.SKIP;
197       }
198       break;
199     case NEXT_ROW:
200       if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
201         return ReturnCode.INCLUDE;
202       }
203       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL)) {
204         return ReturnCode.INCLUDE_AND_NEXT_COL;
205       }
206       if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
207         return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
208       }
209       if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
210         return ReturnCode.SKIP;
211       }
212       if (isInReturnCodes(rc, ReturnCode.NEXT_COL)) {
213         return ReturnCode.NEXT_COL;
214       }
215       if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
216         return ReturnCode.NEXT_ROW;
217       }
218       break;
219     case SEEK_NEXT_USING_HINT:
220       if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
221         ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
222         return ReturnCode.INCLUDE;
223       }
224       if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
225         return ReturnCode.SKIP;
226       }
227       if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) {
228         return ReturnCode.SEEK_NEXT_USING_HINT;
229       }
230       break;
231     }
232     throw new IllegalStateException(
233         "Received code is not valid. rc: " + rc + ", localRC: " + localRC);
234   }
235 
236   private void updatePrevFilterRCList(int index, ReturnCode currentRC) {
237     prevFilterRCList.set(index, currentRC);
238   }
239 
240   private void updatePrevCellList(int index, Cell currentCell, ReturnCode currentRC) {
241     if (currentCell == null || currentRC == ReturnCode.INCLUDE || currentRC == ReturnCode.SKIP) {
242       // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
243       // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
244       // need not save current cell to prevCellList for saving heap memory.
245       prevCellList.set(index, null);
246     } else {
247       prevCellList.set(index, KeyValueUtil.copyToNewKeyValue(currentCell));
248     }
249   }
250 
251   @Override
252   public ReturnCode filterKeyValue(Cell c) throws IOException {
253     if (isEmpty()) {
254       return ReturnCode.INCLUDE;
255     }
256     ReturnCode rc = null;
257     boolean everyFilterReturnHint = true;
258     for (int i = 0, n = filters.size(); i < n; i++) {
259       Filter filter = filters.get(i);
260       subFiltersIncludedCell.set(i, false);
261 
262       Cell prevCell = this.prevCellList.get(i);
263       ReturnCode prevCode = this.prevFilterRCList.get(i);
264       if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, prevCode)) {
265         everyFilterReturnHint = false;
266         continue;
267       }
268 
269       ReturnCode localRC = filter.filterKeyValue(c);
270 
271       // Update previous return code and previous cell for filter[i].
272       updatePrevFilterRCList(i, localRC);
273       updatePrevCellList(i, c, localRC);
274 
275       if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
276         everyFilterReturnHint = false;
277       }
278 
279       rc = mergeReturnCode(rc, localRC);
280 
281       // For INCLUDE* case, we need to update the transformed cell.
282       if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
283         ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
284         subFiltersIncludedCell.set(i, true);
285       }
286     }
287 
288     if (everyFilterReturnHint) {
289       return ReturnCode.SEEK_NEXT_USING_HINT;
290     } else if (rc == null) {
291       // Each sub-filter in filter list got true for filterAllRemaining().
292       return ReturnCode.SKIP;
293     } else {
294       return rc;
295     }
296   }
297 
298   @Override
299   public void reset() throws IOException {
300     for (int i = 0, n = filters.size(); i < n; i++) {
301       filters.get(i).reset();
302       subFiltersIncludedCell.set(i, false);
303       prevFilterRCList.set(i, null);
304       prevCellList.set(i, null);
305     }
306   }
307 
308   @Override
309   public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException {
310     if (isEmpty()) {
311       return super.filterRowKey(rowKey, offset, length);
312     }
313     boolean retVal = true;
314     for (int i = 0, n = filters.size(); i < n; i++) {
315       Filter filter = filters.get(i);
316       if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey, offset, length)) {
317         retVal = false;
318       }
319     }
320     return retVal;
321   }
322 
323   @Override
324   public boolean filterAllRemaining() throws IOException {
325     if (isEmpty()) {
326       return super.filterAllRemaining();
327     }
328     for (int i = 0, n = filters.size(); i < n; i++) {
329       if (!filters.get(i).filterAllRemaining()) {
330         return false;
331       }
332     }
333     return true;
334   }
335 
336   @Override
337   public boolean filterRow() throws IOException {
338     if (isEmpty()) {
339       return super.filterRow();
340     }
341     for (int i = 0, n = filters.size(); i < n; i++) {
342       Filter filter = filters.get(i);
343       if (!filter.filterRow()) {
344         return false;
345       }
346     }
347     return true;
348   }
349 
350   @Override
351   public Cell getNextCellHint(Cell currentCell) throws IOException {
352     if (isEmpty()) {
353       return super.getNextCellHint(currentCell);
354     }
355     Cell minKeyHint = null;
356     // If any condition can pass, we need to keep the min hint
357     for (int i = 0, n = filters.size(); i < n; i++) {
358       if (filters.get(i).filterAllRemaining()) {
359         continue;
360       }
361       Cell curKeyHint = filters.get(i).getNextCellHint(currentCell);
362       if (curKeyHint == null) {
363         // If we ever don't have a hint and this is must-pass-one, then no hint
364         return null;
365       }
366       // If this is the first hint we find, set it
367       if (minKeyHint == null) {
368         minKeyHint = curKeyHint;
369         continue;
370       }
371       if (this.compareCell(minKeyHint, curKeyHint) > 0) {
372         minKeyHint = curKeyHint;
373       }
374     }
375     return minKeyHint;
376   }
377 }