View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.filter;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  
25  import org.apache.hadoop.hbase.util.ByteStringer;
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.classification.InterfaceStability;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.exceptions.DeserializationException;
33  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
34  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
35  import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
36  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.CompareType;
38  import org.apache.hadoop.hbase.util.Bytes;
39  
40  import com.google.common.base.Preconditions;
41  import com.google.protobuf.InvalidProtocolBufferException;
42  
43  /**
44   * This filter is used to filter cells based on value. It takes a {@link CompareFilter.CompareOp}
45   * operator (equal, greater, not equal, etc), and either a byte [] value or
46   * a ByteArrayComparable.
47   * <p>
48   * If we have a byte [] value then we just do a lexicographic compare. For
49   * example, if passed value is 'b' and cell has 'a' and the compare operator
50   * is LESS, then we will filter out this cell (return true).  If this is not
51   * sufficient (eg you want to deserialize a long and then compare it to a fixed
52   * long value), then you can pass in your own comparator instead.
53   * <p>
54   * You must also specify a family and qualifier.  Only the value of this column
55   * will be tested. When using this filter on a 
56   * {@link org.apache.hadoop.hbase.CellScanner} with specified
57   * inputs, the column to be tested should also be added as input (otherwise
58   * the filter will regard the column as missing).
59   * <p>
60   * To prevent the entire row from being emitted if the column is not found
61   * on a row, use {@link #setFilterIfMissing}.
62   * Otherwise, if the column is found, the entire row will be emitted only if
63   * the value passes.  If the value fails, the row will be filtered out.
64   * <p>
65   * In order to test values of previous versions (timestamps), set
66   * {@link #setLatestVersionOnly} to false. The default is true, meaning that
67   * only the latest version's value is tested and all previous versions are ignored.
68   * <p>
69   * To filter based on the value of all scanned columns, use {@link ValueFilter}.
70   */
71  @InterfaceAudience.Public
72  @InterfaceStability.Stable
73  public class SingleColumnValueFilter extends FilterBase {
74    private static final Log LOG = LogFactory.getLog(SingleColumnValueFilter.class);
75  
76    protected byte [] columnFamily;
77    protected byte [] columnQualifier;
78    protected CompareOp compareOp;
79    protected ByteArrayComparable comparator;
80    protected boolean foundColumn = false;
81    protected boolean matchedColumn = false;
82    protected boolean filterIfMissing = false;
83    protected boolean latestVersionOnly = true;
84  
85    /**
86     * Constructor for binary compare of the value of a single column.  If the
87     * column is found and the condition passes, all columns of the row will be
88     * emitted.  If the condition fails, the row will not be emitted.
89     * <p>
90     * Use the filterIfColumnMissing flag to set whether the rest of the columns
91     * in a row will be emitted if the specified column to check is not found in
92     * the row.
93     *
94     * @param family name of column family
95     * @param qualifier name of column qualifier
96     * @param compareOp operator
97     * @param value value to compare column values against
98     */
99    public SingleColumnValueFilter(final byte [] family, final byte [] qualifier,
100       final CompareOp compareOp, final byte[] value) {
101     this(family, qualifier, compareOp, new BinaryComparator(value));
102   }
103 
104   /**
105    * Constructor for binary compare of the value of a single column.  If the
106    * column is found and the condition passes, all columns of the row will be
107    * emitted.  If the condition fails, the row will not be emitted.
108    * <p>
109    * Use the filterIfColumnMissing flag to set whether the rest of the columns
110    * in a row will be emitted if the specified column to check is not found in
111    * the row.
112    *
113    * @param family name of column family
114    * @param qualifier name of column qualifier
115    * @param compareOp operator
116    * @param comparator Comparator to use.
117    */
118   public SingleColumnValueFilter(final byte [] family, final byte [] qualifier,
119       final CompareOp compareOp, final ByteArrayComparable comparator) {
120     this.columnFamily = family;
121     this.columnQualifier = qualifier;
122     this.compareOp = compareOp;
123     this.comparator = comparator;
124   }
125 
126   /**
127    * Constructor for protobuf deserialization only.
128    * @param family
129    * @param qualifier
130    * @param compareOp
131    * @param comparator
132    * @param filterIfMissing
133    * @param latestVersionOnly
134    */
135   protected SingleColumnValueFilter(final byte[] family, final byte[] qualifier,
136       final CompareOp compareOp, ByteArrayComparable comparator, final boolean filterIfMissing,
137       final boolean latestVersionOnly) {
138     this(family, qualifier, compareOp, comparator);
139     this.filterIfMissing = filterIfMissing;
140     this.latestVersionOnly = latestVersionOnly;
141   }
142 
143   /**
144    * @return operator
145    */
146   public CompareOp getOperator() {
147     return compareOp;
148   }
149 
150   /**
151    * @return the comparator
152    */
153   public ByteArrayComparable getComparator() {
154     return comparator;
155   }
156 
157   /**
158    * @return the family
159    */
160   public byte[] getFamily() {
161     return columnFamily;
162   }
163 
164   /**
165    * @return the qualifier
166    */
167   public byte[] getQualifier() {
168     return columnQualifier;
169   }
170 
171   @Override
172   public ReturnCode filterKeyValue(Cell c) {
173     if (this.matchedColumn) {
174       // We already found and matched the single column, all keys now pass
175       return ReturnCode.INCLUDE;
176     } else if (this.latestVersionOnly && this.foundColumn) {
177       // We found but did not match the single column, skip to next row
178       return ReturnCode.NEXT_ROW;
179     }
180     if (!CellUtil.matchingColumn(c, this.columnFamily, this.columnQualifier)) {
181       return ReturnCode.INCLUDE;
182     }
183     foundColumn = true;
184     if (filterColumnValue(c.getValueArray(), c.getValueOffset(), c.getValueLength())) {
185       return this.latestVersionOnly? ReturnCode.NEXT_ROW: ReturnCode.INCLUDE;
186     }
187     this.matchedColumn = true;
188     return ReturnCode.INCLUDE;
189   }
190 
191   // Override here explicitly as the method in super class FilterBase might do a KeyValue recreate.
192   // See HBASE-12068
193   @Override
194   public Cell transformCell(Cell v) {
195     return v;
196   }
197 
198   private boolean filterColumnValue(final byte [] data, final int offset,
199       final int length) {
200     int compareResult = this.comparator.compareTo(data, offset, length);
201     switch (this.compareOp) {
202     case LESS:
203       return compareResult <= 0;
204     case LESS_OR_EQUAL:
205       return compareResult < 0;
206     case EQUAL:
207       return compareResult != 0;
208     case NOT_EQUAL:
209       return compareResult == 0;
210     case GREATER_OR_EQUAL:
211       return compareResult > 0;
212     case GREATER:
213       return compareResult >= 0;
214     default:
215       throw new RuntimeException("Unknown Compare op " + compareOp.name());
216     }
217   }
218 
219   @Override
220   public boolean filterRow() {
221     // If column was found, return false if it was matched, true if it was not
222     // If column not found, return true if we filter if missing, false if not
223     return this.foundColumn? !this.matchedColumn: this.filterIfMissing;
224   }
225   
226   @Override
227   public boolean hasFilterRow() {
228     return true;
229   }
230 
231   @Override
232   public void reset() {
233     foundColumn = false;
234     matchedColumn = false;
235   }
236 
237   /**
238    * Get whether entire row should be filtered if column is not found.
239    * @return true if row should be skipped if column not found, false if row
240    * should be let through anyways
241    */
242   public boolean getFilterIfMissing() {
243     return filterIfMissing;
244   }
245 
246   /**
247    * Set whether entire row should be filtered if column is not found.
248    * <p>
249    * If true, the entire row will be skipped if the column is not found.
250    * <p>
251    * If false, the row will pass if the column is not found.  This is default.
252    * @param filterIfMissing flag
253    */
254   public void setFilterIfMissing(boolean filterIfMissing) {
255     this.filterIfMissing = filterIfMissing;
256   }
257 
258   /**
259    * Get whether only the latest version of the column value should be compared.
260    * If true, the row will be returned if only the latest version of the column
261    * value matches. If false, the row will be returned if any version of the
262    * column value matches. The default is true.
263    * @return return value
264    */
265   public boolean getLatestVersionOnly() {
266     return latestVersionOnly;
267   }
268 
269   /**
270    * Set whether only the latest version of the column value should be compared.
271    * If true, the row will be returned if only the latest version of the column
272    * value matches. If false, the row will be returned if any version of the
273    * column value matches. The default is true.
274    * @param latestVersionOnly flag
275    */
276   public void setLatestVersionOnly(boolean latestVersionOnly) {
277     this.latestVersionOnly = latestVersionOnly;
278   }
279 
280   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) {
281     Preconditions.checkArgument(filterArguments.size() == 4 || filterArguments.size() == 6,
282                                 "Expected 4 or 6 but got: %s", filterArguments.size());
283     byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0));
284     byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1));
285     CompareOp compareOp = ParseFilter.createCompareOp(filterArguments.get(2));
286     ByteArrayComparable comparator = ParseFilter.createComparator(
287       ParseFilter.removeQuotesFromByteArray(filterArguments.get(3)));
288 
289     if (comparator instanceof RegexStringComparator ||
290         comparator instanceof SubstringComparator) {
291       if (compareOp != CompareOp.EQUAL &&
292           compareOp != CompareOp.NOT_EQUAL) {
293         throw new IllegalArgumentException ("A regexstring comparator and substring comparator " +
294                                             "can only be used with EQUAL and NOT_EQUAL");
295       }
296     }
297 
298     SingleColumnValueFilter filter = new SingleColumnValueFilter(family, qualifier,
299                                                                  compareOp, comparator);
300 
301     if (filterArguments.size() == 6) {
302       boolean filterIfMissing = ParseFilter.convertByteArrayToBoolean(filterArguments.get(4));
303       boolean latestVersionOnly = ParseFilter.convertByteArrayToBoolean(filterArguments.get(5));
304       filter.setFilterIfMissing(filterIfMissing);
305       filter.setLatestVersionOnly(latestVersionOnly);
306     }
307     return filter;
308   }
309 
310   FilterProtos.SingleColumnValueFilter convert() {
311     FilterProtos.SingleColumnValueFilter.Builder builder =
312       FilterProtos.SingleColumnValueFilter.newBuilder();
313     if (this.columnFamily != null) {
314       builder.setColumnFamily(ByteStringer.wrap(this.columnFamily));
315     }
316     if (this.columnQualifier != null) {
317       builder.setColumnQualifier(ByteStringer.wrap(this.columnQualifier));
318     }
319     HBaseProtos.CompareType compareOp = CompareType.valueOf(this.compareOp.name());
320     builder.setCompareOp(compareOp);
321     builder.setComparator(ProtobufUtil.toComparator(this.comparator));
322     builder.setFilterIfMissing(this.filterIfMissing);
323     builder.setLatestVersionOnly(this.latestVersionOnly);
324 
325     return builder.build();
326   }
327 
328   /**
329    * @return The filter serialized using pb
330    */
331   @Override
332   public byte [] toByteArray() {
333     return convert().toByteArray();
334   }
335 
336   /**
337    * @param pbBytes A pb serialized {@link SingleColumnValueFilter} instance
338    * @return An instance of {@link SingleColumnValueFilter} made from <code>bytes</code>
339    * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
340    * @see #toByteArray
341    */
342   public static SingleColumnValueFilter parseFrom(final byte [] pbBytes)
343   throws DeserializationException {
344     FilterProtos.SingleColumnValueFilter proto;
345     try {
346       proto = FilterProtos.SingleColumnValueFilter.parseFrom(pbBytes);
347     } catch (InvalidProtocolBufferException e) {
348       throw new DeserializationException(e);
349     }
350 
351     final CompareOp compareOp =
352       CompareOp.valueOf(proto.getCompareOp().name());
353     final ByteArrayComparable comparator;
354     try {
355       comparator = ProtobufUtil.toComparator(proto.getComparator());
356     } catch (IOException ioe) {
357       throw new DeserializationException(ioe);
358     }
359 
360     return new SingleColumnValueFilter(proto.hasColumnFamily() ? proto.getColumnFamily()
361         .toByteArray() : null, proto.hasColumnQualifier() ? proto.getColumnQualifier()
362         .toByteArray() : null, compareOp, comparator, proto.getFilterIfMissing(), proto
363         .getLatestVersionOnly());
364   }
365 
366   /**
367    * @param other
368    * @return true if and only if the fields of the filter that are serialized
369    * are equal to the corresponding fields in other.  Used for testing.
370    */
371   @Override
372   boolean areSerializedFieldsEqual(Filter o) {
373     if (o == this) return true;
374     if (!(o instanceof SingleColumnValueFilter)) return false;
375 
376     SingleColumnValueFilter other = (SingleColumnValueFilter)o;
377     return Bytes.equals(this.getFamily(), other.getFamily())
378       && Bytes.equals(this.getQualifier(), other.getQualifier())
379       && this.compareOp.equals(other.compareOp)
380       && this.getComparator().areSerializedFieldsEqual(other.getComparator())
381       && this.getFilterIfMissing() == other.getFilterIfMissing()
382       && this.getLatestVersionOnly() == other.getLatestVersionOnly();
383   }
384 
385   /**
386    * The only CF this filter needs is given column family. So, it's the only essential
387    * column in whole scan. If filterIfMissing == false, all families are essential,
388    * because of possibility of skipping the rows without any data in filtered CF.
389    */
390   @Override
391   public boolean isFamilyEssential(byte[] name) {
392     return !this.filterIfMissing || Bytes.equals(name, this.columnFamily);
393   }
394 
395   @Override
396   public String toString() {
397     return String.format("%s (%s, %s, %s, %s)",
398         this.getClass().getSimpleName(), Bytes.toStringBinary(this.columnFamily),
399         Bytes.toStringBinary(this.columnQualifier), this.compareOp.name(),
400         Bytes.toStringBinary(this.comparator.getValue()));
401   }
402 }