View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import static org.apache.hadoop.hbase.util.Bytes.getBytes;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.apache.commons.collections.MapUtils;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HRegionLocation;
32  import org.apache.hadoop.hbase.ServerName;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.hbase.Cell;
35  import org.apache.hadoop.hbase.CellUtil;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.client.Append;
38  import org.apache.hadoop.hbase.client.Delete;
39  import org.apache.hadoop.hbase.client.Durability;
40  import org.apache.hadoop.hbase.client.Get;
41  import org.apache.hadoop.hbase.client.HRegionLocator;
42  import org.apache.hadoop.hbase.client.Increment;
43  import org.apache.hadoop.hbase.client.OperationWithAttributes;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.client.RowMutations;
47  import org.apache.hadoop.hbase.client.Scan;
48  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
49  import org.apache.hadoop.hbase.filter.ParseFilter;
50  import org.apache.hadoop.hbase.security.visibility.Authorizations;
51  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
52  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
53  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
54  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
55  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
56  import org.apache.hadoop.hbase.thrift2.generated.TCompareOp;
57  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
58  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
59  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
60  import org.apache.hadoop.hbase.thrift2.generated.TGet;
61  import org.apache.hadoop.hbase.thrift2.generated.THRegionInfo;
62  import org.apache.hadoop.hbase.thrift2.generated.THRegionLocation;
63  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
64  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
65  import org.apache.hadoop.hbase.thrift2.generated.TPut;
66  import org.apache.hadoop.hbase.thrift2.generated.TResult;
67  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
68  import org.apache.hadoop.hbase.thrift2.generated.TScan;
69  import org.apache.hadoop.hbase.thrift2.generated.TServerName;
70  import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
71  import org.apache.hadoop.hbase.util.Bytes;
72  
73  @InterfaceAudience.Private
74  public class ThriftUtilities {
75  
76    private ThriftUtilities() {
77      throw new UnsupportedOperationException("Can't initialize class");
78    }
79  
80    /**
81     * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
82     *
83     * This ignores any timestamps set on {@link TColumn} objects.
84     *
85     * @param in the <code>TGet</code> to convert
86     *
87     * @return <code>Get</code> object
88     *
89     * @throws IOException if an invalid time range or max version parameter is given
90     */
91    public static Get getFromThrift(TGet in) throws IOException {
92      Get out = new Get(in.getRow());
93  
94      // Timestamp overwrites time range if both are set
95      if (in.isSetTimestamp()) {
96        out.setTimeStamp(in.getTimestamp());
97      } else if (in.isSetTimeRange()) {
98        out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
99      }
100 
101     if (in.isSetMaxVersions()) {
102       out.setMaxVersions(in.getMaxVersions());
103     }
104 
105     if (in.isSetFilterString()) {
106       ParseFilter parseFilter = new ParseFilter();
107       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
108     }
109 
110     if (in.isSetAttributes()) {
111       addAttributes(out,in.getAttributes());
112     }
113 
114     if (in.isSetAuthorizations()) {
115       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
116     }
117     
118     if (!in.isSetColumns()) {
119       return out;
120     }
121 
122     for (TColumn column : in.getColumns()) {
123       if (column.isSetQualifier()) {
124         out.addColumn(column.getFamily(), column.getQualifier());
125       } else {
126         out.addFamily(column.getFamily());
127       }
128     }
129 
130     return out;
131   }
132 
133   /**
134    * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase).
135    *
136    * @param in list of <code>TGet</code>s to convert
137    *
138    * @return list of <code>Get</code> objects
139    *
140    * @throws IOException if an invalid time range or max version parameter is given
141    * @see #getFromThrift(TGet)
142    */
143   public static List<Get> getsFromThrift(List<TGet> in) throws IOException {
144     List<Get> out = new ArrayList<Get>(in.size());
145     for (TGet get : in) {
146       out.add(getFromThrift(get));
147     }
148     return out;
149   }
150 
151   /**
152    * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
153    *
154    * @param in the <code>Result</code> to convert
155    *
156    * @return converted result, returns an empty result if the input is <code>null</code>
157    */
158   public static TResult resultFromHBase(Result in) {
159     Cell[] raw = in.rawCells();
160     TResult out = new TResult();
161     byte[] row = in.getRow();
162     if (row != null) {
163       out.setRow(in.getRow());
164     }
165     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
166     for (Cell kv : raw) {
167       TColumnValue col = new TColumnValue();
168       col.setFamily(CellUtil.cloneFamily(kv));
169       col.setQualifier(CellUtil.cloneQualifier(kv));
170       col.setTimestamp(kv.getTimestamp());
171       col.setValue(CellUtil.cloneValue(kv));
172       if (kv.getTagsLength() > 0) {
173         col.setTags(CellUtil.getTagArray(kv));
174       }
175       columnValues.add(col);
176     }
177     out.setColumnValues(columnValues);
178     return out;
179   }
180 
181   /**
182    * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift).
183    *
184    * @param in array of <code>Result</code>s to convert
185    *
186    * @return list of converted <code>TResult</code>s
187    *
188    * @see #resultFromHBase(Result)
189    */
190   public static List<TResult> resultsFromHBase(Result[] in) {
191     List<TResult> out = new ArrayList<TResult>(in.length);
192     for (Result result : in) {
193       out.add(resultFromHBase(result));
194     }
195     return out;
196   }
197 
198   /**
199    * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift)
200    *
201    * @param in the <code>TPut</code> to convert
202    *
203    * @return converted <code>Put</code>
204    */
205   public static Put putFromThrift(TPut in) {
206     Put out;
207 
208     if (in.isSetTimestamp()) {
209       out = new Put(in.getRow(), in.getTimestamp());
210     } else {
211       out = new Put(in.getRow());
212     }
213 
214     if (in.isSetDurability()) {
215       out.setDurability(durabilityFromThrift(in.getDurability()));
216     }
217 
218     for (TColumnValue columnValue : in.getColumnValues()) {
219       if (columnValue.isSetTimestamp()) {
220         out.addImmutable(
221             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(),
222             columnValue.getValue());
223       } else {
224         out.addImmutable(
225             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue());
226       }
227     }
228 
229     if (in.isSetAttributes()) {
230       addAttributes(out,in.getAttributes());
231     }
232     
233     if (in.getCellVisibility() != null) {
234       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
235     }
236 
237     return out;
238   }
239 
240   /**
241    * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase).
242    *
243    * @param in list of <code>TPut</code>s to convert
244    *
245    * @return list of converted <code>Put</code>s
246    *
247    * @see #putFromThrift(TPut)
248    */
249   public static List<Put> putsFromThrift(List<TPut> in) {
250     List<Put> out = new ArrayList<Put>(in.size());
251     for (TPut put : in) {
252       out.add(putFromThrift(put));
253     }
254     return out;
255   }
256 
257   /**
258    * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift).
259    *
260    * @param in the <code>TDelete</code> to convert
261    *
262    * @return converted <code>Delete</code>
263    */
264   public static Delete deleteFromThrift(TDelete in) {
265     Delete out;
266 
267     if (in.isSetColumns()) {
268       out = new Delete(in.getRow());
269       for (TColumn column : in.getColumns()) {
270         if (column.isSetQualifier()) {
271           if (column.isSetTimestamp()) {
272             if (in.isSetDeleteType() &&
273                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
274               out.deleteColumns(column.getFamily(), column.getQualifier(), column.getTimestamp());
275             else
276               out.deleteColumn(column.getFamily(), column.getQualifier(), column.getTimestamp());
277           } else {
278             if (in.isSetDeleteType() &&
279                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
280               out.deleteColumns(column.getFamily(), column.getQualifier());
281             else
282               out.deleteColumn(column.getFamily(), column.getQualifier());
283           }
284 
285         } else {
286           if (column.isSetTimestamp()) {
287             out.deleteFamily(column.getFamily(), column.getTimestamp());
288           } else {
289             out.deleteFamily(column.getFamily());
290           }
291         }
292       }
293     } else {
294       if (in.isSetTimestamp()) {
295         out = new Delete(in.getRow(), in.getTimestamp());
296       } else {
297         out = new Delete(in.getRow());
298       }
299     }
300 
301     if (in.isSetAttributes()) {
302       addAttributes(out,in.getAttributes());
303     }
304 
305     if (in.isSetDurability()) {
306       out.setDurability(durabilityFromThrift(in.getDurability()));
307     }
308 
309     return out;
310   }
311 
312   /**
313    * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase).
314    *
315    * @param in list of <code>TDelete</code>s to convert
316    *
317    * @return list of converted <code>Delete</code>s
318    *
319    * @see #deleteFromThrift(TDelete)
320    */
321 
322   public static List<Delete> deletesFromThrift(List<TDelete> in) {
323     List<Delete> out = new ArrayList<Delete>(in.size());
324     for (TDelete delete : in) {
325       out.add(deleteFromThrift(delete));
326     }
327     return out;
328   }
329 
330   public static TDelete deleteFromHBase(Delete in) {
331     TDelete out = new TDelete(ByteBuffer.wrap(in.getRow()));
332 
333     List<TColumn> columns = new ArrayList<TColumn>();
334     long rowTimestamp = in.getTimeStamp();
335     if (rowTimestamp != HConstants.LATEST_TIMESTAMP) {
336       out.setTimestamp(rowTimestamp);
337     }
338 
339     // Map<family, List<KeyValue>>
340     for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry:
341         in.getFamilyCellMap().entrySet()) {
342       TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey()));
343       for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) {
344         byte[] family = CellUtil.cloneFamily(cell);
345         byte[] qualifier = CellUtil.cloneQualifier(cell);
346         long timestamp = cell.getTimestamp();
347         if (family != null) {
348           column.setFamily(family);
349         }
350         if (qualifier != null) {
351           column.setQualifier(qualifier);
352         }
353         if (timestamp != HConstants.LATEST_TIMESTAMP) {
354           column.setTimestamp(timestamp);
355         }
356       }
357       columns.add(column);
358     }
359     out.setColumns(columns);
360 
361     return out;
362   }
363 
364   /**
365    * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
366    *
367    * @param in the <code>TRowMutations</code> to convert
368    *
369    * @return converted <code>RowMutations</code>
370    */
371   public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
372     RowMutations out = new RowMutations(in.getRow());
373     List<TMutation> mutations = in.getMutations();
374     for (TMutation mutation : mutations) {
375       if (mutation.isSetPut()) {
376         out.add(putFromThrift(mutation.getPut()));
377       }
378       if (mutation.isSetDeleteSingle()) {
379         out.add(deleteFromThrift(mutation.getDeleteSingle()));
380       }
381     }
382     return out;
383   }
384 
385   public static Scan scanFromThrift(TScan in) throws IOException {
386     Scan out = new Scan();
387 
388     if (in.isSetStartRow())
389       out.setStartRow(in.getStartRow());
390     if (in.isSetStopRow())
391       out.setStopRow(in.getStopRow());
392     if (in.isSetCaching())
393       out.setCaching(in.getCaching());
394     if (in.isSetMaxVersions()) {
395       out.setMaxVersions(in.getMaxVersions());
396     }
397 
398     if (in.isSetColumns()) {
399       for (TColumn column : in.getColumns()) {
400         if (column.isSetQualifier()) {
401           out.addColumn(column.getFamily(), column.getQualifier());
402         } else {
403           out.addFamily(column.getFamily());
404         }
405       }
406     }
407 
408     TTimeRange timeRange = in.getTimeRange();
409     if (timeRange != null &&
410         timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
411       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
412     }
413 
414     if (in.isSetBatchSize()) {
415       out.setBatch(in.getBatchSize());
416     }
417 
418     if (in.isSetFilterString()) {
419       ParseFilter parseFilter = new ParseFilter();
420       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
421     }
422 
423     if (in.isSetAttributes()) {
424       addAttributes(out,in.getAttributes());
425     }
426     
427     if (in.isSetAuthorizations()) {
428       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
429     }
430 
431     if (in.isSetReversed()) {
432       out.setReversed(in.isReversed());
433     }
434 
435     if (in.isSetCacheBlocks()) {
436       out.setCacheBlocks(in.isCacheBlocks());
437     }
438 
439     if (in.isSetColFamTimeRangeMap()) {
440       Map<ByteBuffer, TTimeRange> colFamTimeRangeMap = in.getColFamTimeRangeMap();
441       if (MapUtils.isNotEmpty(colFamTimeRangeMap)) {
442         for (Map.Entry<ByteBuffer, TTimeRange> entry : colFamTimeRangeMap.entrySet()) {
443           out.setColumnFamilyTimeRange(Bytes.toBytes(entry.getKey()),
444               entry.getValue().getMinStamp(), entry.getValue().getMaxStamp());
445         }
446       }
447     }
448 
449     if (in.isSetSmall()) {
450       out.setSmall(in.isSmall());
451     }
452 
453     return out;
454   }
455 
456   public static Increment incrementFromThrift(TIncrement in) throws IOException {
457     Increment out = new Increment(in.getRow());
458     for (TColumnIncrement column : in.getColumns()) {
459       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
460     }
461 
462     if (in.isSetAttributes()) {
463       addAttributes(out,in.getAttributes());
464     }
465 
466     if (in.isSetDurability()) {
467       out.setDurability(durabilityFromThrift(in.getDurability()));
468     }
469     
470     if(in.getCellVisibility() != null) {
471       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
472     }
473 
474     return out;
475   }
476 
477   public static Append appendFromThrift(TAppend append) throws IOException {
478     Append out = new Append(append.getRow());
479     for (TColumnValue column : append.getColumns()) {
480       out.add(column.getFamily(), column.getQualifier(), column.getValue());
481     }
482 
483     if (append.isSetAttributes()) {
484       addAttributes(out, append.getAttributes());
485     }
486 
487     if (append.isSetDurability()) {
488       out.setDurability(durabilityFromThrift(append.getDurability()));
489     }
490     
491     if(append.getCellVisibility() != null) {
492       out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
493     }
494 
495     return out;
496   }
497 
498   public static THRegionLocation regionLocationFromHBase(HRegionLocation hrl) {
499     HRegionInfo hri = hrl.getRegionInfo();
500     ServerName serverName = hrl.getServerName();
501 
502     THRegionInfo thRegionInfo = new THRegionInfo();
503     THRegionLocation thRegionLocation = new THRegionLocation();
504     TServerName tServerName = new TServerName();
505 
506     tServerName.setHostName(serverName.getHostname());
507     tServerName.setPort(serverName.getPort());
508     tServerName.setStartCode(serverName.getStartcode());
509 
510     thRegionInfo.setTableName(hri.getTable().getName());
511     thRegionInfo.setEndKey(hri.getEndKey());
512     thRegionInfo.setStartKey(hri.getStartKey());
513     thRegionInfo.setOffline(hri.isOffline());
514     thRegionInfo.setSplit(hri.isSplit());
515     thRegionInfo.setReplicaId(hri.getReplicaId());
516 
517     thRegionLocation.setRegionInfo(thRegionInfo);
518     thRegionLocation.setServerName(tServerName);
519 
520     return thRegionLocation;
521   }
522 
523   public static List<THRegionLocation> regionLocationsFromHBase(List<HRegionLocation> locations) {
524     List<THRegionLocation> tlocations = new ArrayList<THRegionLocation>(locations.size());
525     for (HRegionLocation hrl:locations) {
526       tlocations.add(regionLocationFromHBase(hrl));
527     }
528     return tlocations;
529   }
530 
531   /**
532    * Adds all the attributes into the Operation object
533    */
534   private static void addAttributes(OperationWithAttributes op,
535                                     Map<ByteBuffer, ByteBuffer> attributes) {
536     if (attributes == null || attributes.size() == 0) {
537       return;
538     }
539     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
540       String name = Bytes.toStringBinary(getBytes(entry.getKey()));
541       byte[] value =  getBytes(entry.getValue());
542       op.setAttribute(name, value);
543     }
544   }
545 
546   private static Durability durabilityFromThrift(TDurability tDurability) {
547     switch (tDurability.getValue()) {
548       case 1: return Durability.SKIP_WAL;
549       case 2: return Durability.ASYNC_WAL;
550       case 3: return Durability.SYNC_WAL;
551       case 4: return Durability.FSYNC_WAL;
552       default: return null;
553     }
554   }
555 
556   public static CompareOp compareOpFromThrift(TCompareOp tCompareOp) {
557     switch (tCompareOp.getValue()) {
558       case 0: return CompareOp.LESS;
559       case 1: return CompareOp.LESS_OR_EQUAL;
560       case 2: return CompareOp.EQUAL;
561       case 3: return CompareOp.NOT_EQUAL;
562       case 4: return CompareOp.GREATER_OR_EQUAL;
563       case 5: return CompareOp.GREATER;
564       case 6: return CompareOp.NO_OP;
565       default: return null;
566     }
567   }
568 }