001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.shaded.protobuf;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.nio.ByteBuffer;
025import org.apache.hadoop.hbase.ByteBufferKeyValue;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellBuilderType;
028import org.apache.hadoop.hbase.CellComparatorImpl;
029import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.client.Append;
033import org.apache.hadoop.hbase.client.Delete;
034import org.apache.hadoop.hbase.client.Get;
035import org.apache.hadoop.hbase.client.Increment;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.io.TimeRange;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043
044import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
045import org.apache.hbase.thirdparty.com.google.protobuf.Any;
046import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
047import org.apache.hbase.thirdparty.com.google.protobuf.BytesValue;
048
049import org.apache.hadoop.hbase.shaded.protobuf.generated.CellProtos;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Column;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.DeleteType;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
060
061@Category(SmallTests.class)
062public class TestProtobufUtil {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestProtobufUtil.class);
067
068  public TestProtobufUtil() {
069  }
070
071  @Test
072  public void testException() throws IOException {
073    NameBytesPair.Builder builder = NameBytesPair.newBuilder();
074    final String omg = "OMG!!!";
075    builder.setName("java.io.IOException");
076    builder.setValue(ByteString.copyFrom(Bytes.toBytes(omg)));
077    Throwable t = ProtobufUtil.toException(builder.build());
078    assertEquals(omg, t.getMessage());
079    builder.clear();
080    builder.setName("org.apache.hadoop.ipc.RemoteException");
081    builder.setValue(ByteString.copyFrom(Bytes.toBytes(omg)));
082    t = ProtobufUtil.toException(builder.build());
083    assertEquals(omg, t.getMessage());
084  }
085
086  /**
087   * Test basic Get conversions.
088   *
089   * @throws IOException
090   */
091  @Test
092  public void testGet() throws IOException {
093    ClientProtos.Get.Builder getBuilder = ClientProtos.Get.newBuilder();
094    getBuilder.setRow(ByteString.copyFromUtf8("row"));
095    Column.Builder columnBuilder = Column.newBuilder();
096    columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
097    columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
098    columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
099    getBuilder.addColumn(columnBuilder.build());
100
101    columnBuilder.clear();
102    columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
103    getBuilder.addColumn(columnBuilder.build());
104    getBuilder.setLoadColumnFamiliesOnDemand(true);
105    ClientProtos.Get proto = getBuilder.build();
106    // default fields
107    assertEquals(1, proto.getMaxVersions());
108    assertEquals(true, proto.getCacheBlocks());
109
110    // set the default value for equal comparison
111    getBuilder = ClientProtos.Get.newBuilder(proto);
112    getBuilder.setMaxVersions(1);
113    getBuilder.setCacheBlocks(true);
114    getBuilder.setTimeRange(ProtobufUtil.toTimeRange(TimeRange.allTime()));
115    Get get = ProtobufUtil.toGet(proto);
116    assertEquals(getBuilder.build(), ProtobufUtil.toGet(get));
117  }
118
119  /**
120   * Test Delete Mutate conversions.
121   *
122   * @throws IOException
123   */
124  @Test
125  public void testDelete() throws IOException {
126    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
127    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
128    mutateBuilder.setMutateType(MutationType.DELETE);
129    mutateBuilder.setTimestamp(111111);
130    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
131    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
132    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
133    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
134    qualifierBuilder.setDeleteType(DeleteType.DELETE_ONE_VERSION);
135    qualifierBuilder.setTimestamp(111222);
136    valueBuilder.addQualifierValue(qualifierBuilder.build());
137    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
138    qualifierBuilder.setDeleteType(DeleteType.DELETE_MULTIPLE_VERSIONS);
139    qualifierBuilder.setTimestamp(111333);
140    valueBuilder.addQualifierValue(qualifierBuilder.build());
141    mutateBuilder.addColumnValue(valueBuilder.build());
142
143    MutationProto proto = mutateBuilder.build();
144    // default fields
145    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
146
147    // set the default value for equal comparison
148    mutateBuilder = MutationProto.newBuilder(proto);
149    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
150
151    Delete delete = ProtobufUtil.toDelete(proto);
152
153    // delete always have empty value,
154    // add empty value to the original mutate
155    for (ColumnValue.Builder column:
156        mutateBuilder.getColumnValueBuilderList()) {
157      for (QualifierValue.Builder qualifier:
158          column.getQualifierValueBuilderList()) {
159        qualifier.setValue(ByteString.EMPTY);
160      }
161    }
162    assertEquals(mutateBuilder.build(),
163      ProtobufUtil.toMutation(MutationType.DELETE, delete));
164  }
165
166  /**
167   * Test Put Mutate conversions.
168   *
169   * @throws IOException
170   */
171  @Test
172  public void testPut() throws IOException {
173    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
174    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
175    mutateBuilder.setMutateType(MutationType.PUT);
176    mutateBuilder.setTimestamp(111111);
177    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
178    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
179    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
180    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
181    qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
182    valueBuilder.addQualifierValue(qualifierBuilder.build());
183    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
184    qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
185    qualifierBuilder.setTimestamp(222222);
186    valueBuilder.addQualifierValue(qualifierBuilder.build());
187    mutateBuilder.addColumnValue(valueBuilder.build());
188
189    MutationProto proto = mutateBuilder.build();
190    // default fields
191    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
192
193    // set the default value for equal comparison
194    mutateBuilder = MutationProto.newBuilder(proto);
195    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
196
197    Put put = ProtobufUtil.toPut(proto);
198
199    // put value always use the default timestamp if no
200    // value level timestamp specified,
201    // add the timestamp to the original mutate
202    long timestamp = put.getTimestamp();
203    for (ColumnValue.Builder column:
204        mutateBuilder.getColumnValueBuilderList()) {
205      for (QualifierValue.Builder qualifier:
206          column.getQualifierValueBuilderList()) {
207        if (!qualifier.hasTimestamp()) {
208          qualifier.setTimestamp(timestamp);
209        }
210      }
211    }
212    assertEquals(mutateBuilder.build(),
213      ProtobufUtil.toMutation(MutationType.PUT, put));
214  }
215
216  /**
217   * Test basic Scan conversions.
218   *
219   * @throws IOException
220   */
221  @Test
222  public void testScan() throws IOException {
223    ClientProtos.Scan.Builder scanBuilder = ClientProtos.Scan.newBuilder();
224    scanBuilder.setStartRow(ByteString.copyFromUtf8("row1"));
225    scanBuilder.setStopRow(ByteString.copyFromUtf8("row2"));
226    Column.Builder columnBuilder = Column.newBuilder();
227    columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
228    columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
229    columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
230    scanBuilder.addColumn(columnBuilder.build());
231
232    columnBuilder.clear();
233    columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
234    scanBuilder.addColumn(columnBuilder.build());
235
236    ClientProtos.Scan proto = scanBuilder.build();
237
238    // Verify default values
239    assertEquals(1, proto.getMaxVersions());
240    assertEquals(true, proto.getCacheBlocks());
241
242    // Verify fields survive ClientProtos.Scan -> Scan -> ClientProtos.Scan
243    // conversion
244    scanBuilder = ClientProtos.Scan.newBuilder(proto);
245    scanBuilder.setMaxVersions(2);
246    scanBuilder.setCacheBlocks(false);
247    scanBuilder.setCaching(1024);
248    scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(TimeRange.allTime()));
249    ClientProtos.Scan expectedProto = scanBuilder.build();
250
251    ClientProtos.Scan actualProto = ProtobufUtil.toScan(
252        ProtobufUtil.toScan(expectedProto));
253    assertEquals(expectedProto, actualProto);
254  }
255
256  @Test
257  public void testToCell() throws Exception {
258    KeyValue kv1 =
259        new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
260    KeyValue kv2 =
261        new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
262    KeyValue kv3 =
263        new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
264    byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
265    System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
266    System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
267    System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(),
268      kv3.getLength());
269    ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
270    dbb.put(arr);
271    ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
272    CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV);
273    Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell);
274    assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
275  }
276
277  /**
278   * Test Increment Mutate conversions.
279   *
280   * @throws IOException
281   */
282  @Test
283  public void testIncrement() throws IOException {
284    long timeStamp = 111111;
285    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
286    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
287    mutateBuilder.setMutateType(MutationProto.MutationType.INCREMENT);
288    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
289    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
290    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
291    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
292    qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
293    qualifierBuilder.setTimestamp(timeStamp);
294    valueBuilder.addQualifierValue(qualifierBuilder.build());
295    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
296    qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
297    valueBuilder.addQualifierValue(qualifierBuilder.build());
298    mutateBuilder.addColumnValue(valueBuilder.build());
299
300    MutationProto proto = mutateBuilder.build();
301    // default fields
302    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
303
304    // set the default value for equal comparison
305    mutateBuilder = MutationProto.newBuilder(proto);
306    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
307
308    Increment increment = ProtobufUtil.toIncrement(proto, null);
309    mutateBuilder.setTimestamp(increment.getTimestamp());
310    mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
311    assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
312  }
313
314  /**
315   * Test Append Mutate conversions.
316   *
317   * @throws IOException
318   */
319  @Test
320  public void testAppend() throws IOException {
321    long timeStamp = 111111;
322    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
323    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
324    mutateBuilder.setMutateType(MutationType.APPEND);
325    mutateBuilder.setTimestamp(timeStamp);
326    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
327    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
328    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
329    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
330    qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
331    qualifierBuilder.setTimestamp(timeStamp);
332    valueBuilder.addQualifierValue(qualifierBuilder.build());
333    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
334    qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
335    valueBuilder.addQualifierValue(qualifierBuilder.build());
336    mutateBuilder.addColumnValue(valueBuilder.build());
337
338    MutationProto proto = mutateBuilder.build();
339    // default fields
340    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
341
342    // set the default value for equal comparison
343    mutateBuilder = MutationProto.newBuilder(proto);
344    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
345
346    Append append = ProtobufUtil.toAppend(proto, null);
347
348    // append always use the latest timestamp,
349    // reset the timestamp to the original mutate
350    mutateBuilder.setTimestamp(append.getTimestamp());
351    mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
352    assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
353  }
354
355  private static ProcedureProtos.Procedure.Builder createProcedureBuilder(long procId) {
356    ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder();
357    builder.setProcId(procId);
358    builder.setClassName("java.lang.Object");
359    builder.setSubmittedTime(0);
360    builder.setState(ProcedureProtos.ProcedureState.RUNNABLE);
361    builder.setLastUpdate(0);
362
363    return builder;
364  }
365
366  private static ProcedureProtos.Procedure createProcedure(long procId) {
367    return createProcedureBuilder(procId).build();
368  }
369
370  private static LockServiceProtos.LockedResource createLockedResource(
371      LockServiceProtos.LockedResourceType resourceType, String resourceName,
372      LockServiceProtos.LockType lockType,
373      ProcedureProtos.Procedure exclusiveLockOwnerProcedure, int sharedLockCount) {
374    LockServiceProtos.LockedResource.Builder build = LockServiceProtos.LockedResource.newBuilder();
375    build.setResourceType(resourceType);
376    build.setResourceName(resourceName);
377    build.setLockType(lockType);
378    if (exclusiveLockOwnerProcedure != null) {
379      build.setExclusiveLockOwnerProcedure(exclusiveLockOwnerProcedure);
380    }
381    build.setSharedLockCount(sharedLockCount);
382
383    return build.build();
384  }
385
386  @Test
387  public void testProcedureInfo() {
388    ProcedureProtos.Procedure.Builder builder = createProcedureBuilder(1);
389    ByteString stateBytes = ByteString.copyFrom(new byte[] { 65 });
390    BytesValue state = BytesValue.newBuilder().setValue(stateBytes).build();
391    builder.addStateMessage(Any.pack(state));
392    ProcedureProtos.Procedure procedure = builder.build();
393
394    String procJson = ProtobufUtil.toProcedureJson(Lists.newArrayList(procedure));
395    assertEquals("[{"
396        + "\"className\":\"java.lang.Object\","
397        + "\"procId\":\"1\","
398        + "\"submittedTime\":\"0\","
399        + "\"state\":\"RUNNABLE\","
400        + "\"lastUpdate\":\"0\","
401        + "\"stateMessage\":[{\"value\":\"QQ==\"}]"
402        + "}]", procJson);
403  }
404
405  @Test
406  public void testServerLockInfo() {
407    LockServiceProtos.LockedResource resource = createLockedResource(
408        LockServiceProtos.LockedResourceType.SERVER, "server",
409        LockServiceProtos.LockType.SHARED, null, 2);
410
411    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
412    assertEquals("[{"
413        + "\"resourceType\":\"SERVER\","
414        + "\"resourceName\":\"server\","
415        + "\"lockType\":\"SHARED\","
416        + "\"sharedLockCount\":2"
417        + "}]", lockJson);
418  }
419
420  @Test
421  public void testNamespaceLockInfo() {
422    LockServiceProtos.LockedResource resource = createLockedResource(
423        LockServiceProtos.LockedResourceType.NAMESPACE, "ns",
424        LockServiceProtos.LockType.EXCLUSIVE, createProcedure(2), 0);
425
426    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
427    assertEquals("[{"
428        + "\"resourceType\":\"NAMESPACE\","
429        + "\"resourceName\":\"ns\","
430        + "\"lockType\":\"EXCLUSIVE\","
431        + "\"exclusiveLockOwnerProcedure\":{"
432          + "\"className\":\"java.lang.Object\","
433          + "\"procId\":\"2\","
434          + "\"submittedTime\":\"0\","
435          + "\"state\":\"RUNNABLE\","
436          + "\"lastUpdate\":\"0\""
437        + "},"
438        + "\"sharedLockCount\":0"
439        + "}]", lockJson);
440  }
441
442  @Test
443  public void testTableLockInfo() {
444    LockServiceProtos.LockedResource resource = createLockedResource(
445        LockServiceProtos.LockedResourceType.TABLE, "table",
446        LockServiceProtos.LockType.SHARED, null, 2);
447
448    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
449    assertEquals("[{"
450        + "\"resourceType\":\"TABLE\","
451        + "\"resourceName\":\"table\","
452        + "\"lockType\":\"SHARED\","
453        + "\"sharedLockCount\":2"
454        + "}]", lockJson);
455  }
456
457  @Test
458  public void testRegionLockInfo() {
459    LockServiceProtos.LockedResource resource = createLockedResource(
460        LockServiceProtos.LockedResourceType.REGION, "region",
461        LockServiceProtos.LockType.EXCLUSIVE, createProcedure(3), 0);
462
463    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
464    assertEquals("[{"
465        + "\"resourceType\":\"REGION\","
466        + "\"resourceName\":\"region\","
467        + "\"lockType\":\"EXCLUSIVE\","
468        + "\"exclusiveLockOwnerProcedure\":{"
469          + "\"className\":\"java.lang.Object\","
470          + "\"procId\":\"3\","
471          + "\"submittedTime\":\"0\","
472          + "\"state\":\"RUNNABLE\","
473          + "\"lastUpdate\":\"0\""
474        + "},"
475        + "\"sharedLockCount\":0"
476        + "}]", lockJson);
477  }
478}