001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.ByteArrayInputStream;
021import java.io.ByteArrayOutputStream;
022import java.io.DataInputStream;
023import java.io.DataOutputStream;
024import java.io.IOException;
025import java.math.BigDecimal;
026import java.nio.ByteBuffer;
027import java.util.ArrayList;
028import java.util.Arrays;
029import java.util.List;
030import java.util.Random;
031import junit.framework.TestCase;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.testclassification.MiscTests;
034import org.apache.hadoop.hbase.testclassification.SmallTests;
035import org.apache.hadoop.io.WritableUtils;
036import org.junit.Assert;
037import org.junit.ClassRule;
038import org.junit.experimental.categories.Category;
039
040@Category({MiscTests.class, SmallTests.class})
041public class TestBytes extends TestCase {
042
043  @ClassRule
044  public static final HBaseClassTestRule CLASS_RULE =
045      HBaseClassTestRule.forClass(TestBytes.class);
046
047  public void testNullHashCode() {
048    byte [] b = null;
049    Exception ee = null;
050    try {
051      Bytes.hashCode(b);
052    } catch (Exception e) {
053      ee = e;
054    }
055    assertNotNull(ee);
056  }
057
058  public void testAdd () throws Exception {
059    byte[] a = {0,0,0,0,0,0,0,0,0,0};
060    byte[] b = {1,1,1,1,1,1,1,1,1,1,1};
061    byte[] c = {2,2,2,2,2,2,2,2,2,2,2,2};
062    byte[] d = {3,3,3,3,3,3,3,3,3,3,3,3,3};
063    byte[] result1 = Bytes.add (a, b, c);
064    byte[] result2 = Bytes.add (new byte[][] {a, b, c});
065    assertEquals(0, Bytes.compareTo(result1, result2));
066    byte[] result4 = Bytes.add (result1, d);
067    byte[] result5 = Bytes.add (new byte[][] {result1, d});
068    assertEquals(0, Bytes.compareTo(result1, result2));
069  }
070
071  public void testSplit() throws Exception {
072    byte [] lowest = Bytes.toBytes("AAA");
073    byte [] middle = Bytes.toBytes("CCC");
074    byte [] highest = Bytes.toBytes("EEE");
075    byte [][] parts = Bytes.split(lowest, highest, 1);
076    for (int i = 0; i < parts.length; i++) {
077      System.out.println(Bytes.toString(parts[i]));
078    }
079    assertEquals(3, parts.length);
080    assertTrue(Bytes.equals(parts[1], middle));
081    // Now divide into three parts.  Change highest so split is even.
082    highest = Bytes.toBytes("DDD");
083    parts = Bytes.split(lowest, highest, 2);
084    for (int i = 0; i < parts.length; i++) {
085      System.out.println(Bytes.toString(parts[i]));
086    }
087    assertEquals(4, parts.length);
088    // Assert that 3rd part is 'CCC'.
089    assertTrue(Bytes.equals(parts[2], middle));
090  }
091
092  public void testSplit2() throws Exception {
093    // More split tests.
094    byte [] lowest = Bytes.toBytes("http://A");
095    byte [] highest = Bytes.toBytes("http://z");
096    byte [] middle = Bytes.toBytes("http://]");
097    byte [][] parts = Bytes.split(lowest, highest, 1);
098    for (int i = 0; i < parts.length; i++) {
099      System.out.println(Bytes.toString(parts[i]));
100    }
101    assertEquals(3, parts.length);
102    assertTrue(Bytes.equals(parts[1], middle));
103  }
104
105  public void testSplit3() throws Exception {
106    // Test invalid split cases
107    byte [] low = { 1, 1, 1 };
108    byte [] high = { 1, 1, 3 };
109
110    // If swapped, should throw IAE
111    try {
112      Bytes.split(high, low, 1);
113      assertTrue("Should not be able to split if low > high", false);
114    } catch(IllegalArgumentException iae) {
115      // Correct
116    }
117
118    // Single split should work
119    byte [][] parts = Bytes.split(low, high, 1);
120    for (int i = 0; i < parts.length; i++) {
121      System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i]));
122    }
123    assertTrue("Returned split should have 3 parts but has " + parts.length, parts.length == 3);
124
125    // If split more than once, use additional byte to split
126    parts = Bytes.split(low, high, 2);
127    assertTrue("Split with an additional byte", parts != null);
128    assertEquals(parts.length, low.length + 1);
129
130    // Split 0 times should throw IAE
131    try {
132      parts = Bytes.split(low, high, 0);
133      assertTrue("Should not be able to split 0 times", false);
134    } catch(IllegalArgumentException iae) {
135      // Correct
136    }
137  }
138
139  public void testToInt() throws Exception {
140    int [] ints = {-1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE};
141    for (int i = 0; i < ints.length; i++) {
142      byte [] b = Bytes.toBytes(ints[i]);
143      assertEquals(ints[i], Bytes.toInt(b));
144      byte [] b2 = bytesWithOffset(b);
145      assertEquals(ints[i], Bytes.toInt(b2, 1));
146      assertEquals(ints[i], Bytes.toInt(b2, 1, Bytes.SIZEOF_INT));
147    }
148  }
149
150  public void testToLong() throws Exception {
151    long [] longs = {-1l, 123l, Long.MIN_VALUE, Long.MAX_VALUE};
152    for (int i = 0; i < longs.length; i++) {
153      byte [] b = Bytes.toBytes(longs[i]);
154      assertEquals(longs[i], Bytes.toLong(b));
155      byte [] b2 = bytesWithOffset(b);
156      assertEquals(longs[i], Bytes.toLong(b2, 1));
157      assertEquals(longs[i], Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG));
158    }
159  }
160
161  public void testToFloat() throws Exception {
162    float [] floats = {-1f, 123.123f, Float.MAX_VALUE};
163    for (int i = 0; i < floats.length; i++) {
164      byte [] b = Bytes.toBytes(floats[i]);
165      assertEquals(floats[i], Bytes.toFloat(b), 0.0f);
166      byte [] b2 = bytesWithOffset(b);
167      assertEquals(floats[i], Bytes.toFloat(b2, 1), 0.0f);
168    }
169  }
170
171  public void testToDouble() throws Exception {
172    double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE};
173    for (int i = 0; i < doubles.length; i++) {
174      byte [] b = Bytes.toBytes(doubles[i]);
175      assertEquals(doubles[i], Bytes.toDouble(b), 0.0);
176      byte [] b2 = bytesWithOffset(b);
177      assertEquals(doubles[i], Bytes.toDouble(b2, 1), 0.0);
178    }
179  }
180
181  public void testToBigDecimal() throws Exception {
182    BigDecimal [] decimals = {new BigDecimal("-1"), new BigDecimal("123.123"),
183      new BigDecimal("123123123123")};
184    for (int i = 0; i < decimals.length; i++) {
185      byte [] b = Bytes.toBytes(decimals[i]);
186      assertEquals(decimals[i], Bytes.toBigDecimal(b));
187      byte [] b2 = bytesWithOffset(b);
188      assertEquals(decimals[i], Bytes.toBigDecimal(b2, 1, b.length));
189    }
190  }
191
192  private byte [] bytesWithOffset(byte [] src) {
193    // add one byte in front to test offset
194    byte [] result = new byte[src.length + 1];
195    result[0] = (byte) 0xAA;
196    System.arraycopy(src, 0, result, 1, src.length);
197    return result;
198  }
199
200  public void testToBytesForByteBuffer() {
201    byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
202    ByteBuffer target = ByteBuffer.wrap(array);
203    target.position(2);
204    target.limit(7);
205
206    byte[] actual = Bytes.toBytes(target);
207    byte[] expected = { 0, 1, 2, 3, 4, 5, 6 };
208    assertTrue(Arrays.equals(expected,  actual));
209    assertEquals(2, target.position());
210    assertEquals(7, target.limit());
211
212    ByteBuffer target2 = target.slice();
213    assertEquals(0, target2.position());
214    assertEquals(5, target2.limit());
215
216    byte[] actual2 = Bytes.toBytes(target2);
217    byte[] expected2 = { 2, 3, 4, 5, 6 };
218    assertTrue(Arrays.equals(expected2, actual2));
219    assertEquals(0, target2.position());
220    assertEquals(5, target2.limit());
221  }
222
223  public void testGetBytesForByteBuffer() {
224    byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
225    ByteBuffer target = ByteBuffer.wrap(array);
226    target.position(2);
227    target.limit(7);
228
229    byte[] actual = Bytes.getBytes(target);
230    byte[] expected = { 2, 3, 4, 5, 6 };
231    assertTrue(Arrays.equals(expected,  actual));
232    assertEquals(2, target.position());
233    assertEquals(7, target.limit());
234  }
235
236  public void testReadAsVLong() throws Exception {
237    long [] longs = {-1l, 123l, Long.MIN_VALUE, Long.MAX_VALUE};
238    for (int i = 0; i < longs.length; i++) {
239      ByteArrayOutputStream baos = new ByteArrayOutputStream();
240      DataOutputStream output = new DataOutputStream(baos);
241      WritableUtils.writeVLong(output, longs[i]);
242      byte[] long_bytes_no_offset = baos.toByteArray();
243      assertEquals(longs[i], Bytes.readAsVLong(long_bytes_no_offset, 0));
244      byte[] long_bytes_with_offset = bytesWithOffset(long_bytes_no_offset);
245      assertEquals(longs[i], Bytes.readAsVLong(long_bytes_with_offset, 1));
246    }
247  }
248
249  public void testToStringBinaryForBytes() {
250    byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 };
251    String actual = Bytes.toStringBinary(array);
252    String expected = "09azAZ@\\x01";
253    assertEquals(expected, actual);
254
255    String actual2 = Bytes.toStringBinary(array, 2, 3);
256    String expected2 = "azA";
257    assertEquals(expected2, actual2);
258  }
259
260  public void testToStringBinaryForArrayBasedByteBuffer() {
261    byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 };
262    ByteBuffer target = ByteBuffer.wrap(array);
263    String actual = Bytes.toStringBinary(target);
264    String expected = "09azAZ@\\x01";
265    assertEquals(expected, actual);
266  }
267
268  public void testToStringBinaryForReadOnlyByteBuffer() {
269    byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 };
270    ByteBuffer target = ByteBuffer.wrap(array).asReadOnlyBuffer();
271    String actual = Bytes.toStringBinary(target);
272    String expected = "09azAZ@\\x01";
273    assertEquals(expected, actual);
274  }
275
276  public void testBinarySearch() throws Exception {
277    byte [][] arr = {
278        {1},
279        {3},
280        {5},
281        {7},
282        {9},
283        {11},
284        {13},
285        {15},
286    };
287    byte [] key1 = {3,1};
288    byte [] key2 = {4,9};
289    byte [] key2_2 = {4};
290    byte [] key3 = {5,11};
291    byte [] key4 = {0};
292    byte [] key5 = {2};
293
294    assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1));
295    assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1));
296    assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2,
297      Bytes.BYTES_COMPARATOR));
298    assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1));
299    assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1));
300    assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1));
301    assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1));
302    assertEquals(-1,
303      Bytes.binarySearch(arr, key4, 0, 1));
304    assertEquals(-2,
305      Bytes.binarySearch(arr, key5, 0, 1));
306
307    // Search for values to the left and to the right of each item in the array.
308    for (int i = 0; i < arr.length; ++i) {
309      assertEquals(-(i + 1), Bytes.binarySearch(arr,
310          new byte[] { (byte) (arr[i][0] - 1) }, 0, 1));
311      assertEquals(-(i + 2), Bytes.binarySearch(arr,
312          new byte[] { (byte) (arr[i][0] + 1) }, 0, 1));
313    }
314  }
315
316  public void testToStringBytesBinaryReversible() {
317    //  let's run test with 1000 randomly generated byte arrays
318    Random rand = new Random(System.currentTimeMillis());
319    byte[] randomBytes = new byte[1000];
320    for (int i = 0; i < 1000; i++) {
321      rand.nextBytes(randomBytes);
322      verifyReversibleForBytes(randomBytes);
323    }
324
325    //  some specific cases
326    verifyReversibleForBytes(new  byte[] {});
327    verifyReversibleForBytes(new  byte[] {'\\', 'x', 'A', 'D'});
328    verifyReversibleForBytes(new  byte[] {'\\', 'x', 'A', 'D', '\\'});
329  }
330
331  private void verifyReversibleForBytes(byte[] originalBytes) {
332    String convertedString = Bytes.toStringBinary(originalBytes);
333    byte[] convertedBytes = Bytes.toBytesBinary(convertedString);
334    if (Bytes.compareTo(originalBytes, convertedBytes) != 0) {
335      fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) +
336          ",\nStringBinary: " + convertedString);
337    }
338  }
339
340  public void testStartsWith() {
341    assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h")));
342    assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("")));
343    assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello")));
344    assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld")));
345    assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello")));
346  }
347
348  public void testIncrementBytes() throws IOException {
349
350    assertTrue(checkTestIncrementBytes(10, 1));
351    assertTrue(checkTestIncrementBytes(12, 123435445));
352    assertTrue(checkTestIncrementBytes(124634654, 1));
353    assertTrue(checkTestIncrementBytes(10005460, 5005645));
354    assertTrue(checkTestIncrementBytes(1, -1));
355    assertTrue(checkTestIncrementBytes(10, -1));
356    assertTrue(checkTestIncrementBytes(10, -5));
357    assertTrue(checkTestIncrementBytes(1005435000, -5));
358    assertTrue(checkTestIncrementBytes(10, -43657655));
359    assertTrue(checkTestIncrementBytes(-1, 1));
360    assertTrue(checkTestIncrementBytes(-26, 5034520));
361    assertTrue(checkTestIncrementBytes(-10657200, 5));
362    assertTrue(checkTestIncrementBytes(-12343250, 45376475));
363    assertTrue(checkTestIncrementBytes(-10, -5));
364    assertTrue(checkTestIncrementBytes(-12343250, -5));
365    assertTrue(checkTestIncrementBytes(-12, -34565445));
366    assertTrue(checkTestIncrementBytes(-1546543452, -34565445));
367  }
368
369  private static boolean checkTestIncrementBytes(long val, long amount)
370  throws IOException {
371    byte[] value = Bytes.toBytes(val);
372    byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1};
373    if (value[0] > 0) {
374      testValue = new byte[Bytes.SIZEOF_LONG];
375    }
376    System.arraycopy(value, 0, testValue, testValue.length - value.length,
377        value.length);
378
379    long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount));
380
381    return (Bytes.toLong(testValue) + amount) == incrementResult;
382  }
383
384  public void testFixedSizeString() throws IOException {
385    ByteArrayOutputStream baos = new ByteArrayOutputStream();
386    DataOutputStream dos = new DataOutputStream(baos);
387    Bytes.writeStringFixedSize(dos, "Hello", 5);
388    Bytes.writeStringFixedSize(dos, "World", 18);
389    Bytes.writeStringFixedSize(dos, "", 9);
390
391    try {
392      // Use a long dash which is three bytes in UTF-8. If encoding happens
393      // using ISO-8859-1, this will fail.
394      Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9);
395      fail("Exception expected");
396    } catch (IOException ex) {
397      assertEquals(
398          "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " +
399          "length 9", ex.getMessage());
400    }
401
402    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
403    DataInputStream dis = new DataInputStream(bais);
404    assertEquals("Hello", Bytes.readStringFixedSize(dis, 5));
405    assertEquals("World", Bytes.readStringFixedSize(dis, 18));
406    assertEquals("", Bytes.readStringFixedSize(dis, 9));
407  }
408
409  public void testCopy() throws Exception {
410    byte [] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
411    byte [] copy =  Bytes.copy(bytes);
412    assertFalse(bytes == copy);
413    assertTrue(Bytes.equals(bytes, copy));
414  }
415
416  public void testToBytesBinaryTrailingBackslashes() throws Exception {
417    try {
418      Bytes.toBytesBinary("abc\\x00\\x01\\");
419    } catch (StringIndexOutOfBoundsException ex) {
420      fail("Illegal string access: " + ex.getMessage());
421    }
422  }
423
424  public void testToStringBinary_toBytesBinary_Reversable() throws Exception {
425    String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17));
426    assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0);
427  }
428
429  public void testUnsignedBinarySearch(){
430    byte[] bytes = new byte[]{0,5,123,127,-128,-100,-1};
431    Assert.assertEquals(1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)5));
432    Assert.assertEquals(3, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)127));
433    Assert.assertEquals(4, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-128));
434    Assert.assertEquals(5, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-100));
435    Assert.assertEquals(6, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-1));
436    Assert.assertEquals(-1-1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)2));
437    Assert.assertEquals(-6-1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-5));
438  }
439
440  public void testUnsignedIncrement(){
441    byte[] a = Bytes.toBytes(0);
442    int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0);
443    Assert.assertTrue(a2==1);
444
445    byte[] b = Bytes.toBytes(-1);
446    byte[] actual = Bytes.unsignedCopyAndIncrement(b);
447    Assert.assertNotSame(b, actual);
448    byte[] expected = new byte[]{1,0,0,0,0};
449    Assert.assertArrayEquals(expected, actual);
450
451    byte[] c = Bytes.toBytes(255);//should wrap to the next significant byte
452    int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0);
453    Assert.assertTrue(c2==256);
454  }
455
456  public void testIndexOf() {
457    byte[] array = Bytes.toBytes("hello");
458    assertEquals(1, Bytes.indexOf(array, (byte) 'e'));
459    assertEquals(4, Bytes.indexOf(array, (byte) 'o'));
460    assertEquals(-1, Bytes.indexOf(array, (byte) 'a'));
461    assertEquals(0, Bytes.indexOf(array, Bytes.toBytes("hel")));
462    assertEquals(2, Bytes.indexOf(array, Bytes.toBytes("ll")));
463    assertEquals(-1, Bytes.indexOf(array, Bytes.toBytes("hll")));
464  }
465
466  public void testContains() {
467    byte[] array = Bytes.toBytes("hello world");
468    assertTrue(Bytes.contains(array, (byte) 'e'));
469    assertTrue(Bytes.contains(array, (byte) 'd'));
470    assertFalse( Bytes.contains(array, (byte) 'a'));
471    assertTrue(Bytes.contains(array, Bytes.toBytes("world")));
472    assertTrue(Bytes.contains(array, Bytes.toBytes("ello")));
473    assertFalse(Bytes.contains(array, Bytes.toBytes("owo")));
474  }
475
476  public void testZero() {
477    byte[] array = Bytes.toBytes("hello");
478    Bytes.zero(array);
479    for (int i = 0; i < array.length; i++) {
480      assertEquals(0, array[i]);
481    }
482    array = Bytes.toBytes("hello world");
483    Bytes.zero(array, 2, 7);
484    assertFalse(array[0] == 0);
485    assertFalse(array[1] == 0);
486    for (int i = 2; i < 9; i++) {
487      assertEquals(0, array[i]);
488    }
489    for (int i = 9; i < array.length; i++) {
490      assertFalse(array[i] == 0);
491    }
492  }
493
494  public void testPutBuffer() {
495    byte[] b = new byte[100];
496    for (byte i = 0; i < 100; i++) {
497      Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[]{i}));
498    }
499    for (byte i = 0; i < 100; i++) {
500      Assert.assertEquals(i, b[i]);
501    }
502  }
503
504  public void testToFromHex() {
505    List<String> testStrings = new ArrayList<>(8);
506    testStrings.addAll(Arrays.asList(new String[] {
507        "",
508        "00",
509        "A0",
510        "ff",
511        "FFffFFFFFFFFFF",
512        "12",
513        "0123456789abcdef",
514        "283462839463924623984692834692346ABCDFEDDCA0",
515      }));
516    for (String testString : testStrings)
517    {
518      byte[] byteData = Bytes.fromHex(testString);
519      Assert.assertEquals(testString.length() / 2, byteData.length);
520      String result = Bytes.toHex(byteData);
521      Assert.assertTrue(testString.equalsIgnoreCase(result));
522    }
523
524    List<byte[]> testByteData = new ArrayList<>(5);
525    testByteData.addAll(Arrays.asList(new byte[][] {
526      new byte[0],
527      new byte[1],
528      new byte[10],
529      new byte[] {1, 2, 3, 4, 5},
530      new byte[] {(byte) 0xFF},
531    }));
532    Random r = new Random();
533    for (int i = 0; i < 20; i++)
534    {
535      byte[] bytes = new byte[r.nextInt(100)];
536      r.nextBytes(bytes);
537      testByteData.add(bytes);
538    }
539
540    for (byte[] testData : testByteData)
541    {
542      String hexString = Bytes.toHex(testData);
543      Assert.assertEquals(testData.length * 2, hexString.length());
544      byte[] result = Bytes.fromHex(hexString);
545      Assert.assertArrayEquals(testData, result);
546    }
547  }
548}
549