View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.IOException;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.HRegionLocation;
25  import org.apache.hadoop.hbase.client.HTable;
26  import org.apache.hadoop.hbase.testclassification.LargeTests;
27  import org.junit.Test;
28  import org.junit.experimental.categories.Category;
29  
30  /**
31   * TestTableInputFormatScan part 1.
32   * @see TestTableInputFormatScanBase
33   */
34  @Category(LargeTests.class)
35  public class TestTableInputFormatScan1 extends TestTableInputFormatScanBase {
36  
37    /**
38     * Tests a MR scan using specific start and stop rows.
39     *
40     * @throws IOException
41     * @throws ClassNotFoundException
42     * @throws InterruptedException
43     */
44    @Test
45    public void testScanEmptyToEmpty()
46    throws IOException, InterruptedException, ClassNotFoundException {
47      testScan(null, null, null);
48    }
49  
50    /**
51     * Tests a MR scan using specific start and stop rows.
52     *
53     * @throws IOException
54     * @throws ClassNotFoundException
55     * @throws InterruptedException
56     */
57    @Test
58    public void testScanEmptyToAPP()
59    throws IOException, InterruptedException, ClassNotFoundException {
60      testScan(null, "app", "apo");
61    }
62  
63    /**
64     * Tests a MR scan using specific start and stop rows.
65     *
66     * @throws IOException
67     * @throws ClassNotFoundException
68     * @throws InterruptedException
69     */
70    @Test
71    public void testScanEmptyToBBA()
72    throws IOException, InterruptedException, ClassNotFoundException {
73      testScan(null, "bba", "baz");
74    }
75  
76    /**
77     * Tests a MR scan using specific start and stop rows.
78     *
79     * @throws IOException
80     * @throws ClassNotFoundException
81     * @throws InterruptedException
82     */
83    @Test
84    public void testScanEmptyToBBB()
85    throws IOException, InterruptedException, ClassNotFoundException {
86      testScan(null, "bbb", "bba");
87    }
88  
89    /**
90     * Tests a MR scan using specific start and stop rows.
91     *
92     * @throws IOException
93     * @throws ClassNotFoundException
94     * @throws InterruptedException
95     */
96    @Test
97    public void testScanEmptyToOPP()
98    throws IOException, InterruptedException, ClassNotFoundException {
99      testScan(null, "opp", "opo");
100   }
101 
102   /**
103    * Tests a MR scan using specific number of mappers. The test table has 25 regions,
104    * and all region sizes are set as 0 as default. The average region size is 1 (the smallest
105    * positive). When we set hbase.mapreduce.input.ratio as -1, all regions will be cut into two
106    * MapRedcue input splits, the number of MR input splits should be 50; when we set hbase
107    * .mapreduce.input.ratio as 100, the sum of all region sizes is less then the average region
108    * size, all regions will be combined into 1 MapRedcue input split.
109    *
110    * @throws IOException
111    * @throws ClassNotFoundException
112    * @throws InterruptedException
113    */
114   @Test
115   public void testGetSplits() throws IOException, InterruptedException, ClassNotFoundException {
116     HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
117     List<HRegionLocation> locs = table.getRegionLocator().getAllRegionLocations();
118 
119     testNumOfSplits("-1", locs.size()*2);
120     table.close();
121     testNumOfSplits("100", 1);
122   }
123 
124   /**
125    * Tests the getSplitKey() method in TableInputFormatBase.java
126    *
127    * @throws IOException
128    * @throws ClassNotFoundException
129    * @throws InterruptedException
130    */
131   @Test
132   public void testGetSplitsPoint() throws IOException, InterruptedException,
133   ClassNotFoundException {
134     byte[] start1 = { 'a', 'a', 'a', 'b', 'c', 'd', 'e', 'f' };
135     byte[] end1 = { 'a', 'a', 'a', 'f', 'f' };
136     byte[] splitPoint1 = { 'a', 'a', 'a', 'd', 'd', -78, 50, -77  };
137     testGetSplitKey(start1, end1, splitPoint1, true);
138 
139     byte[] start2 = { '1', '1', '1', '0', '0', '0' };
140     byte[] end2 = { '1', '1', '2', '5', '7', '9', '0' };
141     byte[] splitPoint2 = { '1', '1', '1',  -78, -77, -76, -104 };
142     testGetSplitKey(start2, end2, splitPoint2, true);
143 
144     byte[] start3 = { 'a', 'a', 'a', 'a', 'a', 'a' };
145     byte[] end3 = { 'a', 'a', 'b' };
146     byte[] splitPoint3 = { 'a', 'a', 'a', -80, -80, -80 };
147     testGetSplitKey(start3, end3, splitPoint3, true);
148 
149     byte[] start4 = { 'a', 'a', 'a' };
150     byte[] end4 = { 'a', 'a', 'a', 'z' };
151     byte[] splitPoint4 = { 'a', 'a', 'a', '=' };
152     testGetSplitKey(start4, end4, splitPoint4, true);
153 
154     byte[] start5 = { 'a', 'a', 'a' };
155     byte[] end5 = { 'a', 'a', 'b', 'a' };
156     byte[] splitPoint5 = { 'a', 'a', 'a', -80 };
157     testGetSplitKey(start5, end5, splitPoint5, true);
158 
159     // Test Case 6: empty key and "hhhqqqwww", split point is "h"
160     byte[] start6 = {};
161     byte[] end6 = { 'h', 'h', 'h', 'q', 'q', 'q', 'w', 'w' };
162     byte[] splitPointText6 = { 'h' };
163     byte[] splitPointBinary6 = { 104 };
164     testGetSplitKey(start6, end6, splitPointText6, true);
165     testGetSplitKey(start6, end6, splitPointBinary6, false);
166 
167     // Test Case 7: "ffffaaa" and empty key, split point depends on the mode we choose(text key or
168     // binary key).
169     byte[] start7 = { 'f', 'f', 'f', 'f', 'a', 'a', 'a' };
170     byte[] end7 = {};
171     byte[] splitPointText7 = { 'f', '~', '~', '~', '~', '~', '~'  };
172     byte[] splitPointBinary7 = { 'f', -1, -1, -1, -1, -1, -1  };
173     testGetSplitKey(start7, end7, splitPointText7, true);
174     testGetSplitKey(start7, end7, splitPointBinary7, false);
175 
176     // Test Case 8: both start key and end key are empty. Split point depends on the mode we
177     // choose (text key or binary key).
178     byte[] start8 = {};
179     byte[] end8 = {};
180     byte[] splitPointText8 = { 'O' };
181     byte[] splitPointBinary8 = { 0 };
182     testGetSplitKey(start8, end8, splitPointText8, true);
183     testGetSplitKey(start8, end8, splitPointBinary8, false);
184 
185     // Test Case 9: Binary Key example
186     byte[] start9 = { 13, -19, 126, 127 };
187     byte[] end9 = { 13, -19, 127, 0 };
188     byte[] splitPoint9 = { 13, -19, 126, -65 };
189     testGetSplitKey(start9, end9, splitPoint9, false);
190 
191     // Test Case 10: Binary key split when the start key is an unsigned byte and the end byte is a
192     // signed byte
193     byte[] start10 = { 'x' };
194     byte[] end10 = { -128 };
195     byte[] splitPoint10 = { '|' };
196     testGetSplitKey(start10, end10, splitPoint10, false);
197 
198     // Test Case 11: Binary key split when the start key is an signed byte and the end byte is a
199     // signed byte
200     byte[] start11 = { -100 };
201     byte[] end11 = { -90 };
202     byte[] splitPoint11 = { -95 };
203     testGetSplitKey(start11, end11, splitPoint11, false);
204   }
205 }