View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master.normalizer;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.HBaseIOException;
24  import org.apache.hadoop.hbase.HRegionInfo;
25  import org.apache.hadoop.hbase.RegionLoad;
26  import org.apache.hadoop.hbase.ServerName;
27  import org.apache.hadoop.hbase.TableName;
28  import org.apache.hadoop.hbase.master.MasterRpcServices;
29  import org.apache.hadoop.hbase.master.MasterServices;
30  import org.apache.hadoop.hbase.protobuf.RequestConverter;
31  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
32  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
33  import org.apache.hadoop.hbase.testclassification.SmallTests;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  import org.mockito.Mockito;
39  
40  import com.google.protobuf.RpcController;
41  import com.google.protobuf.ServiceException;
42  
43  import java.util.ArrayList;
44  import java.util.HashMap;
45  import java.util.List;
46  import java.util.Map;
47  
48  import static org.junit.Assert.assertEquals;
49  import static org.junit.Assert.assertTrue;
50  import static org.mockito.Matchers.any;
51  import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
52  import static org.mockito.Mockito.when;
53  
54  /**
55   * Tests logic of {@link SimpleRegionNormalizer}.
56   */
57  @Category(SmallTests.class)
58  public class TestSimpleRegionNormalizer {
59    private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizer.class);
60  
61    private static RegionNormalizer normalizer;
62  
63    // mocks
64    private static MasterServices masterServices;
65    private static MasterRpcServices masterRpcServices;
66  
67    @BeforeClass
68    public static void beforeAllTests() throws Exception {
69      normalizer = new SimpleRegionNormalizer();
70    }
71  
72    @Test
73    public void testNoNormalizationForMetaTable() throws HBaseIOException {
74      TableName testTable = TableName.META_TABLE_NAME;
75      List<HRegionInfo> hris = new ArrayList<>();
76      Map<byte[], Integer> regionSizes = new HashMap<>();
77  
78      setupMocksForNormalizer(regionSizes, hris);
79      List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
80      assertTrue(plans == null);
81    }
82  
83    @Test
84    public void testNoNormalizationIfTooFewRegions() throws HBaseIOException {
85      TableName testTable = TableName.valueOf("testSplitOfSmallRegion");
86      List<HRegionInfo> hris = new ArrayList<>();
87      Map<byte[], Integer> regionSizes = new HashMap<>();
88  
89      HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
90      hris.add(hri1);
91      regionSizes.put(hri1.getRegionName(), 10);
92  
93      HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
94      hris.add(hri2);
95      regionSizes.put(hri2.getRegionName(), 15);
96  
97      setupMocksForNormalizer(regionSizes, hris);
98      List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
99      assertTrue(plans == null);
100   }
101 
102   @Test
103   public void testNoNormalizationOnNormalizedCluster() throws HBaseIOException {
104     TableName testTable = TableName.valueOf("testSplitOfSmallRegion");
105     List<HRegionInfo> hris = new ArrayList<>();
106     Map<byte[], Integer> regionSizes = new HashMap<>();
107 
108     HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
109     hris.add(hri1);
110     regionSizes.put(hri1.getRegionName(), 10);
111 
112     HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
113     hris.add(hri2);
114     regionSizes.put(hri2.getRegionName(), 15);
115 
116     HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
117     hris.add(hri3);
118     regionSizes.put(hri3.getRegionName(), 8);
119 
120     HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
121     hris.add(hri4);
122     regionSizes.put(hri4.getRegionName(), 10);
123 
124     setupMocksForNormalizer(regionSizes, hris);
125     List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
126     assertTrue(plans == null);
127   }
128 
129   @Test
130   public void testMergeOfSmallRegions() throws HBaseIOException {
131     TableName testTable = TableName.valueOf("testMergeOfSmallRegions");
132     List<HRegionInfo> hris = new ArrayList<>();
133     Map<byte[], Integer> regionSizes = new HashMap<>();
134 
135     HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
136     hris.add(hri1);
137     regionSizes.put(hri1.getRegionName(), 15);
138 
139     HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
140     hris.add(hri2);
141     regionSizes.put(hri2.getRegionName(), 5);
142 
143     HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
144     hris.add(hri3);
145     regionSizes.put(hri3.getRegionName(), 5);
146 
147     HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
148     hris.add(hri4);
149     regionSizes.put(hri4.getRegionName(), 15);
150 
151     HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("eee"), Bytes.toBytes("fff"));
152     hris.add(hri5);
153     regionSizes.put(hri5.getRegionName(), 16);
154 
155     setupMocksForNormalizer(regionSizes, hris);
156     List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
157 
158     NormalizationPlan plan = plans.get(0);
159     assertTrue(plan instanceof MergeNormalizationPlan);
160     assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion());
161     assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
162   }
163 
164   // Test for situation illustrated in HBASE-14867
165   @Test
166   public void testMergeOfSecondSmallestRegions() throws HBaseIOException {
167     TableName testTable = TableName.valueOf("testMergeOfSmallRegions");
168     List<HRegionInfo> hris = new ArrayList<>();
169     Map<byte[], Integer> regionSizes = new HashMap<>();
170 
171     HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
172     hris.add(hri1);
173     regionSizes.put(hri1.getRegionName(), 1);
174 
175     HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
176     hris.add(hri2);
177     regionSizes.put(hri2.getRegionName(), 10000);
178 
179     HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
180     hris.add(hri3);
181     regionSizes.put(hri3.getRegionName(), 10000);
182 
183     HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
184     hris.add(hri4);
185     regionSizes.put(hri4.getRegionName(), 10000);
186 
187     HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("eee"), Bytes.toBytes("fff"));
188     hris.add(hri5);
189     regionSizes.put(hri5.getRegionName(), 2700);
190 
191     HRegionInfo hri6 = new HRegionInfo(testTable, Bytes.toBytes("fff"), Bytes.toBytes("ggg"));
192     hris.add(hri6);
193     regionSizes.put(hri6.getRegionName(), 2700);
194 
195     setupMocksForNormalizer(regionSizes, hris);
196     List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
197     NormalizationPlan plan = plans.get(0);
198 
199     assertTrue(plan instanceof MergeNormalizationPlan);
200     assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
201     assertEquals(hri6, ((MergeNormalizationPlan) plan).getSecondRegion());
202   }
203 
204   @Test
205   public void testMergeOfSmallNonAdjacentRegions() throws HBaseIOException {
206     TableName testTable = TableName.valueOf("testMergeOfSmallRegions");
207     List<HRegionInfo> hris = new ArrayList<>();
208     Map<byte[], Integer> regionSizes = new HashMap<>();
209 
210     HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
211     hris.add(hri1);
212     regionSizes.put(hri1.getRegionName(), 15);
213 
214     HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
215     hris.add(hri2);
216     regionSizes.put(hri2.getRegionName(), 5);
217 
218     HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
219     hris.add(hri3);
220     regionSizes.put(hri3.getRegionName(), 16);
221 
222     HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
223     hris.add(hri4);
224     regionSizes.put(hri4.getRegionName(), 15);
225 
226     HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
227     hris.add(hri4);
228     regionSizes.put(hri5.getRegionName(), 5);
229 
230     setupMocksForNormalizer(regionSizes, hris);
231     List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
232 
233     assertTrue(plans == null);
234   }
235 
236   @Test
237   public void testSplitOfLargeRegion() throws HBaseIOException {
238     TableName testTable = TableName.valueOf("testSplitOfLargeRegion");
239     List<HRegionInfo> hris = new ArrayList<>();
240     Map<byte[], Integer> regionSizes = new HashMap<>();
241 
242     HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
243     hris.add(hri1);
244     regionSizes.put(hri1.getRegionName(), 8);
245 
246     HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
247     hris.add(hri2);
248     regionSizes.put(hri2.getRegionName(), 6);
249 
250     HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
251     hris.add(hri3);
252     regionSizes.put(hri3.getRegionName(), 10);
253 
254     HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
255     hris.add(hri4);
256     regionSizes.put(hri4.getRegionName(), 30);
257 
258     setupMocksForNormalizer(regionSizes, hris);
259     List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
260     NormalizationPlan plan = plans.get(0);
261 
262     assertTrue(plan instanceof SplitNormalizationPlan);
263     assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo());
264   }
265 
266   protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
267                                          List<HRegionInfo> hris) {
268     masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
269     masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS);
270 
271     // for simplicity all regions are assumed to be on one server; doesn't matter to us
272     ServerName sn = ServerName.valueOf("localhost", -1, 1L);
273     when(masterServices.getAssignmentManager().getRegionStates().
274       getRegionsOfTable(any(TableName.class))).thenReturn(hris);
275     when(masterServices.getAssignmentManager().getRegionStates().
276       getRegionServerOfRegion(any(HRegionInfo.class))).thenReturn(sn);
277 
278     for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
279       RegionLoad regionLoad = Mockito.mock(RegionLoad.class);
280       when(regionLoad.getName()).thenReturn(region.getKey());
281       when(regionLoad.getStorefileSizeMB()).thenReturn(region.getValue());
282 
283       when(masterServices.getServerManager().getLoad(sn).
284         getRegionsLoad().get(region.getKey())).thenReturn(regionLoad);
285     }
286     try {
287       when(masterRpcServices.isSplitOrMergeEnabled(any(RpcController.class),
288         any(IsSplitOrMergeEnabledRequest.class))).thenReturn(
289           IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build());
290     } catch (ServiceException se) {
291       LOG.debug("error setting isSplitOrMergeEnabled switch", se);
292     }
293 
294     normalizer.setMasterServices(masterServices);
295     normalizer.setMasterRpcServices(masterRpcServices);
296   }
297 }