View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.mockito.Mockito.mock;
25  import static org.mockito.Mockito.when;
26  
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.HashMap;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Map.Entry;
34  import java.util.Queue;
35  import java.util.TreeMap;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.ClusterStatus;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.testclassification.MediumTests;
44  import org.apache.hadoop.hbase.RegionLoad;
45  import org.apache.hadoop.hbase.ServerLoad;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
48  import org.apache.hadoop.hbase.master.RackManager;
49  import org.apache.hadoop.hbase.master.RegionPlan;
50  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.net.DNSToSwitchMapping;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  @Category(MediumTests.class)
58  public class TestStochasticLoadBalancer extends BalancerTestBase {
59    public static final String REGION_KEY = "testRegion";
60    private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
61  
62    int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
63        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
64        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
65        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
67        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
71        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
72        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
73        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
74        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75        0, 0, 0, 0, 0, 56 };
76  
77    // int[testnum][servernumber] -> numregions
78    int[][] clusterStateMocks = new int[][]{
79        // 1 node
80        new int[]{0},
81        new int[]{1},
82        new int[]{10},
83        // 2 node
84        new int[]{0, 0},
85        new int[]{2, 0},
86        new int[]{2, 1},
87        new int[]{2, 2},
88        new int[]{2, 3},
89        new int[]{2, 4},
90        new int[]{1, 1},
91        new int[]{0, 1},
92        new int[]{10, 1},
93        new int[]{514, 1432},
94        new int[]{48, 53},
95        // 3 node
96        new int[]{0, 1, 2},
97        new int[]{1, 2, 3},
98        new int[]{0, 2, 2},
99        new int[]{0, 3, 0},
100       new int[]{0, 4, 0},
101       new int[]{20, 20, 0},
102       // 4 node
103       new int[]{0, 1, 2, 3},
104       new int[]{4, 0, 0, 0},
105       new int[]{5, 0, 0, 0},
106       new int[]{6, 6, 0, 0},
107       new int[]{6, 2, 0, 0},
108       new int[]{6, 1, 0, 0},
109       new int[]{6, 0, 0, 0},
110       new int[]{4, 4, 4, 7},
111       new int[]{4, 4, 4, 8},
112       new int[]{0, 0, 0, 7},
113       // 5 node
114       new int[]{1, 1, 1, 1, 4},
115       // more nodes
116       new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
117       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
118       new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
119       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
120       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
121       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
122       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
123       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
124       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
125       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
126       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
127       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
128       new int[]{10, 7, 12, 8, 11, 10, 9, 14},
129       new int[]{13, 14, 6, 10, 10, 10, 8, 10},
130       new int[]{130, 14, 60, 10, 100, 10, 80, 10},
131       new int[]{130, 140, 60, 100, 100, 100, 80, 100},
132       largeCluster,
133 
134   };
135 
136   @Test
137   public void testKeepRegionLoad() throws Exception {
138 
139     ServerName sn = ServerName.valueOf("test:8080", 100);
140     int numClusterStatusToAdd = 20000;
141     for (int i = 0; i < numClusterStatusToAdd; i++) {
142       ServerLoad sl = mock(ServerLoad.class);
143 
144       RegionLoad rl = mock(RegionLoad.class);
145       when(rl.getStores()).thenReturn(i);
146 
147       Map<byte[], RegionLoad> regionLoadMap =
148           new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
149       regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
150       when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
151 
152       ClusterStatus clusterStatus = mock(ClusterStatus.class);
153       when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
154       when(clusterStatus.getLoad(sn)).thenReturn(sl);
155 
156       loadBalancer.setClusterStatus(clusterStatus);
157     }
158     assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
159     assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
160 
161     Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
162     int i = 0;
163     while(loads.size() > 0) {
164       RegionLoad rl = loads.remove();
165       assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
166       i ++;
167     }
168   }
169 
170   /**
171    * Test the load balancing algorithm.
172    *
173    * Invariant is that all servers should be hosting either floor(average) or
174    * ceiling(average)
175    *
176    * @throws Exception
177    */
178   @Test
179   public void testBalanceCluster() throws Exception {
180 
181     for (int[] mockCluster : clusterStateMocks) {
182       Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
183       List<ServerAndLoad> list = convertToList(servers);
184       LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
185       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
186       List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
187       LOG.info("Mock Balance : " + printMock(balancedCluster));
188       assertClusterAsBalanced(balancedCluster);
189       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(servers);
190       assertNull(secondPlans);
191       for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
192         returnRegions(entry.getValue());
193         returnServer(entry.getKey());
194       }
195     }
196 
197   }
198 
199   @Test
200   public void testSkewCost() {
201     Configuration conf = HBaseConfiguration.create();
202     StochasticLoadBalancer.CostFunction
203         costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
204     for (int[] mockCluster : clusterStateMocks) {
205       costFunction.init(mockCluster(mockCluster));
206       double cost = costFunction.cost();
207       assertTrue(cost >= 0);
208       assertTrue(cost <= 1.01);
209     }
210 
211     costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
212     assertEquals(0,costFunction.cost(), 0.01);
213     costFunction.init(mockCluster(new int[]{0, 0, 0, 1, 1}));
214     assertEquals(0, costFunction.cost(), 0.01);
215     costFunction.init(mockCluster(new int[]{0, 0, 1, 1, 1}));
216     assertEquals(0, costFunction.cost(), 0.01);
217     costFunction.init(mockCluster(new int[]{0, 1, 1, 1, 1}));
218     assertEquals(0, costFunction.cost(), 0.01);
219     costFunction.init(mockCluster(new int[]{1, 1, 1, 1, 1}));
220     assertEquals(0, costFunction.cost(), 0.01);
221     costFunction.init(mockCluster(new int[]{10000, 0, 0, 0, 0}));
222     assertEquals(1, costFunction.cost(), 0.01);
223   }
224 
225   @Test
226   public void testTableSkewCost() {
227     Configuration conf = HBaseConfiguration.create();
228     StochasticLoadBalancer.CostFunction
229         costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
230     for (int[] mockCluster : clusterStateMocks) {
231       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
232       costFunction.init(cluster);
233       double cost = costFunction.cost();
234       assertTrue(cost >= 0);
235       assertTrue(cost <= 1.01);
236     }
237   }
238 
239   @Test
240   public void testCostFromArray() {
241     Configuration conf = HBaseConfiguration.create();
242     StochasticLoadBalancer.CostFromRegionLoadFunction
243         costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
244     costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
245 
246     double[] statOne = new double[100];
247     for (int i =0; i < 100; i++) {
248       statOne[i] = 10;
249     }
250     assertEquals(0, costFunction.costFromArray(statOne), 0.01);
251 
252     double[] statTwo= new double[101];
253     for (int i =0; i < 100; i++) {
254       statTwo[i] = 0;
255     }
256     statTwo[100] = 100;
257     assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
258 
259     double[] statThree = new double[200];
260     for (int i =0; i < 100; i++) {
261       statThree[i] = (0);
262       statThree[i+100] = 100;
263     }
264     assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
265   }
266 
267   @Test(timeout =  60000)
268   public void testLosingRs() throws Exception {
269     int numNodes = 3;
270     int numRegions = 20;
271     int numRegionsPerServer = 3; //all servers except one
272     int replication = 1;
273     int numTables = 2;
274 
275     Map<ServerName, List<HRegionInfo>> serverMap =
276         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
277     List<ServerAndLoad> list = convertToList(serverMap);
278 
279 
280     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
281     assertNotNull(plans);
282 
283     // Apply the plan to the mock cluster.
284     List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
285 
286     assertClusterAsBalanced(balancedCluster);
287 
288     ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
289 
290     ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
291 
292     serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
293 
294     plans = loadBalancer.balanceCluster(serverMap);
295     assertNull(plans);
296   }
297 
298   @Test
299   public void testReplicaCost() {
300     Configuration conf = HBaseConfiguration.create();
301     StochasticLoadBalancer.CostFunction
302         costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
303     for (int[] mockCluster : clusterStateMocks) {
304       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
305       costFunction.init(cluster);
306       double cost = costFunction.cost();
307       assertTrue(cost >= 0);
308       assertTrue(cost <= 1.01);
309     }
310   }
311 
312   @Test
313   public void testReplicaCostForReplicas() {
314     Configuration conf = HBaseConfiguration.create();
315     StochasticLoadBalancer.CostFunction
316         costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
317 
318     int [] servers = new int[] {3,3,3,3,3};
319     TreeMap<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(servers);
320 
321     BaseLoadBalancer.Cluster cluster;
322 
323     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
324     costFunction.init(cluster);
325     double costWithoutReplicas = costFunction.cost();
326     assertEquals(0, costWithoutReplicas, 0);
327 
328     // replicate the region from first server to the last server
329     HRegionInfo replica1 = RegionReplicaUtil.getRegionInfoForReplica(
330       clusterState.firstEntry().getValue().get(0),1);
331     clusterState.lastEntry().getValue().add(replica1);
332 
333     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
334     costFunction.init(cluster);
335     double costWith1ReplicaDifferentServer = costFunction.cost();
336 
337     assertEquals(0, costWith1ReplicaDifferentServer, 0);
338 
339     // add a third replica to the last server
340     HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2);
341     clusterState.lastEntry().getValue().add(replica2);
342 
343     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
344     costFunction.init(cluster);
345     double costWith1ReplicaSameServer = costFunction.cost();
346 
347     assertTrue(costWith1ReplicaDifferentServer < costWith1ReplicaSameServer);
348 
349     // test with replication = 4 for following:
350 
351     HRegionInfo replica3;
352     Iterator<Entry<ServerName, List<HRegionInfo>>> it;
353     Entry<ServerName, List<HRegionInfo>> entry;
354 
355     clusterState = mockClusterServers(servers);
356     it = clusterState.entrySet().iterator();
357     entry = it.next(); //first server
358     HRegionInfo hri = entry.getValue().get(0);
359     replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
360     replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
361     replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
362     entry.getValue().add(replica1);
363     entry.getValue().add(replica2);
364     it.next().getValue().add(replica3); //2nd server
365 
366     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
367     costFunction.init(cluster);
368     double costWith3ReplicasSameServer = costFunction.cost();
369 
370     clusterState = mockClusterServers(servers);
371     hri = clusterState.firstEntry().getValue().get(0);
372     replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
373     replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
374     replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
375 
376     clusterState.firstEntry().getValue().add(replica1);
377     clusterState.lastEntry().getValue().add(replica2);
378     clusterState.lastEntry().getValue().add(replica3);
379 
380     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
381     costFunction.init(cluster);
382     double costWith2ReplicasOnTwoServers = costFunction.cost();
383 
384     assertTrue(costWith2ReplicasOnTwoServers < costWith3ReplicasSameServer);
385   }
386 
387   @Test
388   public void testNeedsBalanceForColocatedReplicas() {
389     // check for the case where there are two hosts and with one rack, and where
390     // both the replicas are hosted on the same server
391     List<HRegionInfo> regions = randomRegions(1);
392     ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
393     ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
394     Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
395     map.put(s1, regions);
396     regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
397     // until the step above s1 holds two replicas of a region
398     regions = randomRegions(1);
399     map.put(s2, regions);
400     assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
401     // check for the case where there are two hosts on the same rack and there are two racks
402     // and both the replicas are on the same rack
403     map.clear();
404     regions = randomRegions(1);
405     List<HRegionInfo> regionsOnS2 = new ArrayList<HRegionInfo>(1);
406     regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
407     map.put(s1, regions);
408     map.put(s2, regionsOnS2);
409     // add another server so that the cluster has some host on another rack
410     map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
411     assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null,
412         new ForTestRackManagerOne())));
413   }
414 
415   @Test (timeout = 60000)
416   public void testSmallCluster() {
417     int numNodes = 10;
418     int numRegions = 1000;
419     int numRegionsPerServer = 40; //all servers except one
420     int replication = 1;
421     int numTables = 10;
422     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
423   }
424 
425   @Test (timeout = 60000)
426   public void testSmallCluster2() {
427     int numNodes = 20;
428     int numRegions = 2000;
429     int numRegionsPerServer = 40; //all servers except one
430     int replication = 1;
431     int numTables = 10;
432     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
433   }
434 
435   @Test (timeout = 60000)
436   public void testSmallCluster3() {
437     int numNodes = 20;
438     int numRegions = 2000;
439     int numRegionsPerServer = 1; // all servers except one
440     int replication = 1;
441     int numTables = 10;
442     /* fails because of max moves */
443     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false);
444   }
445 
446   @Test (timeout = 800000)
447   public void testMidCluster() {
448     int numNodes = 100;
449     int numRegions = 10000;
450     int numRegionsPerServer = 60; // all servers except one
451     int replication = 1;
452     int numTables = 40;
453     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
454   }
455 
456   @Test (timeout = 800000)
457   public void testMidCluster2() {
458     int numNodes = 200;
459     int numRegions = 100000;
460     int numRegionsPerServer = 40; // all servers except one
461     int replication = 1;
462     int numTables = 400;
463     testWithCluster(numNodes,
464         numRegions,
465         numRegionsPerServer,
466         replication,
467         numTables,
468         false, /* num large num regions means may not always get to best balance with one run */
469         false);
470   }
471 
472 
473   @Test (timeout = 800000)
474   public void testMidCluster3() {
475     int numNodes = 100;
476     int numRegions = 2000;
477     int numRegionsPerServer = 9; // all servers except one
478     int replication = 1;
479     int numTables = 110;
480     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
481     // TODO(eclark): Make sure that the tables are well distributed.
482   }
483 
484   @Test
485   public void testLargeCluster() {
486     int numNodes = 1000;
487     int numRegions = 100000; //100 regions per RS
488     int numRegionsPerServer = 80; //all servers except one
489     int numTables = 100;
490     int replication = 1;
491     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
492   }
493 
494   @Test (timeout = 800000)
495   public void testRegionReplicasOnSmallCluster() {
496     int numNodes = 10;
497     int numRegions = 1000;
498     int replication = 3; // 3 replicas per region
499     int numRegionsPerServer = 80; //all regions are mostly balanced
500     int numTables = 10;
501     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
502   }
503 
504   @Test (timeout = 800000)
505   public void testRegionReplicationOnMidClusterSameHosts() {
506     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
507     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
508     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
509     loadBalancer.setConf(conf);
510     int numHosts = 100;
511     int numRegions = 100 * 100;
512     int replication = 3; // 3 replicas per region
513     int numRegionsPerServer = 5;
514     int numTables = 10;
515     Map<ServerName, List<HRegionInfo>> serverMap =
516         createServerMap(numHosts, numRegions, numRegionsPerServer, replication, numTables);
517     int numNodesPerHost = 4;
518 
519     // create a new map with 4 RS per host.
520     Map<ServerName, List<HRegionInfo>> newServerMap = new TreeMap<ServerName, List<HRegionInfo>>(serverMap);
521     for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverMap.entrySet()) {
522       for (int i=1; i < numNodesPerHost; i++) {
523         ServerName s1 = entry.getKey();
524         ServerName s2 = ServerName.valueOf(s1.getHostname(), s1.getPort() + i, 1); // create an RS for the same host
525         newServerMap.put(s2, new ArrayList<HRegionInfo>());
526       }
527     }
528 
529     testWithCluster(newServerMap, null, true, true);
530   }
531 
532   private static class ForTestRackManager extends RackManager {
533     int numRacks;
534     public ForTestRackManager(int numRacks) {
535       this.numRacks = numRacks;
536     }
537     @Override
538     public String getRack(ServerName server) {
539       return "rack_" + (server.hashCode() % numRacks);
540     }
541   }
542 
543   private static class ForTestRackManagerOne extends RackManager {
544   @Override
545     public String getRack(ServerName server) {
546       return server.getHostname().endsWith("1") ? "rack1" : "rack2";
547     }
548   }
549 
550   @Test (timeout = 800000)
551   public void testRegionReplicationOnMidClusterWithRacks() {
552     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
553     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
554     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
555     loadBalancer.setConf(conf);
556     int numNodes = 30;
557     int numRegions = numNodes * 30;
558     int replication = 3; // 3 replicas per region
559     int numRegionsPerServer = 28;
560     int numTables = 10;
561     int numRacks = 4; // all replicas should be on a different rack
562     Map<ServerName, List<HRegionInfo>> serverMap =
563         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
564     RackManager rm = new ForTestRackManager(numRacks);
565 
566     testWithCluster(serverMap, rm, false, true);
567   }
568 }