1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.chaos.actions;
20
21 import java.util.ArrayList;
22 import java.util.HashSet;
23 import java.util.LinkedList;
24 import java.util.List;
25 import java.util.Set;
26
27 import org.apache.commons.lang.math.RandomUtils;
28 import org.apache.hadoop.hbase.ClusterStatus;
29 import org.apache.hadoop.hbase.ServerName;
30 import org.junit.Assert;
31
32
33 public class UnbalanceKillAndRebalanceAction extends Action {
34
35
36 private static final double FRC_SERVERS_THAT_HOARD_AND_LIVE = 0.1;
37 private static final double FRC_SERVERS_THAT_HOARD_AND_DIE = 0.1;
38 private static final double HOARD_FRC_OF_REGIONS = 0.8;
39
40
41 private long waitForUnbalanceMilliSec;
42 private long waitForKillsMilliSec;
43 private long waitAfterBalanceMilliSec;
44 private boolean killMetaRs;
45
46 public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance,
47 boolean killMetaRs) {
48 super();
49 waitForUnbalanceMilliSec = waitUnbalance;
50 waitForKillsMilliSec = waitKill;
51 waitAfterBalanceMilliSec = waitAfterBalance;
52 this.killMetaRs = killMetaRs;
53 }
54
55 @Override
56 public void perform() throws Exception {
57 ClusterStatus status = this.cluster.getClusterStatus();
58 List<ServerName> victimServers = new LinkedList<ServerName>(status.getServers());
59 Set<ServerName> killedServers = new HashSet<ServerName>();
60
61 int liveCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size());
62 int deadCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size());
63 Assert.assertTrue(
64 "There are not enough victim servers: " + victimServers.size(),
65 liveCount + deadCount < victimServers.size());
66 List<ServerName> targetServers = new ArrayList<ServerName>(liveCount);
67 for (int i = 0; i < liveCount + deadCount; ++i) {
68 int victimIx = RandomUtils.nextInt(victimServers.size());
69 targetServers.add(victimServers.remove(victimIx));
70 }
71 unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS);
72 Thread.sleep(waitForUnbalanceMilliSec);
73 ServerName metaServer = cluster.getServerHoldingMeta();
74 for (ServerName targetServer: targetServers) {
75
76
77 if (context.isStopping()) {
78 break;
79 }
80 if (killedServers.size() >= liveCount) {
81 break;
82 }
83
84 if (!killMetaRs && targetServer.equals(metaServer)) {
85 LOG.info("Not killing server because it holds hbase:meta.");
86 } else {
87 killRs(targetServer);
88 killedServers.add(targetServer);
89 }
90 }
91
92 Thread.sleep(waitForKillsMilliSec);
93 forceBalancer();
94 Thread.sleep(waitAfterBalanceMilliSec);
95 for (ServerName server:killedServers) {
96 startRs(server);
97 }
98 }
99 }