View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.chaos.actions;
20  
21  import java.util.ArrayList;
22  import java.util.HashSet;
23  import java.util.LinkedList;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.lang.math.RandomUtils;
28  import org.apache.hadoop.hbase.ClusterStatus;
29  import org.apache.hadoop.hbase.ServerName;
30  import org.junit.Assert;
31  
32  /** This action is too specific to put in ChaosMonkey; put it here */
33  public class UnbalanceKillAndRebalanceAction extends Action {
34    /** Fractions of servers to get regions and live and die respectively; from all other
35     * servers, HOARD_FRC_OF_REGIONS will be removed to the above randomly */
36    private static final double FRC_SERVERS_THAT_HOARD_AND_LIVE = 0.1;
37    private static final double FRC_SERVERS_THAT_HOARD_AND_DIE = 0.1;
38    private static final double HOARD_FRC_OF_REGIONS = 0.8;
39    /** Waits between calling unbalance and killing servers, kills and rebalance, and rebalance
40     * and restarting the servers; to make sure these events have time to impact the cluster. */
41    private long waitForUnbalanceMilliSec;
42    private long waitForKillsMilliSec;
43    private long waitAfterBalanceMilliSec;
44    private boolean killMetaRs;
45  
46    public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance,
47        boolean killMetaRs) {
48      super();
49      waitForUnbalanceMilliSec = waitUnbalance;
50      waitForKillsMilliSec = waitKill;
51      waitAfterBalanceMilliSec = waitAfterBalance;
52      this.killMetaRs = killMetaRs;
53    }
54  
55    @Override
56    public void perform() throws Exception {
57      ClusterStatus status = this.cluster.getClusterStatus();
58      List<ServerName> victimServers = new LinkedList<ServerName>(status.getServers());
59      Set<ServerName> killedServers = new HashSet<ServerName>();
60  
61      int liveCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size());
62      int deadCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size());
63      Assert.assertTrue(
64          "There are not enough victim servers: " + victimServers.size(),
65          liveCount + deadCount < victimServers.size());
66      List<ServerName> targetServers = new ArrayList<ServerName>(liveCount);
67      for (int i = 0; i < liveCount + deadCount; ++i) {
68        int victimIx = RandomUtils.nextInt(victimServers.size());
69        targetServers.add(victimServers.remove(victimIx));
70      }
71      unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS);
72      Thread.sleep(waitForUnbalanceMilliSec);
73      ServerName metaServer = cluster.getServerHoldingMeta();
74      for (ServerName targetServer: targetServers) {
75        // Don't keep killing servers if we're
76        // trying to stop the monkey.
77        if (context.isStopping()) {
78          break;
79        }
80        if (killedServers.size() >= liveCount) {
81          break;
82        }
83  
84        if (!killMetaRs && targetServer.equals(metaServer)) {
85          LOG.info("Not killing server because it holds hbase:meta.");
86        } else {
87          killRs(targetServer);
88          killedServers.add(targetServer);
89        }
90      }
91  
92      Thread.sleep(waitForKillsMilliSec);
93      forceBalancer();
94      Thread.sleep(waitAfterBalanceMilliSec);
95      for (ServerName server:killedServers) {
96        startRs(server);
97      }
98    }
99  }