001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Set;
030
031import org.apache.commons.lang3.StringUtils;
032import org.apache.hadoop.hbase.NamespaceDescriptor;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.constraint.ConstraintException;
037import org.apache.hadoop.hbase.master.HMaster;
038import org.apache.hadoop.hbase.master.LoadBalancer;
039import org.apache.hadoop.hbase.master.MasterServices;
040import org.apache.hadoop.hbase.master.RegionPlan;
041import org.apache.hadoop.hbase.master.RegionState;
042import org.apache.hadoop.hbase.master.ServerManager;
043import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
044import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode;
045import org.apache.hadoop.hbase.net.Address;
046import org.apache.yetus.audience.InterfaceAudience;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
051import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
052
053/**
054 * Service to support Region Server Grouping (HBase-6721).
055 */
056@InterfaceAudience.Private
057public class RSGroupAdminServer implements RSGroupAdmin {
058  private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class);
059
060  private MasterServices master;
061  private final RSGroupInfoManager rsGroupInfoManager;
062
063  public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager)
064      throws IOException {
065    this.master = master;
066    this.rsGroupInfoManager = rsGroupInfoManager;
067  }
068
069  @Override
070  public RSGroupInfo getRSGroupInfo(String groupName) throws IOException {
071    return rsGroupInfoManager.getRSGroup(groupName);
072  }
073
074  @Override
075  public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException {
076    // We are reading across two Maps in the below with out synchronizing across
077    // them; should be safe most of the time.
078    String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName);
079    return groupName == null? null: rsGroupInfoManager.getRSGroup(groupName);
080  }
081
082  private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException {
083    // This uglyness is because we only have Address, not ServerName.
084    // Online servers are keyed by ServerName.
085    Set<Address> onlineServers = new HashSet<>();
086    for(ServerName server: master.getServerManager().getOnlineServers().keySet()) {
087      onlineServers.add(server.getAddress());
088    }
089    for (Address address: servers) {
090      if (!onlineServers.contains(address)) {
091        throw new ConstraintException(
092            "Server " + address + " is not an online server in 'default' RSGroup.");
093      }
094    }
095  }
096
097  /**
098   * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found.
099   * @return The RSGroupInfo named <code>name</code>
100   */
101  private RSGroupInfo getAndCheckRSGroupInfo(String name)
102  throws IOException {
103    if (StringUtils.isEmpty(name)) {
104      throw new ConstraintException("RSGroup cannot be null.");
105    }
106    RSGroupInfo rsGroupInfo = getRSGroupInfo(name);
107    if (rsGroupInfo == null) {
108      throw new ConstraintException("RSGroup does not exist: " + name);
109    }
110    return rsGroupInfo;
111  }
112
113  /**
114   * @return List of Regions associated with this <code>server</code>.
115   */
116  private List<RegionInfo> getRegions(final Address server) {
117    LinkedList<RegionInfo> regions = new LinkedList<>();
118    for (Map.Entry<RegionInfo, ServerName> el :
119        master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
120      if (el.getValue() == null) {
121        continue;
122      }
123
124      if (el.getValue().getAddress().equals(server)) {
125        addRegion(regions, el.getKey());
126      }
127    }
128    for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) {
129      if (state.getRegionLocation().getAddress().equals(server)) {
130        addRegion(regions, state.getRegionInfo());
131      }
132    }
133    return regions;
134  }
135
136  private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) {
137    // If meta, move it last otherwise other unassigns fail because meta is not
138    // online for them to update state in. This is dodgy. Needs to be made more
139    // robust. See TODO below.
140    if (hri.isMetaRegion()) {
141      regions.addLast(hri);
142    } else {
143      regions.addFirst(hri);
144    }
145  }
146
147  /**
148   * Check servers and tables.
149   *
150   * @param servers servers to move
151   * @param tables tables to move
152   * @param targetGroupName target group name
153   * @throws IOException if nulls or if servers and tables not belong to the same group
154   */
155  private void checkServersAndTables(Set<Address> servers, Set<TableName> tables,
156                                     String targetGroupName) throws IOException {
157    // Presume first server's source group. Later ensure all servers are from this group.
158    Address firstServer = servers.iterator().next();
159    RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
160    if (tmpSrcGrp == null) {
161      // Be careful. This exception message is tested for in TestRSGroupsBase...
162      throw new ConstraintException("Source RSGroup for server " + firstServer
163              + " does not exist.");
164    }
165    RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
166    if (srcGrp.getName().equals(targetGroupName)) {
167      throw new ConstraintException("Target RSGroup " + targetGroupName +
168              " is same as source " + srcGrp.getName() + " RSGroup.");
169    }
170    // Only move online servers
171    checkOnlineServersOnly(servers);
172
173    // Ensure all servers are of same rsgroup.
174    for (Address server: servers) {
175      String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
176      if (!tmpGroup.equals(srcGrp.getName())) {
177        throw new ConstraintException("Move server request should only come from one source " +
178                "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
179      }
180    }
181
182    // Ensure all tables and servers are of same rsgroup.
183    for (TableName table : tables) {
184      String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
185      if (!tmpGroup.equals(srcGrp.getName())) {
186        throw new ConstraintException("Move table request should only come from one source " +
187                "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
188      }
189    }
190
191    if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
192      throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
193              " that contains tables without servers to host them.");
194    }
195  }
196
197  /**
198   * Moves every region from servers which are currently located on these servers,
199   * but should not be located there.
200   * @param servers the servers that will move to new group
201   * @param tables these tables will be kept on the servers, others will be moved
202   * @param targetGroupName the target group name
203   * @throws IOException if moving the server and tables fail
204   */
205  private void moveRegionsFromServers(Set<Address> servers, Set<TableName> tables,
206      String targetGroupName) throws IOException {
207    boolean foundRegionsToMove;
208    RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName);
209    Set<Address> allSevers = new HashSet<>(servers);
210    do {
211      foundRegionsToMove = false;
212      for (Iterator<Address> iter = allSevers.iterator(); iter.hasNext();) {
213        Address rs = iter.next();
214        // Get regions that are associated with this server and filter regions by tables.
215        List<RegionInfo> regions = new ArrayList<>();
216        for (RegionInfo region : getRegions(rs)) {
217          if (!tables.contains(region.getTable())) {
218            regions.add(region);
219          }
220        }
221
222        LOG.info("Moving " + regions.size() + " region(s) from " + rs +
223            " for server move to " + targetGroupName);
224        if (!regions.isEmpty()) {
225          for (RegionInfo region: regions) {
226            // Regions might get assigned from tables of target group so we need to filter
227            if (!targetGrp.containsTable(region.getTable())) {
228              this.master.getAssignmentManager().move(region);
229              if (master.getAssignmentManager().getRegionStates().
230                  getRegionState(region).isFailedOpen()) {
231                continue;
232              }
233              foundRegionsToMove = true;
234            }
235          }
236        }
237        if (!foundRegionsToMove) {
238          iter.remove();
239        }
240      }
241      try {
242        rsGroupInfoManager.wait(1000);
243      } catch (InterruptedException e) {
244        LOG.warn("Sleep interrupted", e);
245        Thread.currentThread().interrupt();
246      }
247    } while (foundRegionsToMove);
248  }
249
250  /**
251   * Moves every region of tables which should be kept on the servers,
252   * but currently they are located on other servers.
253   * @param servers the regions of these servers will be kept on the servers, others will be moved
254   * @param tables the tables that will move to new group
255   * @param targetGroupName the target group name
256   * @throws IOException if moving the region fails
257   */
258  private void moveRegionsToServers(Set<Address> servers, Set<TableName> tables,
259      String targetGroupName) throws IOException {
260    for (TableName table: tables) {
261      LOG.info("Moving region(s) from " + table + " for table move to " + targetGroupName);
262      for (RegionInfo region : master.getAssignmentManager().getRegionStates()
263          .getRegionsOfTable(table)) {
264        ServerName sn = master.getAssignmentManager().getRegionStates()
265            .getRegionServerOfRegion(region);
266        if (!servers.contains(sn.getAddress())) {
267          master.getAssignmentManager().move(region);
268        }
269      }
270    }
271  }
272
273  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
274      value="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
275      justification="Ignoring complaint because don't know what it is complaining about")
276  @Override
277  public void moveServers(Set<Address> servers, String targetGroupName)
278  throws IOException {
279    if (servers == null) {
280      throw new ConstraintException("The list of servers to move cannot be null.");
281    }
282    if (servers.isEmpty()) {
283      // For some reason this difference between null servers and isEmpty is important distinction.
284      // TODO. Why? Stuff breaks if I equate them.
285      return;
286    }
287    RSGroupInfo targetGrp = getAndCheckRSGroupInfo(targetGroupName);
288
289    // Hold a lock on the manager instance while moving servers to prevent
290    // another writer changing our state while we are working.
291    synchronized (rsGroupInfoManager) {
292      if (master.getMasterCoprocessorHost() != null) {
293        master.getMasterCoprocessorHost().preMoveServers(servers, targetGroupName);
294      }
295      // Presume first server's source group. Later ensure all servers are from this group.
296      Address firstServer = servers.iterator().next();
297      RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
298      if (srcGrp == null) {
299        // Be careful. This exception message is tested for in TestRSGroupsBase...
300        throw new ConstraintException("Source RSGroup for server " + firstServer
301            + " does not exist.");
302      }
303      if (srcGrp.getName().equals(targetGroupName)) {
304        throw new ConstraintException("Target RSGroup " + targetGroupName +
305            " is same as source " + srcGrp + " RSGroup.");
306      }
307      // Only move online servers (when moving from 'default') or servers from other
308      // groups. This prevents bogus servers from entering groups
309      if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
310        checkOnlineServersOnly(servers);
311      }
312      // Ensure all servers are of same rsgroup.
313      for (Address server: servers) {
314        String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
315        if (!tmpGroup.equals(srcGrp.getName())) {
316          throw new ConstraintException("Move server request should only come from one source " +
317              "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
318        }
319      }
320      if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
321        throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
322            " that contains tables without servers to host them.");
323      }
324
325      // MovedServers may be < passed in 'servers'.
326      Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(),
327          targetGroupName);
328      List<Address> editableMovedServers = Lists.newArrayList(movedServers);
329      boolean foundRegionsToMove;
330      do {
331        foundRegionsToMove = false;
332        for (Iterator<Address> iter = editableMovedServers.iterator(); iter.hasNext();) {
333          Address rs = iter.next();
334          // Get regions that are associated with this server.
335          List<RegionInfo> regions = getRegions(rs);
336
337          LOG.info("Moving " + regions.size() + " region(s) from " + rs +
338              " for server move to " + targetGroupName);
339
340          for (RegionInfo region: regions) {
341            // Regions might get assigned from tables of target group so we need to filter
342            if (targetGrp.containsTable(region.getTable())) {
343              continue;
344            }
345            LOG.info("Moving region " + region.getShortNameToLog());
346            this.master.getAssignmentManager().move(region);
347            if (master.getAssignmentManager().getRegionStates().
348                getRegionState(region).isFailedOpen()) {
349              // If region is in FAILED_OPEN state, it won't recover, not without
350              // operator intervention... in hbase-2.0.0 at least. Continue rather
351              // than mark region as 'foundRegionsToMove'.
352              continue;
353            }
354            foundRegionsToMove = true;
355          }
356          if (!foundRegionsToMove) {
357            iter.remove();
358          }
359        }
360        try {
361          rsGroupInfoManager.wait(1000);
362        } catch (InterruptedException e) {
363          LOG.warn("Sleep interrupted", e);
364          Thread.currentThread().interrupt();
365        }
366      } while (foundRegionsToMove);
367
368      if (master.getMasterCoprocessorHost() != null) {
369        master.getMasterCoprocessorHost().postMoveServers(servers, targetGroupName);
370      }
371      LOG.info("Move server done: " + srcGrp.getName() + "=>" + targetGroupName);
372    }
373  }
374
375  @Override
376  public void moveTables(Set<TableName> tables, String targetGroup) throws IOException {
377    if (tables == null) {
378      throw new ConstraintException("The list of servers cannot be null.");
379    }
380    if (tables.size() < 1) {
381      LOG.debug("moveTables() passed an empty set. Ignoring.");
382      return;
383    }
384
385    // Hold a lock on the manager instance while moving servers to prevent
386    // another writer changing our state while we are working.
387    synchronized (rsGroupInfoManager) {
388      if (master.getMasterCoprocessorHost() != null) {
389        master.getMasterCoprocessorHost().preMoveTables(tables, targetGroup);
390      }
391      if(targetGroup != null) {
392        RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup);
393        if(destGroup == null) {
394          throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist.");
395        }
396        if(destGroup.getServers().size() < 1) {
397          throw new ConstraintException("Target RSGroup must have at least one server.");
398        }
399      }
400
401      for (TableName table : tables) {
402        String srcGroup = rsGroupInfoManager.getRSGroupOfTable(table);
403        if(srcGroup != null && srcGroup.equals(targetGroup)) {
404          throw new ConstraintException(
405              "Source RSGroup " + srcGroup + " is same as target " + targetGroup +
406              " RSGroup for table " + table);
407        }
408        LOG.info("Moving table " + table.getNameAsString() + " to RSGroup " + targetGroup);
409      }
410      rsGroupInfoManager.moveTables(tables, targetGroup);
411
412      // targetGroup is null when a table is being deleted. In this case no further
413      // action is required.
414      if (targetGroup != null) {
415        for (TableName table: tables) {
416          if (master.getAssignmentManager().isTableDisabled(table)) {
417            LOG.debug("Skipping move regions because the table" + table + " is disabled.");
418            continue;
419          }
420          for (RegionInfo region :
421              master.getAssignmentManager().getRegionStates().getRegionsOfTable(table)) {
422            LOG.info("Moving region " + region.getShortNameToLog() +
423                " to RSGroup " + targetGroup);
424            master.getAssignmentManager().move(region);
425          }
426        }
427      }
428
429      if (master.getMasterCoprocessorHost() != null) {
430        master.getMasterCoprocessorHost().postMoveTables(tables, targetGroup);
431      }
432    }
433  }
434
435  @Override
436  public void addRSGroup(String name) throws IOException {
437    if (master.getMasterCoprocessorHost() != null) {
438      master.getMasterCoprocessorHost().preAddRSGroup(name);
439    }
440    rsGroupInfoManager.addRSGroup(new RSGroupInfo(name));
441    if (master.getMasterCoprocessorHost() != null) {
442      master.getMasterCoprocessorHost().postAddRSGroup(name);
443    }
444  }
445
446  @Override
447  public void removeRSGroup(String name) throws IOException {
448    // Hold a lock on the manager instance while moving servers to prevent
449    // another writer changing our state while we are working.
450    synchronized (rsGroupInfoManager) {
451      if (master.getMasterCoprocessorHost() != null) {
452        master.getMasterCoprocessorHost().preRemoveRSGroup(name);
453      }
454      RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name);
455      if (rsGroupInfo == null) {
456        throw new ConstraintException("RSGroup " + name + " does not exist");
457      }
458      int tableCount = rsGroupInfo.getTables().size();
459      if (tableCount > 0) {
460        throw new ConstraintException("RSGroup " + name + " has " + tableCount +
461            " tables; you must remove these tables from the rsgroup before " +
462            "the rsgroup can be removed.");
463      }
464      int serverCount = rsGroupInfo.getServers().size();
465      if (serverCount > 0) {
466        throw new ConstraintException("RSGroup " + name + " has " + serverCount +
467            " servers; you must remove these servers from the RSGroup before" +
468            "the RSGroup can be removed.");
469      }
470      for (NamespaceDescriptor ns: master.getClusterSchema().getNamespaces()) {
471        String nsGroup = ns.getConfigurationValue(rsGroupInfo.NAMESPACE_DESC_PROP_GROUP);
472        if (nsGroup != null &&  nsGroup.equals(name)) {
473          throw new ConstraintException("RSGroup " + name + " is referenced by namespace: " +
474              ns.getName());
475        }
476      }
477      rsGroupInfoManager.removeRSGroup(name);
478      if (master.getMasterCoprocessorHost() != null) {
479        master.getMasterCoprocessorHost().postRemoveRSGroup(name);
480      }
481    }
482  }
483
484  @Override
485  public boolean balanceRSGroup(String groupName) throws IOException {
486    ServerManager serverManager = master.getServerManager();
487    AssignmentManager assignmentManager = master.getAssignmentManager();
488    LoadBalancer balancer = master.getLoadBalancer();
489
490    synchronized (balancer) {
491      // If balance not true, don't run balancer.
492      if (!((HMaster) master).isBalancerOn()) {
493        return false;
494      }
495
496      if (master.getMasterCoprocessorHost() != null) {
497        master.getMasterCoprocessorHost().preBalanceRSGroup(groupName);
498      }
499      if (getRSGroupInfo(groupName) == null) {
500        throw new ConstraintException("RSGroup does not exist: "+groupName);
501      }
502      // Only allow one balance run at at time.
503      Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
504      if (groupRIT.size() > 0) {
505        LOG.debug("Not running balancer because " + groupRIT.size() + " region(s) in transition: " +
506          StringUtils.abbreviate(
507              master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(),
508              256));
509        return false;
510      }
511      if (serverManager.areDeadServersInProgress()) {
512        LOG.debug("Not running balancer because processing dead regionserver(s): " +
513            serverManager.getDeadServers());
514        return false;
515      }
516
517      //We balance per group instead of per table
518      List<RegionPlan> plans = new ArrayList<>();
519      for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> tableMap:
520          getRSGroupAssignmentsByTable(groupName).entrySet()) {
521        LOG.info("Creating partial plan for table " + tableMap.getKey() + ": "
522            + tableMap.getValue());
523        List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue());
524        LOG.info("Partial plan for table " + tableMap.getKey() + ": " + partialPlans);
525        if (partialPlans != null) {
526          plans.addAll(partialPlans);
527        }
528      }
529      long startTime = System.currentTimeMillis();
530      boolean balancerRan = !plans.isEmpty();
531      if (balancerRan) {
532        LOG.info("RSGroup balance " + groupName + " starting with plan count: " + plans.size());
533        for (RegionPlan plan: plans) {
534          LOG.info("balance " + plan);
535          assignmentManager.moveAsync(plan);
536        }
537        LOG.info("RSGroup balance " + groupName + " completed after " +
538            (System.currentTimeMillis()-startTime) + " seconds");
539      }
540      if (master.getMasterCoprocessorHost() != null) {
541        master.getMasterCoprocessorHost().postBalanceRSGroup(groupName, balancerRan);
542      }
543      return balancerRan;
544    }
545  }
546
547  @Override
548  public List<RSGroupInfo> listRSGroups() throws IOException {
549    return rsGroupInfoManager.listRSGroups();
550  }
551
552  @Override
553  public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException {
554    return rsGroupInfoManager.getRSGroupOfServer(hostPort);
555  }
556
557  @Override
558  public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup)
559      throws IOException {
560    if (servers == null || servers.isEmpty()) {
561      throw new ConstraintException("The list of servers to move cannot be null or empty.");
562    }
563    if (tables == null || tables.isEmpty()) {
564      throw new ConstraintException("The list of tables to move cannot be null or empty.");
565    }
566
567    //check target group
568    getAndCheckRSGroupInfo(targetGroup);
569
570    // Hold a lock on the manager instance while moving servers and tables to prevent
571    // another writer changing our state while we are working.
572    synchronized (rsGroupInfoManager) {
573      if (master.getMasterCoprocessorHost() != null) {
574        master.getMasterCoprocessorHost().preMoveServersAndTables(servers, tables, targetGroup);
575      }
576      //check servers and tables status
577      checkServersAndTables(servers, tables, targetGroup);
578
579      //Move servers and tables to a new group.
580      String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName();
581      rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup);
582
583      //move regions which should not belong to these tables
584      moveRegionsFromServers(servers, tables, targetGroup);
585      //move regions which should belong to these servers
586      moveRegionsToServers(servers, tables, targetGroup);
587
588      if (master.getMasterCoprocessorHost() != null) {
589        master.getMasterCoprocessorHost().postMoveServersAndTables(servers, tables, targetGroup);
590      }
591    }
592    LOG.info("Move servers and tables done. Severs :"
593            + servers + " , Tables : " + tables + " => " +  targetGroup);
594  }
595
596  @Override
597  public void removeServers(Set<Address> servers) throws IOException {
598    {
599      if (servers == null || servers.isEmpty()) {
600        throw new ConstraintException("The set of servers to remove cannot be null or empty.");
601      }
602      // Hold a lock on the manager instance while moving servers to prevent
603      // another writer changing our state while we are working.
604      synchronized (rsGroupInfoManager) {
605        if (master.getMasterCoprocessorHost() != null) {
606          master.getMasterCoprocessorHost().preRemoveServers(servers);
607        }
608        //check the set of servers
609        checkForDeadOrOnlineServers(servers);
610        rsGroupInfoManager.removeServers(servers);
611        if (master.getMasterCoprocessorHost() != null) {
612          master.getMasterCoprocessorHost().postRemoveServers(servers);
613        }
614        LOG.info("Remove decommissioned servers " + servers + " from rsgroup done.");
615      }
616    }
617  }
618
619  private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName)
620      throws IOException {
621    Map<String, RegionState> rit = Maps.newTreeMap();
622    AssignmentManager am = master.getAssignmentManager();
623    for(TableName tableName : getRSGroupInfo(groupName).getTables()) {
624      for(RegionInfo regionInfo: am.getRegionStates().getRegionsOfTable(tableName)) {
625        RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo);
626        if(state != null) {
627          rit.put(regionInfo.getEncodedName(), state);
628        }
629      }
630    }
631    return rit;
632  }
633
634  private Map<TableName, Map<ServerName, List<RegionInfo>>>
635      getRSGroupAssignmentsByTable(String groupName) throws IOException {
636    Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
637    RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
638    Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap();
639    for(Map.Entry<RegionInfo, ServerName> entry:
640        master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
641      TableName currTable = entry.getKey().getTable();
642      ServerName currServer = entry.getValue();
643      RegionInfo currRegion = entry.getKey();
644      if (rsGroupInfo.getTables().contains(currTable)) {
645        assignments.putIfAbsent(currTable, new HashMap<>());
646        assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>());
647        assignments.get(currTable).get(currServer).add(currRegion);
648      }
649    }
650
651    Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap();
652    for(ServerName serverName: master.getServerManager().getOnlineServers().keySet()) {
653      if(rsGroupInfo.getServers().contains(serverName.getAddress())) {
654        serverMap.put(serverName, Collections.emptyList());
655      }
656    }
657
658    // add all tables that are members of the group
659    for(TableName tableName : rsGroupInfo.getTables()) {
660      if(assignments.containsKey(tableName)) {
661        result.put(tableName, new HashMap<>());
662        result.get(tableName).putAll(serverMap);
663        result.get(tableName).putAll(assignments.get(tableName));
664        LOG.debug("Adding assignments for " + tableName + ": " + assignments.get(tableName));
665      }
666    }
667
668    return result;
669  }
670
671  /**
672   * Check if the set of servers are belong to dead servers list or online servers list.
673   * @param servers servers to remove
674   */
675  private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException {
676    // This uglyness is because we only have Address, not ServerName.
677    Set<Address> onlineServers = new HashSet<>();
678    List<ServerName> drainingServers = master.getServerManager().getDrainingServersList();
679    for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
680      // Only online but not decommissioned servers are really online
681      if (!drainingServers.contains(server)) {
682        onlineServers.add(server.getAddress());
683      }
684    }
685
686    Set<Address> deadServers = new HashSet<>();
687    for(ServerName server: master.getServerManager().getDeadServers().copyServerNames()) {
688      deadServers.add(server.getAddress());
689    }
690
691    for (Address address: servers) {
692      if (onlineServers.contains(address)) {
693        throw new ConstraintException(
694            "Server " + address + " is an online server, not allowed to remove.");
695      }
696      if (deadServers.contains(address)) {
697        throw new ConstraintException(
698            "Server " + address + " is on the dead servers list,"
699                + " Maybe it will come back again, not allowed to remove.");
700      }
701    }
702  }
703}