001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rsgroup; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.Collections; 023import java.util.HashMap; 024import java.util.HashSet; 025import java.util.Iterator; 026import java.util.LinkedList; 027import java.util.List; 028import java.util.Map; 029import java.util.Set; 030 031import org.apache.commons.lang3.StringUtils; 032import org.apache.hadoop.hbase.NamespaceDescriptor; 033import org.apache.hadoop.hbase.ServerName; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.constraint.ConstraintException; 037import org.apache.hadoop.hbase.master.HMaster; 038import org.apache.hadoop.hbase.master.LoadBalancer; 039import org.apache.hadoop.hbase.master.MasterServices; 040import org.apache.hadoop.hbase.master.RegionPlan; 041import org.apache.hadoop.hbase.master.RegionState; 042import org.apache.hadoop.hbase.master.ServerManager; 043import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 044import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode; 045import org.apache.hadoop.hbase.net.Address; 046import org.apache.yetus.audience.InterfaceAudience; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 051import org.apache.hbase.thirdparty.com.google.common.collect.Maps; 052 053/** 054 * Service to support Region Server Grouping (HBase-6721). 055 */ 056@InterfaceAudience.Private 057public class RSGroupAdminServer implements RSGroupAdmin { 058 private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class); 059 060 private MasterServices master; 061 private final RSGroupInfoManager rsGroupInfoManager; 062 063 public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager) 064 throws IOException { 065 this.master = master; 066 this.rsGroupInfoManager = rsGroupInfoManager; 067 } 068 069 @Override 070 public RSGroupInfo getRSGroupInfo(String groupName) throws IOException { 071 return rsGroupInfoManager.getRSGroup(groupName); 072 } 073 074 @Override 075 public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException { 076 // We are reading across two Maps in the below with out synchronizing across 077 // them; should be safe most of the time. 078 String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName); 079 return groupName == null? null: rsGroupInfoManager.getRSGroup(groupName); 080 } 081 082 private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException { 083 // This uglyness is because we only have Address, not ServerName. 084 // Online servers are keyed by ServerName. 085 Set<Address> onlineServers = new HashSet<>(); 086 for(ServerName server: master.getServerManager().getOnlineServers().keySet()) { 087 onlineServers.add(server.getAddress()); 088 } 089 for (Address address: servers) { 090 if (!onlineServers.contains(address)) { 091 throw new ConstraintException( 092 "Server " + address + " is not an online server in 'default' RSGroup."); 093 } 094 } 095 } 096 097 /** 098 * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found. 099 * @return The RSGroupInfo named <code>name</code> 100 */ 101 private RSGroupInfo getAndCheckRSGroupInfo(String name) 102 throws IOException { 103 if (StringUtils.isEmpty(name)) { 104 throw new ConstraintException("RSGroup cannot be null."); 105 } 106 RSGroupInfo rsGroupInfo = getRSGroupInfo(name); 107 if (rsGroupInfo == null) { 108 throw new ConstraintException("RSGroup does not exist: " + name); 109 } 110 return rsGroupInfo; 111 } 112 113 /** 114 * @return List of Regions associated with this <code>server</code>. 115 */ 116 private List<RegionInfo> getRegions(final Address server) { 117 LinkedList<RegionInfo> regions = new LinkedList<>(); 118 for (Map.Entry<RegionInfo, ServerName> el : 119 master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) { 120 if (el.getValue() == null) { 121 continue; 122 } 123 124 if (el.getValue().getAddress().equals(server)) { 125 addRegion(regions, el.getKey()); 126 } 127 } 128 for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) { 129 if (state.getRegionLocation().getAddress().equals(server)) { 130 addRegion(regions, state.getRegionInfo()); 131 } 132 } 133 return regions; 134 } 135 136 private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) { 137 // If meta, move it last otherwise other unassigns fail because meta is not 138 // online for them to update state in. This is dodgy. Needs to be made more 139 // robust. See TODO below. 140 if (hri.isMetaRegion()) { 141 regions.addLast(hri); 142 } else { 143 regions.addFirst(hri); 144 } 145 } 146 147 /** 148 * Check servers and tables. 149 * 150 * @param servers servers to move 151 * @param tables tables to move 152 * @param targetGroupName target group name 153 * @throws IOException if nulls or if servers and tables not belong to the same group 154 */ 155 private void checkServersAndTables(Set<Address> servers, Set<TableName> tables, 156 String targetGroupName) throws IOException { 157 // Presume first server's source group. Later ensure all servers are from this group. 158 Address firstServer = servers.iterator().next(); 159 RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer); 160 if (tmpSrcGrp == null) { 161 // Be careful. This exception message is tested for in TestRSGroupsBase... 162 throw new ConstraintException("Source RSGroup for server " + firstServer 163 + " does not exist."); 164 } 165 RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp); 166 if (srcGrp.getName().equals(targetGroupName)) { 167 throw new ConstraintException("Target RSGroup " + targetGroupName + 168 " is same as source " + srcGrp.getName() + " RSGroup."); 169 } 170 // Only move online servers 171 checkOnlineServersOnly(servers); 172 173 // Ensure all servers are of same rsgroup. 174 for (Address server: servers) { 175 String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName(); 176 if (!tmpGroup.equals(srcGrp.getName())) { 177 throw new ConstraintException("Move server request should only come from one source " + 178 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup); 179 } 180 } 181 182 // Ensure all tables and servers are of same rsgroup. 183 for (TableName table : tables) { 184 String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table); 185 if (!tmpGroup.equals(srcGrp.getName())) { 186 throw new ConstraintException("Move table request should only come from one source " + 187 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup); 188 } 189 } 190 191 if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) { 192 throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() + 193 " that contains tables without servers to host them."); 194 } 195 } 196 197 /** 198 * Moves every region from servers which are currently located on these servers, 199 * but should not be located there. 200 * @param servers the servers that will move to new group 201 * @param tables these tables will be kept on the servers, others will be moved 202 * @param targetGroupName the target group name 203 * @throws IOException if moving the server and tables fail 204 */ 205 private void moveRegionsFromServers(Set<Address> servers, Set<TableName> tables, 206 String targetGroupName) throws IOException { 207 boolean foundRegionsToMove; 208 RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName); 209 Set<Address> allSevers = new HashSet<>(servers); 210 do { 211 foundRegionsToMove = false; 212 for (Iterator<Address> iter = allSevers.iterator(); iter.hasNext();) { 213 Address rs = iter.next(); 214 // Get regions that are associated with this server and filter regions by tables. 215 List<RegionInfo> regions = new ArrayList<>(); 216 for (RegionInfo region : getRegions(rs)) { 217 if (!tables.contains(region.getTable())) { 218 regions.add(region); 219 } 220 } 221 222 LOG.info("Moving " + regions.size() + " region(s) from " + rs + 223 " for server move to " + targetGroupName); 224 if (!regions.isEmpty()) { 225 for (RegionInfo region: regions) { 226 // Regions might get assigned from tables of target group so we need to filter 227 if (!targetGrp.containsTable(region.getTable())) { 228 this.master.getAssignmentManager().move(region); 229 if (master.getAssignmentManager().getRegionStates(). 230 getRegionState(region).isFailedOpen()) { 231 continue; 232 } 233 foundRegionsToMove = true; 234 } 235 } 236 } 237 if (!foundRegionsToMove) { 238 iter.remove(); 239 } 240 } 241 try { 242 rsGroupInfoManager.wait(1000); 243 } catch (InterruptedException e) { 244 LOG.warn("Sleep interrupted", e); 245 Thread.currentThread().interrupt(); 246 } 247 } while (foundRegionsToMove); 248 } 249 250 /** 251 * Moves every region of tables which should be kept on the servers, 252 * but currently they are located on other servers. 253 * @param servers the regions of these servers will be kept on the servers, others will be moved 254 * @param tables the tables that will move to new group 255 * @param targetGroupName the target group name 256 * @throws IOException if moving the region fails 257 */ 258 private void moveRegionsToServers(Set<Address> servers, Set<TableName> tables, 259 String targetGroupName) throws IOException { 260 for (TableName table: tables) { 261 LOG.info("Moving region(s) from " + table + " for table move to " + targetGroupName); 262 for (RegionInfo region : master.getAssignmentManager().getRegionStates() 263 .getRegionsOfTable(table)) { 264 ServerName sn = master.getAssignmentManager().getRegionStates() 265 .getRegionServerOfRegion(region); 266 if (!servers.contains(sn.getAddress())) { 267 master.getAssignmentManager().move(region); 268 } 269 } 270 } 271 } 272 273 @edu.umd.cs.findbugs.annotations.SuppressWarnings( 274 value="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", 275 justification="Ignoring complaint because don't know what it is complaining about") 276 @Override 277 public void moveServers(Set<Address> servers, String targetGroupName) 278 throws IOException { 279 if (servers == null) { 280 throw new ConstraintException("The list of servers to move cannot be null."); 281 } 282 if (servers.isEmpty()) { 283 // For some reason this difference between null servers and isEmpty is important distinction. 284 // TODO. Why? Stuff breaks if I equate them. 285 return; 286 } 287 RSGroupInfo targetGrp = getAndCheckRSGroupInfo(targetGroupName); 288 289 // Hold a lock on the manager instance while moving servers to prevent 290 // another writer changing our state while we are working. 291 synchronized (rsGroupInfoManager) { 292 if (master.getMasterCoprocessorHost() != null) { 293 master.getMasterCoprocessorHost().preMoveServers(servers, targetGroupName); 294 } 295 // Presume first server's source group. Later ensure all servers are from this group. 296 Address firstServer = servers.iterator().next(); 297 RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer); 298 if (srcGrp == null) { 299 // Be careful. This exception message is tested for in TestRSGroupsBase... 300 throw new ConstraintException("Source RSGroup for server " + firstServer 301 + " does not exist."); 302 } 303 if (srcGrp.getName().equals(targetGroupName)) { 304 throw new ConstraintException("Target RSGroup " + targetGroupName + 305 " is same as source " + srcGrp + " RSGroup."); 306 } 307 // Only move online servers (when moving from 'default') or servers from other 308 // groups. This prevents bogus servers from entering groups 309 if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) { 310 checkOnlineServersOnly(servers); 311 } 312 // Ensure all servers are of same rsgroup. 313 for (Address server: servers) { 314 String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName(); 315 if (!tmpGroup.equals(srcGrp.getName())) { 316 throw new ConstraintException("Move server request should only come from one source " + 317 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup); 318 } 319 } 320 if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) { 321 throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() + 322 " that contains tables without servers to host them."); 323 } 324 325 // MovedServers may be < passed in 'servers'. 326 Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(), 327 targetGroupName); 328 List<Address> editableMovedServers = Lists.newArrayList(movedServers); 329 boolean foundRegionsToMove; 330 do { 331 foundRegionsToMove = false; 332 for (Iterator<Address> iter = editableMovedServers.iterator(); iter.hasNext();) { 333 Address rs = iter.next(); 334 // Get regions that are associated with this server. 335 List<RegionInfo> regions = getRegions(rs); 336 337 LOG.info("Moving " + regions.size() + " region(s) from " + rs + 338 " for server move to " + targetGroupName); 339 340 for (RegionInfo region: regions) { 341 // Regions might get assigned from tables of target group so we need to filter 342 if (targetGrp.containsTable(region.getTable())) { 343 continue; 344 } 345 LOG.info("Moving region " + region.getShortNameToLog()); 346 this.master.getAssignmentManager().move(region); 347 if (master.getAssignmentManager().getRegionStates(). 348 getRegionState(region).isFailedOpen()) { 349 // If region is in FAILED_OPEN state, it won't recover, not without 350 // operator intervention... in hbase-2.0.0 at least. Continue rather 351 // than mark region as 'foundRegionsToMove'. 352 continue; 353 } 354 foundRegionsToMove = true; 355 } 356 if (!foundRegionsToMove) { 357 iter.remove(); 358 } 359 } 360 try { 361 rsGroupInfoManager.wait(1000); 362 } catch (InterruptedException e) { 363 LOG.warn("Sleep interrupted", e); 364 Thread.currentThread().interrupt(); 365 } 366 } while (foundRegionsToMove); 367 368 if (master.getMasterCoprocessorHost() != null) { 369 master.getMasterCoprocessorHost().postMoveServers(servers, targetGroupName); 370 } 371 LOG.info("Move server done: " + srcGrp.getName() + "=>" + targetGroupName); 372 } 373 } 374 375 @Override 376 public void moveTables(Set<TableName> tables, String targetGroup) throws IOException { 377 if (tables == null) { 378 throw new ConstraintException("The list of servers cannot be null."); 379 } 380 if (tables.size() < 1) { 381 LOG.debug("moveTables() passed an empty set. Ignoring."); 382 return; 383 } 384 385 // Hold a lock on the manager instance while moving servers to prevent 386 // another writer changing our state while we are working. 387 synchronized (rsGroupInfoManager) { 388 if (master.getMasterCoprocessorHost() != null) { 389 master.getMasterCoprocessorHost().preMoveTables(tables, targetGroup); 390 } 391 if(targetGroup != null) { 392 RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup); 393 if(destGroup == null) { 394 throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist."); 395 } 396 if(destGroup.getServers().size() < 1) { 397 throw new ConstraintException("Target RSGroup must have at least one server."); 398 } 399 } 400 401 for (TableName table : tables) { 402 String srcGroup = rsGroupInfoManager.getRSGroupOfTable(table); 403 if(srcGroup != null && srcGroup.equals(targetGroup)) { 404 throw new ConstraintException( 405 "Source RSGroup " + srcGroup + " is same as target " + targetGroup + 406 " RSGroup for table " + table); 407 } 408 LOG.info("Moving table " + table.getNameAsString() + " to RSGroup " + targetGroup); 409 } 410 rsGroupInfoManager.moveTables(tables, targetGroup); 411 412 // targetGroup is null when a table is being deleted. In this case no further 413 // action is required. 414 if (targetGroup != null) { 415 for (TableName table: tables) { 416 if (master.getAssignmentManager().isTableDisabled(table)) { 417 LOG.debug("Skipping move regions because the table" + table + " is disabled."); 418 continue; 419 } 420 for (RegionInfo region : 421 master.getAssignmentManager().getRegionStates().getRegionsOfTable(table)) { 422 LOG.info("Moving region " + region.getShortNameToLog() + 423 " to RSGroup " + targetGroup); 424 master.getAssignmentManager().move(region); 425 } 426 } 427 } 428 429 if (master.getMasterCoprocessorHost() != null) { 430 master.getMasterCoprocessorHost().postMoveTables(tables, targetGroup); 431 } 432 } 433 } 434 435 @Override 436 public void addRSGroup(String name) throws IOException { 437 if (master.getMasterCoprocessorHost() != null) { 438 master.getMasterCoprocessorHost().preAddRSGroup(name); 439 } 440 rsGroupInfoManager.addRSGroup(new RSGroupInfo(name)); 441 if (master.getMasterCoprocessorHost() != null) { 442 master.getMasterCoprocessorHost().postAddRSGroup(name); 443 } 444 } 445 446 @Override 447 public void removeRSGroup(String name) throws IOException { 448 // Hold a lock on the manager instance while moving servers to prevent 449 // another writer changing our state while we are working. 450 synchronized (rsGroupInfoManager) { 451 if (master.getMasterCoprocessorHost() != null) { 452 master.getMasterCoprocessorHost().preRemoveRSGroup(name); 453 } 454 RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name); 455 if (rsGroupInfo == null) { 456 throw new ConstraintException("RSGroup " + name + " does not exist"); 457 } 458 int tableCount = rsGroupInfo.getTables().size(); 459 if (tableCount > 0) { 460 throw new ConstraintException("RSGroup " + name + " has " + tableCount + 461 " tables; you must remove these tables from the rsgroup before " + 462 "the rsgroup can be removed."); 463 } 464 int serverCount = rsGroupInfo.getServers().size(); 465 if (serverCount > 0) { 466 throw new ConstraintException("RSGroup " + name + " has " + serverCount + 467 " servers; you must remove these servers from the RSGroup before" + 468 "the RSGroup can be removed."); 469 } 470 for (NamespaceDescriptor ns: master.getClusterSchema().getNamespaces()) { 471 String nsGroup = ns.getConfigurationValue(rsGroupInfo.NAMESPACE_DESC_PROP_GROUP); 472 if (nsGroup != null && nsGroup.equals(name)) { 473 throw new ConstraintException("RSGroup " + name + " is referenced by namespace: " + 474 ns.getName()); 475 } 476 } 477 rsGroupInfoManager.removeRSGroup(name); 478 if (master.getMasterCoprocessorHost() != null) { 479 master.getMasterCoprocessorHost().postRemoveRSGroup(name); 480 } 481 } 482 } 483 484 @Override 485 public boolean balanceRSGroup(String groupName) throws IOException { 486 ServerManager serverManager = master.getServerManager(); 487 AssignmentManager assignmentManager = master.getAssignmentManager(); 488 LoadBalancer balancer = master.getLoadBalancer(); 489 490 synchronized (balancer) { 491 // If balance not true, don't run balancer. 492 if (!((HMaster) master).isBalancerOn()) { 493 return false; 494 } 495 496 if (master.getMasterCoprocessorHost() != null) { 497 master.getMasterCoprocessorHost().preBalanceRSGroup(groupName); 498 } 499 if (getRSGroupInfo(groupName) == null) { 500 throw new ConstraintException("RSGroup does not exist: "+groupName); 501 } 502 // Only allow one balance run at at time. 503 Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName); 504 if (groupRIT.size() > 0) { 505 LOG.debug("Not running balancer because " + groupRIT.size() + " region(s) in transition: " + 506 StringUtils.abbreviate( 507 master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(), 508 256)); 509 return false; 510 } 511 if (serverManager.areDeadServersInProgress()) { 512 LOG.debug("Not running balancer because processing dead regionserver(s): " + 513 serverManager.getDeadServers()); 514 return false; 515 } 516 517 //We balance per group instead of per table 518 List<RegionPlan> plans = new ArrayList<>(); 519 for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> tableMap: 520 getRSGroupAssignmentsByTable(groupName).entrySet()) { 521 LOG.info("Creating partial plan for table " + tableMap.getKey() + ": " 522 + tableMap.getValue()); 523 List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue()); 524 LOG.info("Partial plan for table " + tableMap.getKey() + ": " + partialPlans); 525 if (partialPlans != null) { 526 plans.addAll(partialPlans); 527 } 528 } 529 long startTime = System.currentTimeMillis(); 530 boolean balancerRan = !plans.isEmpty(); 531 if (balancerRan) { 532 LOG.info("RSGroup balance " + groupName + " starting with plan count: " + plans.size()); 533 for (RegionPlan plan: plans) { 534 LOG.info("balance " + plan); 535 assignmentManager.moveAsync(plan); 536 } 537 LOG.info("RSGroup balance " + groupName + " completed after " + 538 (System.currentTimeMillis()-startTime) + " seconds"); 539 } 540 if (master.getMasterCoprocessorHost() != null) { 541 master.getMasterCoprocessorHost().postBalanceRSGroup(groupName, balancerRan); 542 } 543 return balancerRan; 544 } 545 } 546 547 @Override 548 public List<RSGroupInfo> listRSGroups() throws IOException { 549 return rsGroupInfoManager.listRSGroups(); 550 } 551 552 @Override 553 public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException { 554 return rsGroupInfoManager.getRSGroupOfServer(hostPort); 555 } 556 557 @Override 558 public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup) 559 throws IOException { 560 if (servers == null || servers.isEmpty()) { 561 throw new ConstraintException("The list of servers to move cannot be null or empty."); 562 } 563 if (tables == null || tables.isEmpty()) { 564 throw new ConstraintException("The list of tables to move cannot be null or empty."); 565 } 566 567 //check target group 568 getAndCheckRSGroupInfo(targetGroup); 569 570 // Hold a lock on the manager instance while moving servers and tables to prevent 571 // another writer changing our state while we are working. 572 synchronized (rsGroupInfoManager) { 573 if (master.getMasterCoprocessorHost() != null) { 574 master.getMasterCoprocessorHost().preMoveServersAndTables(servers, tables, targetGroup); 575 } 576 //check servers and tables status 577 checkServersAndTables(servers, tables, targetGroup); 578 579 //Move servers and tables to a new group. 580 String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName(); 581 rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup); 582 583 //move regions which should not belong to these tables 584 moveRegionsFromServers(servers, tables, targetGroup); 585 //move regions which should belong to these servers 586 moveRegionsToServers(servers, tables, targetGroup); 587 588 if (master.getMasterCoprocessorHost() != null) { 589 master.getMasterCoprocessorHost().postMoveServersAndTables(servers, tables, targetGroup); 590 } 591 } 592 LOG.info("Move servers and tables done. Severs :" 593 + servers + " , Tables : " + tables + " => " + targetGroup); 594 } 595 596 @Override 597 public void removeServers(Set<Address> servers) throws IOException { 598 { 599 if (servers == null || servers.isEmpty()) { 600 throw new ConstraintException("The set of servers to remove cannot be null or empty."); 601 } 602 // Hold a lock on the manager instance while moving servers to prevent 603 // another writer changing our state while we are working. 604 synchronized (rsGroupInfoManager) { 605 if (master.getMasterCoprocessorHost() != null) { 606 master.getMasterCoprocessorHost().preRemoveServers(servers); 607 } 608 //check the set of servers 609 checkForDeadOrOnlineServers(servers); 610 rsGroupInfoManager.removeServers(servers); 611 if (master.getMasterCoprocessorHost() != null) { 612 master.getMasterCoprocessorHost().postRemoveServers(servers); 613 } 614 LOG.info("Remove decommissioned servers " + servers + " from rsgroup done."); 615 } 616 } 617 } 618 619 private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName) 620 throws IOException { 621 Map<String, RegionState> rit = Maps.newTreeMap(); 622 AssignmentManager am = master.getAssignmentManager(); 623 for(TableName tableName : getRSGroupInfo(groupName).getTables()) { 624 for(RegionInfo regionInfo: am.getRegionStates().getRegionsOfTable(tableName)) { 625 RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo); 626 if(state != null) { 627 rit.put(regionInfo.getEncodedName(), state); 628 } 629 } 630 } 631 return rit; 632 } 633 634 private Map<TableName, Map<ServerName, List<RegionInfo>>> 635 getRSGroupAssignmentsByTable(String groupName) throws IOException { 636 Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap(); 637 RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName); 638 Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap(); 639 for(Map.Entry<RegionInfo, ServerName> entry: 640 master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) { 641 TableName currTable = entry.getKey().getTable(); 642 ServerName currServer = entry.getValue(); 643 RegionInfo currRegion = entry.getKey(); 644 if (rsGroupInfo.getTables().contains(currTable)) { 645 assignments.putIfAbsent(currTable, new HashMap<>()); 646 assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>()); 647 assignments.get(currTable).get(currServer).add(currRegion); 648 } 649 } 650 651 Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap(); 652 for(ServerName serverName: master.getServerManager().getOnlineServers().keySet()) { 653 if(rsGroupInfo.getServers().contains(serverName.getAddress())) { 654 serverMap.put(serverName, Collections.emptyList()); 655 } 656 } 657 658 // add all tables that are members of the group 659 for(TableName tableName : rsGroupInfo.getTables()) { 660 if(assignments.containsKey(tableName)) { 661 result.put(tableName, new HashMap<>()); 662 result.get(tableName).putAll(serverMap); 663 result.get(tableName).putAll(assignments.get(tableName)); 664 LOG.debug("Adding assignments for " + tableName + ": " + assignments.get(tableName)); 665 } 666 } 667 668 return result; 669 } 670 671 /** 672 * Check if the set of servers are belong to dead servers list or online servers list. 673 * @param servers servers to remove 674 */ 675 private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException { 676 // This uglyness is because we only have Address, not ServerName. 677 Set<Address> onlineServers = new HashSet<>(); 678 List<ServerName> drainingServers = master.getServerManager().getDrainingServersList(); 679 for (ServerName server : master.getServerManager().getOnlineServers().keySet()) { 680 // Only online but not decommissioned servers are really online 681 if (!drainingServers.contains(server)) { 682 onlineServers.add(server.getAddress()); 683 } 684 } 685 686 Set<Address> deadServers = new HashSet<>(); 687 for(ServerName server: master.getServerManager().getDeadServers().copyServerNames()) { 688 deadServers.add(server.getAddress()); 689 } 690 691 for (Address address: servers) { 692 if (onlineServers.contains(address)) { 693 throw new ConstraintException( 694 "Server " + address + " is an online server, not allowed to remove."); 695 } 696 if (deadServers.contains(address)) { 697 throw new ConstraintException( 698 "Server " + address + " is on the dead servers list," 699 + " Maybe it will come back again, not allowed to remove."); 700 } 701 } 702 } 703}