View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import com.google.common.annotations.VisibleForTesting;
22  import com.google.common.cache.Cache;
23  import com.google.common.cache.CacheBuilder;
24  import com.google.common.collect.Lists;
25  import com.google.common.collect.Maps;
26  import com.google.protobuf.Descriptors;
27  import com.google.protobuf.Service;
28  
29  import java.io.IOException;
30  import java.io.InterruptedIOException;
31  import java.lang.reflect.Constructor;
32  import java.lang.reflect.InvocationTargetException;
33  import java.net.InetAddress;
34  import java.net.InetSocketAddress;
35  import java.net.UnknownHostException;
36  import java.util.ArrayList;
37  import java.util.Collection;
38  import java.util.Collections;
39  import java.util.Comparator;
40  import java.util.HashSet;
41  import java.util.Iterator;
42  import java.util.List;
43  import java.util.Map;
44  import java.util.Map.Entry;
45  import java.util.Set;
46  import java.util.concurrent.CountDownLatch;
47  import java.util.concurrent.TimeUnit;
48  import java.util.concurrent.atomic.AtomicInteger;
49  import java.util.concurrent.atomic.AtomicReference;
50  import java.util.regex.Pattern;
51  
52  import javax.servlet.ServletException;
53  import javax.servlet.http.HttpServlet;
54  import javax.servlet.http.HttpServletRequest;
55  import javax.servlet.http.HttpServletResponse;
56  
57  import org.apache.commons.logging.Log;
58  import org.apache.commons.logging.LogFactory;
59  import org.apache.hadoop.conf.Configuration;
60  import org.apache.hadoop.fs.FileSystem;
61  import org.apache.hadoop.fs.Path;
62  import org.apache.hadoop.hbase.ClusterStatus;
63  import org.apache.hadoop.hbase.CoordinatedStateException;
64  import org.apache.hadoop.hbase.CoordinatedStateManager;
65  import org.apache.hadoop.hbase.DoNotRetryIOException;
66  import org.apache.hadoop.hbase.HBaseIOException;
67  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
68  import org.apache.hadoop.hbase.HColumnDescriptor;
69  import org.apache.hadoop.hbase.HConstants;
70  import org.apache.hadoop.hbase.HRegionInfo;
71  import org.apache.hadoop.hbase.HTableDescriptor;
72  import org.apache.hadoop.hbase.MasterNotRunningException;
73  import org.apache.hadoop.hbase.MetaMigrationConvertingToPB;
74  import org.apache.hadoop.hbase.MetaTableAccessor;
75  import org.apache.hadoop.hbase.NamespaceDescriptor;
76  import org.apache.hadoop.hbase.NamespaceNotFoundException;
77  import org.apache.hadoop.hbase.PleaseHoldException;
78  import org.apache.hadoop.hbase.ProcedureInfo;
79  import org.apache.hadoop.hbase.ScheduledChore;
80  import org.apache.hadoop.hbase.Server;
81  import org.apache.hadoop.hbase.ServerLoad;
82  import org.apache.hadoop.hbase.ServerName;
83  import org.apache.hadoop.hbase.TableDescriptors;
84  import org.apache.hadoop.hbase.TableName;
85  import org.apache.hadoop.hbase.TableNotDisabledException;
86  import org.apache.hadoop.hbase.TableNotFoundException;
87  import org.apache.hadoop.hbase.TableStateManager;
88  import org.apache.hadoop.hbase.UnknownRegionException;
89  import org.apache.hadoop.hbase.backup.BackupType;
90  import org.apache.hadoop.hbase.backup.HBackupFileSystem;
91  import org.apache.hadoop.hbase.backup.impl.BackupManager;
92  import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
93  import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
94  import org.apache.hadoop.hbase.backup.master.FullTableBackupProcedure;
95  import org.apache.hadoop.hbase.backup.master.IncrementalTableBackupProcedure;
96  import org.apache.hadoop.hbase.classification.InterfaceAudience;
97  import org.apache.hadoop.hbase.client.Admin;
98  import org.apache.hadoop.hbase.client.MetaScanner;
99  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
102 import org.apache.hadoop.hbase.client.Result;
103 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
104 import org.apache.hadoop.hbase.exceptions.DeserializationException;
105 import org.apache.hadoop.hbase.executor.ExecutorType;
106 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
107 import org.apache.hadoop.hbase.ipc.RpcServer;
108 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
109 import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
110 import org.apache.hadoop.hbase.master.RegionState.State;
111 import org.apache.hadoop.hbase.master.balancer.BalancerChore;
112 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
113 import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
114 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
115 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
116 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
117 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
118 import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
119 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
120 import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
121 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
122 import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
123 import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
124 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
125 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
126 import org.apache.hadoop.hbase.master.handler.TruncateTableHandler;
127 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
128 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
129 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
130 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
131 import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
132 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
133 import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
134 import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
135 import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
136 import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
137 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
138 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
139 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
140 import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
141 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
142 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
143 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
144 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
145 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
146 import org.apache.hadoop.hbase.mob.MobConstants;
147 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
148 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
149 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
150 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
151 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
152 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
153 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
154 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
155 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
156 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
157 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy;
158 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
159 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
160 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
161 import org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver;
162 import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
163 import org.apache.hadoop.hbase.quotas.QuotaUtil;
164 import org.apache.hadoop.hbase.quotas.RegionStateListener;
165 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifier;
166 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory;
167 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
168 import org.apache.hadoop.hbase.regionserver.HRegionServer;
169 import org.apache.hadoop.hbase.regionserver.HStore;
170 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
171 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
172 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
173 import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
174 import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
175 import org.apache.hadoop.hbase.replication.regionserver.Replication;
176 import org.apache.hadoop.hbase.security.AccessDeniedException;
177 import org.apache.hadoop.hbase.security.UserProvider;
178 import org.apache.hadoop.hbase.util.Addressing;
179 import org.apache.hadoop.hbase.util.Bytes;
180 import org.apache.hadoop.hbase.util.CompressionTest;
181 import org.apache.hadoop.hbase.util.ConfigUtil;
182 import org.apache.hadoop.hbase.util.EncryptionTest;
183 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
184 import org.apache.hadoop.hbase.util.FSUtils;
185 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
186 import org.apache.hadoop.hbase.util.HasThread;
187 import org.apache.hadoop.hbase.util.IdLock;
188 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
189 import org.apache.hadoop.hbase.util.Pair;
190 import org.apache.hadoop.hbase.util.Threads;
191 import org.apache.hadoop.hbase.util.VersionInfo;
192 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
193 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
194 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
195 import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
196 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
197 import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
198 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
199 import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
200 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
201 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
202 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
203 import org.apache.zookeeper.KeeperException;
204 import org.mortbay.jetty.Connector;
205 import org.mortbay.jetty.nio.SelectChannelConnector;
206 import org.mortbay.jetty.servlet.Context;
207 
208 /**
209  * HMaster is the "master server" for HBase. An HBase cluster has one active
210  * master.  If many masters are started, all compete.  Whichever wins goes on to
211  * run the cluster.  All others park themselves in their constructor until
212  * master or cluster shutdown or until the active master loses its lease in
213  * zookeeper.  Thereafter, all running master jostle to take over master role.
214  *
215  * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
216  * this case it will tell all regionservers to go down and then wait on them
217  * all reporting in that they are down.  This master will then shut itself down.
218  *
219  * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
220  *
221  * @see org.apache.zookeeper.Watcher
222  */
223 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
224 @SuppressWarnings("deprecation")
225 public class HMaster extends HRegionServer implements MasterServices, Server {
226   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
227 
228   /**
229    * Protection against zombie master. Started once Master accepts active responsibility and
230    * starts taking over responsibilities. Allows a finite time window before giving up ownership.
231    */
232   private static class InitializationMonitor extends HasThread {
233     /** The amount of time in milliseconds to sleep before checking initialization status. */
234     public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
235     public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
236 
237     /**
238      * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
239      * true, do nothing otherwise.
240      */
241     public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
242     public static final boolean HALT_DEFAULT = false;
243 
244     private final HMaster master;
245     private final long timeout;
246     private final boolean haltOnTimeout;
247 
248     /** Creates a Thread that monitors the {@link #isInitialized()} state. */
249     InitializationMonitor(HMaster master) {
250       super("MasterInitializationMonitor");
251       this.master = master;
252       this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
253       this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
254       this.setDaemon(true);
255     }
256 
257     @Override
258     public void run() {
259       try {
260         while (!master.isStopped() && master.isActiveMaster()) {
261           Thread.sleep(timeout);
262           if (master.isInitialized()) {
263             LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
264           } else {
265             LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
266                 + " consider submitting a bug report including a thread dump of this process.");
267             if (haltOnTimeout) {
268               LOG.error("Zombie Master exiting. Thread dump to stdout");
269               Threads.printThreadInfo(System.out, "Zombie HMaster");
270               System.exit(-1);
271             }
272           }
273         }
274       } catch (InterruptedException ie) {
275         LOG.trace("InitMonitor thread interrupted. Existing.");
276       }
277     }
278   }
279 
280   // MASTER is name of the webapp and the attribute name used stuffing this
281   //instance into web context.
282   public static final String MASTER = "master";
283 
284   // Manager and zk listener for master election
285   private final ActiveMasterManager activeMasterManager;
286   // Region server tracker
287   RegionServerTracker regionServerTracker;
288   // Draining region server tracker
289   private DrainingServerTracker drainingServerTracker;
290   // Tracker for load balancer state
291   LoadBalancerTracker loadBalancerTracker;
292 
293   // Tracker for split and merge state
294   private SplitOrMergeTracker splitOrMergeTracker;
295 
296   // Tracker for region normalizer state
297   private RegionNormalizerTracker regionNormalizerTracker;
298 
299   /** Namespace stuff */
300   private TableNamespaceManager tableNamespaceManager;
301 
302   //Tracker for master maintenance mode setting
303   private MasterMaintenanceModeTracker maintenanceModeTracker;
304 
305   // Metrics for the HMaster
306   final MetricsMaster metricsMaster;
307   // file system manager for the master FS operations
308   private MasterFileSystem fileSystemManager;
309 
310   // server manager to deal with region server info
311   volatile ServerManager serverManager;
312 
313   // manager of assignment nodes in zookeeper
314   AssignmentManager assignmentManager;
315 
316   // buffer for "fatal error" notices from region servers
317   // in the cluster. This is only used for assisting
318   // operations/debugging.
319   MemoryBoundedLogMessageBuffer rsFatals;
320 
321   // flag set after we become the active master (used for testing)
322   private volatile boolean isActiveMaster = false;
323 
324   // flag set after we complete initialization once active,
325   // it is not private since it's used in unit tests
326   volatile boolean initialized = false;
327 
328   // flag set after master services are started,
329   // initialization may have not completed yet.
330   volatile boolean serviceStarted = false;
331 
332   // flag set after we complete assignMeta.
333   private volatile boolean serverShutdownHandlerEnabled = false;
334 
335   LoadBalancer balancer;
336   private RegionNormalizer normalizer;
337   private BalancerChore balancerChore;
338   private RegionNormalizerChore normalizerChore;
339   private ClusterStatusChore clusterStatusChore;
340   private ClusterStatusPublisher clusterStatusPublisherChore = null;
341   private PeriodicDoMetrics periodicDoMetricsChore = null;
342 
343   CatalogJanitor catalogJanitorChore;
344   private LogCleaner logCleaner;
345   private HFileCleaner hfileCleaner;
346   private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
347   private MobCompactionChore mobCompactChore;
348   private MasterMobCompactionThread mobCompactThread;
349   // used to synchronize the mobCompactionStates
350   private final IdLock mobCompactionLock = new IdLock();
351   // save the information of mob compactions in tables.
352   // the key is table name, the value is the number of compactions in that table.
353   private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
354 
355   MasterCoprocessorHost cpHost;
356 
357   private final boolean preLoadTableDescriptors;
358 
359   // Time stamps for when a hmaster became active
360   private long masterActiveTime;
361 
362   //should we check the compression codec type at master side, default true, HBASE-6370
363   private final boolean masterCheckCompression;
364 
365   //should we check encryption settings at master side, default true
366   private final boolean masterCheckEncryption;
367 
368   // This is for fallback to use the code from 1.0 release.
369   private enum ProcedureConf {
370     PROCEDURE_ENABLED, // default
371     HANDLER_USED, // handler code executed in DDL, procedure executor still start
372     PROCEDURE_FULLY_DISABLED, // procedure fully disabled
373   }
374   private final ProcedureConf procedureConf;
375 
376   Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
377 
378   // monitor for snapshot of hbase tables
379   SnapshotManager snapshotManager;
380   // monitor for distributed procedures
381   private MasterProcedureManagerHost mpmHost;
382 
383   // it is assigned after 'initialized' guard set to true, so should be volatile
384   private volatile MasterQuotaManager quotaManager;
385   private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier;
386   private QuotaObserverChore quotaObserverChore;
387 
388   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
389   private WALProcedureStore procedureStore;
390 
391   /** flag used in test cases in order to simulate RS failures during master initialization */
392   private volatile boolean initializationBeforeMetaAssignment = false;
393 
394   /** jetty server for master to redirect requests to regionserver infoServer */
395   private org.mortbay.jetty.Server masterJettyServer;
396 
397   public static class RedirectServlet extends HttpServlet {
398     private static final long serialVersionUID = 2894774810058302472L;
399     private static int regionServerInfoPort;
400 
401     @Override
402     public void doGet(HttpServletRequest request,
403         HttpServletResponse response) throws ServletException, IOException {
404       String redirectUrl = request.getScheme() + "://"
405         + request.getServerName() + ":" + regionServerInfoPort
406         + request.getRequestURI();
407       response.sendRedirect(redirectUrl);
408     }
409   }
410 
411   private static class PeriodicDoMetrics extends ScheduledChore {
412     private final HMaster server;
413     public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
414       super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval);
415       this.server = server;
416     }
417 
418     @Override
419     protected void chore() {
420       server.doMetrics();
421     }
422   }
423 
424   /**
425    * Initializes the HMaster. The steps are as follows:
426    * <p>
427    * <ol>
428    * <li>Initialize the local HRegionServer
429    * <li>Start the ActiveMasterManager.
430    * </ol>
431    * <p>
432    * Remaining steps of initialization occur in
433    * #finishActiveMasterInitialization(MonitoredTask) after
434    * the master becomes the active one.
435    *
436    * @throws InterruptedException
437    * @throws KeeperException
438    * @throws IOException
439    */
440   public HMaster(final Configuration conf, CoordinatedStateManager csm)
441       throws IOException, KeeperException, InterruptedException {
442     super(conf, csm);
443     this.rsFatals = new MemoryBoundedLogMessageBuffer(
444       conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
445 
446     LOG.info("hbase.rootdir=" + getRootDir() +
447       ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
448 
449     // Disable usage of meta replicas in the master
450     this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
451 
452     Replication.decorateMasterConfiguration(this.conf);
453     BackupManager.decorateMasterConfiguration(this.conf);
454 
455     // Hack! Maps DFSClient => Master for logs.  HDFS made this
456     // config param for task trackers, but we can piggyback off of it.
457     if (this.conf.get("mapreduce.task.attempt.id") == null) {
458       this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
459     }
460 
461     // should we check the compression codec type at master side, default true, HBASE-6370
462     this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
463 
464     // should we check encryption settings at master side, default true
465     this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
466 
467     this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
468 
469     // Check configuration to see whether procedure is disabled (not execute at all),
470     // unused (not used to execute DDL, but executor starts to complete unfinished operations
471     // in procedure store, or enabled (default behavior).
472     String procedureConfString = conf.get("hbase.master.procedure.tableddl", "enabled");
473     if (procedureConfString.equalsIgnoreCase("disabled")) {
474       LOG.info("Master will use handler for new table DDL"
475         + " and all unfinished table DDLs in procedure store will be disgarded.");
476       this.procedureConf = ProcedureConf.PROCEDURE_FULLY_DISABLED;
477     } else if (procedureConfString.equalsIgnoreCase("unused")) {
478       LOG.info("Master will use handler for new table DDL"
479         + " and all unfinished table DDLs in procedure store will continue to execute.");
480       this.procedureConf = ProcedureConf.HANDLER_USED;
481     } else {
482       this.procedureConf = ProcedureConf.PROCEDURE_ENABLED;
483     }
484     // preload table descriptor at startup
485     this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
486 
487     // Do we publish the status?
488 
489     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
490         HConstants.STATUS_PUBLISHED_DEFAULT);
491     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
492         conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
493             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
494             ClusterStatusPublisher.Publisher.class);
495 
496     if (shouldPublish) {
497       if (publisherClass == null) {
498         LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
499             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
500             " is not set - not publishing status");
501       } else {
502         clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
503         getChoreService().scheduleChore(clusterStatusPublisherChore);
504       }
505     }
506 
507     // Some unit tests don't need a cluster, so no zookeeper at all
508     if (!conf.getBoolean("hbase.testing.nocluster", false)) {
509       setInitLatch(new CountDownLatch(1));
510       activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
511       int infoPort = putUpJettyServer();
512       startActiveMasterManager(infoPort);
513     } else {
514       activeMasterManager = null;
515     }
516   }
517 
518   // return the actual infoPort, -1 means disable info server.
519   private int putUpJettyServer() throws IOException {
520     if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
521       return -1;
522     }
523     int infoPort = conf.getInt("hbase.master.info.port.orig",
524       HConstants.DEFAULT_MASTER_INFOPORT);
525     // -1 is for disabling info server, so no redirecting
526     if (infoPort < 0 || infoServer == null) {
527       return -1;
528     }
529     String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
530     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
531       String msg =
532           "Failed to start redirecting jetty server. Address " + addr
533               + " does not belong to this host. Correct configuration parameter: "
534               + "hbase.master.info.bindAddress";
535       LOG.error(msg);
536       throw new IOException(msg);
537     }
538 
539     RedirectServlet.regionServerInfoPort = infoServer.getPort();
540     if(RedirectServlet.regionServerInfoPort == infoPort) {
541       return infoPort;
542     }
543     masterJettyServer = new org.mortbay.jetty.Server();
544     Connector connector = new SelectChannelConnector();
545     connector.setHost(addr);
546     connector.setPort(infoPort);
547     masterJettyServer.addConnector(connector);
548     masterJettyServer.setStopAtShutdown(true);
549     Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
550     context.addServlet(RedirectServlet.class, "/*");
551     try {
552       masterJettyServer.start();
553     } catch (Exception e) {
554       throw new IOException("Failed to start redirecting jetty server", e);
555     }
556     return connector.getLocalPort();
557   }
558 
559   /**
560    * For compatibility, if failed with regionserver credentials, try the master one
561    */
562   @Override
563   protected void login(UserProvider user, String host) throws IOException {
564     try {
565       super.login(user, host);
566     } catch (IOException ie) {
567       user.login("hbase.master.keytab.file",
568         "hbase.master.kerberos.principal", host);
569     }
570   }
571 
572   /**
573    * If configured to put regions on active master,
574    * wait till a backup master becomes active.
575    * Otherwise, loop till the server is stopped or aborted.
576    */
577   @Override
578   protected void waitForMasterActive(){
579     boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
580     while (!(tablesOnMaster && isActiveMaster)
581         && !isStopped() && !isAborted()) {
582       sleeper.sleep();
583     }
584   }
585 
586   @VisibleForTesting
587   public MasterRpcServices getMasterRpcServices() {
588     return (MasterRpcServices)rpcServices;
589   }
590 
591   public boolean balanceSwitch(final boolean b) throws IOException {
592     return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
593   }
594 
595   @Override
596   protected String getProcessName() {
597     return MASTER;
598   }
599 
600   @Override
601   protected boolean canCreateBaseZNode() {
602     return true;
603   }
604 
605   @Override
606   protected boolean canUpdateTableDescriptor() {
607     return true;
608   }
609 
610   @Override
611   protected RSRpcServices createRpcServices() throws IOException {
612     return new MasterRpcServices(this);
613   }
614 
615   @Override
616   protected void configureInfoServer() {
617     infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
618     infoServer.setAttribute(MASTER, this);
619     if (BaseLoadBalancer.tablesOnMaster(conf)) {
620       super.configureInfoServer();
621     }
622   }
623 
624   @Override
625   protected Class<? extends HttpServlet> getDumpServlet() {
626     return MasterDumpServlet.class;
627   }
628 
629   /**
630    * Emit the HMaster metrics, such as region in transition metrics.
631    * Surrounding in a try block just to be sure metrics doesn't abort HMaster.
632    */
633   private void doMetrics() {
634     try {
635       if (assignmentManager != null) {
636         assignmentManager.updateRegionsInTransitionMetrics();
637       }
638     } catch (Throwable e) {
639       LOG.error("Couldn't update metrics: " + e.getMessage());
640     }
641   }
642 
643   MetricsMaster getMasterMetrics() {
644     return metricsMaster;
645   }
646 
647   /**
648    * Initialize all ZK based system trackers.
649    * @throws IOException
650    * @throws InterruptedException
651    * @throws KeeperException
652    * @throws CoordinatedStateException
653    */
654   void initializeZKBasedSystemTrackers() throws IOException,
655       InterruptedException, KeeperException, CoordinatedStateException {
656     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
657     this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
658     this.normalizer.setMasterServices(this);
659     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
660     this.loadBalancerTracker.start();
661 
662     this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this);
663     this.regionNormalizerTracker.start();
664 
665     this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
666     this.splitOrMergeTracker.start();
667 
668     this.assignmentManager = new AssignmentManager(this, serverManager,
669       this.balancer, this.service, this.metricsMaster,
670       this.tableLockManager);
671     zooKeeper.registerListenerFirst(assignmentManager);
672 
673     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
674         this.serverManager);
675     this.regionServerTracker.start();
676 
677     this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
678       this.serverManager);
679     this.drainingServerTracker.start();
680 
681     this.maintenanceModeTracker = new MasterMaintenanceModeTracker(zooKeeper);
682     this.maintenanceModeTracker.start();
683 
684     // Set the cluster as up.  If new RSs, they'll be waiting on this before
685     // going ahead with their startup.
686     boolean wasUp = this.clusterStatusTracker.isClusterUp();
687     if (!wasUp) this.clusterStatusTracker.setClusterUp();
688 
689     LOG.info("Server active/primary master=" + this.serverName +
690         ", sessionid=0x" +
691         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
692         ", setting cluster-up flag (Was=" + wasUp + ")");
693 
694     // create/initialize the snapshot manager and other procedure managers
695     this.snapshotManager = new SnapshotManager();
696     this.mpmHost = new MasterProcedureManagerHost();
697     this.mpmHost.register(this.snapshotManager);
698     this.mpmHost.register(new MasterFlushTableProcedureManager());
699     this.mpmHost.loadProcedures(conf);
700     this.mpmHost.initialize(this, this.metricsMaster);
701   }
702 
703   /**
704    * Finish initialization of HMaster after becoming the primary master.
705    *
706    * <ol>
707    * <li>Initialize master components - file system manager, server manager,
708    *     assignment manager, region server tracker, etc</li>
709    * <li>Start necessary service threads - balancer, catalog janior,
710    *     executor services, etc</li>
711    * <li>Set cluster as UP in ZooKeeper</li>
712    * <li>Wait for RegionServers to check-in</li>
713    * <li>Split logs and perform data recovery, if necessary</li>
714    * <li>Ensure assignment of meta/namespace regions<li>
715    * <li>Handle either fresh cluster start or master failover</li>
716    * </ol>
717    *
718    * @throws IOException
719    * @throws InterruptedException
720    * @throws KeeperException
721    * @throws CoordinatedStateException
722    */
723   private void finishActiveMasterInitialization(MonitoredTask status)
724       throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
725 
726     isActiveMaster = true;
727     Thread zombieDetector = new Thread(new InitializationMonitor(this));
728     zombieDetector.start();
729 
730     /*
731      * We are active master now... go initialize components we need to run.
732      * Note, there may be dross in zk from previous runs; it'll get addressed
733      * below after we determine if cluster startup or failover.
734      */
735 
736     status.setStatus("Initializing Master file system");
737 
738     this.masterActiveTime = System.currentTimeMillis();
739     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
740     this.fileSystemManager = new MasterFileSystem(this, this);
741 
742     // enable table descriptors cache
743     this.tableDescriptors.setCacheOn();
744     // set the META's descriptor to the correct replication
745     this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
746         conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
747     // warm-up HTDs cache on master initialization
748     if (preLoadTableDescriptors) {
749       status.setStatus("Pre-loading table descriptors");
750       this.tableDescriptors.getAll();
751     }
752 
753     // publish cluster ID
754     status.setStatus("Publishing Cluster ID in ZooKeeper");
755     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
756     this.initLatch.countDown();
757     this.serverManager = createServerManager(this, this);
758 
759     setupClusterConnection();
760 
761     // Invalidate all write locks held previously
762     this.tableLockManager.reapWriteLocks();
763 
764     status.setStatus("Initializing ZK system trackers");
765     initializeZKBasedSystemTrackers();
766 
767     // Add the Observer to delete space quotas on table deletion before starting all CPs by
768     // default with quota support, avoiding if user specifically asks to not load this Observer.
769     if (QuotaUtil.isQuotaEnabled(conf)) {
770       updateConfigurationForSpaceQuotaObserver(conf);
771     }
772     // initialize master side coprocessors before we start handling requests
773     status.setStatus("Initializing master coprocessors");
774     this.cpHost = new MasterCoprocessorHost(this, this.conf);
775 
776     // start up all service threads.
777     status.setStatus("Initializing master service threads");
778     startServiceThreads();
779 
780     // Wake up this server to check in
781     sleeper.skipSleepCycle();
782 
783     // Wait for region servers to report in
784     this.serverManager.waitForRegionServers(status);
785     // Check zk for region servers that are up but didn't register
786     for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
787       // The isServerOnline check is opportunistic, correctness is handled inside
788       if (!this.serverManager.isServerOnline(sn)
789           && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
790         LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
791       }
792     }
793 
794     // get a list for previously failed RS which need log splitting work
795     // we recover hbase:meta region servers inside master initialization and
796     // handle other failed servers in SSH in order to start up master node ASAP
797     Set<ServerName> previouslyFailedServers = this.fileSystemManager
798         .getFailedServersFromLogFolders();
799 
800     // remove stale recovering regions from previous run
801     this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);
802 
803     // log splitting for hbase:meta server
804     ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
805     if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
806       splitMetaLogBeforeAssignment(oldMetaServerLocation);
807       // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it
808       // may also host user regions
809     }
810     Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
811     // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers
812     // instead of previouslyFailedMetaRSs alone to address the following two situations:
813     // 1) the chained failure situation(recovery failed multiple times in a row).
814     // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the
815     // same server still has non-meta wals to be replayed so that
816     // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region
817     // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,
818     // there is no op for the server.
819     previouslyFailedMetaRSs.addAll(previouslyFailedServers);
820 
821     this.initializationBeforeMetaAssignment = true;
822 
823     // Wait for regionserver to finish initialization.
824     if (BaseLoadBalancer.tablesOnMaster(conf)) {
825       waitForServerOnline();
826     }
827 
828     //initialize load balancer
829     this.balancer.setClusterStatus(getClusterStatus());
830     this.balancer.setMasterServices(this);
831     this.balancer.initialize();
832 
833     // Check if master is shutting down because of some issue
834     // in initializing the regionserver or the balancer.
835     if(isStopped()) return;
836 
837     // Make sure meta assigned before proceeding.
838     status.setStatus("Assigning Meta Region");
839     assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID);
840     // check if master is shutting down because above assignMeta could return even hbase:meta isn't
841     // assigned when master is shutting down
842     if(isStopped()) return;
843 
844     status.setStatus("Submitting log splitting work for previously failed region servers");
845     // Master has recovered hbase:meta region server and we put
846     // other failed region servers in a queue to be handled later by SSH
847     for (ServerName tmpServer : previouslyFailedServers) {
848       this.serverManager.processDeadServer(tmpServer, true);
849     }
850 
851     // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization
852     // in meta. This must happen before we assign all user regions or else the assignment will
853     // fail.
854     if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {
855       MetaMigrationConvertingToPB.updateMetaIfNecessary(this);
856     }
857 
858     // Fix up assignment manager status
859     status.setStatus("Starting assignment manager");
860     this.assignmentManager.joinCluster();
861 
862     //set cluster status again after user regions are assigned
863     this.balancer.setClusterStatus(getClusterStatus());
864 
865     // Start balancer and meta catalog janitor after meta and regions have
866     // been assigned.
867     status.setStatus("Starting balancer and catalog janitor");
868     this.clusterStatusChore = new ClusterStatusChore(this, balancer);
869     getChoreService().scheduleChore(clusterStatusChore);
870     this.balancerChore = new BalancerChore(this);
871     getChoreService().scheduleChore(balancerChore);
872     this.normalizerChore = new RegionNormalizerChore(this);
873     getChoreService().scheduleChore(normalizerChore);
874     this.catalogJanitorChore = new CatalogJanitor(this, this);
875     getChoreService().scheduleChore(catalogJanitorChore);
876 
877     // Do Metrics periodically
878     periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
879     getChoreService().scheduleChore(periodicDoMetricsChore);
880 
881     status.setStatus("Starting namespace manager and quota manager");
882     initNamespaceAndQuotaManager();
883 
884     if (this.cpHost != null) {
885       try {
886         this.cpHost.preMasterInitialization();
887       } catch (IOException e) {
888         LOG.error("Coprocessor preMasterInitialization() hook failed", e);
889       }
890     }
891 
892     status.markComplete("Initialization successful");
893     LOG.info("Master has completed initialization");
894     configurationManager.registerObserver(this.balancer);
895     //configurationManager.registerObserver(this.hfileCleaner);
896     initialized = true;
897 
898     status.setStatus("Starting quota manager");
899     if (QuotaUtil.isQuotaEnabled(conf)) {
900       // Create the quota snapshot notifier
901       spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
902       spaceQuotaSnapshotNotifier.initialize(this.clusterConnection);
903       this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
904       // Start the chore to read the region FS space reports and act on them
905       getChoreService().scheduleChore(quotaObserverChore);
906     }
907 
908     // assign the meta replicas
909     Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
910     int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
911            HConstants.DEFAULT_META_REPLICA_NUM);
912     for (int i = 1; i < numReplicas; i++) {
913       assignMeta(status, EMPTY_SET, i);
914     }
915     unassignExcessMetaReplica(zooKeeper, numReplicas);
916 
917     // clear the dead servers with same host name and port of online server because we are not
918     // removing dead server with same hostname and port of rs which is trying to check in before
919     // master initialization. See HBASE-5916.
920     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
921 
922     // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
923     status.setStatus("Checking ZNode ACLs");
924     zooKeeper.checkAndSetZNodeAcls();
925 
926     status.setStatus("Calling postStartMaster coprocessors");
927 
928     this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);
929     getChoreService().scheduleChore(expiredMobFileCleanerChore);
930 
931     int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
932       MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);
933     if (mobCompactionPeriod > 0) {
934       this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);
935       getChoreService().scheduleChore(mobCompactChore);
936     } else {
937       LOG
938         .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");
939     }
940     this.mobCompactThread = new MasterMobCompactionThread(this);
941 
942     if (this.cpHost != null) {
943       // don't let cp initialization errors kill the master
944       try {
945         this.cpHost.postStartMaster();
946       } catch (IOException ioe) {
947         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
948       }
949     }
950 
951     zombieDetector.interrupt();
952   }
953 
954   /**
955    * Adds the {@code MasterSpaceQuotaObserver} to the list of configured Master observers to
956    * automatically remove space quotas for a table when that table is deleted.
957    */
958   @VisibleForTesting
959   public void updateConfigurationForSpaceQuotaObserver(Configuration conf) {
960     // We're configured to not delete quotas on table deletion, so we don't need to add the obs.
961     if (!conf.getBoolean(
962           MasterSpaceQuotaObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
963           MasterSpaceQuotaObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
964       return;
965     }
966     String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
967     final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
968     String[] updatedCoprocs = new String[length + 1];
969     if (length > 0) {
970       System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
971     }
972     updatedCoprocs[length] = MasterSpaceQuotaObserver.class.getName();
973     conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
974   }
975 
976   private void initQuotaManager() throws IOException {
977     quotaManager = new MasterQuotaManager(this);
978     this.assignmentManager.setRegionStateListener((RegionStateListener) quotaManager);
979     quotaManager.start();
980   }
981 
982   /**
983    * Create a {@link ServerManager} instance.
984    * @param master
985    * @param services
986    * @return An instance of {@link ServerManager}
987    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
988    * @throws IOException
989    */
990   ServerManager createServerManager(final Server master,
991       final MasterServices services)
992   throws IOException {
993     // We put this out here in a method so can do a Mockito.spy and stub it out
994     // w/ a mocked up ServerManager.
995     return new ServerManager(master, services);
996   }
997 
998   private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicasConfigured) {
999     // unassign the unneeded replicas (for e.g., if the previous master was configured
1000     // with a replication of 3 and now it is 2, we need to unassign the 1 unneeded replica)
1001     try {
1002       List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
1003       for (String metaReplicaZnode : metaReplicaZnodes) {
1004         int replicaId = zooKeeper.getMetaReplicaIdFromZnode(metaReplicaZnode);
1005         if (replicaId >= numMetaReplicasConfigured) {
1006           RegionState r = MetaTableLocator.getMetaRegionState(zkw, replicaId);
1007           LOG.info("Closing excess replica of meta region " + r.getRegion());
1008           // send a close and wait for a max of 30 seconds
1009           ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(),
1010               r.getRegion(), 30000);
1011           ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId));
1012         }
1013       }
1014     } catch (Exception ex) {
1015       // ignore the exception since we don't want the master to be wedged due to potential
1016       // issues in the cleanup of the extra regions. We can do that cleanup via hbck or manually
1017       LOG.warn("Ignoring exception " + ex);
1018     }
1019   }
1020 
1021   /**
1022    * Check <code>hbase:meta</code> is assigned. If not, assign it.
1023    * @param status MonitoredTask
1024    * @param previouslyFailedMetaRSs
1025    * @param replicaId
1026    * @throws InterruptedException
1027    * @throws IOException
1028    * @throws KeeperException
1029    */
1030   void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
1031       throws InterruptedException, IOException, KeeperException {
1032     // Work on meta region
1033     int assigned = 0;
1034     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
1035     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
1036       status.setStatus("Assigning hbase:meta region");
1037     } else {
1038       status.setStatus("Assigning hbase:meta region, replicaId " + replicaId);
1039     }
1040     // Get current meta state from zk.
1041     RegionStates regionStates = assignmentManager.getRegionStates();
1042     RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper(), replicaId);
1043     HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
1044         replicaId);
1045     ServerName currentMetaServer = metaState.getServerName();
1046     if (!ConfigUtil.useZKForAssignment(conf)) {
1047       regionStates.createRegionState(hri, metaState.getState(),
1048         currentMetaServer, null);
1049     } else {
1050       regionStates.createRegionState(hri);
1051     }
1052     boolean rit = this.assignmentManager.
1053       processRegionInTransitionAndBlockUntilAssigned(hri);
1054     boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation(
1055       this.getConnection(), this.getZooKeeper(), timeout, replicaId);
1056     if (!metaRegionLocation || !metaState.isOpened()) {
1057       // Meta location is not verified. It should be in transition, or offline.
1058       // We will wait for it to be assigned in enableSSHandWaitForMeta below.
1059       assigned++;
1060       if (!ConfigUtil.useZKForAssignment(conf)) {
1061         assignMetaZkLess(regionStates, metaState, timeout, previouslyFailedMetaRSs);
1062       } else if (!rit) {
1063         // Assign meta since not already in transition
1064         if (currentMetaServer != null) {
1065           // If the meta server is not known to be dead or online,
1066           // just split the meta log, and don't expire it since this
1067           // could be a full cluster restart. Otherwise, we will think
1068           // this is a failover and lose previous region locations.
1069           // If it is really a failover case, AM will find out in rebuilding
1070           // user regions. Otherwise, we are good since all logs are split
1071           // or known to be replayed before user regions are assigned.
1072           if (serverManager.isServerOnline(currentMetaServer)) {
1073             LOG.info("Forcing expire of " + currentMetaServer);
1074             serverManager.expireServer(currentMetaServer);
1075           }
1076           if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
1077             splitMetaLogBeforeAssignment(currentMetaServer);
1078             previouslyFailedMetaRSs.add(currentMetaServer);
1079           }
1080         }
1081         assignmentManager.assignMeta(hri);
1082       }
1083     } else {
1084       // Region already assigned. We didn't assign it. Add to in-memory state.
1085       regionStates.updateRegionState(hri, State.OPEN, currentMetaServer);
1086       this.assignmentManager.regionOnline(hri, currentMetaServer);
1087     }
1088 
1089     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableMeta(TableName.META_TABLE_NAME);
1090 
1091     if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
1092         && (!previouslyFailedMetaRSs.isEmpty())) {
1093       // replay WAL edits mode need new hbase:meta RS is assigned firstly
1094       status.setStatus("replaying log for Meta Region");
1095       this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
1096     }
1097 
1098     // Make sure a hbase:meta location is set. We need to enable SSH here since
1099     // if the meta region server is died at this time, we need it to be re-assigned
1100     // by SSH so that system tables can be assigned.
1101     // No need to wait for meta is assigned = 0 when meta is just verified.
1102     if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableServerShutdownHandler(assigned != 0);
1103     LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", rit=" + rit +
1104       ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId));
1105     status.setStatus("META assigned.");
1106   }
1107 
1108   private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout,
1109       Set<ServerName> previouslyFailedRs) throws IOException, KeeperException {
1110     ServerName currentServer = regionState.getServerName();
1111     if (serverManager.isServerOnline(currentServer)) {
1112       LOG.info("Meta was in transition on " + currentServer);
1113       assignmentManager.processRegionInTransitionZkLess();
1114     } else {
1115       if (currentServer != null) {
1116         if (regionState.getRegion().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1117           splitMetaLogBeforeAssignment(currentServer);
1118           regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
1119           previouslyFailedRs.add(currentServer);
1120         }
1121       }
1122       LOG.info("Re-assigning hbase:meta, it was on " + currentServer);
1123       regionStates.updateRegionState(regionState.getRegion(), State.OFFLINE);
1124       assignmentManager.assignMeta(regionState.getRegion());
1125     }
1126   }
1127 
1128   /*
1129    * The main purpose is to start namespace manager and quota manager async to	   
1130    * unblock overall master initialization
1131    *
1132    * @throws IOException
1133    */
1134   private void initNamespaceAndQuotaManager() throws IOException {
1135     //create namespace manager
1136 	tableNamespaceManager = new TableNamespaceManager(this);
1137 
1138     //create quota manager
1139 	this.quotaManager = new MasterQuotaManager(this);
1140 	this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
1141 
1142     if (conf.getBoolean("hbase.master.start.wait.for.namespacemanager", false)) {
1143       // If being asked not to async start namespace manager, then just block
1144 	  // master service starting until namespace manager is ready.
1145 	  //
1146       // Note: Quota manager depends on namespace manager.  Therefore, its starting
1147       // method has to be in-sync with namespace manager.
1148       LOG.info("Starting namespace manager and quota manager synchronously");
1149 	
1150       tableNamespaceManager.start();
1151       LOG.info("Namespace manager started successfully.");
1152 
1153 	  quotaManager.start();
1154 	  LOG.info("Quota manager started successfully.");
1155 	} else { // else asynchronously start namespace manager and quota manager
1156       LOG.info("Starting namespace manager and quota manager asynchronously");
1157       Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1158         @Override
1159         public void run() {
1160           // Start namespace manager and wait to it to be fully started.
1161           try {
1162             tableNamespaceManager.start();
1163             LOG.info("Namespace manager started successfully.");
1164           } catch (IOException e) {
1165         	LOG.error("Namespace manager failed to start. ", e);
1166             abort("Shutdown Master due to namespace manager failed to start. ", e);
1167           }
1168           // Quota Manager depends on Namespace manager to be fully initialized.
1169           try {
1170             quotaManager.start();
1171             LOG.info("Quota manager started successfully.");
1172           } catch (IOException ie) {
1173             LOG.error("Quota Manager failed to start. ", ie);
1174             abort("Shutdown Master due to Quota Manager failure to start. ", ie);
1175           }
1176 	    }
1177 	  }, "Init Namespace Manager and Quota Manager Async"));
1178 	}
1179   }
1180 
1181   SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
1182     SpaceQuotaSnapshotNotifier notifier =
1183         SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());
1184     notifier.initialize(getConnection());
1185     return notifier;
1186   }
1187 
1188   boolean isCatalogJanitorEnabled() {
1189     return catalogJanitorChore != null ?
1190       catalogJanitorChore.getEnabled() : false;
1191   }
1192 
1193   private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
1194     if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
1195       // In log replay mode, we mark hbase:meta region as recovering in ZK
1196       Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
1197       regions.add(HRegionInfo.FIRST_META_REGIONINFO);
1198       this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
1199     } else {
1200       // In recovered.edits mode: create recovered edits file for hbase:meta server
1201       this.fileSystemManager.splitMetaLog(currentMetaServer);
1202     }
1203   }
1204 
1205   private void enableServerShutdownHandler(
1206       final boolean waitForMeta) throws IOException, InterruptedException {
1207     // If ServerShutdownHandler is disabled, we enable it and expire those dead
1208     // but not expired servers. This is required so that if meta is assigning to
1209     // a server which dies after assignMeta starts assignment,
1210     // SSH can re-assign it. Otherwise, we will be
1211     // stuck here waiting forever if waitForMeta is specified.
1212     if (!serverShutdownHandlerEnabled) {
1213       serverShutdownHandlerEnabled = true;
1214       this.serverManager.processQueuedDeadServers();
1215     }
1216 
1217     if (waitForMeta) {
1218       metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
1219       // Above check waits for general meta availability but this does not
1220       // guarantee that the transition has completed
1221       this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
1222     }
1223   }
1224 
1225   private void enableMeta(TableName metaTableName) {
1226     if (!this.assignmentManager.getTableStateManager().isTableState(metaTableName,
1227         ZooKeeperProtos.Table.State.ENABLED)) {
1228       this.assignmentManager.setEnabledTable(metaTableName);
1229     }
1230   }
1231 
1232   /**
1233    * This function returns a set of region server names under hbase:meta recovering region ZK node
1234    * @return Set of meta server names which were recorded in ZK
1235    * @throws KeeperException
1236    */
1237   private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
1238     Set<ServerName> result = new HashSet<ServerName>();
1239     String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
1240       HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1241     List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
1242     if (regionFailedServers == null) return result;
1243 
1244     for(String failedServer : regionFailedServers) {
1245       ServerName server = ServerName.parseServerName(failedServer);
1246       result.add(server);
1247     }
1248     return result;
1249   }
1250 
1251   @Override
1252   public TableDescriptors getTableDescriptors() {
1253     return this.tableDescriptors;
1254   }
1255 
1256   @Override
1257   public ServerManager getServerManager() {
1258     return this.serverManager;
1259   }
1260 
1261   @Override
1262   public MasterFileSystem getMasterFileSystem() {
1263     return this.fileSystemManager;
1264   }
1265 
1266   /*
1267    * Start up all services. If any of these threads gets an unhandled exception
1268    * then they just die with a logged message.  This should be fine because
1269    * in general, we do not expect the master to get such unhandled exceptions
1270    *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
1271    *  need to install an unexpected exception handler.
1272    */
1273   private void startServiceThreads() throws IOException{
1274    // Start the executor service pools
1275    this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
1276       conf.getInt("hbase.master.executor.openregion.threads", 5));
1277    this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
1278       conf.getInt("hbase.master.executor.closeregion.threads", 5));
1279    this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1280       conf.getInt("hbase.master.executor.serverops.threads", 5));
1281    this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1282       conf.getInt("hbase.master.executor.serverops.threads", 5));
1283    this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
1284       conf.getInt("hbase.master.executor.logreplayops.threads", 10));
1285 
1286    // We depend on there being only one instance of this executor running
1287    // at a time.  To do concurrency, would need fencing of enable/disable of
1288    // tables.
1289    // Any time changing this maxThreads to > 1, pls see the comment at
1290    // AccessController#postCreateTableHandler
1291    this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1292    startProcedureExecutor();
1293 
1294    // Start log cleaner thread
1295    int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
1296    this.logCleaner =
1297       new LogCleaner(cleanerInterval,
1298          this, conf, getMasterFileSystem().getOldLogDir().getFileSystem(conf),
1299          getMasterFileSystem().getOldLogDir());
1300     getChoreService().scheduleChore(logCleaner);
1301 
1302    //start the hfile archive cleaner thread
1303     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1304     this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
1305         .getFileSystem(), archiveDir);
1306     getChoreService().scheduleChore(hfileCleaner);
1307     serviceStarted = true;
1308     if (LOG.isTraceEnabled()) {
1309       LOG.trace("Started service threads");
1310     }
1311   }
1312 
1313   @Override
1314   protected void sendShutdownInterrupt() {
1315     super.sendShutdownInterrupt();
1316     stopProcedureExecutor();
1317   }
1318 
1319   @Override
1320   protected void stopServiceThreads() {
1321     if (masterJettyServer != null) {
1322       LOG.info("Stopping master jetty server");
1323       try {
1324         masterJettyServer.stop();
1325       } catch (Exception e) {
1326         LOG.error("Failed to stop master jetty server", e);
1327       }
1328     }
1329     super.stopServiceThreads();
1330     stopChores();
1331 
1332     // Wait for all the remaining region servers to report in IFF we were
1333     // running a cluster shutdown AND we were NOT aborting.
1334     if (!isAborted() && this.serverManager != null &&
1335         this.serverManager.isClusterShutdown()) {
1336       this.serverManager.letRegionServersShutdown();
1337     }
1338     if (LOG.isDebugEnabled()) {
1339       LOG.debug("Stopping service threads");
1340     }
1341     // Clean up and close up shop
1342     if (this.logCleaner != null) this.logCleaner.cancel(true);
1343     if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
1344     if (this.quotaManager != null) this.quotaManager.stop();
1345     if (this.activeMasterManager != null) this.activeMasterManager.stop();
1346     if (this.serverManager != null) this.serverManager.stop();
1347     if (this.assignmentManager != null) this.assignmentManager.stop();
1348     if (this.fileSystemManager != null) this.fileSystemManager.stop();
1349     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1350   }
1351 
1352   /**
1353    * Check whether the procedure executor is enabled
1354    */
1355   @Override
1356   public boolean isMasterProcedureExecutorEnabled() {
1357     return (this.procedureConf == ProcedureConf.PROCEDURE_ENABLED);
1358   }
1359 
1360   private void startProcedureExecutor() throws IOException {
1361     final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1362     final Path walDir = new Path(FSUtils.getWALRootDir(this.conf),
1363         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
1364 
1365     if (this.procedureConf == ProcedureConf.PROCEDURE_FULLY_DISABLED) {
1366       // Clean up the procedure store so that we will in a clean state when procedure
1367       // is enabled later.
1368       // Note: hbck might needed for uncompleted procedures.
1369       try {
1370         fs.delete(walDir, true);
1371         LOG.warn("Procedure executor is disabled from configuartion. " +
1372             "All the state logs from procedure store were removed." +
1373             "You should check the cluster state using HBCK.");
1374       } catch (Exception e) {
1375         // Ignore exception and move on.
1376         LOG.error("Removing all the state logs from procedure store failed." +
1377             "You should check the cluster state using HBCK.");
1378       }
1379       return;
1380     }
1381 
1382     procedureStore = new WALProcedureStore(conf, walDir.getFileSystem(conf), walDir,
1383         new MasterProcedureEnv.WALStoreLeaseRecovery(this));
1384 
1385     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
1386     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
1387         procEnv.getProcedureQueue());
1388 
1389     final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
1390         Math.max(Runtime.getRuntime().availableProcessors(),
1391           MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1392     procedureStore.start(numThreads);
1393     procedureExecutor.start(numThreads);
1394   }
1395 
1396   private void stopProcedureExecutor() {
1397     if (procedureExecutor != null) {
1398       procedureExecutor.stop();
1399     }
1400 
1401     if (procedureStore != null) {
1402       procedureStore.stop(isAborted());
1403     }
1404   }
1405 
1406   private void stopChores() {
1407     if (this.expiredMobFileCleanerChore != null) {
1408       this.expiredMobFileCleanerChore.cancel(true);
1409     }
1410     if (this.mobCompactChore != null) {
1411       this.mobCompactChore.cancel(true);
1412     }
1413     if (this.balancerChore != null) {
1414       this.balancerChore.cancel(true);
1415     }
1416     if (this.normalizerChore != null) {
1417       this.normalizerChore.cancel(true);
1418     }
1419     if (this.clusterStatusChore != null) {
1420       this.clusterStatusChore.cancel(true);
1421     }
1422     if (this.catalogJanitorChore != null) {
1423       this.catalogJanitorChore.cancel(true);
1424     }
1425     if (this.clusterStatusPublisherChore != null){
1426       clusterStatusPublisherChore.cancel(true);
1427     }
1428     if (this.mobCompactThread != null) {
1429       this.mobCompactThread.close();
1430     }
1431     if (this.quotaObserverChore != null) {
1432       quotaObserverChore.cancel();
1433     }
1434     if (this.periodicDoMetricsChore != null) {
1435       periodicDoMetricsChore.cancel();
1436     }
1437   }
1438 
1439   /**
1440    * @return Get remote side's InetAddress
1441    * @throws UnknownHostException
1442    */
1443   InetAddress getRemoteInetAddress(final int port,
1444       final long serverStartCode) throws UnknownHostException {
1445     // Do it out here in its own little method so can fake an address when
1446     // mocking up in tests.
1447     InetAddress ia = RpcServer.getRemoteIp();
1448 
1449     // The call could be from the local regionserver,
1450     // in which case, there is no remote address.
1451     if (ia == null && serverStartCode == startcode) {
1452       InetSocketAddress isa = rpcServices.getSocketAddress();
1453       if (isa != null && isa.getPort() == port) {
1454         ia = isa.getAddress();
1455       }
1456     }
1457     return ia;
1458   }
1459 
1460   /**
1461    * @return Maximum time we should run balancer for
1462    */
1463   private int getBalancerCutoffTime() {
1464     int balancerCutoffTime =
1465       getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1466     if (balancerCutoffTime == -1) {
1467       // No time period set so create one
1468       int balancerPeriod =
1469         getConfiguration().getInt("hbase.balancer.period", 300000);
1470       balancerCutoffTime = balancerPeriod;
1471       // If nonsense period, set it to balancerPeriod
1472       if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;
1473     }
1474     return balancerCutoffTime;
1475   }
1476 
1477   public boolean balance() throws IOException {
1478     return balance(false);
1479   }
1480 
1481   public boolean balance(boolean force) throws IOException {
1482     // if master not initialized, don't run balancer.
1483     if (!this.initialized) {
1484       LOG.debug("Master has not been initialized, don't run balancer.");
1485       return false;
1486     }
1487 
1488     if (isInMaintenanceMode()) {
1489       LOG.info("Master is in maintenanceMode mode, don't run balancer.");
1490       return false;
1491     }
1492 
1493     // Do this call outside of synchronized block.
1494     int maximumBalanceTime = getBalancerCutoffTime();
1495     synchronized (this.balancer) {
1496       // If balance not true, don't run balancer.
1497       if (!this.loadBalancerTracker.isBalancerOn()) return false;
1498       // Only allow one balance run at at time.
1499       if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1500         Map<String, RegionState> regionsInTransition =
1501           this.assignmentManager.getRegionStates().getRegionsInTransition();
1502         // if hbase:meta region is in transition, result of assignment cannot be recorded
1503         // ignore the force flag in that case
1504         boolean metaInTransition = assignmentManager.getRegionStates().isMetaRegionInTransition();
1505         String prefix = force && !metaInTransition ? "R" : "Not r";
1506         LOG.debug(prefix + "unning balancer because " + regionsInTransition.size() +
1507           " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1508             abbreviate(regionsInTransition.toString(), 256));
1509         if (!force || metaInTransition) return false;
1510       }
1511       if (this.serverManager.areDeadServersInProgress()) {
1512         LOG.debug("Not running balancer because processing dead regionserver(s): " +
1513           this.serverManager.getDeadServers());
1514         return false;
1515       }
1516 
1517       if (this.cpHost != null) {
1518         try {
1519           if (this.cpHost.preBalance()) {
1520             LOG.debug("Coprocessor bypassing balancer request");
1521             return false;
1522           }
1523         } catch (IOException ioe) {
1524           LOG.error("Error invoking master coprocessor preBalance()", ioe);
1525           return false;
1526         }
1527       }
1528 
1529       Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1530         this.assignmentManager.getRegionStates().getAssignmentsByTable();
1531 
1532       List<RegionPlan> plans = new ArrayList<RegionPlan>();
1533 
1534       //Give the balancer the current cluster state.
1535       this.balancer.setClusterStatus(getClusterStatus());
1536       for (Entry<TableName, Map<ServerName, List<HRegionInfo>>> e : assignmentsByTable.entrySet()) {
1537         List<RegionPlan> partialPlans = this.balancer.balanceCluster(e.getKey(), e.getValue());
1538         if (partialPlans != null) plans.addAll(partialPlans);
1539       }
1540 
1541       long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1542       int rpCount = 0;  // number of RegionPlans balanced so far
1543       long totalRegPlanExecTime = 0;
1544       if (plans != null && !plans.isEmpty()) {
1545         for (RegionPlan plan: plans) {
1546           LOG.info("balance " + plan);
1547           long balStartTime = System.currentTimeMillis();
1548           //TODO: bulk assign
1549           this.assignmentManager.balance(plan);
1550           totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1551           rpCount++;
1552           if (rpCount < plans.size() &&
1553               // if performing next balance exceeds cutoff time, exit the loop
1554               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1555             //TODO: After balance, there should not be a cutoff time (keeping it as a security net for now)
1556             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1557               maximumBalanceTime);
1558             break;
1559           }
1560         }
1561       }
1562       if (this.cpHost != null) {
1563         try {
1564           this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1565         } catch (IOException ioe) {
1566           // balancing already succeeded so don't change the result
1567           LOG.error("Error invoking master coprocessor postBalance()", ioe);
1568         }
1569       }
1570     }
1571     // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1572     // Return true indicating a success.
1573     return true;
1574   }
1575 
1576   /**
1577    * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
1578    *
1579    * @return true if normalization step was performed successfully, false otherwise
1580    *   (specifically, if HMaster hasn't been initialized properly or normalization
1581    *   is globally disabled)
1582    * @throws IOException, CoordinatedStateException
1583    */
1584   public boolean normalizeRegions() throws IOException, CoordinatedStateException {
1585     if (!this.initialized) {
1586       LOG.debug("Master has not been initialized, don't run region normalizer.");
1587       return false;
1588     }
1589 
1590     if (isInMaintenanceMode()) {
1591       LOG.info("Master is in maintenance mode, don't run region normalizer.");
1592       return false;
1593     }
1594 
1595     if (!this.regionNormalizerTracker.isNormalizerOn()) {
1596       LOG.debug("Region normalization is disabled, don't run region normalizer.");
1597       return false;
1598     }
1599 
1600     synchronized (this.normalizer) {
1601       // Don't run the normalizer concurrently
1602       List<TableName> allEnabledTables = new ArrayList<>(
1603         this.assignmentManager.getTableStateManager().getTablesInStates(
1604           ZooKeeperProtos.Table.State.ENABLED));
1605 
1606       Collections.shuffle(allEnabledTables);
1607 
1608       for (TableName table : allEnabledTables) {
1609         if (isInMaintenanceMode()) {
1610           LOG.debug("Master is in maintenance mode, stop running region normalizer.");
1611           return false;
1612         }
1613 
1614         if (table.isSystemTable() || (getTableDescriptors().get(table) != null &&
1615             !getTableDescriptors().get(table).isNormalizationEnabled())) {
1616           LOG.debug("Skipping normalization for table: " + table + ", as it's either system"
1617               + " table or doesn't have auto normalization turned on");
1618           continue;
1619         }
1620         List<NormalizationPlan> plans = this.normalizer.computePlanForTable(table);
1621         if (plans != null) {
1622           for (NormalizationPlan plan : plans) {
1623             plan.execute(clusterConnection.getAdmin());
1624           }
1625         }
1626       }
1627     }
1628     // If Region did not generate any plans, it means the cluster is already balanced.
1629     // Return true indicating a success.
1630     return true;
1631   }
1632 
1633   /**
1634    * @return Client info for use as prefix on an audit log string; who did an action
1635    */
1636   String getClientIdAuditPrefix() {
1637     return "Client=" + RpcServer.getRequestUserName() + "/" + RpcServer.getRemoteAddress();
1638   }
1639 
1640   /**
1641    * Switch for the background CatalogJanitor thread.
1642    * Used for testing.  The thread will continue to run.  It will just be a noop
1643    * if disabled.
1644    * @param b If false, the catalog janitor won't do anything.
1645    */
1646   public void setCatalogJanitorEnabled(final boolean b) {
1647     this.catalogJanitorChore.setEnabled(b);
1648   }
1649 
1650   @Override
1651   public void dispatchMergingRegions(final HRegionInfo region_a,
1652       final HRegionInfo region_b, final boolean forcible) throws IOException {
1653     checkInitialized();
1654     this.service.submit(new DispatchMergingRegionHandler(this,
1655       this.catalogJanitorChore, region_a, region_b, forcible));
1656   }
1657 
1658   void move(final byte[] encodedRegionName,
1659       final byte[] destServerName) throws HBaseIOException {
1660     RegionState regionState = assignmentManager.getRegionStates().
1661       getRegionState(Bytes.toString(encodedRegionName));
1662 
1663     HRegionInfo hri;
1664     if (regionState != null) {
1665       hri = regionState.getRegion();
1666     } else {
1667       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1668     }
1669 
1670     ServerName dest;
1671     if (destServerName == null || destServerName.length == 0) {
1672       LOG.info("Passed destination servername is null/empty so " +
1673         "choosing a server at random");
1674       final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1675         regionState.getServerName());
1676       dest = balancer.randomAssignment(hri, destServers);
1677       if (dest == null) {
1678         LOG.debug("Unable to determine a plan to assign " + hri);
1679         return;
1680       }
1681     } else {
1682       ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));
1683       dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));
1684       if (dest == null) {
1685         LOG.debug("Unable to determine a plan to assign " + hri);
1686         return;
1687       }
1688       if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1689           && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1690         // To avoid unnecessary region moving later by balancer. Don't put user
1691         // regions on master. Regions on master could be put on other region
1692         // server intentionally by test however.
1693         LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1694           + " to avoid unnecessary region moving later by load balancer,"
1695           + " because it should not be on master");
1696         return;
1697       }
1698     }
1699 
1700     if (dest.equals(regionState.getServerName())) {
1701       LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1702         + " because region already assigned to the same server " + dest + ".");
1703       return;
1704     }
1705 
1706     // Now we can do the move
1707     RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1708 
1709     try {
1710       checkInitialized();
1711       if (this.cpHost != null) {
1712         if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1713           return;
1714         }
1715       }
1716       // warmup the region on the destination before initiating the move. this call
1717       // is synchronous and takes some time. doing it before the source region gets
1718       // closed
1719       serverManager.sendRegionWarmup(rp.getDestination(), hri);
1720 
1721       LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1722       this.assignmentManager.balance(rp);
1723       if (this.cpHost != null) {
1724         this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1725       }
1726     } catch (IOException ioe) {
1727       if (ioe instanceof HBaseIOException) {
1728         throw (HBaseIOException)ioe;
1729       }
1730       throw new HBaseIOException(ioe);
1731     }
1732   }
1733 
1734   @Override
1735   public long createTable(
1736       final HTableDescriptor hTableDescriptor,
1737       final byte [][] splitKeys,
1738       final long nonceGroup,
1739       final long nonce) throws IOException {
1740     if (isStopped()) {
1741       throw new MasterNotRunningException();
1742     }
1743 
1744     TableName tableName = hTableDescriptor.getTableName();
1745     String namespace = tableName.getNamespaceAsString();
1746     ensureNamespaceExists(namespace);
1747 
1748     final HRegionInfo[] newRegions =
1749         ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
1750     checkInitialized();
1751     sanityCheckTableDescriptor(hTableDescriptor);
1752 
1753     if (isMasterProcedureExecutorEnabled()) {
1754       return MasterProcedureUtil.submitProcedure(
1755         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
1756         @Override
1757         protected void run() throws IOException {
1758           getMaster().getMasterCoprocessorHost().preCreateTable(hTableDescriptor, newRegions);
1759 
1760           LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1761 
1762           // TODO: We can handle/merge duplicate requests, and differentiate the case of
1763           //       TableExistsException by saying if the schema is the same or not.
1764           ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1765           submitProcedure(new CreateTableProcedure(
1766             procedureExecutor.getEnvironment(), hTableDescriptor, newRegions, latch));
1767           latch.await();
1768 
1769           getMaster().getMasterCoprocessorHost().postCreateTable(hTableDescriptor, newRegions);
1770         }
1771 
1772         @Override
1773         protected String getDescription() {
1774           return "CreateTableProcedure";
1775         }
1776       });
1777     } else {
1778       try {
1779         this.quotaManager.checkNamespaceTableAndRegionQuota(tableName, newRegions.length);
1780         if (cpHost != null) {
1781           cpHost.preCreateTable(hTableDescriptor, newRegions);
1782         }
1783         LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1784         this.service.submit(new CreateTableHandler(this, this.fileSystemManager, hTableDescriptor,
1785             conf, newRegions, this).prepare());
1786         if (cpHost != null) {
1787           cpHost.postCreateTable(hTableDescriptor, newRegions);
1788         }
1789       } catch (IOException e) {
1790         this.quotaManager.removeTableFromNamespaceQuota(tableName);
1791         LOG.error("Exception occurred while creating the table " + tableName.getNameAsString(), e);
1792         throw e;
1793       }
1794       return -1;
1795     }
1796   }
1797 
1798   /**
1799    * Checks whether the table conforms to some sane limits, and configured
1800    * values (compression, etc) work. Throws an exception if something is wrong.
1801    * @throws IOException
1802    */
1803   private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1804     final String CONF_KEY = "hbase.table.sanity.checks";
1805     boolean logWarn = false;
1806     if (!conf.getBoolean(CONF_KEY, true)) {
1807       logWarn = true;
1808     }
1809     String tableVal = htd.getConfigurationValue(CONF_KEY);
1810     if (tableVal != null && !Boolean.valueOf(tableVal)) {
1811       logWarn = true;
1812     }
1813 
1814     // check max file size
1815     long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit
1816     long maxFileSize = htd.getMaxFileSize();
1817     if (maxFileSize < 0) {
1818       maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1819     }
1820     if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1821       String message = "MAX_FILESIZE for table descriptor or "
1822           + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1823           + ") is too small, which might cause over splitting into unmanageable "
1824           + "number of regions.";
1825       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1826     }
1827 
1828     // check flush size
1829     long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit
1830     long flushSize = htd.getMemStoreFlushSize();
1831     if (flushSize < 0) {
1832       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1833     }
1834     if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1835       String message = "MEMSTORE_FLUSHSIZE for table descriptor or "
1836           + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1837           + " very frequent flushing.";
1838       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1839     }
1840 
1841     // check that coprocessors and other specified plugin classes can be loaded
1842     try {
1843       checkClassLoading(conf, htd);
1844     } catch (Exception ex) {
1845       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);
1846     }
1847 
1848     // check compression can be loaded
1849     try {
1850       checkCompression(htd);
1851     } catch (IOException e) {
1852       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1853     }
1854 
1855     // check encryption can be loaded
1856     try {
1857       checkEncryption(conf, htd);
1858     } catch (IOException e) {
1859       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1860     }
1861     // Verify compaction policy
1862     try{
1863       checkCompactionPolicy(conf, htd);
1864     } catch(IOException e){
1865       warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);
1866     }
1867     // check that we have at least 1 CF
1868     if (htd.getColumnFamilies().length == 0) {
1869       String message = "Table should have at least one column family.";
1870       warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1871     }
1872 
1873     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1874       if (hcd.getTimeToLive() <= 0) {
1875         String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";
1876         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1877       }
1878 
1879       // check blockSize
1880       if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1881         String message = "Block size for column family " + hcd.getNameAsString()
1882             + "  must be between 1K and 16MB.";
1883         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1884       }
1885 
1886       // check versions
1887       if (hcd.getMinVersions() < 0) {
1888         String message = "Min versions for column family " + hcd.getNameAsString()
1889           + "  must be positive.";
1890         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1891       }
1892       // max versions already being checked
1893 
1894       // HBASE-13776 Setting illegal versions for HColumnDescriptor
1895       //  does not throw IllegalArgumentException
1896       // check minVersions <= maxVerions
1897       if (hcd.getMinVersions() > hcd.getMaxVersions()) {
1898         String message = "Min versions for column family " + hcd.getNameAsString()
1899             + " must be less than the Max versions.";
1900         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1901       }
1902 
1903       // check replication scope
1904       if (hcd.getScope() < 0) {
1905         String message = "Replication scope for column family "
1906           + hcd.getNameAsString() + "  must be positive.";
1907         warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1908       }
1909 
1910       // TODO: should we check coprocessors and encryption ?
1911     }
1912   }
1913 
1914   private void checkCompactionPolicy(Configuration conf, HTableDescriptor htd)
1915       throws IOException {
1916     // FIFO compaction has some requirements
1917     // Actually FCP ignores periodic major compactions
1918     String className =
1919         htd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1920     if (className == null) {
1921       className =
1922           conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
1923             ExploringCompactionPolicy.class.getName());
1924     }
1925 
1926 
1927     int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;
1928     String sv = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1929     if (sv != null) {
1930       blockingFileCount = Integer.parseInt(sv);
1931     } else {
1932       blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);
1933     }
1934 
1935     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1936       String compactionPolicy =
1937           hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1938       if (compactionPolicy == null) {
1939         compactionPolicy = className;
1940       }
1941       if (compactionPolicy.equals(FIFOCompactionPolicy.class.getName()) == false) {
1942         continue;
1943       }
1944       // FIFOCompaction
1945       String message = null;
1946 
1947       // 1. Check TTL
1948       if (hcd.getTimeToLive() == HColumnDescriptor.DEFAULT_TTL) {
1949         message = "Default TTL is not supported for FIFO compaction";
1950         throw new IOException(message);
1951       }
1952 
1953       // 2. Check min versions
1954       if (hcd.getMinVersions() > 0) {
1955         message = "MIN_VERSION > 0 is not supported for FIFO compaction";
1956         throw new IOException(message);
1957       }
1958 
1959       // 3. blocking file count
1960       String sbfc = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1961       if (sbfc != null) {
1962         blockingFileCount = Integer.parseInt(sbfc);
1963       }
1964       if (blockingFileCount < 1000) {
1965         message =
1966             "blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount
1967                 + " is below recommended minimum of 1000";
1968         throw new IOException(message);
1969       }
1970     }
1971   }
1972 
1973   // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.
1974   private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,
1975       String message, Exception cause) throws IOException {
1976     if (!logWarn) {
1977       throw new DoNotRetryIOException(message + " Set " + confKey +
1978           " to false at conf or table descriptor if you want to bypass sanity checks", cause);
1979     }
1980     LOG.warn(message);
1981   }
1982 
1983   private void startActiveMasterManager(int infoPort) throws KeeperException {
1984     String backupZNode = ZKUtil.joinZNode(
1985       zooKeeper.backupMasterAddressesZNode, serverName.toString());
1986     /*
1987     * Add a ZNode for ourselves in the backup master directory since we
1988     * may not become the active master. If so, we want the actual active
1989     * master to know we are backup masters, so that it won't assign
1990     * regions to us if so configured.
1991     *
1992     * If we become the active master later, ActiveMasterManager will delete
1993     * this node explicitly.  If we crash before then, ZooKeeper will delete
1994     * this node for us since it is ephemeral.
1995     */
1996     LOG.info("Adding backup master ZNode " + backupZNode);
1997     if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1998         serverName, infoPort)) {
1999       LOG.warn("Failed create of " + backupZNode + " by " + serverName);
2000     }
2001 
2002     activeMasterManager.setInfoPort(infoPort);
2003     // Start a thread to try to become the active master, so we won't block here
2004     Threads.setDaemonThreadRunning(new Thread(new Runnable() {
2005       @Override
2006       public void run() {
2007         int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
2008           HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
2009         // If we're a backup master, stall until a primary to writes his address
2010         if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
2011           HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
2012           LOG.debug("HMaster started in backup mode. "
2013             + "Stalling until master znode is written.");
2014           // This will only be a minute or so while the cluster starts up,
2015           // so don't worry about setting watches on the parent znode
2016           while (!activeMasterManager.hasActiveMaster()) {
2017             LOG.debug("Waiting for master address ZNode to be written "
2018               + "(Also watching cluster state node)");
2019             Threads.sleep(timeout);
2020           }
2021         }
2022         MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
2023         status.setDescription("Master startup");
2024         try {
2025           if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
2026             finishActiveMasterInitialization(status);
2027           }
2028         } catch (Throwable t) {
2029           status.setStatus("Failed to become active: " + t.getMessage());
2030           LOG.fatal("Failed to become active master", t);
2031           // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
2032           if (t instanceof NoClassDefFoundError &&
2033               t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
2034             // improved error message for this special case
2035             abort("HBase is having a problem with its Hadoop jars.  You may need to "
2036               + "recompile HBase against Hadoop version "
2037               +  org.apache.hadoop.util.VersionInfo.getVersion()
2038               + " or change your hadoop jars to start properly", t);
2039           } else {
2040             abort("Unhandled exception. Starting shutdown.", t);
2041           }
2042         } finally {
2043           status.cleanup();
2044         }
2045       }
2046     }, getServerName().toShortString() + ".activeMasterManager"));
2047   }
2048 
2049   private void checkCompression(final HTableDescriptor htd)
2050   throws IOException {
2051     if (!this.masterCheckCompression) return;
2052     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
2053       checkCompression(hcd);
2054     }
2055   }
2056 
2057   private void checkCompression(final HColumnDescriptor hcd)
2058   throws IOException {
2059     if (!this.masterCheckCompression) return;
2060     CompressionTest.testCompression(hcd.getCompression());
2061     CompressionTest.testCompression(hcd.getCompactionCompression());
2062   }
2063 
2064   private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
2065   throws IOException {
2066     if (!this.masterCheckEncryption) return;
2067     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
2068       checkEncryption(conf, hcd);
2069     }
2070   }
2071 
2072   private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
2073   throws IOException {
2074     if (!this.masterCheckEncryption) return;
2075     EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
2076   }
2077 
2078   private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
2079   throws IOException {
2080     RegionSplitPolicy.getSplitPolicyClass(htd, conf);
2081     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
2082   }
2083 
2084   private static boolean isCatalogTable(final TableName tableName) {
2085     return tableName.equals(TableName.META_TABLE_NAME);
2086   }
2087 
2088   @Override
2089   public long deleteTable(
2090       final TableName tableName,
2091       final long nonceGroup,
2092       final long nonce) throws IOException {
2093     checkInitialized();
2094 
2095     if (isMasterProcedureExecutorEnabled()) {
2096       return MasterProcedureUtil.submitProcedure(
2097         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2098         @Override
2099         protected void run() throws IOException {
2100           getMaster().getMasterCoprocessorHost().preDeleteTable(tableName);
2101 
2102           LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2103 
2104           // TODO: We can handle/merge duplicate request
2105           ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
2106           submitProcedure(new DeleteTableProcedure(procedureExecutor.getEnvironment(),
2107               tableName, latch));
2108           latch.await();
2109 
2110           getMaster().getMasterCoprocessorHost().postDeleteTable(tableName);
2111         }
2112 
2113         @Override
2114         protected String getDescription() {
2115           return "DeleteTableProcedure";
2116         }
2117       });
2118     } else {
2119       if (cpHost != null) {
2120         cpHost.preDeleteTable(tableName);
2121       }
2122       LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2123       this.service.submit(new DeleteTableHandler(tableName, this, this).prepare());
2124       if (cpHost != null) {
2125         cpHost.postDeleteTable(tableName);
2126       }
2127       return -1;
2128     }
2129   }
2130 
2131   @Override
2132   public void truncateTable(
2133       final TableName tableName,
2134       final boolean preserveSplits,
2135       final long nonceGroup,
2136       final long nonce) throws IOException {
2137     checkInitialized();
2138 
2139     if (isMasterProcedureExecutorEnabled()) {
2140       MasterProcedureUtil.submitProcedure(
2141         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2142         @Override
2143         protected void run() throws IOException {
2144           getMaster().getMasterCoprocessorHost().preTruncateTable(tableName);
2145 
2146           LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2147 
2148           long procId = submitProcedure(new TruncateTableProcedure(procedureExecutor.getEnvironment(),
2149               tableName, preserveSplits));
2150           ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2151 
2152           getMaster().getMasterCoprocessorHost().postTruncateTable(tableName);
2153         }
2154 
2155         @Override
2156         protected String getDescription() {
2157           return "TruncateTableProcedure";
2158         }
2159       });
2160     } else {
2161       if (cpHost != null) {
2162         cpHost.preTruncateTable(tableName);
2163       }
2164       LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2165       TruncateTableHandler handler =
2166           new TruncateTableHandler(tableName, this, this, preserveSplits);
2167       handler.prepare();
2168       handler.process();
2169       if (cpHost != null) {
2170         cpHost.postTruncateTable(tableName);
2171       }
2172     }
2173   }
2174 
2175   @Override
2176   public void addColumn(
2177       final TableName tableName,
2178       final HColumnDescriptor columnDescriptor,
2179       final long nonceGroup,
2180       final long nonce)
2181       throws IOException {
2182     checkInitialized();
2183     checkCompression(columnDescriptor);
2184     checkEncryption(conf, columnDescriptor);
2185 
2186     if (isMasterProcedureExecutorEnabled()) {
2187       MasterProcedureUtil.submitProcedure(
2188         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2189         @Override
2190         protected void run() throws IOException {
2191           if (getMaster().getMasterCoprocessorHost().preAddColumn(tableName, columnDescriptor)) {
2192             return;
2193           }
2194 
2195           LOG.info(getClientIdAuditPrefix() + " add " + columnDescriptor);
2196           // Execute the operation synchronously, wait for the operation to complete before
2197           // continuing
2198           long procId = submitProcedure(new AddColumnFamilyProcedure(
2199             procedureExecutor.getEnvironment(), tableName, columnDescriptor));
2200           ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2201 
2202           getMaster().getMasterCoprocessorHost().postAddColumn(tableName, columnDescriptor);
2203         }
2204         @Override
2205         protected String getDescription() {
2206           return "AddColumnFamilyProcedure";
2207         }
2208       });
2209     } else {
2210       if (cpHost != null) {
2211         if (cpHost.preAddColumn(tableName, columnDescriptor)) {
2212           return;
2213         }
2214       }
2215       LOG.info(getClientIdAuditPrefix() + " add " + columnDescriptor);
2216       new TableAddFamilyHandler(tableName, columnDescriptor, this, this).prepare().process();
2217       if (cpHost != null) {
2218         cpHost.postAddColumn(tableName, columnDescriptor);
2219       }
2220     }
2221   }
2222 
2223   @Override
2224   public void modifyColumn(
2225       final TableName tableName,
2226       final HColumnDescriptor descriptor,
2227       final long nonceGroup,
2228       final long nonce)
2229       throws IOException {
2230     checkInitialized();
2231     checkCompression(descriptor);
2232     checkEncryption(conf, descriptor);
2233 
2234     if (isMasterProcedureExecutorEnabled()) {
2235       MasterProcedureUtil.submitProcedure(
2236         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2237         @Override
2238         protected void run() throws IOException {
2239           if (getMaster().getMasterCoprocessorHost().preModifyColumn(tableName, descriptor)) {
2240             return;
2241           }
2242 
2243           LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2244 
2245           // Execute the operation synchronously - wait for the operation to complete before
2246           // continuing.
2247           long procId = submitProcedure(new ModifyColumnFamilyProcedure(
2248             procedureExecutor.getEnvironment(), tableName, descriptor));
2249           ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2250 
2251           getMaster().getMasterCoprocessorHost().postModifyColumn(tableName, descriptor);
2252         }
2253 
2254         @Override
2255         protected String getDescription() {
2256           return "ModifyColumnFamilyProcedure";
2257         }
2258       });
2259     } else {
2260       if (cpHost != null) {
2261         if (cpHost.preModifyColumn(tableName, descriptor)) {
2262           return;
2263         }
2264       }
2265       LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2266       new TableModifyFamilyHandler(tableName, descriptor, this, this).prepare().process();
2267       if (cpHost != null) {
2268         cpHost.postModifyColumn(tableName, descriptor);
2269       }
2270     }
2271   }
2272 
2273   @Override
2274   public void deleteColumn(
2275       final TableName tableName,
2276       final byte[] columnName,
2277       final long nonceGroup,
2278       final long nonce)
2279       throws IOException {
2280     checkInitialized();
2281 
2282     if (isMasterProcedureExecutorEnabled()) {
2283       MasterProcedureUtil.submitProcedure(
2284         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2285         @Override
2286         protected void run() throws IOException {
2287           if (getMaster().getMasterCoprocessorHost().preDeleteColumn(tableName, columnName)) {
2288             return;
2289           }
2290 
2291           LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
2292 
2293           // Execute the operation synchronously - wait for the operation to complete before
2294           // continuing.
2295           long procId = submitProcedure(new DeleteColumnFamilyProcedure(
2296             procedureExecutor.getEnvironment(), tableName, columnName));
2297           ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2298 
2299           getMaster().getMasterCoprocessorHost().postDeleteColumn(tableName, columnName);
2300         }
2301 
2302         @Override
2303         protected String getDescription() {
2304           return "DeleteColumnFamilyProcedure";
2305         }
2306       });
2307     } else {
2308       if (cpHost != null) {
2309         if (cpHost.preDeleteColumn(tableName, columnName)) {
2310           return;
2311         }
2312       }
2313       LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
2314       new TableDeleteFamilyHandler(tableName, columnName, this, this).prepare().process();
2315       if (cpHost != null) {
2316         cpHost.postDeleteColumn(tableName, columnName);
2317       }
2318     }
2319   }
2320 
2321   @Override
2322   public long enableTable(final TableName tableName, final long nonceGroup, final long nonce)
2323       throws IOException {
2324     checkInitialized();
2325     if (cpHost != null) {
2326       cpHost.preEnableTable(tableName);
2327     }
2328 
2329     // Normally, it would make sense for this authorization check to exist inside AccessController,
2330     // but because the authorization check is done based on internal state (rather than explicit
2331     // permissions) we'll do the check here instead of in the coprocessor.
2332     MasterQuotaManager quotaManager = getMasterQuotaManager();
2333     if (quotaManager != null) {
2334       if (quotaManager.isQuotaInitialized()) {
2335         Quotas quotaForTable = QuotaUtil.getTableQuota(getConnection(), tableName);
2336         if (quotaForTable != null && quotaForTable.hasSpace()) {
2337           SpaceViolationPolicy policy = quotaForTable.getSpace().getViolationPolicy();
2338           if (SpaceViolationPolicy.DISABLE == policy) {
2339             throw new AccessDeniedException("Enabling the table '" + tableName
2340                 + "' is disallowed due to a violated space quota.");
2341           }
2342         }
2343       } else if (LOG.isTraceEnabled()) {
2344         LOG.trace("Unable to check for space quotas as the manager is not enabled");
2345       }
2346     }
2347 
2348     LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2349 
2350     if (isMasterProcedureExecutorEnabled()) {
2351       return MasterProcedureUtil.submitProcedure(
2352         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2353         @Override
2354         protected void run() throws IOException {
2355           getMaster().getMasterCoprocessorHost().preEnableTable(tableName);
2356 
2357           LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2358 
2359           // Execute the operation asynchronously - client will check the progress of the operation
2360           // In case the request is from a <1.1 client before returning,
2361           // we want to make sure that the table is prepared to be
2362           // enabled (the table is locked and the table state is set).
2363           // Note: if the procedure throws exception, we will catch it and rethrow.
2364           final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2365           submitProcedure(new EnableTableProcedure(
2366             procedureExecutor.getEnvironment(), tableName, false, prepareLatch));
2367           // Before returning to client, we want to make sure that the table is prepared to be
2368           // enabled (the table is locked and the table state is set).
2369           //
2370           // Note: if the procedure throws exception, we will catch it and rethrow.
2371           prepareLatch.await();
2372 
2373           getMaster().getMasterCoprocessorHost().postEnableTable(tableName);
2374         }
2375 
2376         @Override
2377         protected String getDescription() {
2378           return "EnableTableProcedure";
2379         }
2380       });
2381     } else {
2382       if (cpHost != null) {
2383         cpHost.preEnableTable(tableName);
2384       }
2385       LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2386       this.service.submit(new EnableTableHandler(this, tableName,
2387         assignmentManager, tableLockManager, false).prepare());
2388       if (cpHost != null) {
2389         cpHost.postEnableTable(tableName);
2390       }
2391       return -1;
2392     }
2393   }
2394 
2395   @Override
2396   public long disableTable(final TableName tableName, final long nonceGroup, final long nonce)
2397       throws IOException {
2398     checkInitialized();
2399 
2400     if (isMasterProcedureExecutorEnabled()) {
2401       return MasterProcedureUtil.submitProcedure(
2402         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2403         @Override
2404         protected void run() throws IOException {
2405           getMaster().getMasterCoprocessorHost().preDisableTable(tableName);
2406 
2407           LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2408 
2409           // Execute the operation asynchronously - client will check the progress of the operation
2410           // In case the request is from a <1.1 client before returning,
2411           // we want to make sure that the table is prepared to be
2412           // enabled (the table is locked and the table state is set).
2413           // Note: if the procedure throws exception, we will catch it and rethrow.
2414           final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2415           submitProcedure(new DisableTableProcedure(procedureExecutor.getEnvironment(),
2416             tableName, false, prepareLatch));
2417           prepareLatch.await();
2418 
2419           getMaster().getMasterCoprocessorHost().postDisableTable(tableName);
2420         }
2421 
2422         @Override
2423         protected String getDescription() {
2424           return "DisableTableProcedure";
2425         }
2426       });
2427     } else {
2428       if (cpHost != null) {
2429         cpHost.preDisableTable(tableName);
2430       }
2431       LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2432       this.service.submit(new DisableTableHandler(this, tableName,
2433         assignmentManager, tableLockManager, false).prepare());
2434       if (cpHost != null) {
2435         cpHost.postDisableTable(tableName);
2436       }
2437       return -1;
2438     }
2439   }
2440 
2441   /**
2442    * Return the region and current deployment for the region containing
2443    * the given row. If the region cannot be found, returns null. If it
2444    * is found, but not currently deployed, the second element of the pair
2445    * may be null.
2446    */
2447   @VisibleForTesting // Used by TestMaster.
2448   Pair<HRegionInfo, ServerName> getTableRegionForRow(
2449       final TableName tableName, final byte [] rowKey)
2450   throws IOException {
2451     final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2452       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2453 
2454     MetaScannerVisitor visitor =
2455       new MetaScannerVisitorBase() {
2456         @Override
2457         public boolean processRow(Result data) throws IOException {
2458           if (data == null || data.size() <= 0) {
2459             return true;
2460           }
2461           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
2462           if (pair == null) {
2463             return false;
2464           }
2465           if (!pair.getFirst().getTable().equals(tableName)) {
2466             return false;
2467           }
2468           result.set(pair);
2469           return true;
2470         }
2471     };
2472 
2473     MetaScanner.metaScan(clusterConnection, visitor, tableName, rowKey, 1);
2474     return result.get();
2475   }
2476 
2477   @Override
2478   public void modifyTable(
2479       final TableName tableName,
2480       final HTableDescriptor descriptor,
2481       final long nonceGroup,
2482       final long nonce)
2483       throws IOException {
2484     checkInitialized();
2485     sanityCheckTableDescriptor(descriptor);
2486 
2487     if (isMasterProcedureExecutorEnabled()) {
2488       MasterProcedureUtil.submitProcedure(
2489         new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2490         @Override
2491         protected void run() throws IOException {
2492           getMaster().getMasterCoprocessorHost().preModifyTable(tableName, descriptor);
2493           LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
2494 
2495           // Execute the operation synchronously - wait for the operation completes before continuing.
2496           long procId = submitProcedure(new ModifyTableProcedure(
2497             procedureExecutor.getEnvironment(), descriptor));
2498           ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2499 
2500           getMaster().getMasterCoprocessorHost().postModifyTable(tableName, descriptor);
2501         }
2502 
2503         @Override
2504         protected String getDescription() {
2505           return "ModifyTableProcedure";
2506         }
2507       });
2508     } else {
2509       if (cpHost != null) {
2510         cpHost.preModifyTable(tableName, descriptor);
2511       }
2512       LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
2513       new ModifyTableHandler(tableName, descriptor, this, this).prepare().process();
2514       if (cpHost != null) {
2515         cpHost.postModifyTable(tableName, descriptor);
2516       }
2517     }
2518   }
2519 
2520   @Override
2521   public void checkTableModifiable(final TableName tableName)
2522       throws IOException, TableNotFoundException, TableNotDisabledException {
2523     if (isCatalogTable(tableName)) {
2524       throw new IOException("Can't modify catalog tables");
2525     }
2526     if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
2527       throw new TableNotFoundException(tableName);
2528     }
2529     if (!getAssignmentManager().getTableStateManager().
2530         isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)) {
2531       throw new TableNotDisabledException(tableName);
2532     }
2533   }
2534 
2535   /**
2536    * @return cluster status
2537    */
2538   public ClusterStatus getClusterStatus() throws InterruptedIOException {
2539     // Build Set of backup masters from ZK nodes
2540     List<String> backupMasterStrings;
2541     try {
2542       backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
2543         this.zooKeeper.backupMasterAddressesZNode);
2544     } catch (KeeperException e) {
2545       LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
2546       backupMasterStrings = null;
2547     }
2548 
2549     List<ServerName> backupMasters = null;
2550     if (backupMasterStrings != null && !backupMasterStrings.isEmpty()) {
2551       backupMasters = new ArrayList<ServerName>(backupMasterStrings.size());
2552       for (String s: backupMasterStrings) {
2553         try {
2554           byte [] bytes;
2555           try {
2556             bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
2557                 this.zooKeeper.backupMasterAddressesZNode, s));
2558           } catch (InterruptedException e) {
2559             throw new InterruptedIOException();
2560           }
2561           if (bytes != null) {
2562             ServerName sn;
2563             try {
2564               sn = ServerName.parseFrom(bytes);
2565             } catch (DeserializationException e) {
2566               LOG.warn("Failed parse, skipping registering backup server", e);
2567               continue;
2568             }
2569             backupMasters.add(sn);
2570           }
2571         } catch (KeeperException e) {
2572           LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
2573                    "backup servers"), e);
2574         }
2575       }
2576       Collections.sort(backupMasters, new Comparator<ServerName>() {
2577         @Override
2578         public int compare(ServerName s1, ServerName s2) {
2579           return s1.getServerName().compareTo(s2.getServerName());
2580         }});
2581     }
2582 
2583     String clusterId = fileSystemManager != null ?
2584       fileSystemManager.getClusterId().toString() : null;
2585     Map<String, RegionState> regionsInTransition = assignmentManager != null ?
2586       assignmentManager.getRegionStates().getRegionsInTransition() : null;
2587     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
2588     boolean balancerOn = loadBalancerTracker != null ?
2589       loadBalancerTracker.isBalancerOn() : false;
2590     Map<ServerName, ServerLoad> onlineServers = null;
2591     Set<ServerName> deadServers = null;
2592     if (serverManager != null) {
2593       deadServers = serverManager.getDeadServers().copyServerNames();
2594       onlineServers = serverManager.getOnlineServers();
2595     }
2596     return new ClusterStatus(VersionInfo.getVersion(), clusterId,
2597       onlineServers, deadServers, serverName, backupMasters,
2598       regionsInTransition, coprocessors, balancerOn);
2599   }
2600 
2601   /**
2602    * The set of loaded coprocessors is stored in a static set. Since it's
2603    * statically allocated, it does not require that HMaster's cpHost be
2604    * initialized prior to accessing it.
2605    * @return a String representation of the set of names of the loaded
2606    * coprocessors.
2607    */
2608   public static String getLoadedCoprocessors() {
2609     return CoprocessorHost.getLoadedCoprocessors().toString();
2610   }
2611 
2612   /**
2613    * @return timestamp in millis when HMaster was started.
2614    */
2615   public long getMasterStartTime() {
2616     return startcode;
2617   }
2618 
2619   /**
2620    * @return timestamp in millis when HMaster became the active master.
2621    */
2622   public long getMasterActiveTime() {
2623     return masterActiveTime;
2624   }
2625 
2626   public int getNumWALFiles() {
2627     return procedureStore != null ? procedureStore.getActiveLogs().size() : 0;
2628   }
2629 
2630   public WALProcedureStore getWalProcedureStore() {
2631     return procedureStore;
2632   }
2633 
2634   public int getRegionServerInfoPort(final ServerName sn) {
2635     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
2636     if (info == null || info.getInfoPort() == 0) {
2637       return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2638         HConstants.DEFAULT_REGIONSERVER_INFOPORT);
2639     }
2640     return info.getInfoPort();
2641   }
2642 
2643   /**
2644    * @return array of coprocessor SimpleNames.
2645    */
2646   public String[] getMasterCoprocessors() {
2647     Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2648     return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2649   }
2650 
2651   @Override
2652   public void abort(final String msg, final Throwable t) {
2653     if (isAborted() || isStopped()) {
2654       return;
2655     }
2656     if (cpHost != null) {
2657       // HBASE-4014: dump a list of loaded coprocessors.
2658       LOG.fatal("Master server abort: loaded coprocessors are: " +
2659           getLoadedCoprocessors());
2660     }
2661     if (t != null) LOG.fatal(msg, t);
2662     stop(msg);
2663   }
2664 
2665   @Override
2666   public ZooKeeperWatcher getZooKeeper() {
2667     return zooKeeper;
2668   }
2669 
2670   @Override
2671   public MasterCoprocessorHost getMasterCoprocessorHost() {
2672     return cpHost;
2673   }
2674 
2675   @Override
2676   public MasterQuotaManager getMasterQuotaManager() {
2677     return quotaManager;
2678   }
2679 
2680   @Override
2681   public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2682     return procedureExecutor;
2683   }
2684 
2685   @Override
2686   public ServerName getServerName() {
2687     return this.serverName;
2688   }
2689 
2690   @Override
2691   public AssignmentManager getAssignmentManager() {
2692     return this.assignmentManager;
2693   }
2694 
2695   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2696     return rsFatals;
2697   }
2698 
2699   public void shutdown() throws IOException {
2700     if (cpHost != null) {
2701       cpHost.preShutdown();
2702     }
2703 
2704     if (this.serverManager != null) {
2705       this.serverManager.shutdownCluster();
2706     }
2707     if (this.clusterStatusTracker != null){
2708       try {
2709         this.clusterStatusTracker.setClusterDown();
2710       } catch (KeeperException e) {
2711         LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2712       }
2713     }
2714   }
2715 
2716   public void stopMaster() throws IOException {
2717     if (cpHost != null) {
2718       cpHost.preStopMaster();
2719     }
2720     stop("Stopped by " + Thread.currentThread().getName());
2721   }
2722 
2723   void checkServiceStarted() throws ServerNotRunningYetException {
2724     if (!serviceStarted) {
2725       throw new ServerNotRunningYetException("Server is not running yet");
2726     }
2727   }
2728 
2729   void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
2730     checkServiceStarted();
2731     if (!this.initialized) {
2732       throw new PleaseHoldException("Master is initializing");
2733     }
2734   }
2735 
2736   void checkNamespaceManagerReady() throws IOException {
2737     checkInitialized();
2738 	if (tableNamespaceManager == null) {
2739 	  throw new IOException("Table Namespace Manager not ready yet, try again later");
2740 	} else if (!tableNamespaceManager.isTableAvailableAndInitialized()) {
2741 	  try {
2742 		// Wait some time.
2743 	    long startTime = EnvironmentEdgeManager.currentTime();
2744 	    int timeout = conf.getInt("hbase.master.namespace.wait.for.ready", 30000);
2745 	    while (!tableNamespaceManager.isTableNamespaceManagerStarted() &&
2746 	      EnvironmentEdgeManager.currentTime() - startTime < timeout) {
2747 	      Thread.sleep(100);
2748 	    }
2749 	  } catch (InterruptedException e) {
2750 	    throw (InterruptedIOException) new InterruptedIOException().initCause(e);
2751 	  }
2752 	  if (!tableNamespaceManager.isTableNamespaceManagerStarted()) {
2753 		throw new IOException("Table Namespace Manager not fully initialized, try again later");
2754 	  }
2755     }
2756   }
2757 
2758   /**
2759    * Report whether this master is currently the active master or not.
2760    * If not active master, we are parked on ZK waiting to become active.
2761    *
2762    * This method is used for testing.
2763    *
2764    * @return true if active master, false if not.
2765    */
2766   public boolean isActiveMaster() {
2767     return isActiveMaster;
2768   }
2769 
2770   /**
2771    * Report whether this master has completed with its initialization and is
2772    * ready.  If ready, the master is also the active master.  A standby master
2773    * is never ready.
2774    *
2775    * This method is used for testing.
2776    *
2777    * @return true if master is ready to go, false if not.
2778    */
2779   @Override
2780   public boolean isInitialized() {
2781     return initialized;
2782   }
2783 
2784   /**
2785    * Report whether this master has completed with its initialization and
2786    * namespace manager is ready
2787    *
2788    * This method is used for testing.
2789    *
2790    * @return true if master is fully initialized
2791    */
2792   @Override
2793   public boolean isNamespaceManagerInitialized() {
2794     return initialized & tableNamespaceManager != null
2795       && tableNamespaceManager.isTableNamespaceManagerStarted();
2796   }
2797 
2798   /**
2799    * Report whether this master is in maintenance mode.
2800    *
2801    * @return true if master is in maintenanceMode
2802    */
2803   @Override
2804   public boolean isInMaintenanceMode() {
2805     return maintenanceModeTracker.isInMaintenanceMode();
2806   }
2807 
2808   /**
2809    * ServerShutdownHandlerEnabled is set false before completing
2810    * assignMeta to prevent processing of ServerShutdownHandler.
2811    * @return true if assignMeta has completed;
2812    */
2813   @Override
2814   public boolean isServerShutdownHandlerEnabled() {
2815     return this.serverShutdownHandlerEnabled;
2816   }
2817 
2818   /**
2819    * Report whether this master has started initialization and is about to do meta region assignment
2820    * @return true if master is in initialization & about to assign hbase:meta regions
2821    */
2822   public boolean isInitializationStartsMetaRegionAssignment() {
2823     return this.initializationBeforeMetaAssignment;
2824   }
2825 
2826   public void assignRegion(HRegionInfo hri) {
2827     assignmentManager.assign(hri, true);
2828   }
2829 
2830   /**
2831    * Compute the average load across all region servers.
2832    * Currently, this uses a very naive computation - just uses the number of
2833    * regions being served, ignoring stats about number of requests.
2834    * @return the average load
2835    */
2836   public double getAverageLoad() {
2837     if (this.assignmentManager == null) {
2838       return 0;
2839     }
2840 
2841     RegionStates regionStates = this.assignmentManager.getRegionStates();
2842     if (regionStates == null) {
2843       return 0;
2844     }
2845     return regionStates.getAverageLoad();
2846   }
2847 
2848   @Override
2849   public boolean registerService(Service instance) {
2850     /*
2851      * No stacking of instances is allowed for a single service name
2852      */
2853     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2854     String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
2855     if (coprocessorServiceHandlers.containsKey(serviceName)) {
2856       LOG.error("Coprocessor service "+serviceName+
2857           " already registered, rejecting request from "+instance
2858       );
2859       return false;
2860     }
2861 
2862     coprocessorServiceHandlers.put(serviceName, instance);
2863     if (LOG.isDebugEnabled()) {
2864       LOG.debug("Registered master coprocessor service: service="+serviceName);
2865     }
2866     return true;
2867   }
2868 
2869   /**
2870    * Utility for constructing an instance of the passed HMaster class.
2871    * @param masterClass
2872    * @param conf
2873    * @return HMaster instance.
2874    */
2875   public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2876       final Configuration conf, final CoordinatedStateManager cp)  {
2877     try {
2878       Constructor<? extends HMaster> c =
2879         masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
2880       return c.newInstance(conf, cp);
2881     } catch (InvocationTargetException ite) {
2882       Throwable target = ite.getTargetException() != null?
2883         ite.getTargetException(): ite;
2884       if (target.getCause() != null) target = target.getCause();
2885       throw new RuntimeException("Failed construction of Master: " +
2886         masterClass.toString(), target);
2887     } catch (Exception e) {
2888       throw new RuntimeException("Failed construction of Master: " +
2889         masterClass.toString() + ((e.getCause() != null)?
2890           e.getCause().getMessage(): ""), e);
2891     }
2892   }
2893 
2894   /**
2895    * @see org.apache.hadoop.hbase.master.HMasterCommandLine
2896    */
2897   public static void main(String [] args) {
2898     VersionInfo.logVersion();
2899     new HMasterCommandLine(HMaster.class).doMain(args);
2900   }
2901 
2902   public HFileCleaner getHFileCleaner() {
2903     return this.hfileCleaner;
2904   }
2905 
2906   /**
2907    * @return the underlying snapshot manager
2908    */
2909   public SnapshotManager getSnapshotManager() {
2910     return this.snapshotManager;
2911   }
2912 
2913   /**
2914    * @return the underlying MasterProcedureManagerHost
2915    */
2916   public MasterProcedureManagerHost getMasterProcedureManagerHost() {
2917     return mpmHost;
2918   }
2919 
2920   @Override
2921   public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2922     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2923     checkNamespaceManagerReady();
2924     if (cpHost != null) {
2925       if (cpHost.preCreateNamespace(descriptor)) {
2926         return;
2927       }
2928     }
2929     LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2930     tableNamespaceManager.create(descriptor);
2931     if (cpHost != null) {
2932       cpHost.postCreateNamespace(descriptor);
2933     }
2934   }
2935 
2936   @Override
2937   public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2938     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2939     checkNamespaceManagerReady();
2940     if (cpHost != null) {
2941       if (cpHost.preModifyNamespace(descriptor)) {
2942         return;
2943       }
2944     }
2945     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2946     tableNamespaceManager.update(descriptor);
2947     if (cpHost != null) {
2948       cpHost.postModifyNamespace(descriptor);
2949     }
2950   }
2951 
2952   @Override
2953   public void deleteNamespace(String name) throws IOException {
2954     checkNamespaceManagerReady();
2955     if (cpHost != null) {
2956       if (cpHost.preDeleteNamespace(name)) {
2957         return;
2958       }
2959     }
2960     LOG.info(getClientIdAuditPrefix() + " delete " + name);
2961     tableNamespaceManager.remove(name);
2962     if (cpHost != null) {
2963       cpHost.postDeleteNamespace(name);
2964     }
2965   }
2966 
2967   /**
2968    * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
2969    *
2970    * @param name the namespace to check
2971    * @throws IOException if the namespace manager is not ready yet.
2972    * @throws NamespaceNotFoundException if the namespace does not exists
2973    */
2974   private void ensureNamespaceExists(final String name)
2975       throws IOException, NamespaceNotFoundException {
2976     checkNamespaceManagerReady();
2977     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2978     if (nsd == null) {
2979       throw new NamespaceNotFoundException(name);
2980     }
2981   }
2982 
2983   @Override
2984   public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2985     checkNamespaceManagerReady();
2986 
2987     if (cpHost != null) {
2988       cpHost.preGetNamespaceDescriptor(name);
2989     }
2990 
2991     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2992     if (nsd == null) {
2993       throw new NamespaceNotFoundException(name);
2994     }
2995 
2996     if (cpHost != null) {
2997       cpHost.postGetNamespaceDescriptor(nsd);
2998     }
2999 
3000     return nsd;
3001   }
3002 
3003   @Override
3004   public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
3005     checkNamespaceManagerReady();
3006 
3007     final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
3008     boolean bypass = false;
3009     if (cpHost != null) {
3010       bypass = cpHost.preListNamespaceDescriptors(descriptors);
3011     }
3012 
3013     if (!bypass) {
3014       descriptors.addAll(tableNamespaceManager.list());
3015 
3016       if (cpHost != null) {
3017         cpHost.postListNamespaceDescriptors(descriptors);
3018       }
3019     }
3020     return descriptors;
3021   }
3022 
3023   @Override
3024   public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
3025       throws IOException {
3026     if (cpHost != null) {
3027       cpHost.preAbortProcedure(this.procedureExecutor, procId);
3028     }
3029 
3030     final boolean result = this.procedureExecutor.abort(procId, mayInterruptIfRunning);
3031 
3032     if (cpHost != null) {
3033       cpHost.postAbortProcedure();
3034     }
3035 
3036     return result;
3037   }
3038 
3039   @Override
3040   public List<ProcedureInfo> listProcedures() throws IOException {
3041     if (cpHost != null) {
3042       cpHost.preListProcedures();
3043     }
3044 
3045     final List<ProcedureInfo> procInfoList = this.procedureExecutor.listProcedures();
3046 
3047     if (cpHost != null) {
3048       cpHost.postListProcedures(procInfoList);
3049     }
3050 
3051     return procInfoList;
3052   }
3053 
3054   @Override
3055   public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
3056     ensureNamespaceExists(name);
3057     return listTableDescriptors(name, null, null, true);
3058   }
3059 
3060   @Override
3061   public List<TableName> listTableNamesByNamespace(String name) throws IOException {
3062     ensureNamespaceExists(name);
3063     return listTableNames(name, null, true);
3064   }
3065 
3066   @Override
3067   public Pair<Long, String> backupTables(final BackupType type,
3068         List<TableName> tableList, final String targetRootDir, final int workers,
3069         final long bandwidth) throws IOException {
3070     long procId;
3071     String backupId = BackupRestoreConstants.BACKUPID_PREFIX + 
3072         EnvironmentEdgeManager.currentTime();
3073     if (type == BackupType.INCREMENTAL) {
3074       Set<TableName> incrTableSet = null;
3075       try (BackupSystemTable table = new BackupSystemTable(getConnection())) {
3076         incrTableSet = table.getIncrementalBackupTableSet(targetRootDir);
3077       }
3078          
3079       if (incrTableSet.isEmpty()) {
3080         LOG.warn("Incremental backup table set contains no table.\n"
3081             + "Use 'backup create full' or 'backup stop' to \n "
3082             + "change the tables covered by incremental backup.");
3083         throw new DoNotRetryIOException("No table covered by incremental backup.");
3084       }
3085 
3086       LOG.info("Incremental backup for the following table set: " + incrTableSet);
3087       tableList = Lists.newArrayList(incrTableSet);
3088     }
3089     if (tableList != null && !tableList.isEmpty()) {
3090       for (TableName table : tableList) {
3091         String targetTableBackupDir =
3092             HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
3093         Path targetTableBackupDirPath = new Path(targetTableBackupDir);
3094         FileSystem outputFs = FileSystem.get(targetTableBackupDirPath.toUri(), conf);
3095         if (outputFs.exists(targetTableBackupDirPath)) {
3096           throw new DoNotRetryIOException("Target backup directory " + targetTableBackupDir
3097             + " exists already.");
3098         }
3099       }
3100       ArrayList<TableName> nonExistingTableList = null;
3101       for (TableName tableName : tableList) {
3102         if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
3103           if (nonExistingTableList == null) {
3104             nonExistingTableList = new ArrayList<>();
3105           }
3106           nonExistingTableList.add(tableName);
3107         }
3108       }
3109       if (nonExistingTableList != null) {
3110         if (type == BackupType.INCREMENTAL ) {
3111           LOG.warn("Incremental backup table set contains non-exising table: "
3112               + nonExistingTableList);
3113           // Update incremental backup set 
3114           tableList = excludeNonExistingTables(tableList, nonExistingTableList);
3115         } else {
3116           // Throw exception only in full mode - we try to backup non-existing table
3117           throw new DoNotRetryIOException("Non-existing tables found in the table list: "
3118               + nonExistingTableList);
3119         }
3120       }
3121     }
3122     if (type == BackupType.FULL) {
3123       procId = this.procedureExecutor.submitProcedure(
3124         new FullTableBackupProcedure(procedureExecutor.getEnvironment(), backupId,
3125           tableList, targetRootDir, workers, bandwidth));
3126     } else {
3127       procId = this.procedureExecutor.submitProcedure(
3128         new IncrementalTableBackupProcedure(procedureExecutor.getEnvironment(), backupId,
3129           tableList, targetRootDir, workers, bandwidth));
3130     }
3131     return new Pair<>(procId, backupId);
3132   }
3133 
3134   private List<TableName> excludeNonExistingTables(List<TableName> tableList,
3135       List<TableName> nonExistingTableList) {
3136     
3137     for(TableName table: nonExistingTableList) {
3138       tableList.remove(table);
3139     }
3140     return tableList;
3141   }
3142 
3143   /**
3144    * Returns the list of table descriptors that match the specified request
3145    *
3146    * @param namespace the namespace to query, or null if querying for all
3147    * @param regex The regular expression to match against, or null if querying for all
3148    * @param tableNameList the list of table names, or null if querying for all
3149    * @param includeSysTables False to match only against userspace tables
3150    * @return the list of table descriptors
3151    */
3152   public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
3153       final List<TableName> tableNameList, final boolean includeSysTables)
3154       throws IOException {
3155     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
3156 
3157     boolean bypass = false;
3158     if (cpHost != null) {
3159       bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors);
3160       // method required for AccessController.
3161       bypass |= cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
3162     }
3163 
3164     if (!bypass) {
3165       if (tableNameList == null || tableNameList.size() == 0) {
3166         // request for all TableDescriptors
3167         Collection<HTableDescriptor> htds;
3168         if (namespace != null && namespace.length() > 0) {
3169           htds = tableDescriptors.getByNamespace(namespace).values();
3170         } else {
3171           htds = tableDescriptors.getAll().values();
3172         }
3173 
3174         for (HTableDescriptor desc: htds) {
3175           if (includeSysTables || !desc.getTableName().isSystemTable()) {
3176             descriptors.add(desc);
3177           }
3178         }
3179       } else {
3180         for (TableName s: tableNameList) {
3181           HTableDescriptor desc = tableDescriptors.get(s);
3182           if (desc != null) {
3183             descriptors.add(desc);
3184           }
3185         }
3186         if (tableNameList.size() == 1) { // For debugging
3187           LOG.info(getClientIdAuditPrefix() + " List Table Descriptor for the " + tableNameList.get(0)
3188             + " table " + ((descriptors.size() == 1) ? "succeeds" : "fails"));
3189         }
3190       }
3191 
3192       // Retains only those matched by regular expression.
3193       if (regex != null) {
3194         filterTablesByRegex(descriptors, Pattern.compile(regex));
3195       }
3196 
3197       if (cpHost != null) {
3198         cpHost.postGetTableDescriptors(descriptors);
3199         // method required for AccessController.
3200         cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
3201       }
3202     }
3203     return descriptors;
3204   }
3205 
3206   /**
3207    * Returns the list of table names that match the specified request
3208    * @param regex The regular expression to match against, or null if querying for all
3209    * @param namespace the namespace to query, or null if querying for all
3210    * @param includeSysTables False to match only against userspace tables
3211    * @return the list of table names
3212    */
3213   public List<TableName> listTableNames(final String namespace, final String regex,
3214       final boolean includeSysTables) throws IOException {
3215     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
3216 
3217     boolean bypass = false;
3218     if (cpHost != null) {
3219       bypass = cpHost.preGetTableNames(descriptors, regex);
3220     }
3221 
3222     if (!bypass) {
3223       // get all descriptors
3224       Collection<HTableDescriptor> htds;
3225       if (namespace != null && namespace.length() > 0) {
3226         htds = tableDescriptors.getByNamespace(namespace).values();
3227       } else {
3228         htds = tableDescriptors.getAll().values();
3229       }
3230 
3231       for (HTableDescriptor htd: htds) {
3232         if (includeSysTables || !htd.getTableName().isSystemTable()) {
3233           descriptors.add(htd);
3234         }
3235       }
3236 
3237       // Retains only those matched by regular expression.
3238       if (regex != null) {
3239         filterTablesByRegex(descriptors, Pattern.compile(regex));
3240       }
3241 
3242       if (cpHost != null) {
3243         cpHost.postGetTableNames(descriptors, regex);
3244       }
3245     }
3246 
3247     List<TableName> result = new ArrayList<TableName>(descriptors.size());
3248     for (HTableDescriptor htd: descriptors) {
3249       result.add(htd.getTableName());
3250     }
3251     return result;
3252   }
3253 
3254 
3255   /**
3256    * Removes the table descriptors that don't match the pattern.
3257    * @param descriptors list of table descriptors to filter
3258    * @param pattern the regex to use
3259    */
3260   private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
3261       final Pattern pattern) {
3262     final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
3263     Iterator<HTableDescriptor> itr = descriptors.iterator();
3264     while (itr.hasNext()) {
3265       HTableDescriptor htd = itr.next();
3266       String tableName = htd.getTableName().getNameAsString();
3267       boolean matched = pattern.matcher(tableName).matches();
3268       if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
3269         matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
3270       }
3271       if (!matched) {
3272         itr.remove();
3273       }
3274     }
3275   }
3276 
3277   @Override
3278   public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
3279     return getClusterStatus().getLastMajorCompactionTsForTable(table);
3280   }
3281 
3282   @Override
3283   public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
3284     return getClusterStatus().getLastMajorCompactionTsForRegion(regionName);
3285   }
3286 
3287   /**
3288    * Gets the mob file compaction state for a specific table.
3289    * Whether all the mob files are selected is known during the compaction execution, but
3290    * the statistic is done just before compaction starts, it is hard to know the compaction
3291    * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
3292    * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
3293    * @param tableName The current table name.
3294    * @return If a given table is in mob file compaction now.
3295    */
3296   public CompactionState getMobCompactionState(TableName tableName) {
3297     AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3298     if (compactionsCount != null && compactionsCount.get() != 0) {
3299       return CompactionState.MAJOR_AND_MINOR;
3300     }
3301     return CompactionState.NONE;
3302   }
3303 
3304   public void reportMobCompactionStart(TableName tableName) throws IOException {
3305     IdLock.Entry lockEntry = null;
3306     try {
3307       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3308       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3309       if (compactionsCount == null) {
3310         compactionsCount = new AtomicInteger(0);
3311         mobCompactionStates.put(tableName, compactionsCount);
3312       }
3313       compactionsCount.incrementAndGet();
3314     } finally {
3315       if (lockEntry != null) {
3316         mobCompactionLock.releaseLockEntry(lockEntry);
3317       }
3318     }
3319   }
3320 
3321   public void reportMobCompactionEnd(TableName tableName) throws IOException {
3322     IdLock.Entry lockEntry = null;
3323     try {
3324       lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3325       AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3326       if (compactionsCount != null) {
3327         int count = compactionsCount.decrementAndGet();
3328         // remove the entry if the count is 0.
3329         if (count == 0) {
3330           mobCompactionStates.remove(tableName);
3331         }
3332       }
3333     } finally {
3334       if (lockEntry != null) {
3335         mobCompactionLock.releaseLockEntry(lockEntry);
3336       }
3337     }
3338   }
3339 
3340   /**
3341    * Requests mob compaction.
3342    * @param tableName The table the compact.
3343    * @param columns The compacted columns.
3344    * @param allFiles Whether add all mob files into the compaction.
3345    */
3346   public void requestMobCompaction(TableName tableName,
3347     List<HColumnDescriptor> columns, boolean allFiles) throws IOException {
3348     mobCompactThread.requestMobCompaction(conf, fs, tableName, columns,
3349       tableLockManager, allFiles);
3350   }
3351 
3352   /**
3353    * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
3354    * false is returned.
3355    *
3356    * @return The state of the load balancer, or false if the load balancer isn't defined.
3357    */
3358   public boolean isBalancerOn() {
3359     if (null == loadBalancerTracker || isInMaintenanceMode()) {
3360       return false;
3361     }
3362     return loadBalancerTracker.isBalancerOn();
3363   }
3364 
3365   /**
3366    * Queries the state of the {@link RegionNormalizerTracker}. If it's not initialized,
3367    * false is returned.
3368    */
3369   public boolean isNormalizerOn() {
3370     return (null == regionNormalizerTracker || isInMaintenanceMode()) ?
3371         false: regionNormalizerTracker.isNormalizerOn();
3372   }
3373 
3374   /**
3375    * Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
3376    * false is returned. If switchType is illegal, false will return.
3377    * @param switchType see {@link org.apache.hadoop.hbase.client.Admin.MasterSwitchType}
3378    * @return The state of the switch
3379    */
3380   public boolean isSplitOrMergeEnabled(Admin.MasterSwitchType switchType) {
3381     if (null == splitOrMergeTracker || isInMaintenanceMode()) {
3382       return false;
3383     }
3384     return splitOrMergeTracker.isSplitOrMergeEnabled(switchType);
3385   }
3386 
3387   /**
3388    * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
3389    *
3390    * @return The name of the {@link LoadBalancer} in use.
3391    */
3392   public String getLoadBalancerClassName() {
3393     return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
3394         .getDefaultLoadBalancerClass().getName());
3395   }
3396 
3397   /**
3398    * @return RegionNormalizerTracker instance
3399    */
3400   public RegionNormalizerTracker getRegionNormalizerTracker() {
3401     return regionNormalizerTracker;
3402   }
3403 
3404   public SplitOrMergeTracker getSplitOrMergeTracker() {
3405     return splitOrMergeTracker;
3406   }
3407 
3408   @Override
3409   public LoadBalancer getLoadBalancer() {
3410     return balancer;
3411   }
3412 
3413   @Override
3414   public TableStateManager getTableStateManager() {
3415     return assignmentManager.getTableStateManager();
3416   }
3417 
3418   public QuotaObserverChore getQuotaObserverChore() {
3419     return this.quotaObserverChore;
3420   }
3421 
3422   public SpaceQuotaSnapshotNotifier getSpaceQuotaSnapshotNotifier() {
3423     return this.spaceQuotaSnapshotNotifier;
3424   }
3425 }