1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import com.google.common.annotations.VisibleForTesting;
22 import com.google.common.cache.Cache;
23 import com.google.common.cache.CacheBuilder;
24 import com.google.common.collect.Lists;
25 import com.google.common.collect.Maps;
26 import com.google.protobuf.Descriptors;
27 import com.google.protobuf.Service;
28
29 import java.io.IOException;
30 import java.io.InterruptedIOException;
31 import java.lang.reflect.Constructor;
32 import java.lang.reflect.InvocationTargetException;
33 import java.net.InetAddress;
34 import java.net.InetSocketAddress;
35 import java.net.UnknownHostException;
36 import java.util.ArrayList;
37 import java.util.Collection;
38 import java.util.Collections;
39 import java.util.Comparator;
40 import java.util.HashSet;
41 import java.util.Iterator;
42 import java.util.List;
43 import java.util.Map;
44 import java.util.Map.Entry;
45 import java.util.Set;
46 import java.util.concurrent.CountDownLatch;
47 import java.util.concurrent.TimeUnit;
48 import java.util.concurrent.atomic.AtomicInteger;
49 import java.util.concurrent.atomic.AtomicReference;
50 import java.util.regex.Pattern;
51
52 import javax.servlet.ServletException;
53 import javax.servlet.http.HttpServlet;
54 import javax.servlet.http.HttpServletRequest;
55 import javax.servlet.http.HttpServletResponse;
56
57 import org.apache.commons.logging.Log;
58 import org.apache.commons.logging.LogFactory;
59 import org.apache.hadoop.conf.Configuration;
60 import org.apache.hadoop.fs.FileSystem;
61 import org.apache.hadoop.fs.Path;
62 import org.apache.hadoop.hbase.ClusterStatus;
63 import org.apache.hadoop.hbase.CoordinatedStateException;
64 import org.apache.hadoop.hbase.CoordinatedStateManager;
65 import org.apache.hadoop.hbase.DoNotRetryIOException;
66 import org.apache.hadoop.hbase.HBaseIOException;
67 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
68 import org.apache.hadoop.hbase.HColumnDescriptor;
69 import org.apache.hadoop.hbase.HConstants;
70 import org.apache.hadoop.hbase.HRegionInfo;
71 import org.apache.hadoop.hbase.HTableDescriptor;
72 import org.apache.hadoop.hbase.MasterNotRunningException;
73 import org.apache.hadoop.hbase.MetaMigrationConvertingToPB;
74 import org.apache.hadoop.hbase.MetaTableAccessor;
75 import org.apache.hadoop.hbase.NamespaceDescriptor;
76 import org.apache.hadoop.hbase.NamespaceNotFoundException;
77 import org.apache.hadoop.hbase.PleaseHoldException;
78 import org.apache.hadoop.hbase.ProcedureInfo;
79 import org.apache.hadoop.hbase.ScheduledChore;
80 import org.apache.hadoop.hbase.Server;
81 import org.apache.hadoop.hbase.ServerLoad;
82 import org.apache.hadoop.hbase.ServerName;
83 import org.apache.hadoop.hbase.TableDescriptors;
84 import org.apache.hadoop.hbase.TableName;
85 import org.apache.hadoop.hbase.TableNotDisabledException;
86 import org.apache.hadoop.hbase.TableNotFoundException;
87 import org.apache.hadoop.hbase.TableStateManager;
88 import org.apache.hadoop.hbase.UnknownRegionException;
89 import org.apache.hadoop.hbase.backup.BackupType;
90 import org.apache.hadoop.hbase.backup.HBackupFileSystem;
91 import org.apache.hadoop.hbase.backup.impl.BackupManager;
92 import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
93 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
94 import org.apache.hadoop.hbase.backup.master.FullTableBackupProcedure;
95 import org.apache.hadoop.hbase.backup.master.IncrementalTableBackupProcedure;
96 import org.apache.hadoop.hbase.classification.InterfaceAudience;
97 import org.apache.hadoop.hbase.client.Admin;
98 import org.apache.hadoop.hbase.client.MetaScanner;
99 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
102 import org.apache.hadoop.hbase.client.Result;
103 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
104 import org.apache.hadoop.hbase.exceptions.DeserializationException;
105 import org.apache.hadoop.hbase.executor.ExecutorType;
106 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
107 import org.apache.hadoop.hbase.ipc.RpcServer;
108 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
109 import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
110 import org.apache.hadoop.hbase.master.RegionState.State;
111 import org.apache.hadoop.hbase.master.balancer.BalancerChore;
112 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
113 import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
114 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
115 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
116 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
117 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
118 import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
119 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
120 import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
121 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
122 import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
123 import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
124 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
125 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
126 import org.apache.hadoop.hbase.master.handler.TruncateTableHandler;
127 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
128 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
129 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
130 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
131 import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
132 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
133 import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
134 import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
135 import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
136 import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
137 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
138 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
139 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
140 import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
141 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
142 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
143 import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
144 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
145 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
146 import org.apache.hadoop.hbase.mob.MobConstants;
147 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
148 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
149 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
150 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
151 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
152 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
153 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
154 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
155 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
156 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
157 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy;
158 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
159 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
160 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
161 import org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver;
162 import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
163 import org.apache.hadoop.hbase.quotas.QuotaUtil;
164 import org.apache.hadoop.hbase.quotas.RegionStateListener;
165 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifier;
166 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory;
167 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
168 import org.apache.hadoop.hbase.regionserver.HRegionServer;
169 import org.apache.hadoop.hbase.regionserver.HStore;
170 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
171 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
172 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
173 import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
174 import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
175 import org.apache.hadoop.hbase.replication.regionserver.Replication;
176 import org.apache.hadoop.hbase.security.AccessDeniedException;
177 import org.apache.hadoop.hbase.security.UserProvider;
178 import org.apache.hadoop.hbase.util.Addressing;
179 import org.apache.hadoop.hbase.util.Bytes;
180 import org.apache.hadoop.hbase.util.CompressionTest;
181 import org.apache.hadoop.hbase.util.ConfigUtil;
182 import org.apache.hadoop.hbase.util.EncryptionTest;
183 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
184 import org.apache.hadoop.hbase.util.FSUtils;
185 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
186 import org.apache.hadoop.hbase.util.HasThread;
187 import org.apache.hadoop.hbase.util.IdLock;
188 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
189 import org.apache.hadoop.hbase.util.Pair;
190 import org.apache.hadoop.hbase.util.Threads;
191 import org.apache.hadoop.hbase.util.VersionInfo;
192 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
193 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
194 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
195 import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
196 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
197 import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
198 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
199 import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
200 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
201 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
202 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
203 import org.apache.zookeeper.KeeperException;
204 import org.mortbay.jetty.Connector;
205 import org.mortbay.jetty.nio.SelectChannelConnector;
206 import org.mortbay.jetty.servlet.Context;
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
224 @SuppressWarnings("deprecation")
225 public class HMaster extends HRegionServer implements MasterServices, Server {
226 private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
227
228
229
230
231
232 private static class InitializationMonitor extends HasThread {
233
234 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
235 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
236
237
238
239
240
241 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
242 public static final boolean HALT_DEFAULT = false;
243
244 private final HMaster master;
245 private final long timeout;
246 private final boolean haltOnTimeout;
247
248
249 InitializationMonitor(HMaster master) {
250 super("MasterInitializationMonitor");
251 this.master = master;
252 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
253 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
254 this.setDaemon(true);
255 }
256
257 @Override
258 public void run() {
259 try {
260 while (!master.isStopped() && master.isActiveMaster()) {
261 Thread.sleep(timeout);
262 if (master.isInitialized()) {
263 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
264 } else {
265 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
266 + " consider submitting a bug report including a thread dump of this process.");
267 if (haltOnTimeout) {
268 LOG.error("Zombie Master exiting. Thread dump to stdout");
269 Threads.printThreadInfo(System.out, "Zombie HMaster");
270 System.exit(-1);
271 }
272 }
273 }
274 } catch (InterruptedException ie) {
275 LOG.trace("InitMonitor thread interrupted. Existing.");
276 }
277 }
278 }
279
280
281
282 public static final String MASTER = "master";
283
284
285 private final ActiveMasterManager activeMasterManager;
286
287 RegionServerTracker regionServerTracker;
288
289 private DrainingServerTracker drainingServerTracker;
290
291 LoadBalancerTracker loadBalancerTracker;
292
293
294 private SplitOrMergeTracker splitOrMergeTracker;
295
296
297 private RegionNormalizerTracker regionNormalizerTracker;
298
299
300 private TableNamespaceManager tableNamespaceManager;
301
302
303 private MasterMaintenanceModeTracker maintenanceModeTracker;
304
305
306 final MetricsMaster metricsMaster;
307
308 private MasterFileSystem fileSystemManager;
309
310
311 volatile ServerManager serverManager;
312
313
314 AssignmentManager assignmentManager;
315
316
317
318
319 MemoryBoundedLogMessageBuffer rsFatals;
320
321
322 private volatile boolean isActiveMaster = false;
323
324
325
326 volatile boolean initialized = false;
327
328
329
330 volatile boolean serviceStarted = false;
331
332
333 private volatile boolean serverShutdownHandlerEnabled = false;
334
335 LoadBalancer balancer;
336 private RegionNormalizer normalizer;
337 private BalancerChore balancerChore;
338 private RegionNormalizerChore normalizerChore;
339 private ClusterStatusChore clusterStatusChore;
340 private ClusterStatusPublisher clusterStatusPublisherChore = null;
341 private PeriodicDoMetrics periodicDoMetricsChore = null;
342
343 CatalogJanitor catalogJanitorChore;
344 private LogCleaner logCleaner;
345 private HFileCleaner hfileCleaner;
346 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
347 private MobCompactionChore mobCompactChore;
348 private MasterMobCompactionThread mobCompactThread;
349
350 private final IdLock mobCompactionLock = new IdLock();
351
352
353 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
354
355 MasterCoprocessorHost cpHost;
356
357 private final boolean preLoadTableDescriptors;
358
359
360 private long masterActiveTime;
361
362
363 private final boolean masterCheckCompression;
364
365
366 private final boolean masterCheckEncryption;
367
368
369 private enum ProcedureConf {
370 PROCEDURE_ENABLED,
371 HANDLER_USED,
372 PROCEDURE_FULLY_DISABLED,
373 }
374 private final ProcedureConf procedureConf;
375
376 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
377
378
379 SnapshotManager snapshotManager;
380
381 private MasterProcedureManagerHost mpmHost;
382
383
384 private volatile MasterQuotaManager quotaManager;
385 private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier;
386 private QuotaObserverChore quotaObserverChore;
387
388 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
389 private WALProcedureStore procedureStore;
390
391
392 private volatile boolean initializationBeforeMetaAssignment = false;
393
394
395 private org.mortbay.jetty.Server masterJettyServer;
396
397 public static class RedirectServlet extends HttpServlet {
398 private static final long serialVersionUID = 2894774810058302472L;
399 private static int regionServerInfoPort;
400
401 @Override
402 public void doGet(HttpServletRequest request,
403 HttpServletResponse response) throws ServletException, IOException {
404 String redirectUrl = request.getScheme() + "://"
405 + request.getServerName() + ":" + regionServerInfoPort
406 + request.getRequestURI();
407 response.sendRedirect(redirectUrl);
408 }
409 }
410
411 private static class PeriodicDoMetrics extends ScheduledChore {
412 private final HMaster server;
413 public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
414 super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval);
415 this.server = server;
416 }
417
418 @Override
419 protected void chore() {
420 server.doMetrics();
421 }
422 }
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440 public HMaster(final Configuration conf, CoordinatedStateManager csm)
441 throws IOException, KeeperException, InterruptedException {
442 super(conf, csm);
443 this.rsFatals = new MemoryBoundedLogMessageBuffer(
444 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
445
446 LOG.info("hbase.rootdir=" + getRootDir() +
447 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
448
449
450 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
451
452 Replication.decorateMasterConfiguration(this.conf);
453 BackupManager.decorateMasterConfiguration(this.conf);
454
455
456
457 if (this.conf.get("mapreduce.task.attempt.id") == null) {
458 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
459 }
460
461
462 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
463
464
465 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
466
467 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
468
469
470
471
472 String procedureConfString = conf.get("hbase.master.procedure.tableddl", "enabled");
473 if (procedureConfString.equalsIgnoreCase("disabled")) {
474 LOG.info("Master will use handler for new table DDL"
475 + " and all unfinished table DDLs in procedure store will be disgarded.");
476 this.procedureConf = ProcedureConf.PROCEDURE_FULLY_DISABLED;
477 } else if (procedureConfString.equalsIgnoreCase("unused")) {
478 LOG.info("Master will use handler for new table DDL"
479 + " and all unfinished table DDLs in procedure store will continue to execute.");
480 this.procedureConf = ProcedureConf.HANDLER_USED;
481 } else {
482 this.procedureConf = ProcedureConf.PROCEDURE_ENABLED;
483 }
484
485 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
486
487
488
489 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
490 HConstants.STATUS_PUBLISHED_DEFAULT);
491 Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
492 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
493 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
494 ClusterStatusPublisher.Publisher.class);
495
496 if (shouldPublish) {
497 if (publisherClass == null) {
498 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
499 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
500 " is not set - not publishing status");
501 } else {
502 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
503 getChoreService().scheduleChore(clusterStatusPublisherChore);
504 }
505 }
506
507
508 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
509 setInitLatch(new CountDownLatch(1));
510 activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
511 int infoPort = putUpJettyServer();
512 startActiveMasterManager(infoPort);
513 } else {
514 activeMasterManager = null;
515 }
516 }
517
518
519 private int putUpJettyServer() throws IOException {
520 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
521 return -1;
522 }
523 int infoPort = conf.getInt("hbase.master.info.port.orig",
524 HConstants.DEFAULT_MASTER_INFOPORT);
525
526 if (infoPort < 0 || infoServer == null) {
527 return -1;
528 }
529 String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
530 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
531 String msg =
532 "Failed to start redirecting jetty server. Address " + addr
533 + " does not belong to this host. Correct configuration parameter: "
534 + "hbase.master.info.bindAddress";
535 LOG.error(msg);
536 throw new IOException(msg);
537 }
538
539 RedirectServlet.regionServerInfoPort = infoServer.getPort();
540 if(RedirectServlet.regionServerInfoPort == infoPort) {
541 return infoPort;
542 }
543 masterJettyServer = new org.mortbay.jetty.Server();
544 Connector connector = new SelectChannelConnector();
545 connector.setHost(addr);
546 connector.setPort(infoPort);
547 masterJettyServer.addConnector(connector);
548 masterJettyServer.setStopAtShutdown(true);
549 Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
550 context.addServlet(RedirectServlet.class, "/*");
551 try {
552 masterJettyServer.start();
553 } catch (Exception e) {
554 throw new IOException("Failed to start redirecting jetty server", e);
555 }
556 return connector.getLocalPort();
557 }
558
559
560
561
562 @Override
563 protected void login(UserProvider user, String host) throws IOException {
564 try {
565 super.login(user, host);
566 } catch (IOException ie) {
567 user.login("hbase.master.keytab.file",
568 "hbase.master.kerberos.principal", host);
569 }
570 }
571
572
573
574
575
576
577 @Override
578 protected void waitForMasterActive(){
579 boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
580 while (!(tablesOnMaster && isActiveMaster)
581 && !isStopped() && !isAborted()) {
582 sleeper.sleep();
583 }
584 }
585
586 @VisibleForTesting
587 public MasterRpcServices getMasterRpcServices() {
588 return (MasterRpcServices)rpcServices;
589 }
590
591 public boolean balanceSwitch(final boolean b) throws IOException {
592 return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
593 }
594
595 @Override
596 protected String getProcessName() {
597 return MASTER;
598 }
599
600 @Override
601 protected boolean canCreateBaseZNode() {
602 return true;
603 }
604
605 @Override
606 protected boolean canUpdateTableDescriptor() {
607 return true;
608 }
609
610 @Override
611 protected RSRpcServices createRpcServices() throws IOException {
612 return new MasterRpcServices(this);
613 }
614
615 @Override
616 protected void configureInfoServer() {
617 infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
618 infoServer.setAttribute(MASTER, this);
619 if (BaseLoadBalancer.tablesOnMaster(conf)) {
620 super.configureInfoServer();
621 }
622 }
623
624 @Override
625 protected Class<? extends HttpServlet> getDumpServlet() {
626 return MasterDumpServlet.class;
627 }
628
629
630
631
632
633 private void doMetrics() {
634 try {
635 if (assignmentManager != null) {
636 assignmentManager.updateRegionsInTransitionMetrics();
637 }
638 } catch (Throwable e) {
639 LOG.error("Couldn't update metrics: " + e.getMessage());
640 }
641 }
642
643 MetricsMaster getMasterMetrics() {
644 return metricsMaster;
645 }
646
647
648
649
650
651
652
653
654 void initializeZKBasedSystemTrackers() throws IOException,
655 InterruptedException, KeeperException, CoordinatedStateException {
656 this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
657 this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
658 this.normalizer.setMasterServices(this);
659 this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
660 this.loadBalancerTracker.start();
661
662 this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this);
663 this.regionNormalizerTracker.start();
664
665 this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
666 this.splitOrMergeTracker.start();
667
668 this.assignmentManager = new AssignmentManager(this, serverManager,
669 this.balancer, this.service, this.metricsMaster,
670 this.tableLockManager);
671 zooKeeper.registerListenerFirst(assignmentManager);
672
673 this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
674 this.serverManager);
675 this.regionServerTracker.start();
676
677 this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
678 this.serverManager);
679 this.drainingServerTracker.start();
680
681 this.maintenanceModeTracker = new MasterMaintenanceModeTracker(zooKeeper);
682 this.maintenanceModeTracker.start();
683
684
685
686 boolean wasUp = this.clusterStatusTracker.isClusterUp();
687 if (!wasUp) this.clusterStatusTracker.setClusterUp();
688
689 LOG.info("Server active/primary master=" + this.serverName +
690 ", sessionid=0x" +
691 Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
692 ", setting cluster-up flag (Was=" + wasUp + ")");
693
694
695 this.snapshotManager = new SnapshotManager();
696 this.mpmHost = new MasterProcedureManagerHost();
697 this.mpmHost.register(this.snapshotManager);
698 this.mpmHost.register(new MasterFlushTableProcedureManager());
699 this.mpmHost.loadProcedures(conf);
700 this.mpmHost.initialize(this, this.metricsMaster);
701 }
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723 private void finishActiveMasterInitialization(MonitoredTask status)
724 throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
725
726 isActiveMaster = true;
727 Thread zombieDetector = new Thread(new InitializationMonitor(this));
728 zombieDetector.start();
729
730
731
732
733
734
735
736 status.setStatus("Initializing Master file system");
737
738 this.masterActiveTime = System.currentTimeMillis();
739
740 this.fileSystemManager = new MasterFileSystem(this, this);
741
742
743 this.tableDescriptors.setCacheOn();
744
745 this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
746 conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
747
748 if (preLoadTableDescriptors) {
749 status.setStatus("Pre-loading table descriptors");
750 this.tableDescriptors.getAll();
751 }
752
753
754 status.setStatus("Publishing Cluster ID in ZooKeeper");
755 ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
756 this.initLatch.countDown();
757 this.serverManager = createServerManager(this, this);
758
759 setupClusterConnection();
760
761
762 this.tableLockManager.reapWriteLocks();
763
764 status.setStatus("Initializing ZK system trackers");
765 initializeZKBasedSystemTrackers();
766
767
768
769 if (QuotaUtil.isQuotaEnabled(conf)) {
770 updateConfigurationForSpaceQuotaObserver(conf);
771 }
772
773 status.setStatus("Initializing master coprocessors");
774 this.cpHost = new MasterCoprocessorHost(this, this.conf);
775
776
777 status.setStatus("Initializing master service threads");
778 startServiceThreads();
779
780
781 sleeper.skipSleepCycle();
782
783
784 this.serverManager.waitForRegionServers(status);
785
786 for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
787
788 if (!this.serverManager.isServerOnline(sn)
789 && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
790 LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
791 }
792 }
793
794
795
796
797 Set<ServerName> previouslyFailedServers = this.fileSystemManager
798 .getFailedServersFromLogFolders();
799
800
801 this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);
802
803
804 ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
805 if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
806 splitMetaLogBeforeAssignment(oldMetaServerLocation);
807
808
809 }
810 Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
811
812
813
814
815
816
817
818
819 previouslyFailedMetaRSs.addAll(previouslyFailedServers);
820
821 this.initializationBeforeMetaAssignment = true;
822
823
824 if (BaseLoadBalancer.tablesOnMaster(conf)) {
825 waitForServerOnline();
826 }
827
828
829 this.balancer.setClusterStatus(getClusterStatus());
830 this.balancer.setMasterServices(this);
831 this.balancer.initialize();
832
833
834
835 if(isStopped()) return;
836
837
838 status.setStatus("Assigning Meta Region");
839 assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID);
840
841
842 if(isStopped()) return;
843
844 status.setStatus("Submitting log splitting work for previously failed region servers");
845
846
847 for (ServerName tmpServer : previouslyFailedServers) {
848 this.serverManager.processDeadServer(tmpServer, true);
849 }
850
851
852
853
854 if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {
855 MetaMigrationConvertingToPB.updateMetaIfNecessary(this);
856 }
857
858
859 status.setStatus("Starting assignment manager");
860 this.assignmentManager.joinCluster();
861
862
863 this.balancer.setClusterStatus(getClusterStatus());
864
865
866
867 status.setStatus("Starting balancer and catalog janitor");
868 this.clusterStatusChore = new ClusterStatusChore(this, balancer);
869 getChoreService().scheduleChore(clusterStatusChore);
870 this.balancerChore = new BalancerChore(this);
871 getChoreService().scheduleChore(balancerChore);
872 this.normalizerChore = new RegionNormalizerChore(this);
873 getChoreService().scheduleChore(normalizerChore);
874 this.catalogJanitorChore = new CatalogJanitor(this, this);
875 getChoreService().scheduleChore(catalogJanitorChore);
876
877
878 periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
879 getChoreService().scheduleChore(periodicDoMetricsChore);
880
881 status.setStatus("Starting namespace manager and quota manager");
882 initNamespaceAndQuotaManager();
883
884 if (this.cpHost != null) {
885 try {
886 this.cpHost.preMasterInitialization();
887 } catch (IOException e) {
888 LOG.error("Coprocessor preMasterInitialization() hook failed", e);
889 }
890 }
891
892 status.markComplete("Initialization successful");
893 LOG.info("Master has completed initialization");
894 configurationManager.registerObserver(this.balancer);
895
896 initialized = true;
897
898 status.setStatus("Starting quota manager");
899 if (QuotaUtil.isQuotaEnabled(conf)) {
900
901 spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
902 spaceQuotaSnapshotNotifier.initialize(this.clusterConnection);
903 this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
904
905 getChoreService().scheduleChore(quotaObserverChore);
906 }
907
908
909 Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
910 int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
911 HConstants.DEFAULT_META_REPLICA_NUM);
912 for (int i = 1; i < numReplicas; i++) {
913 assignMeta(status, EMPTY_SET, i);
914 }
915 unassignExcessMetaReplica(zooKeeper, numReplicas);
916
917
918
919
920 this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
921
922
923 status.setStatus("Checking ZNode ACLs");
924 zooKeeper.checkAndSetZNodeAcls();
925
926 status.setStatus("Calling postStartMaster coprocessors");
927
928 this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);
929 getChoreService().scheduleChore(expiredMobFileCleanerChore);
930
931 int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
932 MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);
933 if (mobCompactionPeriod > 0) {
934 this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);
935 getChoreService().scheduleChore(mobCompactChore);
936 } else {
937 LOG
938 .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");
939 }
940 this.mobCompactThread = new MasterMobCompactionThread(this);
941
942 if (this.cpHost != null) {
943
944 try {
945 this.cpHost.postStartMaster();
946 } catch (IOException ioe) {
947 LOG.error("Coprocessor postStartMaster() hook failed", ioe);
948 }
949 }
950
951 zombieDetector.interrupt();
952 }
953
954
955
956
957
958 @VisibleForTesting
959 public void updateConfigurationForSpaceQuotaObserver(Configuration conf) {
960
961 if (!conf.getBoolean(
962 MasterSpaceQuotaObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
963 MasterSpaceQuotaObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
964 return;
965 }
966 String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
967 final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
968 String[] updatedCoprocs = new String[length + 1];
969 if (length > 0) {
970 System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
971 }
972 updatedCoprocs[length] = MasterSpaceQuotaObserver.class.getName();
973 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
974 }
975
976 private void initQuotaManager() throws IOException {
977 quotaManager = new MasterQuotaManager(this);
978 this.assignmentManager.setRegionStateListener((RegionStateListener) quotaManager);
979 quotaManager.start();
980 }
981
982
983
984
985
986
987
988
989
990 ServerManager createServerManager(final Server master,
991 final MasterServices services)
992 throws IOException {
993
994
995 return new ServerManager(master, services);
996 }
997
998 private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicasConfigured) {
999
1000
1001 try {
1002 List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
1003 for (String metaReplicaZnode : metaReplicaZnodes) {
1004 int replicaId = zooKeeper.getMetaReplicaIdFromZnode(metaReplicaZnode);
1005 if (replicaId >= numMetaReplicasConfigured) {
1006 RegionState r = MetaTableLocator.getMetaRegionState(zkw, replicaId);
1007 LOG.info("Closing excess replica of meta region " + r.getRegion());
1008
1009 ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(),
1010 r.getRegion(), 30000);
1011 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId));
1012 }
1013 }
1014 } catch (Exception ex) {
1015
1016
1017 LOG.warn("Ignoring exception " + ex);
1018 }
1019 }
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030 void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
1031 throws InterruptedException, IOException, KeeperException {
1032
1033 int assigned = 0;
1034 long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
1035 if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
1036 status.setStatus("Assigning hbase:meta region");
1037 } else {
1038 status.setStatus("Assigning hbase:meta region, replicaId " + replicaId);
1039 }
1040
1041 RegionStates regionStates = assignmentManager.getRegionStates();
1042 RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper(), replicaId);
1043 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO,
1044 replicaId);
1045 ServerName currentMetaServer = metaState.getServerName();
1046 if (!ConfigUtil.useZKForAssignment(conf)) {
1047 regionStates.createRegionState(hri, metaState.getState(),
1048 currentMetaServer, null);
1049 } else {
1050 regionStates.createRegionState(hri);
1051 }
1052 boolean rit = this.assignmentManager.
1053 processRegionInTransitionAndBlockUntilAssigned(hri);
1054 boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation(
1055 this.getConnection(), this.getZooKeeper(), timeout, replicaId);
1056 if (!metaRegionLocation || !metaState.isOpened()) {
1057
1058
1059 assigned++;
1060 if (!ConfigUtil.useZKForAssignment(conf)) {
1061 assignMetaZkLess(regionStates, metaState, timeout, previouslyFailedMetaRSs);
1062 } else if (!rit) {
1063
1064 if (currentMetaServer != null) {
1065
1066
1067
1068
1069
1070
1071
1072 if (serverManager.isServerOnline(currentMetaServer)) {
1073 LOG.info("Forcing expire of " + currentMetaServer);
1074 serverManager.expireServer(currentMetaServer);
1075 }
1076 if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
1077 splitMetaLogBeforeAssignment(currentMetaServer);
1078 previouslyFailedMetaRSs.add(currentMetaServer);
1079 }
1080 }
1081 assignmentManager.assignMeta(hri);
1082 }
1083 } else {
1084
1085 regionStates.updateRegionState(hri, State.OPEN, currentMetaServer);
1086 this.assignmentManager.regionOnline(hri, currentMetaServer);
1087 }
1088
1089 if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableMeta(TableName.META_TABLE_NAME);
1090
1091 if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
1092 && (!previouslyFailedMetaRSs.isEmpty())) {
1093
1094 status.setStatus("replaying log for Meta Region");
1095 this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
1096 }
1097
1098
1099
1100
1101
1102 if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableServerShutdownHandler(assigned != 0);
1103 LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", rit=" + rit +
1104 ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId));
1105 status.setStatus("META assigned.");
1106 }
1107
1108 private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout,
1109 Set<ServerName> previouslyFailedRs) throws IOException, KeeperException {
1110 ServerName currentServer = regionState.getServerName();
1111 if (serverManager.isServerOnline(currentServer)) {
1112 LOG.info("Meta was in transition on " + currentServer);
1113 assignmentManager.processRegionInTransitionZkLess();
1114 } else {
1115 if (currentServer != null) {
1116 if (regionState.getRegion().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1117 splitMetaLogBeforeAssignment(currentServer);
1118 regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
1119 previouslyFailedRs.add(currentServer);
1120 }
1121 }
1122 LOG.info("Re-assigning hbase:meta, it was on " + currentServer);
1123 regionStates.updateRegionState(regionState.getRegion(), State.OFFLINE);
1124 assignmentManager.assignMeta(regionState.getRegion());
1125 }
1126 }
1127
1128
1129
1130
1131
1132
1133
1134 private void initNamespaceAndQuotaManager() throws IOException {
1135
1136 tableNamespaceManager = new TableNamespaceManager(this);
1137
1138
1139 this.quotaManager = new MasterQuotaManager(this);
1140 this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager);
1141
1142 if (conf.getBoolean("hbase.master.start.wait.for.namespacemanager", false)) {
1143
1144
1145
1146
1147
1148 LOG.info("Starting namespace manager and quota manager synchronously");
1149
1150 tableNamespaceManager.start();
1151 LOG.info("Namespace manager started successfully.");
1152
1153 quotaManager.start();
1154 LOG.info("Quota manager started successfully.");
1155 } else {
1156 LOG.info("Starting namespace manager and quota manager asynchronously");
1157 Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1158 @Override
1159 public void run() {
1160
1161 try {
1162 tableNamespaceManager.start();
1163 LOG.info("Namespace manager started successfully.");
1164 } catch (IOException e) {
1165 LOG.error("Namespace manager failed to start. ", e);
1166 abort("Shutdown Master due to namespace manager failed to start. ", e);
1167 }
1168
1169 try {
1170 quotaManager.start();
1171 LOG.info("Quota manager started successfully.");
1172 } catch (IOException ie) {
1173 LOG.error("Quota Manager failed to start. ", ie);
1174 abort("Shutdown Master due to Quota Manager failure to start. ", ie);
1175 }
1176 }
1177 }, "Init Namespace Manager and Quota Manager Async"));
1178 }
1179 }
1180
1181 SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
1182 SpaceQuotaSnapshotNotifier notifier =
1183 SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());
1184 notifier.initialize(getConnection());
1185 return notifier;
1186 }
1187
1188 boolean isCatalogJanitorEnabled() {
1189 return catalogJanitorChore != null ?
1190 catalogJanitorChore.getEnabled() : false;
1191 }
1192
1193 private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
1194 if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
1195
1196 Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
1197 regions.add(HRegionInfo.FIRST_META_REGIONINFO);
1198 this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
1199 } else {
1200
1201 this.fileSystemManager.splitMetaLog(currentMetaServer);
1202 }
1203 }
1204
1205 private void enableServerShutdownHandler(
1206 final boolean waitForMeta) throws IOException, InterruptedException {
1207
1208
1209
1210
1211
1212 if (!serverShutdownHandlerEnabled) {
1213 serverShutdownHandlerEnabled = true;
1214 this.serverManager.processQueuedDeadServers();
1215 }
1216
1217 if (waitForMeta) {
1218 metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
1219
1220
1221 this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
1222 }
1223 }
1224
1225 private void enableMeta(TableName metaTableName) {
1226 if (!this.assignmentManager.getTableStateManager().isTableState(metaTableName,
1227 ZooKeeperProtos.Table.State.ENABLED)) {
1228 this.assignmentManager.setEnabledTable(metaTableName);
1229 }
1230 }
1231
1232
1233
1234
1235
1236
1237 private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
1238 Set<ServerName> result = new HashSet<ServerName>();
1239 String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
1240 HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1241 List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
1242 if (regionFailedServers == null) return result;
1243
1244 for(String failedServer : regionFailedServers) {
1245 ServerName server = ServerName.parseServerName(failedServer);
1246 result.add(server);
1247 }
1248 return result;
1249 }
1250
1251 @Override
1252 public TableDescriptors getTableDescriptors() {
1253 return this.tableDescriptors;
1254 }
1255
1256 @Override
1257 public ServerManager getServerManager() {
1258 return this.serverManager;
1259 }
1260
1261 @Override
1262 public MasterFileSystem getMasterFileSystem() {
1263 return this.fileSystemManager;
1264 }
1265
1266
1267
1268
1269
1270
1271
1272
1273 private void startServiceThreads() throws IOException{
1274
1275 this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
1276 conf.getInt("hbase.master.executor.openregion.threads", 5));
1277 this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
1278 conf.getInt("hbase.master.executor.closeregion.threads", 5));
1279 this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
1280 conf.getInt("hbase.master.executor.serverops.threads", 5));
1281 this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
1282 conf.getInt("hbase.master.executor.serverops.threads", 5));
1283 this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
1284 conf.getInt("hbase.master.executor.logreplayops.threads", 10));
1285
1286
1287
1288
1289
1290
1291 this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
1292 startProcedureExecutor();
1293
1294
1295 int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
1296 this.logCleaner =
1297 new LogCleaner(cleanerInterval,
1298 this, conf, getMasterFileSystem().getOldLogDir().getFileSystem(conf),
1299 getMasterFileSystem().getOldLogDir());
1300 getChoreService().scheduleChore(logCleaner);
1301
1302
1303 Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
1304 this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
1305 .getFileSystem(), archiveDir);
1306 getChoreService().scheduleChore(hfileCleaner);
1307 serviceStarted = true;
1308 if (LOG.isTraceEnabled()) {
1309 LOG.trace("Started service threads");
1310 }
1311 }
1312
1313 @Override
1314 protected void sendShutdownInterrupt() {
1315 super.sendShutdownInterrupt();
1316 stopProcedureExecutor();
1317 }
1318
1319 @Override
1320 protected void stopServiceThreads() {
1321 if (masterJettyServer != null) {
1322 LOG.info("Stopping master jetty server");
1323 try {
1324 masterJettyServer.stop();
1325 } catch (Exception e) {
1326 LOG.error("Failed to stop master jetty server", e);
1327 }
1328 }
1329 super.stopServiceThreads();
1330 stopChores();
1331
1332
1333
1334 if (!isAborted() && this.serverManager != null &&
1335 this.serverManager.isClusterShutdown()) {
1336 this.serverManager.letRegionServersShutdown();
1337 }
1338 if (LOG.isDebugEnabled()) {
1339 LOG.debug("Stopping service threads");
1340 }
1341
1342 if (this.logCleaner != null) this.logCleaner.cancel(true);
1343 if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
1344 if (this.quotaManager != null) this.quotaManager.stop();
1345 if (this.activeMasterManager != null) this.activeMasterManager.stop();
1346 if (this.serverManager != null) this.serverManager.stop();
1347 if (this.assignmentManager != null) this.assignmentManager.stop();
1348 if (this.fileSystemManager != null) this.fileSystemManager.stop();
1349 if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1350 }
1351
1352
1353
1354
1355 @Override
1356 public boolean isMasterProcedureExecutorEnabled() {
1357 return (this.procedureConf == ProcedureConf.PROCEDURE_ENABLED);
1358 }
1359
1360 private void startProcedureExecutor() throws IOException {
1361 final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
1362 final Path walDir = new Path(FSUtils.getWALRootDir(this.conf),
1363 MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
1364
1365 if (this.procedureConf == ProcedureConf.PROCEDURE_FULLY_DISABLED) {
1366
1367
1368
1369 try {
1370 fs.delete(walDir, true);
1371 LOG.warn("Procedure executor is disabled from configuartion. " +
1372 "All the state logs from procedure store were removed." +
1373 "You should check the cluster state using HBCK.");
1374 } catch (Exception e) {
1375
1376 LOG.error("Removing all the state logs from procedure store failed." +
1377 "You should check the cluster state using HBCK.");
1378 }
1379 return;
1380 }
1381
1382 procedureStore = new WALProcedureStore(conf, walDir.getFileSystem(conf), walDir,
1383 new MasterProcedureEnv.WALStoreLeaseRecovery(this));
1384
1385 procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
1386 procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
1387 procEnv.getProcedureQueue());
1388
1389 final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
1390 Math.max(Runtime.getRuntime().availableProcessors(),
1391 MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
1392 procedureStore.start(numThreads);
1393 procedureExecutor.start(numThreads);
1394 }
1395
1396 private void stopProcedureExecutor() {
1397 if (procedureExecutor != null) {
1398 procedureExecutor.stop();
1399 }
1400
1401 if (procedureStore != null) {
1402 procedureStore.stop(isAborted());
1403 }
1404 }
1405
1406 private void stopChores() {
1407 if (this.expiredMobFileCleanerChore != null) {
1408 this.expiredMobFileCleanerChore.cancel(true);
1409 }
1410 if (this.mobCompactChore != null) {
1411 this.mobCompactChore.cancel(true);
1412 }
1413 if (this.balancerChore != null) {
1414 this.balancerChore.cancel(true);
1415 }
1416 if (this.normalizerChore != null) {
1417 this.normalizerChore.cancel(true);
1418 }
1419 if (this.clusterStatusChore != null) {
1420 this.clusterStatusChore.cancel(true);
1421 }
1422 if (this.catalogJanitorChore != null) {
1423 this.catalogJanitorChore.cancel(true);
1424 }
1425 if (this.clusterStatusPublisherChore != null){
1426 clusterStatusPublisherChore.cancel(true);
1427 }
1428 if (this.mobCompactThread != null) {
1429 this.mobCompactThread.close();
1430 }
1431 if (this.quotaObserverChore != null) {
1432 quotaObserverChore.cancel();
1433 }
1434 if (this.periodicDoMetricsChore != null) {
1435 periodicDoMetricsChore.cancel();
1436 }
1437 }
1438
1439
1440
1441
1442
1443 InetAddress getRemoteInetAddress(final int port,
1444 final long serverStartCode) throws UnknownHostException {
1445
1446
1447 InetAddress ia = RpcServer.getRemoteIp();
1448
1449
1450
1451 if (ia == null && serverStartCode == startcode) {
1452 InetSocketAddress isa = rpcServices.getSocketAddress();
1453 if (isa != null && isa.getPort() == port) {
1454 ia = isa.getAddress();
1455 }
1456 }
1457 return ia;
1458 }
1459
1460
1461
1462
1463 private int getBalancerCutoffTime() {
1464 int balancerCutoffTime =
1465 getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1466 if (balancerCutoffTime == -1) {
1467
1468 int balancerPeriod =
1469 getConfiguration().getInt("hbase.balancer.period", 300000);
1470 balancerCutoffTime = balancerPeriod;
1471
1472 if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;
1473 }
1474 return balancerCutoffTime;
1475 }
1476
1477 public boolean balance() throws IOException {
1478 return balance(false);
1479 }
1480
1481 public boolean balance(boolean force) throws IOException {
1482
1483 if (!this.initialized) {
1484 LOG.debug("Master has not been initialized, don't run balancer.");
1485 return false;
1486 }
1487
1488 if (isInMaintenanceMode()) {
1489 LOG.info("Master is in maintenanceMode mode, don't run balancer.");
1490 return false;
1491 }
1492
1493
1494 int maximumBalanceTime = getBalancerCutoffTime();
1495 synchronized (this.balancer) {
1496
1497 if (!this.loadBalancerTracker.isBalancerOn()) return false;
1498
1499 if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1500 Map<String, RegionState> regionsInTransition =
1501 this.assignmentManager.getRegionStates().getRegionsInTransition();
1502
1503
1504 boolean metaInTransition = assignmentManager.getRegionStates().isMetaRegionInTransition();
1505 String prefix = force && !metaInTransition ? "R" : "Not r";
1506 LOG.debug(prefix + "unning balancer because " + regionsInTransition.size() +
1507 " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1508 abbreviate(regionsInTransition.toString(), 256));
1509 if (!force || metaInTransition) return false;
1510 }
1511 if (this.serverManager.areDeadServersInProgress()) {
1512 LOG.debug("Not running balancer because processing dead regionserver(s): " +
1513 this.serverManager.getDeadServers());
1514 return false;
1515 }
1516
1517 if (this.cpHost != null) {
1518 try {
1519 if (this.cpHost.preBalance()) {
1520 LOG.debug("Coprocessor bypassing balancer request");
1521 return false;
1522 }
1523 } catch (IOException ioe) {
1524 LOG.error("Error invoking master coprocessor preBalance()", ioe);
1525 return false;
1526 }
1527 }
1528
1529 Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1530 this.assignmentManager.getRegionStates().getAssignmentsByTable();
1531
1532 List<RegionPlan> plans = new ArrayList<RegionPlan>();
1533
1534
1535 this.balancer.setClusterStatus(getClusterStatus());
1536 for (Entry<TableName, Map<ServerName, List<HRegionInfo>>> e : assignmentsByTable.entrySet()) {
1537 List<RegionPlan> partialPlans = this.balancer.balanceCluster(e.getKey(), e.getValue());
1538 if (partialPlans != null) plans.addAll(partialPlans);
1539 }
1540
1541 long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1542 int rpCount = 0;
1543 long totalRegPlanExecTime = 0;
1544 if (plans != null && !plans.isEmpty()) {
1545 for (RegionPlan plan: plans) {
1546 LOG.info("balance " + plan);
1547 long balStartTime = System.currentTimeMillis();
1548
1549 this.assignmentManager.balance(plan);
1550 totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1551 rpCount++;
1552 if (rpCount < plans.size() &&
1553
1554 (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1555
1556 LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1557 maximumBalanceTime);
1558 break;
1559 }
1560 }
1561 }
1562 if (this.cpHost != null) {
1563 try {
1564 this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1565 } catch (IOException ioe) {
1566
1567 LOG.error("Error invoking master coprocessor postBalance()", ioe);
1568 }
1569 }
1570 }
1571
1572
1573 return true;
1574 }
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584 public boolean normalizeRegions() throws IOException, CoordinatedStateException {
1585 if (!this.initialized) {
1586 LOG.debug("Master has not been initialized, don't run region normalizer.");
1587 return false;
1588 }
1589
1590 if (isInMaintenanceMode()) {
1591 LOG.info("Master is in maintenance mode, don't run region normalizer.");
1592 return false;
1593 }
1594
1595 if (!this.regionNormalizerTracker.isNormalizerOn()) {
1596 LOG.debug("Region normalization is disabled, don't run region normalizer.");
1597 return false;
1598 }
1599
1600 synchronized (this.normalizer) {
1601
1602 List<TableName> allEnabledTables = new ArrayList<>(
1603 this.assignmentManager.getTableStateManager().getTablesInStates(
1604 ZooKeeperProtos.Table.State.ENABLED));
1605
1606 Collections.shuffle(allEnabledTables);
1607
1608 for (TableName table : allEnabledTables) {
1609 if (isInMaintenanceMode()) {
1610 LOG.debug("Master is in maintenance mode, stop running region normalizer.");
1611 return false;
1612 }
1613
1614 if (table.isSystemTable() || (getTableDescriptors().get(table) != null &&
1615 !getTableDescriptors().get(table).isNormalizationEnabled())) {
1616 LOG.debug("Skipping normalization for table: " + table + ", as it's either system"
1617 + " table or doesn't have auto normalization turned on");
1618 continue;
1619 }
1620 List<NormalizationPlan> plans = this.normalizer.computePlanForTable(table);
1621 if (plans != null) {
1622 for (NormalizationPlan plan : plans) {
1623 plan.execute(clusterConnection.getAdmin());
1624 }
1625 }
1626 }
1627 }
1628
1629
1630 return true;
1631 }
1632
1633
1634
1635
1636 String getClientIdAuditPrefix() {
1637 return "Client=" + RpcServer.getRequestUserName() + "/" + RpcServer.getRemoteAddress();
1638 }
1639
1640
1641
1642
1643
1644
1645
1646 public void setCatalogJanitorEnabled(final boolean b) {
1647 this.catalogJanitorChore.setEnabled(b);
1648 }
1649
1650 @Override
1651 public void dispatchMergingRegions(final HRegionInfo region_a,
1652 final HRegionInfo region_b, final boolean forcible) throws IOException {
1653 checkInitialized();
1654 this.service.submit(new DispatchMergingRegionHandler(this,
1655 this.catalogJanitorChore, region_a, region_b, forcible));
1656 }
1657
1658 void move(final byte[] encodedRegionName,
1659 final byte[] destServerName) throws HBaseIOException {
1660 RegionState regionState = assignmentManager.getRegionStates().
1661 getRegionState(Bytes.toString(encodedRegionName));
1662
1663 HRegionInfo hri;
1664 if (regionState != null) {
1665 hri = regionState.getRegion();
1666 } else {
1667 throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1668 }
1669
1670 ServerName dest;
1671 if (destServerName == null || destServerName.length == 0) {
1672 LOG.info("Passed destination servername is null/empty so " +
1673 "choosing a server at random");
1674 final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1675 regionState.getServerName());
1676 dest = balancer.randomAssignment(hri, destServers);
1677 if (dest == null) {
1678 LOG.debug("Unable to determine a plan to assign " + hri);
1679 return;
1680 }
1681 } else {
1682 ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));
1683 dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));
1684 if (dest == null) {
1685 LOG.debug("Unable to determine a plan to assign " + hri);
1686 return;
1687 }
1688 if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1689 && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1690
1691
1692
1693 LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1694 + " to avoid unnecessary region moving later by load balancer,"
1695 + " because it should not be on master");
1696 return;
1697 }
1698 }
1699
1700 if (dest.equals(regionState.getServerName())) {
1701 LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1702 + " because region already assigned to the same server " + dest + ".");
1703 return;
1704 }
1705
1706
1707 RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1708
1709 try {
1710 checkInitialized();
1711 if (this.cpHost != null) {
1712 if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1713 return;
1714 }
1715 }
1716
1717
1718
1719 serverManager.sendRegionWarmup(rp.getDestination(), hri);
1720
1721 LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1722 this.assignmentManager.balance(rp);
1723 if (this.cpHost != null) {
1724 this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1725 }
1726 } catch (IOException ioe) {
1727 if (ioe instanceof HBaseIOException) {
1728 throw (HBaseIOException)ioe;
1729 }
1730 throw new HBaseIOException(ioe);
1731 }
1732 }
1733
1734 @Override
1735 public long createTable(
1736 final HTableDescriptor hTableDescriptor,
1737 final byte [][] splitKeys,
1738 final long nonceGroup,
1739 final long nonce) throws IOException {
1740 if (isStopped()) {
1741 throw new MasterNotRunningException();
1742 }
1743
1744 TableName tableName = hTableDescriptor.getTableName();
1745 String namespace = tableName.getNamespaceAsString();
1746 ensureNamespaceExists(namespace);
1747
1748 final HRegionInfo[] newRegions =
1749 ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
1750 checkInitialized();
1751 sanityCheckTableDescriptor(hTableDescriptor);
1752
1753 if (isMasterProcedureExecutorEnabled()) {
1754 return MasterProcedureUtil.submitProcedure(
1755 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
1756 @Override
1757 protected void run() throws IOException {
1758 getMaster().getMasterCoprocessorHost().preCreateTable(hTableDescriptor, newRegions);
1759
1760 LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1761
1762
1763
1764 ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
1765 submitProcedure(new CreateTableProcedure(
1766 procedureExecutor.getEnvironment(), hTableDescriptor, newRegions, latch));
1767 latch.await();
1768
1769 getMaster().getMasterCoprocessorHost().postCreateTable(hTableDescriptor, newRegions);
1770 }
1771
1772 @Override
1773 protected String getDescription() {
1774 return "CreateTableProcedure";
1775 }
1776 });
1777 } else {
1778 try {
1779 this.quotaManager.checkNamespaceTableAndRegionQuota(tableName, newRegions.length);
1780 if (cpHost != null) {
1781 cpHost.preCreateTable(hTableDescriptor, newRegions);
1782 }
1783 LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1784 this.service.submit(new CreateTableHandler(this, this.fileSystemManager, hTableDescriptor,
1785 conf, newRegions, this).prepare());
1786 if (cpHost != null) {
1787 cpHost.postCreateTable(hTableDescriptor, newRegions);
1788 }
1789 } catch (IOException e) {
1790 this.quotaManager.removeTableFromNamespaceQuota(tableName);
1791 LOG.error("Exception occurred while creating the table " + tableName.getNameAsString(), e);
1792 throw e;
1793 }
1794 return -1;
1795 }
1796 }
1797
1798
1799
1800
1801
1802
1803 private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1804 final String CONF_KEY = "hbase.table.sanity.checks";
1805 boolean logWarn = false;
1806 if (!conf.getBoolean(CONF_KEY, true)) {
1807 logWarn = true;
1808 }
1809 String tableVal = htd.getConfigurationValue(CONF_KEY);
1810 if (tableVal != null && !Boolean.valueOf(tableVal)) {
1811 logWarn = true;
1812 }
1813
1814
1815 long maxFileSizeLowerLimit = 2 * 1024 * 1024L;
1816 long maxFileSize = htd.getMaxFileSize();
1817 if (maxFileSize < 0) {
1818 maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1819 }
1820 if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1821 String message = "MAX_FILESIZE for table descriptor or "
1822 + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1823 + ") is too small, which might cause over splitting into unmanageable "
1824 + "number of regions.";
1825 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1826 }
1827
1828
1829 long flushSizeLowerLimit = 1024 * 1024L;
1830 long flushSize = htd.getMemStoreFlushSize();
1831 if (flushSize < 0) {
1832 flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1833 }
1834 if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1835 String message = "MEMSTORE_FLUSHSIZE for table descriptor or "
1836 + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1837 + " very frequent flushing.";
1838 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1839 }
1840
1841
1842 try {
1843 checkClassLoading(conf, htd);
1844 } catch (Exception ex) {
1845 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);
1846 }
1847
1848
1849 try {
1850 checkCompression(htd);
1851 } catch (IOException e) {
1852 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1853 }
1854
1855
1856 try {
1857 checkEncryption(conf, htd);
1858 } catch (IOException e) {
1859 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);
1860 }
1861
1862 try{
1863 checkCompactionPolicy(conf, htd);
1864 } catch(IOException e){
1865 warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);
1866 }
1867
1868 if (htd.getColumnFamilies().length == 0) {
1869 String message = "Table should have at least one column family.";
1870 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1871 }
1872
1873 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1874 if (hcd.getTimeToLive() <= 0) {
1875 String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";
1876 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1877 }
1878
1879
1880 if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1881 String message = "Block size for column family " + hcd.getNameAsString()
1882 + " must be between 1K and 16MB.";
1883 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1884 }
1885
1886
1887 if (hcd.getMinVersions() < 0) {
1888 String message = "Min versions for column family " + hcd.getNameAsString()
1889 + " must be positive.";
1890 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1891 }
1892
1893
1894
1895
1896
1897 if (hcd.getMinVersions() > hcd.getMaxVersions()) {
1898 String message = "Min versions for column family " + hcd.getNameAsString()
1899 + " must be less than the Max versions.";
1900 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1901 }
1902
1903
1904 if (hcd.getScope() < 0) {
1905 String message = "Replication scope for column family "
1906 + hcd.getNameAsString() + " must be positive.";
1907 warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);
1908 }
1909
1910
1911 }
1912 }
1913
1914 private void checkCompactionPolicy(Configuration conf, HTableDescriptor htd)
1915 throws IOException {
1916
1917
1918 String className =
1919 htd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1920 if (className == null) {
1921 className =
1922 conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
1923 ExploringCompactionPolicy.class.getName());
1924 }
1925
1926
1927 int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;
1928 String sv = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1929 if (sv != null) {
1930 blockingFileCount = Integer.parseInt(sv);
1931 } else {
1932 blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);
1933 }
1934
1935 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1936 String compactionPolicy =
1937 hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);
1938 if (compactionPolicy == null) {
1939 compactionPolicy = className;
1940 }
1941 if (compactionPolicy.equals(FIFOCompactionPolicy.class.getName()) == false) {
1942 continue;
1943 }
1944
1945 String message = null;
1946
1947
1948 if (hcd.getTimeToLive() == HColumnDescriptor.DEFAULT_TTL) {
1949 message = "Default TTL is not supported for FIFO compaction";
1950 throw new IOException(message);
1951 }
1952
1953
1954 if (hcd.getMinVersions() > 0) {
1955 message = "MIN_VERSION > 0 is not supported for FIFO compaction";
1956 throw new IOException(message);
1957 }
1958
1959
1960 String sbfc = htd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);
1961 if (sbfc != null) {
1962 blockingFileCount = Integer.parseInt(sbfc);
1963 }
1964 if (blockingFileCount < 1000) {
1965 message =
1966 "blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount
1967 + " is below recommended minimum of 1000";
1968 throw new IOException(message);
1969 }
1970 }
1971 }
1972
1973
1974 private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,
1975 String message, Exception cause) throws IOException {
1976 if (!logWarn) {
1977 throw new DoNotRetryIOException(message + " Set " + confKey +
1978 " to false at conf or table descriptor if you want to bypass sanity checks", cause);
1979 }
1980 LOG.warn(message);
1981 }
1982
1983 private void startActiveMasterManager(int infoPort) throws KeeperException {
1984 String backupZNode = ZKUtil.joinZNode(
1985 zooKeeper.backupMasterAddressesZNode, serverName.toString());
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996 LOG.info("Adding backup master ZNode " + backupZNode);
1997 if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1998 serverName, infoPort)) {
1999 LOG.warn("Failed create of " + backupZNode + " by " + serverName);
2000 }
2001
2002 activeMasterManager.setInfoPort(infoPort);
2003
2004 Threads.setDaemonThreadRunning(new Thread(new Runnable() {
2005 @Override
2006 public void run() {
2007 int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
2008 HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
2009
2010 if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
2011 HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
2012 LOG.debug("HMaster started in backup mode. "
2013 + "Stalling until master znode is written.");
2014
2015
2016 while (!activeMasterManager.hasActiveMaster()) {
2017 LOG.debug("Waiting for master address ZNode to be written "
2018 + "(Also watching cluster state node)");
2019 Threads.sleep(timeout);
2020 }
2021 }
2022 MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
2023 status.setDescription("Master startup");
2024 try {
2025 if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
2026 finishActiveMasterInitialization(status);
2027 }
2028 } catch (Throwable t) {
2029 status.setStatus("Failed to become active: " + t.getMessage());
2030 LOG.fatal("Failed to become active master", t);
2031
2032 if (t instanceof NoClassDefFoundError &&
2033 t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
2034
2035 abort("HBase is having a problem with its Hadoop jars. You may need to "
2036 + "recompile HBase against Hadoop version "
2037 + org.apache.hadoop.util.VersionInfo.getVersion()
2038 + " or change your hadoop jars to start properly", t);
2039 } else {
2040 abort("Unhandled exception. Starting shutdown.", t);
2041 }
2042 } finally {
2043 status.cleanup();
2044 }
2045 }
2046 }, getServerName().toShortString() + ".activeMasterManager"));
2047 }
2048
2049 private void checkCompression(final HTableDescriptor htd)
2050 throws IOException {
2051 if (!this.masterCheckCompression) return;
2052 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
2053 checkCompression(hcd);
2054 }
2055 }
2056
2057 private void checkCompression(final HColumnDescriptor hcd)
2058 throws IOException {
2059 if (!this.masterCheckCompression) return;
2060 CompressionTest.testCompression(hcd.getCompression());
2061 CompressionTest.testCompression(hcd.getCompactionCompression());
2062 }
2063
2064 private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
2065 throws IOException {
2066 if (!this.masterCheckEncryption) return;
2067 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
2068 checkEncryption(conf, hcd);
2069 }
2070 }
2071
2072 private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
2073 throws IOException {
2074 if (!this.masterCheckEncryption) return;
2075 EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
2076 }
2077
2078 private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
2079 throws IOException {
2080 RegionSplitPolicy.getSplitPolicyClass(htd, conf);
2081 RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
2082 }
2083
2084 private static boolean isCatalogTable(final TableName tableName) {
2085 return tableName.equals(TableName.META_TABLE_NAME);
2086 }
2087
2088 @Override
2089 public long deleteTable(
2090 final TableName tableName,
2091 final long nonceGroup,
2092 final long nonce) throws IOException {
2093 checkInitialized();
2094
2095 if (isMasterProcedureExecutorEnabled()) {
2096 return MasterProcedureUtil.submitProcedure(
2097 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2098 @Override
2099 protected void run() throws IOException {
2100 getMaster().getMasterCoprocessorHost().preDeleteTable(tableName);
2101
2102 LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2103
2104
2105 ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
2106 submitProcedure(new DeleteTableProcedure(procedureExecutor.getEnvironment(),
2107 tableName, latch));
2108 latch.await();
2109
2110 getMaster().getMasterCoprocessorHost().postDeleteTable(tableName);
2111 }
2112
2113 @Override
2114 protected String getDescription() {
2115 return "DeleteTableProcedure";
2116 }
2117 });
2118 } else {
2119 if (cpHost != null) {
2120 cpHost.preDeleteTable(tableName);
2121 }
2122 LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
2123 this.service.submit(new DeleteTableHandler(tableName, this, this).prepare());
2124 if (cpHost != null) {
2125 cpHost.postDeleteTable(tableName);
2126 }
2127 return -1;
2128 }
2129 }
2130
2131 @Override
2132 public void truncateTable(
2133 final TableName tableName,
2134 final boolean preserveSplits,
2135 final long nonceGroup,
2136 final long nonce) throws IOException {
2137 checkInitialized();
2138
2139 if (isMasterProcedureExecutorEnabled()) {
2140 MasterProcedureUtil.submitProcedure(
2141 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2142 @Override
2143 protected void run() throws IOException {
2144 getMaster().getMasterCoprocessorHost().preTruncateTable(tableName);
2145
2146 LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2147
2148 long procId = submitProcedure(new TruncateTableProcedure(procedureExecutor.getEnvironment(),
2149 tableName, preserveSplits));
2150 ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2151
2152 getMaster().getMasterCoprocessorHost().postTruncateTable(tableName);
2153 }
2154
2155 @Override
2156 protected String getDescription() {
2157 return "TruncateTableProcedure";
2158 }
2159 });
2160 } else {
2161 if (cpHost != null) {
2162 cpHost.preTruncateTable(tableName);
2163 }
2164 LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
2165 TruncateTableHandler handler =
2166 new TruncateTableHandler(tableName, this, this, preserveSplits);
2167 handler.prepare();
2168 handler.process();
2169 if (cpHost != null) {
2170 cpHost.postTruncateTable(tableName);
2171 }
2172 }
2173 }
2174
2175 @Override
2176 public void addColumn(
2177 final TableName tableName,
2178 final HColumnDescriptor columnDescriptor,
2179 final long nonceGroup,
2180 final long nonce)
2181 throws IOException {
2182 checkInitialized();
2183 checkCompression(columnDescriptor);
2184 checkEncryption(conf, columnDescriptor);
2185
2186 if (isMasterProcedureExecutorEnabled()) {
2187 MasterProcedureUtil.submitProcedure(
2188 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2189 @Override
2190 protected void run() throws IOException {
2191 if (getMaster().getMasterCoprocessorHost().preAddColumn(tableName, columnDescriptor)) {
2192 return;
2193 }
2194
2195 LOG.info(getClientIdAuditPrefix() + " add " + columnDescriptor);
2196
2197
2198 long procId = submitProcedure(new AddColumnFamilyProcedure(
2199 procedureExecutor.getEnvironment(), tableName, columnDescriptor));
2200 ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2201
2202 getMaster().getMasterCoprocessorHost().postAddColumn(tableName, columnDescriptor);
2203 }
2204 @Override
2205 protected String getDescription() {
2206 return "AddColumnFamilyProcedure";
2207 }
2208 });
2209 } else {
2210 if (cpHost != null) {
2211 if (cpHost.preAddColumn(tableName, columnDescriptor)) {
2212 return;
2213 }
2214 }
2215 LOG.info(getClientIdAuditPrefix() + " add " + columnDescriptor);
2216 new TableAddFamilyHandler(tableName, columnDescriptor, this, this).prepare().process();
2217 if (cpHost != null) {
2218 cpHost.postAddColumn(tableName, columnDescriptor);
2219 }
2220 }
2221 }
2222
2223 @Override
2224 public void modifyColumn(
2225 final TableName tableName,
2226 final HColumnDescriptor descriptor,
2227 final long nonceGroup,
2228 final long nonce)
2229 throws IOException {
2230 checkInitialized();
2231 checkCompression(descriptor);
2232 checkEncryption(conf, descriptor);
2233
2234 if (isMasterProcedureExecutorEnabled()) {
2235 MasterProcedureUtil.submitProcedure(
2236 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2237 @Override
2238 protected void run() throws IOException {
2239 if (getMaster().getMasterCoprocessorHost().preModifyColumn(tableName, descriptor)) {
2240 return;
2241 }
2242
2243 LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2244
2245
2246
2247 long procId = submitProcedure(new ModifyColumnFamilyProcedure(
2248 procedureExecutor.getEnvironment(), tableName, descriptor));
2249 ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2250
2251 getMaster().getMasterCoprocessorHost().postModifyColumn(tableName, descriptor);
2252 }
2253
2254 @Override
2255 protected String getDescription() {
2256 return "ModifyColumnFamilyProcedure";
2257 }
2258 });
2259 } else {
2260 if (cpHost != null) {
2261 if (cpHost.preModifyColumn(tableName, descriptor)) {
2262 return;
2263 }
2264 }
2265 LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2266 new TableModifyFamilyHandler(tableName, descriptor, this, this).prepare().process();
2267 if (cpHost != null) {
2268 cpHost.postModifyColumn(tableName, descriptor);
2269 }
2270 }
2271 }
2272
2273 @Override
2274 public void deleteColumn(
2275 final TableName tableName,
2276 final byte[] columnName,
2277 final long nonceGroup,
2278 final long nonce)
2279 throws IOException {
2280 checkInitialized();
2281
2282 if (isMasterProcedureExecutorEnabled()) {
2283 MasterProcedureUtil.submitProcedure(
2284 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2285 @Override
2286 protected void run() throws IOException {
2287 if (getMaster().getMasterCoprocessorHost().preDeleteColumn(tableName, columnName)) {
2288 return;
2289 }
2290
2291 LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
2292
2293
2294
2295 long procId = submitProcedure(new DeleteColumnFamilyProcedure(
2296 procedureExecutor.getEnvironment(), tableName, columnName));
2297 ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2298
2299 getMaster().getMasterCoprocessorHost().postDeleteColumn(tableName, columnName);
2300 }
2301
2302 @Override
2303 protected String getDescription() {
2304 return "DeleteColumnFamilyProcedure";
2305 }
2306 });
2307 } else {
2308 if (cpHost != null) {
2309 if (cpHost.preDeleteColumn(tableName, columnName)) {
2310 return;
2311 }
2312 }
2313 LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
2314 new TableDeleteFamilyHandler(tableName, columnName, this, this).prepare().process();
2315 if (cpHost != null) {
2316 cpHost.postDeleteColumn(tableName, columnName);
2317 }
2318 }
2319 }
2320
2321 @Override
2322 public long enableTable(final TableName tableName, final long nonceGroup, final long nonce)
2323 throws IOException {
2324 checkInitialized();
2325 if (cpHost != null) {
2326 cpHost.preEnableTable(tableName);
2327 }
2328
2329
2330
2331
2332 MasterQuotaManager quotaManager = getMasterQuotaManager();
2333 if (quotaManager != null) {
2334 if (quotaManager.isQuotaInitialized()) {
2335 Quotas quotaForTable = QuotaUtil.getTableQuota(getConnection(), tableName);
2336 if (quotaForTable != null && quotaForTable.hasSpace()) {
2337 SpaceViolationPolicy policy = quotaForTable.getSpace().getViolationPolicy();
2338 if (SpaceViolationPolicy.DISABLE == policy) {
2339 throw new AccessDeniedException("Enabling the table '" + tableName
2340 + "' is disallowed due to a violated space quota.");
2341 }
2342 }
2343 } else if (LOG.isTraceEnabled()) {
2344 LOG.trace("Unable to check for space quotas as the manager is not enabled");
2345 }
2346 }
2347
2348 LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2349
2350 if (isMasterProcedureExecutorEnabled()) {
2351 return MasterProcedureUtil.submitProcedure(
2352 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2353 @Override
2354 protected void run() throws IOException {
2355 getMaster().getMasterCoprocessorHost().preEnableTable(tableName);
2356
2357 LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2358
2359
2360
2361
2362
2363
2364 final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2365 submitProcedure(new EnableTableProcedure(
2366 procedureExecutor.getEnvironment(), tableName, false, prepareLatch));
2367
2368
2369
2370
2371 prepareLatch.await();
2372
2373 getMaster().getMasterCoprocessorHost().postEnableTable(tableName);
2374 }
2375
2376 @Override
2377 protected String getDescription() {
2378 return "EnableTableProcedure";
2379 }
2380 });
2381 } else {
2382 if (cpHost != null) {
2383 cpHost.preEnableTable(tableName);
2384 }
2385 LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
2386 this.service.submit(new EnableTableHandler(this, tableName,
2387 assignmentManager, tableLockManager, false).prepare());
2388 if (cpHost != null) {
2389 cpHost.postEnableTable(tableName);
2390 }
2391 return -1;
2392 }
2393 }
2394
2395 @Override
2396 public long disableTable(final TableName tableName, final long nonceGroup, final long nonce)
2397 throws IOException {
2398 checkInitialized();
2399
2400 if (isMasterProcedureExecutorEnabled()) {
2401 return MasterProcedureUtil.submitProcedure(
2402 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2403 @Override
2404 protected void run() throws IOException {
2405 getMaster().getMasterCoprocessorHost().preDisableTable(tableName);
2406
2407 LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2408
2409
2410
2411
2412
2413
2414 final ProcedurePrepareLatch prepareLatch = ProcedurePrepareLatch.createLatch();
2415 submitProcedure(new DisableTableProcedure(procedureExecutor.getEnvironment(),
2416 tableName, false, prepareLatch));
2417 prepareLatch.await();
2418
2419 getMaster().getMasterCoprocessorHost().postDisableTable(tableName);
2420 }
2421
2422 @Override
2423 protected String getDescription() {
2424 return "DisableTableProcedure";
2425 }
2426 });
2427 } else {
2428 if (cpHost != null) {
2429 cpHost.preDisableTable(tableName);
2430 }
2431 LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
2432 this.service.submit(new DisableTableHandler(this, tableName,
2433 assignmentManager, tableLockManager, false).prepare());
2434 if (cpHost != null) {
2435 cpHost.postDisableTable(tableName);
2436 }
2437 return -1;
2438 }
2439 }
2440
2441
2442
2443
2444
2445
2446
2447 @VisibleForTesting
2448 Pair<HRegionInfo, ServerName> getTableRegionForRow(
2449 final TableName tableName, final byte [] rowKey)
2450 throws IOException {
2451 final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2452 new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2453
2454 MetaScannerVisitor visitor =
2455 new MetaScannerVisitorBase() {
2456 @Override
2457 public boolean processRow(Result data) throws IOException {
2458 if (data == null || data.size() <= 0) {
2459 return true;
2460 }
2461 Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
2462 if (pair == null) {
2463 return false;
2464 }
2465 if (!pair.getFirst().getTable().equals(tableName)) {
2466 return false;
2467 }
2468 result.set(pair);
2469 return true;
2470 }
2471 };
2472
2473 MetaScanner.metaScan(clusterConnection, visitor, tableName, rowKey, 1);
2474 return result.get();
2475 }
2476
2477 @Override
2478 public void modifyTable(
2479 final TableName tableName,
2480 final HTableDescriptor descriptor,
2481 final long nonceGroup,
2482 final long nonce)
2483 throws IOException {
2484 checkInitialized();
2485 sanityCheckTableDescriptor(descriptor);
2486
2487 if (isMasterProcedureExecutorEnabled()) {
2488 MasterProcedureUtil.submitProcedure(
2489 new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
2490 @Override
2491 protected void run() throws IOException {
2492 getMaster().getMasterCoprocessorHost().preModifyTable(tableName, descriptor);
2493 LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
2494
2495
2496 long procId = submitProcedure(new ModifyTableProcedure(
2497 procedureExecutor.getEnvironment(), descriptor));
2498 ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
2499
2500 getMaster().getMasterCoprocessorHost().postModifyTable(tableName, descriptor);
2501 }
2502
2503 @Override
2504 protected String getDescription() {
2505 return "ModifyTableProcedure";
2506 }
2507 });
2508 } else {
2509 if (cpHost != null) {
2510 cpHost.preModifyTable(tableName, descriptor);
2511 }
2512 LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
2513 new ModifyTableHandler(tableName, descriptor, this, this).prepare().process();
2514 if (cpHost != null) {
2515 cpHost.postModifyTable(tableName, descriptor);
2516 }
2517 }
2518 }
2519
2520 @Override
2521 public void checkTableModifiable(final TableName tableName)
2522 throws IOException, TableNotFoundException, TableNotDisabledException {
2523 if (isCatalogTable(tableName)) {
2524 throw new IOException("Can't modify catalog tables");
2525 }
2526 if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
2527 throw new TableNotFoundException(tableName);
2528 }
2529 if (!getAssignmentManager().getTableStateManager().
2530 isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)) {
2531 throw new TableNotDisabledException(tableName);
2532 }
2533 }
2534
2535
2536
2537
2538 public ClusterStatus getClusterStatus() throws InterruptedIOException {
2539
2540 List<String> backupMasterStrings;
2541 try {
2542 backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
2543 this.zooKeeper.backupMasterAddressesZNode);
2544 } catch (KeeperException e) {
2545 LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
2546 backupMasterStrings = null;
2547 }
2548
2549 List<ServerName> backupMasters = null;
2550 if (backupMasterStrings != null && !backupMasterStrings.isEmpty()) {
2551 backupMasters = new ArrayList<ServerName>(backupMasterStrings.size());
2552 for (String s: backupMasterStrings) {
2553 try {
2554 byte [] bytes;
2555 try {
2556 bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
2557 this.zooKeeper.backupMasterAddressesZNode, s));
2558 } catch (InterruptedException e) {
2559 throw new InterruptedIOException();
2560 }
2561 if (bytes != null) {
2562 ServerName sn;
2563 try {
2564 sn = ServerName.parseFrom(bytes);
2565 } catch (DeserializationException e) {
2566 LOG.warn("Failed parse, skipping registering backup server", e);
2567 continue;
2568 }
2569 backupMasters.add(sn);
2570 }
2571 } catch (KeeperException e) {
2572 LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
2573 "backup servers"), e);
2574 }
2575 }
2576 Collections.sort(backupMasters, new Comparator<ServerName>() {
2577 @Override
2578 public int compare(ServerName s1, ServerName s2) {
2579 return s1.getServerName().compareTo(s2.getServerName());
2580 }});
2581 }
2582
2583 String clusterId = fileSystemManager != null ?
2584 fileSystemManager.getClusterId().toString() : null;
2585 Map<String, RegionState> regionsInTransition = assignmentManager != null ?
2586 assignmentManager.getRegionStates().getRegionsInTransition() : null;
2587 String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
2588 boolean balancerOn = loadBalancerTracker != null ?
2589 loadBalancerTracker.isBalancerOn() : false;
2590 Map<ServerName, ServerLoad> onlineServers = null;
2591 Set<ServerName> deadServers = null;
2592 if (serverManager != null) {
2593 deadServers = serverManager.getDeadServers().copyServerNames();
2594 onlineServers = serverManager.getOnlineServers();
2595 }
2596 return new ClusterStatus(VersionInfo.getVersion(), clusterId,
2597 onlineServers, deadServers, serverName, backupMasters,
2598 regionsInTransition, coprocessors, balancerOn);
2599 }
2600
2601
2602
2603
2604
2605
2606
2607
2608 public static String getLoadedCoprocessors() {
2609 return CoprocessorHost.getLoadedCoprocessors().toString();
2610 }
2611
2612
2613
2614
2615 public long getMasterStartTime() {
2616 return startcode;
2617 }
2618
2619
2620
2621
2622 public long getMasterActiveTime() {
2623 return masterActiveTime;
2624 }
2625
2626 public int getNumWALFiles() {
2627 return procedureStore != null ? procedureStore.getActiveLogs().size() : 0;
2628 }
2629
2630 public WALProcedureStore getWalProcedureStore() {
2631 return procedureStore;
2632 }
2633
2634 public int getRegionServerInfoPort(final ServerName sn) {
2635 RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
2636 if (info == null || info.getInfoPort() == 0) {
2637 return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
2638 HConstants.DEFAULT_REGIONSERVER_INFOPORT);
2639 }
2640 return info.getInfoPort();
2641 }
2642
2643
2644
2645
2646 public String[] getMasterCoprocessors() {
2647 Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
2648 return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
2649 }
2650
2651 @Override
2652 public void abort(final String msg, final Throwable t) {
2653 if (isAborted() || isStopped()) {
2654 return;
2655 }
2656 if (cpHost != null) {
2657
2658 LOG.fatal("Master server abort: loaded coprocessors are: " +
2659 getLoadedCoprocessors());
2660 }
2661 if (t != null) LOG.fatal(msg, t);
2662 stop(msg);
2663 }
2664
2665 @Override
2666 public ZooKeeperWatcher getZooKeeper() {
2667 return zooKeeper;
2668 }
2669
2670 @Override
2671 public MasterCoprocessorHost getMasterCoprocessorHost() {
2672 return cpHost;
2673 }
2674
2675 @Override
2676 public MasterQuotaManager getMasterQuotaManager() {
2677 return quotaManager;
2678 }
2679
2680 @Override
2681 public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
2682 return procedureExecutor;
2683 }
2684
2685 @Override
2686 public ServerName getServerName() {
2687 return this.serverName;
2688 }
2689
2690 @Override
2691 public AssignmentManager getAssignmentManager() {
2692 return this.assignmentManager;
2693 }
2694
2695 public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
2696 return rsFatals;
2697 }
2698
2699 public void shutdown() throws IOException {
2700 if (cpHost != null) {
2701 cpHost.preShutdown();
2702 }
2703
2704 if (this.serverManager != null) {
2705 this.serverManager.shutdownCluster();
2706 }
2707 if (this.clusterStatusTracker != null){
2708 try {
2709 this.clusterStatusTracker.setClusterDown();
2710 } catch (KeeperException e) {
2711 LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
2712 }
2713 }
2714 }
2715
2716 public void stopMaster() throws IOException {
2717 if (cpHost != null) {
2718 cpHost.preStopMaster();
2719 }
2720 stop("Stopped by " + Thread.currentThread().getName());
2721 }
2722
2723 void checkServiceStarted() throws ServerNotRunningYetException {
2724 if (!serviceStarted) {
2725 throw new ServerNotRunningYetException("Server is not running yet");
2726 }
2727 }
2728
2729 void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
2730 checkServiceStarted();
2731 if (!this.initialized) {
2732 throw new PleaseHoldException("Master is initializing");
2733 }
2734 }
2735
2736 void checkNamespaceManagerReady() throws IOException {
2737 checkInitialized();
2738 if (tableNamespaceManager == null) {
2739 throw new IOException("Table Namespace Manager not ready yet, try again later");
2740 } else if (!tableNamespaceManager.isTableAvailableAndInitialized()) {
2741 try {
2742
2743 long startTime = EnvironmentEdgeManager.currentTime();
2744 int timeout = conf.getInt("hbase.master.namespace.wait.for.ready", 30000);
2745 while (!tableNamespaceManager.isTableNamespaceManagerStarted() &&
2746 EnvironmentEdgeManager.currentTime() - startTime < timeout) {
2747 Thread.sleep(100);
2748 }
2749 } catch (InterruptedException e) {
2750 throw (InterruptedIOException) new InterruptedIOException().initCause(e);
2751 }
2752 if (!tableNamespaceManager.isTableNamespaceManagerStarted()) {
2753 throw new IOException("Table Namespace Manager not fully initialized, try again later");
2754 }
2755 }
2756 }
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766 public boolean isActiveMaster() {
2767 return isActiveMaster;
2768 }
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779 @Override
2780 public boolean isInitialized() {
2781 return initialized;
2782 }
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792 @Override
2793 public boolean isNamespaceManagerInitialized() {
2794 return initialized & tableNamespaceManager != null
2795 && tableNamespaceManager.isTableNamespaceManagerStarted();
2796 }
2797
2798
2799
2800
2801
2802
2803 @Override
2804 public boolean isInMaintenanceMode() {
2805 return maintenanceModeTracker.isInMaintenanceMode();
2806 }
2807
2808
2809
2810
2811
2812
2813 @Override
2814 public boolean isServerShutdownHandlerEnabled() {
2815 return this.serverShutdownHandlerEnabled;
2816 }
2817
2818
2819
2820
2821
2822 public boolean isInitializationStartsMetaRegionAssignment() {
2823 return this.initializationBeforeMetaAssignment;
2824 }
2825
2826 public void assignRegion(HRegionInfo hri) {
2827 assignmentManager.assign(hri, true);
2828 }
2829
2830
2831
2832
2833
2834
2835
2836 public double getAverageLoad() {
2837 if (this.assignmentManager == null) {
2838 return 0;
2839 }
2840
2841 RegionStates regionStates = this.assignmentManager.getRegionStates();
2842 if (regionStates == null) {
2843 return 0;
2844 }
2845 return regionStates.getAverageLoad();
2846 }
2847
2848 @Override
2849 public boolean registerService(Service instance) {
2850
2851
2852
2853 Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
2854 String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
2855 if (coprocessorServiceHandlers.containsKey(serviceName)) {
2856 LOG.error("Coprocessor service "+serviceName+
2857 " already registered, rejecting request from "+instance
2858 );
2859 return false;
2860 }
2861
2862 coprocessorServiceHandlers.put(serviceName, instance);
2863 if (LOG.isDebugEnabled()) {
2864 LOG.debug("Registered master coprocessor service: service="+serviceName);
2865 }
2866 return true;
2867 }
2868
2869
2870
2871
2872
2873
2874
2875 public static HMaster constructMaster(Class<? extends HMaster> masterClass,
2876 final Configuration conf, final CoordinatedStateManager cp) {
2877 try {
2878 Constructor<? extends HMaster> c =
2879 masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
2880 return c.newInstance(conf, cp);
2881 } catch (InvocationTargetException ite) {
2882 Throwable target = ite.getTargetException() != null?
2883 ite.getTargetException(): ite;
2884 if (target.getCause() != null) target = target.getCause();
2885 throw new RuntimeException("Failed construction of Master: " +
2886 masterClass.toString(), target);
2887 } catch (Exception e) {
2888 throw new RuntimeException("Failed construction of Master: " +
2889 masterClass.toString() + ((e.getCause() != null)?
2890 e.getCause().getMessage(): ""), e);
2891 }
2892 }
2893
2894
2895
2896
2897 public static void main(String [] args) {
2898 VersionInfo.logVersion();
2899 new HMasterCommandLine(HMaster.class).doMain(args);
2900 }
2901
2902 public HFileCleaner getHFileCleaner() {
2903 return this.hfileCleaner;
2904 }
2905
2906
2907
2908
2909 public SnapshotManager getSnapshotManager() {
2910 return this.snapshotManager;
2911 }
2912
2913
2914
2915
2916 public MasterProcedureManagerHost getMasterProcedureManagerHost() {
2917 return mpmHost;
2918 }
2919
2920 @Override
2921 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2922 TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2923 checkNamespaceManagerReady();
2924 if (cpHost != null) {
2925 if (cpHost.preCreateNamespace(descriptor)) {
2926 return;
2927 }
2928 }
2929 LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2930 tableNamespaceManager.create(descriptor);
2931 if (cpHost != null) {
2932 cpHost.postCreateNamespace(descriptor);
2933 }
2934 }
2935
2936 @Override
2937 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2938 TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2939 checkNamespaceManagerReady();
2940 if (cpHost != null) {
2941 if (cpHost.preModifyNamespace(descriptor)) {
2942 return;
2943 }
2944 }
2945 LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2946 tableNamespaceManager.update(descriptor);
2947 if (cpHost != null) {
2948 cpHost.postModifyNamespace(descriptor);
2949 }
2950 }
2951
2952 @Override
2953 public void deleteNamespace(String name) throws IOException {
2954 checkNamespaceManagerReady();
2955 if (cpHost != null) {
2956 if (cpHost.preDeleteNamespace(name)) {
2957 return;
2958 }
2959 }
2960 LOG.info(getClientIdAuditPrefix() + " delete " + name);
2961 tableNamespaceManager.remove(name);
2962 if (cpHost != null) {
2963 cpHost.postDeleteNamespace(name);
2964 }
2965 }
2966
2967
2968
2969
2970
2971
2972
2973
2974 private void ensureNamespaceExists(final String name)
2975 throws IOException, NamespaceNotFoundException {
2976 checkNamespaceManagerReady();
2977 NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2978 if (nsd == null) {
2979 throw new NamespaceNotFoundException(name);
2980 }
2981 }
2982
2983 @Override
2984 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2985 checkNamespaceManagerReady();
2986
2987 if (cpHost != null) {
2988 cpHost.preGetNamespaceDescriptor(name);
2989 }
2990
2991 NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2992 if (nsd == null) {
2993 throw new NamespaceNotFoundException(name);
2994 }
2995
2996 if (cpHost != null) {
2997 cpHost.postGetNamespaceDescriptor(nsd);
2998 }
2999
3000 return nsd;
3001 }
3002
3003 @Override
3004 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
3005 checkNamespaceManagerReady();
3006
3007 final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
3008 boolean bypass = false;
3009 if (cpHost != null) {
3010 bypass = cpHost.preListNamespaceDescriptors(descriptors);
3011 }
3012
3013 if (!bypass) {
3014 descriptors.addAll(tableNamespaceManager.list());
3015
3016 if (cpHost != null) {
3017 cpHost.postListNamespaceDescriptors(descriptors);
3018 }
3019 }
3020 return descriptors;
3021 }
3022
3023 @Override
3024 public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
3025 throws IOException {
3026 if (cpHost != null) {
3027 cpHost.preAbortProcedure(this.procedureExecutor, procId);
3028 }
3029
3030 final boolean result = this.procedureExecutor.abort(procId, mayInterruptIfRunning);
3031
3032 if (cpHost != null) {
3033 cpHost.postAbortProcedure();
3034 }
3035
3036 return result;
3037 }
3038
3039 @Override
3040 public List<ProcedureInfo> listProcedures() throws IOException {
3041 if (cpHost != null) {
3042 cpHost.preListProcedures();
3043 }
3044
3045 final List<ProcedureInfo> procInfoList = this.procedureExecutor.listProcedures();
3046
3047 if (cpHost != null) {
3048 cpHost.postListProcedures(procInfoList);
3049 }
3050
3051 return procInfoList;
3052 }
3053
3054 @Override
3055 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
3056 ensureNamespaceExists(name);
3057 return listTableDescriptors(name, null, null, true);
3058 }
3059
3060 @Override
3061 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
3062 ensureNamespaceExists(name);
3063 return listTableNames(name, null, true);
3064 }
3065
3066 @Override
3067 public Pair<Long, String> backupTables(final BackupType type,
3068 List<TableName> tableList, final String targetRootDir, final int workers,
3069 final long bandwidth) throws IOException {
3070 long procId;
3071 String backupId = BackupRestoreConstants.BACKUPID_PREFIX +
3072 EnvironmentEdgeManager.currentTime();
3073 if (type == BackupType.INCREMENTAL) {
3074 Set<TableName> incrTableSet = null;
3075 try (BackupSystemTable table = new BackupSystemTable(getConnection())) {
3076 incrTableSet = table.getIncrementalBackupTableSet(targetRootDir);
3077 }
3078
3079 if (incrTableSet.isEmpty()) {
3080 LOG.warn("Incremental backup table set contains no table.\n"
3081 + "Use 'backup create full' or 'backup stop' to \n "
3082 + "change the tables covered by incremental backup.");
3083 throw new DoNotRetryIOException("No table covered by incremental backup.");
3084 }
3085
3086 LOG.info("Incremental backup for the following table set: " + incrTableSet);
3087 tableList = Lists.newArrayList(incrTableSet);
3088 }
3089 if (tableList != null && !tableList.isEmpty()) {
3090 for (TableName table : tableList) {
3091 String targetTableBackupDir =
3092 HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
3093 Path targetTableBackupDirPath = new Path(targetTableBackupDir);
3094 FileSystem outputFs = FileSystem.get(targetTableBackupDirPath.toUri(), conf);
3095 if (outputFs.exists(targetTableBackupDirPath)) {
3096 throw new DoNotRetryIOException("Target backup directory " + targetTableBackupDir
3097 + " exists already.");
3098 }
3099 }
3100 ArrayList<TableName> nonExistingTableList = null;
3101 for (TableName tableName : tableList) {
3102 if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
3103 if (nonExistingTableList == null) {
3104 nonExistingTableList = new ArrayList<>();
3105 }
3106 nonExistingTableList.add(tableName);
3107 }
3108 }
3109 if (nonExistingTableList != null) {
3110 if (type == BackupType.INCREMENTAL ) {
3111 LOG.warn("Incremental backup table set contains non-exising table: "
3112 + nonExistingTableList);
3113
3114 tableList = excludeNonExistingTables(tableList, nonExistingTableList);
3115 } else {
3116
3117 throw new DoNotRetryIOException("Non-existing tables found in the table list: "
3118 + nonExistingTableList);
3119 }
3120 }
3121 }
3122 if (type == BackupType.FULL) {
3123 procId = this.procedureExecutor.submitProcedure(
3124 new FullTableBackupProcedure(procedureExecutor.getEnvironment(), backupId,
3125 tableList, targetRootDir, workers, bandwidth));
3126 } else {
3127 procId = this.procedureExecutor.submitProcedure(
3128 new IncrementalTableBackupProcedure(procedureExecutor.getEnvironment(), backupId,
3129 tableList, targetRootDir, workers, bandwidth));
3130 }
3131 return new Pair<>(procId, backupId);
3132 }
3133
3134 private List<TableName> excludeNonExistingTables(List<TableName> tableList,
3135 List<TableName> nonExistingTableList) {
3136
3137 for(TableName table: nonExistingTableList) {
3138 tableList.remove(table);
3139 }
3140 return tableList;
3141 }
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152 public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
3153 final List<TableName> tableNameList, final boolean includeSysTables)
3154 throws IOException {
3155 final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
3156
3157 boolean bypass = false;
3158 if (cpHost != null) {
3159 bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors);
3160
3161 bypass |= cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
3162 }
3163
3164 if (!bypass) {
3165 if (tableNameList == null || tableNameList.size() == 0) {
3166
3167 Collection<HTableDescriptor> htds;
3168 if (namespace != null && namespace.length() > 0) {
3169 htds = tableDescriptors.getByNamespace(namespace).values();
3170 } else {
3171 htds = tableDescriptors.getAll().values();
3172 }
3173
3174 for (HTableDescriptor desc: htds) {
3175 if (includeSysTables || !desc.getTableName().isSystemTable()) {
3176 descriptors.add(desc);
3177 }
3178 }
3179 } else {
3180 for (TableName s: tableNameList) {
3181 HTableDescriptor desc = tableDescriptors.get(s);
3182 if (desc != null) {
3183 descriptors.add(desc);
3184 }
3185 }
3186 if (tableNameList.size() == 1) {
3187 LOG.info(getClientIdAuditPrefix() + " List Table Descriptor for the " + tableNameList.get(0)
3188 + " table " + ((descriptors.size() == 1) ? "succeeds" : "fails"));
3189 }
3190 }
3191
3192
3193 if (regex != null) {
3194 filterTablesByRegex(descriptors, Pattern.compile(regex));
3195 }
3196
3197 if (cpHost != null) {
3198 cpHost.postGetTableDescriptors(descriptors);
3199
3200 cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
3201 }
3202 }
3203 return descriptors;
3204 }
3205
3206
3207
3208
3209
3210
3211
3212
3213 public List<TableName> listTableNames(final String namespace, final String regex,
3214 final boolean includeSysTables) throws IOException {
3215 final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
3216
3217 boolean bypass = false;
3218 if (cpHost != null) {
3219 bypass = cpHost.preGetTableNames(descriptors, regex);
3220 }
3221
3222 if (!bypass) {
3223
3224 Collection<HTableDescriptor> htds;
3225 if (namespace != null && namespace.length() > 0) {
3226 htds = tableDescriptors.getByNamespace(namespace).values();
3227 } else {
3228 htds = tableDescriptors.getAll().values();
3229 }
3230
3231 for (HTableDescriptor htd: htds) {
3232 if (includeSysTables || !htd.getTableName().isSystemTable()) {
3233 descriptors.add(htd);
3234 }
3235 }
3236
3237
3238 if (regex != null) {
3239 filterTablesByRegex(descriptors, Pattern.compile(regex));
3240 }
3241
3242 if (cpHost != null) {
3243 cpHost.postGetTableNames(descriptors, regex);
3244 }
3245 }
3246
3247 List<TableName> result = new ArrayList<TableName>(descriptors.size());
3248 for (HTableDescriptor htd: descriptors) {
3249 result.add(htd.getTableName());
3250 }
3251 return result;
3252 }
3253
3254
3255
3256
3257
3258
3259
3260 private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
3261 final Pattern pattern) {
3262 final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
3263 Iterator<HTableDescriptor> itr = descriptors.iterator();
3264 while (itr.hasNext()) {
3265 HTableDescriptor htd = itr.next();
3266 String tableName = htd.getTableName().getNameAsString();
3267 boolean matched = pattern.matcher(tableName).matches();
3268 if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
3269 matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
3270 }
3271 if (!matched) {
3272 itr.remove();
3273 }
3274 }
3275 }
3276
3277 @Override
3278 public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
3279 return getClusterStatus().getLastMajorCompactionTsForTable(table);
3280 }
3281
3282 @Override
3283 public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
3284 return getClusterStatus().getLastMajorCompactionTsForRegion(regionName);
3285 }
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296 public CompactionState getMobCompactionState(TableName tableName) {
3297 AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3298 if (compactionsCount != null && compactionsCount.get() != 0) {
3299 return CompactionState.MAJOR_AND_MINOR;
3300 }
3301 return CompactionState.NONE;
3302 }
3303
3304 public void reportMobCompactionStart(TableName tableName) throws IOException {
3305 IdLock.Entry lockEntry = null;
3306 try {
3307 lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3308 AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3309 if (compactionsCount == null) {
3310 compactionsCount = new AtomicInteger(0);
3311 mobCompactionStates.put(tableName, compactionsCount);
3312 }
3313 compactionsCount.incrementAndGet();
3314 } finally {
3315 if (lockEntry != null) {
3316 mobCompactionLock.releaseLockEntry(lockEntry);
3317 }
3318 }
3319 }
3320
3321 public void reportMobCompactionEnd(TableName tableName) throws IOException {
3322 IdLock.Entry lockEntry = null;
3323 try {
3324 lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
3325 AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
3326 if (compactionsCount != null) {
3327 int count = compactionsCount.decrementAndGet();
3328
3329 if (count == 0) {
3330 mobCompactionStates.remove(tableName);
3331 }
3332 }
3333 } finally {
3334 if (lockEntry != null) {
3335 mobCompactionLock.releaseLockEntry(lockEntry);
3336 }
3337 }
3338 }
3339
3340
3341
3342
3343
3344
3345
3346 public void requestMobCompaction(TableName tableName,
3347 List<HColumnDescriptor> columns, boolean allFiles) throws IOException {
3348 mobCompactThread.requestMobCompaction(conf, fs, tableName, columns,
3349 tableLockManager, allFiles);
3350 }
3351
3352
3353
3354
3355
3356
3357
3358 public boolean isBalancerOn() {
3359 if (null == loadBalancerTracker || isInMaintenanceMode()) {
3360 return false;
3361 }
3362 return loadBalancerTracker.isBalancerOn();
3363 }
3364
3365
3366
3367
3368
3369 public boolean isNormalizerOn() {
3370 return (null == regionNormalizerTracker || isInMaintenanceMode()) ?
3371 false: regionNormalizerTracker.isNormalizerOn();
3372 }
3373
3374
3375
3376
3377
3378
3379
3380 public boolean isSplitOrMergeEnabled(Admin.MasterSwitchType switchType) {
3381 if (null == splitOrMergeTracker || isInMaintenanceMode()) {
3382 return false;
3383 }
3384 return splitOrMergeTracker.isSplitOrMergeEnabled(switchType);
3385 }
3386
3387
3388
3389
3390
3391
3392 public String getLoadBalancerClassName() {
3393 return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
3394 .getDefaultLoadBalancerClass().getName());
3395 }
3396
3397
3398
3399
3400 public RegionNormalizerTracker getRegionNormalizerTracker() {
3401 return regionNormalizerTracker;
3402 }
3403
3404 public SplitOrMergeTracker getSplitOrMergeTracker() {
3405 return splitOrMergeTracker;
3406 }
3407
3408 @Override
3409 public LoadBalancer getLoadBalancer() {
3410 return balancer;
3411 }
3412
3413 @Override
3414 public TableStateManager getTableStateManager() {
3415 return assignmentManager.getTableStateManager();
3416 }
3417
3418 public QuotaObserverChore getQuotaObserverChore() {
3419 return this.quotaObserverChore;
3420 }
3421
3422 public SpaceQuotaSnapshotNotifier getSpaceQuotaSnapshotNotifier() {
3423 return this.spaceQuotaSnapshotNotifier;
3424 }
3425 }