1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Random;
29 import java.util.SortedMap;
30 import java.util.SortedSet;
31 import java.util.TreeSet;
32 import java.util.UUID;
33 import java.util.concurrent.ConcurrentHashMap;
34 import java.util.concurrent.CopyOnWriteArrayList;
35 import java.util.concurrent.LinkedBlockingQueue;
36 import java.util.concurrent.RejectedExecutionException;
37 import java.util.concurrent.ThreadPoolExecutor;
38 import java.util.concurrent.TimeUnit;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FileSystem;
44 import org.apache.hadoop.fs.Path;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.Server;
47 import org.apache.hadoop.hbase.TableDescriptors;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.classification.InterfaceAudience;
50 import org.apache.hadoop.hbase.regionserver.HRegionServer;
51 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
52 import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
53 import org.apache.hadoop.hbase.replication.ReplicationException;
54 import org.apache.hadoop.hbase.replication.ReplicationListener;
55 import org.apache.hadoop.hbase.replication.ReplicationPeer;
56 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
57 import org.apache.hadoop.hbase.replication.ReplicationPeers;
58 import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
59 import org.apache.hadoop.hbase.replication.ReplicationQueues;
60 import org.apache.hadoop.hbase.replication.ReplicationTracker;
61 import org.apache.hadoop.hbase.util.Pair;
62
63 import com.google.common.util.concurrent.ThreadFactoryBuilder;
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 @InterfaceAudience.Private
80 public class ReplicationSourceManager implements ReplicationListener {
81 private static final Log LOG =
82 LogFactory.getLog(ReplicationSourceManager.class);
83
84 private final List<ReplicationSourceInterface> sources;
85
86 private final List<ReplicationSourceInterface> oldsources;
87 private final ReplicationQueues replicationQueues;
88 private final ReplicationTracker replicationTracker;
89 private final ReplicationPeers replicationPeers;
90
91 private final UUID clusterId;
92
93 private final Server server;
94
95 private final Map<String, SortedSet<String>> walsById;
96
97 private final Map<String, SortedSet<String>> walsByIdRecoveredQueues;
98 private final Configuration conf;
99 private final FileSystem fs;
100
101 private Path latestPath;
102
103 private final Path logDir;
104
105 private final Path oldLogDir;
106
107 private final long sleepBeforeFailover;
108
109 private final ThreadPoolExecutor executor;
110
111 private final Random rand;
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126 public ReplicationSourceManager(final ReplicationQueues replicationQueues,
127 final ReplicationPeers replicationPeers, final ReplicationTracker replicationTracker,
128 final Configuration conf, final Server server, final FileSystem fs, final Path logDir,
129 final Path oldLogDir, final UUID clusterId) {
130
131
132 this.sources = new CopyOnWriteArrayList<ReplicationSourceInterface>();
133 this.replicationQueues = replicationQueues;
134 this.replicationPeers = replicationPeers;
135 this.replicationTracker = replicationTracker;
136 this.server = server;
137 this.walsById = new HashMap<String, SortedSet<String>>();
138 this.walsByIdRecoveredQueues = new ConcurrentHashMap<String, SortedSet<String>>();
139 this.oldsources = new CopyOnWriteArrayList<ReplicationSourceInterface>();
140 this.conf = conf;
141 this.fs = fs;
142 this.logDir = logDir;
143 this.oldLogDir = oldLogDir;
144 this.sleepBeforeFailover =
145 conf.getLong("replication.sleep.before.failover", 30000);
146 this.clusterId = clusterId;
147 this.replicationTracker.registerListener(this);
148 this.replicationPeers.getAllPeerIds();
149
150
151 int nbWorkers = conf.getInt("replication.executor.workers", 1);
152
153
154 this.executor = new ThreadPoolExecutor(nbWorkers, nbWorkers,
155 100, TimeUnit.MILLISECONDS,
156 new LinkedBlockingQueue<Runnable>());
157 ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
158 tfb.setNameFormat("ReplicationExecutor-%d");
159 tfb.setDaemon(true);
160 this.executor.setThreadFactory(tfb.build());
161 this.rand = new Random();
162 }
163
164
165
166
167
168
169
170
171
172
173
174
175 public void logPositionAndCleanOldLogs(Path log, String id, long position,
176 boolean queueRecovered, boolean holdLogInZK) {
177 String fileName = log.getName();
178 this.replicationQueues.setLogPosition(id, fileName, position);
179 if (holdLogInZK) {
180 return;
181 }
182 cleanOldLogs(fileName, id, queueRecovered);
183 }
184
185
186
187
188
189
190
191
192 public void cleanOldLogs(String key, String id, boolean queueRecovered) {
193 if (queueRecovered) {
194 SortedSet<String> wals = walsByIdRecoveredQueues.get(id);
195 if (wals != null && !wals.first().equals(key)) {
196 cleanOldLogs(wals, key, id);
197 }
198 } else {
199 synchronized (this.walsById) {
200 SortedSet<String> wals = walsById.get(id);
201 if (!wals.first().equals(key)) {
202 cleanOldLogs(wals, key, id);
203 }
204 }
205 }
206 }
207
208 private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
209 SortedSet<String> walSet = wals.headSet(key);
210 LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
211 for (String wal : walSet) {
212 this.replicationQueues.removeLog(id, wal);
213 }
214 walSet.clear();
215 }
216
217
218
219
220
221 protected void init() throws IOException, ReplicationException {
222 boolean replicationForBulkLoadDataEnabled =
223 conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
224 HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
225 for (String id : this.replicationPeers.getPeerIds()) {
226 addSource(id);
227 if (replicationForBulkLoadDataEnabled) {
228
229
230 this.replicationQueues.addPeerToHFileRefs(id);
231 }
232 }
233 List<String> currentReplicators = this.replicationQueues.getListOfReplicators();
234 if (currentReplicators == null || currentReplicators.size() == 0) {
235 return;
236 }
237 List<String> otherRegionServers = replicationTracker.getListOfRegionServers();
238 LOG.info("Current list of replicators: " + currentReplicators + " other RSs: "
239 + otherRegionServers);
240
241
242 for (String rs : currentReplicators) {
243 if (!otherRegionServers.contains(rs)) {
244 transferQueues(rs);
245 }
246 }
247 }
248
249
250
251
252
253
254
255 protected ReplicationSourceInterface addSource(String id) throws IOException,
256 ReplicationException {
257 ReplicationPeerConfig peerConfig
258 = replicationPeers.getReplicationPeerConfig(id);
259 ReplicationPeer peer = replicationPeers.getPeer(id);
260 ReplicationSourceInterface src =
261 getReplicationSource(this.conf, this.fs, this, this.replicationQueues,
262 this.replicationPeers, server, id, this.clusterId, peerConfig, peer);
263 synchronized (this.walsById) {
264 this.sources.add(src);
265 this.walsById.put(id, new TreeSet<String>());
266
267 if (this.latestPath != null) {
268 String name = this.latestPath.getName();
269 this.walsById.get(id).add(name);
270 try {
271 this.replicationQueues.addLog(src.getPeerClusterZnode(), name);
272 } catch (ReplicationException e) {
273 String message =
274 "Cannot add log to queue when creating a new source, queueId="
275 + src.getPeerClusterZnode() + ", filename=" + name;
276 server.stop(message);
277 throw e;
278 }
279 src.enqueueLog(this.latestPath);
280 }
281 }
282 src.startup();
283 return src;
284 }
285
286
287
288
289
290 public void deleteSource(String peerId, boolean closeConnection) {
291 this.replicationQueues.removeQueue(peerId);
292 if (closeConnection) {
293 this.replicationPeers.peerRemoved(peerId);
294 }
295 }
296
297
298
299
300 public void join() {
301 this.executor.shutdown();
302 if (this.sources.size() == 0) {
303 this.replicationQueues.removeAllQueues();
304 }
305 for (ReplicationSourceInterface source : this.sources) {
306 source.terminate("Region server is closing");
307 }
308 }
309
310
311
312
313
314 protected Map<String, SortedSet<String>> getWALs() {
315 return Collections.unmodifiableMap(walsById);
316 }
317
318
319
320
321
322 protected Map<String, SortedSet<String>> getWalsByIdRecoveredQueues() {
323 return Collections.unmodifiableMap(walsByIdRecoveredQueues);
324 }
325
326
327
328
329
330 public List<ReplicationSourceInterface> getSources() {
331 return this.sources;
332 }
333
334
335
336
337
338 public List<ReplicationSourceInterface> getOldSources() {
339 return this.oldsources;
340 }
341
342 void preLogRoll(Path newLog) throws IOException {
343 synchronized (this.walsById) {
344 String name = newLog.getName();
345 for (ReplicationSourceInterface source : this.sources) {
346 try {
347 this.replicationQueues.addLog(source.getPeerClusterZnode(), name);
348 } catch (ReplicationException e) {
349 throw new IOException("Cannot add log to replication queue with id="
350 + source.getPeerClusterZnode() + ", filename=" + name, e);
351 }
352 }
353 for (SortedSet<String> wals : this.walsById.values()) {
354 if (this.sources.isEmpty()) {
355
356
357 wals.clear();
358 }
359 wals.add(name);
360 }
361 }
362
363 this.latestPath = newLog;
364 }
365
366 void postLogRoll(Path newLog) throws IOException {
367
368 for (ReplicationSourceInterface source : this.sources) {
369 source.enqueueLog(newLog);
370 }
371 }
372
373
374
375
376
377
378
379
380
381
382
383 protected ReplicationSourceInterface getReplicationSource(final Configuration conf,
384 final FileSystem fs, final ReplicationSourceManager manager,
385 final ReplicationQueues replicationQueues, final ReplicationPeers replicationPeers,
386 final Server server, final String peerId, final UUID clusterId,
387 final ReplicationPeerConfig peerConfig, final ReplicationPeer replicationPeer)
388 throws IOException {
389 RegionServerCoprocessorHost rsServerHost = null;
390 TableDescriptors tableDescriptors = null;
391 if (server instanceof HRegionServer) {
392 rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
393 tableDescriptors = ((HRegionServer) server).getTableDescriptors();
394 }
395 ReplicationSourceInterface src;
396 try {
397 @SuppressWarnings("rawtypes")
398 Class c = Class.forName(conf.get("replication.replicationsource.implementation",
399 ReplicationSource.class.getCanonicalName()));
400 src = (ReplicationSourceInterface) c.newInstance();
401 } catch (Exception e) {
402 LOG.warn("Passed replication source implementation throws errors, " +
403 "defaulting to ReplicationSource", e);
404 src = new ReplicationSource();
405 }
406
407 ReplicationEndpoint replicationEndpoint = null;
408 try {
409 String replicationEndpointImpl = peerConfig.getReplicationEndpointImpl();
410 if (replicationEndpointImpl == null) {
411
412 replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
413 }
414 @SuppressWarnings("rawtypes")
415 Class c = Class.forName(replicationEndpointImpl);
416 replicationEndpoint = (ReplicationEndpoint) c.newInstance();
417 if(rsServerHost != null) {
418 ReplicationEndpoint newReplicationEndPoint = rsServerHost
419 .postCreateReplicationEndPoint(replicationEndpoint);
420 if(newReplicationEndPoint != null) {
421
422 replicationEndpoint = newReplicationEndPoint;
423 }
424 }
425 } catch (Exception e) {
426 LOG.warn("Passed replication endpoint implementation throws errors", e);
427 throw new IOException(e);
428 }
429
430 MetricsSource metrics = new MetricsSource(peerId);
431
432 src.init(conf, fs, manager, replicationQueues, replicationPeers, server, peerId,
433 clusterId, replicationEndpoint, metrics);
434
435
436 replicationEndpoint.init(new ReplicationEndpoint.Context(replicationPeer.getConfiguration(),
437 fs, peerConfig, peerId, clusterId, replicationPeer, metrics, tableDescriptors));
438
439 return src;
440 }
441
442
443
444
445
446
447
448
449
450 private void transferQueues(String rsZnode) {
451 NodeFailoverWorker transfer =
452 new NodeFailoverWorker(rsZnode, this.replicationQueues, this.replicationPeers,
453 this.clusterId);
454 try {
455 this.executor.execute(transfer);
456 } catch (RejectedExecutionException ex) {
457 LOG.info("Cancelling the transfer of " + rsZnode + " because of " + ex.getMessage());
458 }
459 }
460
461
462
463
464
465 public void closeRecoveredQueue(ReplicationSourceInterface src) {
466 LOG.info("Done with the recovered queue " + src.getPeerClusterZnode());
467 this.oldsources.remove(src);
468 deleteSource(src.getPeerClusterZnode(), false);
469 this.walsByIdRecoveredQueues.remove(src.getPeerClusterZnode());
470 }
471
472
473
474
475
476
477 public void removePeer(String id) {
478 LOG.info("Closing the following queue " + id + ", currently have "
479 + sources.size() + " and another "
480 + oldsources.size() + " that were recovered");
481 String terminateMessage = "Replication stream was removed by a user";
482 ReplicationSourceInterface srcToRemove = null;
483 List<ReplicationSourceInterface> oldSourcesToDelete =
484 new ArrayList<ReplicationSourceInterface>();
485
486 for (ReplicationSourceInterface src : oldsources) {
487 if (id.equals(src.getPeerClusterId())) {
488 oldSourcesToDelete.add(src);
489 }
490 }
491 for (ReplicationSourceInterface src : oldSourcesToDelete) {
492 src.terminate(terminateMessage);
493 closeRecoveredQueue((src));
494 }
495 LOG.info("Number of deleted recovered sources for " + id + ": "
496 + oldSourcesToDelete.size());
497
498 for (ReplicationSourceInterface src : this.sources) {
499 if (id.equals(src.getPeerClusterId())) {
500 srcToRemove = src;
501 break;
502 }
503 }
504 if (srcToRemove == null) {
505 LOG.error("The queue we wanted to close is missing " + id);
506 return;
507 }
508 srcToRemove.terminate(terminateMessage);
509 this.sources.remove(srcToRemove);
510 deleteSource(id, true);
511 }
512
513 @Override
514 public void regionServerRemoved(String regionserver) {
515 transferQueues(regionserver);
516 }
517
518 @Override
519 public void peerRemoved(String peerId) {
520 removePeer(peerId);
521 }
522
523 @Override
524 public void peerListChanged(List<String> peerIds) {
525 for (String id : peerIds) {
526 try {
527 boolean added = this.replicationPeers.peerAdded(id);
528 if (added) {
529 addSource(id);
530 }
531 } catch (Exception e) {
532 LOG.error("Error while adding a new peer", e);
533 }
534 }
535 }
536
537
538
539
540
541 class NodeFailoverWorker extends Thread {
542
543 private String rsZnode;
544 private final ReplicationQueues rq;
545 private final ReplicationPeers rp;
546 private final UUID clusterId;
547
548
549
550
551
552 public NodeFailoverWorker(String rsZnode, final ReplicationQueues replicationQueues,
553 final ReplicationPeers replicationPeers, final UUID clusterId) {
554 super("Failover-for-"+rsZnode);
555 this.rsZnode = rsZnode;
556 this.rq = replicationQueues;
557 this.rp = replicationPeers;
558 this.clusterId = clusterId;
559 }
560
561 @Override
562 public void run() {
563 if (this.rq.isThisOurZnode(rsZnode)) {
564 return;
565 }
566
567
568 try {
569 Thread.sleep(sleepBeforeFailover + (long) (rand.nextFloat() * sleepBeforeFailover));
570 } catch (InterruptedException e) {
571 LOG.warn("Interrupted while waiting before transferring a queue.");
572 Thread.currentThread().interrupt();
573 }
574
575 if (server.isStopped()) {
576 LOG.info("Not transferring queue since we are shutting down");
577 return;
578 }
579 SortedMap<String, SortedSet<String>> newQueues = null;
580
581 newQueues = this.rq.claimQueues(rsZnode);
582
583
584 if (newQueues.isEmpty()) {
585
586
587 return;
588 }
589
590 for (Map.Entry<String, SortedSet<String>> entry : newQueues.entrySet()) {
591 String peerId = entry.getKey();
592 try {
593
594 ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
595 String actualPeerId = replicationQueueInfo.getPeerId();
596 ReplicationPeer peer = replicationPeers.getPeer(actualPeerId);
597 ReplicationPeerConfig peerConfig = null;
598 try {
599 peerConfig = replicationPeers.getReplicationPeerConfig(actualPeerId);
600 } catch (ReplicationException ex) {
601 LOG.warn("Received exception while getting replication peer config, skipping replay"
602 + ex);
603 }
604 if (peer == null || peerConfig == null) {
605 LOG.warn("Skipping failover for peer:" + actualPeerId + " of node" + rsZnode);
606 continue;
607 }
608
609 ReplicationSourceInterface src =
610 getReplicationSource(conf, fs, ReplicationSourceManager.this, this.rq, this.rp,
611 server, peerId, this.clusterId, peerConfig, peer);
612 if (!this.rp.getPeerIds().contains((src.getPeerClusterId()))) {
613 src.terminate("Recovered queue doesn't belong to any current peer");
614 break;
615 }
616 oldsources.add(src);
617 SortedSet<String> walsSet = entry.getValue();
618 for (String wal : walsSet) {
619 src.enqueueLog(new Path(oldLogDir, wal));
620 }
621 src.startup();
622 walsByIdRecoveredQueues.put(peerId, walsSet);
623 } catch (IOException e) {
624
625 LOG.error("Failed creating a source", e);
626 }
627 }
628 }
629 }
630
631
632
633
634
635 public Path getOldLogDir() {
636 return this.oldLogDir;
637 }
638
639
640
641
642
643 public Path getLogDir() {
644 return this.logDir;
645 }
646
647
648
649
650
651 public FileSystem getFs() {
652 return this.fs;
653 }
654
655
656
657
658 public String getStats() {
659 StringBuffer stats = new StringBuffer();
660 for (ReplicationSourceInterface source : sources) {
661 stats.append("Normal source for cluster " + source.getPeerClusterId() + ": ");
662 stats.append(source.getStats() + "\n");
663 }
664 for (ReplicationSourceInterface oldSource : oldsources) {
665 stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerClusterId()+": ");
666 stats.append(oldSource.getStats()+ "\n");
667 }
668 return stats.toString();
669 }
670
671 public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)
672 throws ReplicationException {
673 for (ReplicationSourceInterface source : this.sources) {
674 source.addHFileRefs(tableName, family, pairs);
675 }
676 }
677
678 public void cleanUpHFileRefs(String peerId, List<String> files) {
679 this.replicationQueues.removeHFileRefs(peerId, files);
680 }
681 }