View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.HashMap;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.Random;
29  import java.util.SortedMap;
30  import java.util.SortedSet;
31  import java.util.TreeSet;
32  import java.util.UUID;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.CopyOnWriteArrayList;
35  import java.util.concurrent.LinkedBlockingQueue;
36  import java.util.concurrent.RejectedExecutionException;
37  import java.util.concurrent.ThreadPoolExecutor;
38  import java.util.concurrent.TimeUnit;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.fs.FileSystem;
44  import org.apache.hadoop.fs.Path;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.Server;
47  import org.apache.hadoop.hbase.TableDescriptors;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.classification.InterfaceAudience;
50  import org.apache.hadoop.hbase.regionserver.HRegionServer;
51  import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
52  import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
53  import org.apache.hadoop.hbase.replication.ReplicationException;
54  import org.apache.hadoop.hbase.replication.ReplicationListener;
55  import org.apache.hadoop.hbase.replication.ReplicationPeer;
56  import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
57  import org.apache.hadoop.hbase.replication.ReplicationPeers;
58  import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
59  import org.apache.hadoop.hbase.replication.ReplicationQueues;
60  import org.apache.hadoop.hbase.replication.ReplicationTracker;
61  import org.apache.hadoop.hbase.util.Pair;
62  
63  import com.google.common.util.concurrent.ThreadFactoryBuilder;
64  
65  /**
66   * This class is responsible to manage all the replication
67   * sources. There are two classes of sources:
68   * <li> Normal sources are persistent and one per peer cluster</li>
69   * <li> Old sources are recovered from a failed region server and our
70   * only goal is to finish replicating the WAL queue it had up in ZK</li>
71   *
72   * When a region server dies, this class uses a watcher to get notified and it
73   * tries to grab a lock in order to transfer all the queues in a local
74   * old source.
75   *
76   * This class implements the ReplicationListener interface so that it can track changes in
77   * replication state.
78   */
79  @InterfaceAudience.Private
80  public class ReplicationSourceManager implements ReplicationListener {
81    private static final Log LOG =
82        LogFactory.getLog(ReplicationSourceManager.class);
83    // List of all the sources that read this RS's logs
84    private final List<ReplicationSourceInterface> sources;
85    // List of all the sources we got from died RSs
86    private final List<ReplicationSourceInterface> oldsources;
87    private final ReplicationQueues replicationQueues;
88    private final ReplicationTracker replicationTracker;
89    private final ReplicationPeers replicationPeers;
90    // UUID for this cluster
91    private final UUID clusterId;
92    // All about stopping
93    private final Server server;
94    // All logs we are currently tracking
95    private final Map<String, SortedSet<String>> walsById;
96    // Logs for recovered sources we are currently tracking
97    private final Map<String, SortedSet<String>> walsByIdRecoveredQueues;
98    private final Configuration conf;
99    private final FileSystem fs;
100   // The path to the latest log we saw, for new coming sources
101   private Path latestPath;
102   // Path to the wals directories
103   private final Path logDir;
104   // Path to the wal archive
105   private final Path oldLogDir;
106   // The number of ms that we wait before moving znodes, HBASE-3596
107   private final long sleepBeforeFailover;
108   // Homemade executer service for replication
109   private final ThreadPoolExecutor executor;
110 
111   private final Random rand;
112 
113 
114   /**
115    * Creates a replication manager and sets the watch on all the other registered region servers
116    * @param replicationQueues the interface for manipulating replication queues
117    * @param replicationPeers
118    * @param replicationTracker
119    * @param conf the configuration to use
120    * @param server the server for this region server
121    * @param fs the file system to use
122    * @param logDir the directory that contains all wal directories of live RSs
123    * @param oldLogDir the directory where old logs are archived
124    * @param clusterId
125    */
126   public ReplicationSourceManager(final ReplicationQueues replicationQueues,
127       final ReplicationPeers replicationPeers, final ReplicationTracker replicationTracker,
128       final Configuration conf, final Server server, final FileSystem fs, final Path logDir,
129       final Path oldLogDir, final UUID clusterId) {
130     //CopyOnWriteArrayList is thread-safe.
131     //Generally, reading is more than modifying.
132     this.sources = new CopyOnWriteArrayList<ReplicationSourceInterface>();
133     this.replicationQueues = replicationQueues;
134     this.replicationPeers = replicationPeers;
135     this.replicationTracker = replicationTracker;
136     this.server = server;
137     this.walsById = new HashMap<String, SortedSet<String>>();
138     this.walsByIdRecoveredQueues = new ConcurrentHashMap<String, SortedSet<String>>();
139     this.oldsources = new CopyOnWriteArrayList<ReplicationSourceInterface>();
140     this.conf = conf;
141     this.fs = fs;
142     this.logDir = logDir;
143     this.oldLogDir = oldLogDir;
144     this.sleepBeforeFailover =
145         conf.getLong("replication.sleep.before.failover", 30000); // 30 seconds
146     this.clusterId = clusterId;
147     this.replicationTracker.registerListener(this);
148     this.replicationPeers.getAllPeerIds();
149     // It's preferable to failover 1 RS at a time, but with good zk servers
150     // more could be processed at the same time.
151     int nbWorkers = conf.getInt("replication.executor.workers", 1);
152     // use a short 100ms sleep since this could be done inline with a RS startup
153     // even if we fail, other region servers can take care of it
154     this.executor = new ThreadPoolExecutor(nbWorkers, nbWorkers,
155         100, TimeUnit.MILLISECONDS,
156         new LinkedBlockingQueue<Runnable>());
157     ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
158     tfb.setNameFormat("ReplicationExecutor-%d");
159     tfb.setDaemon(true);
160     this.executor.setThreadFactory(tfb.build());
161     this.rand = new Random();
162   }
163 
164   /**
165    * Provide the id of the peer and a log key and this method will figure which
166    * wal it belongs to and will log, for this region server, the current
167    * position. It will also clean old logs from the queue.
168    * @param log Path to the log currently being replicated from
169    * replication status in zookeeper. It will also delete older entries.
170    * @param id id of the peer cluster
171    * @param position current location in the log
172    * @param queueRecovered indicates if this queue comes from another region server
173    * @param holdLogInZK if true then the log is retained in ZK
174    */
175   public void logPositionAndCleanOldLogs(Path log, String id, long position,
176       boolean queueRecovered, boolean holdLogInZK) {
177     String fileName = log.getName();
178     this.replicationQueues.setLogPosition(id, fileName, position);
179     if (holdLogInZK) {
180      return;
181     }
182     cleanOldLogs(fileName, id, queueRecovered);
183   }
184 
185   /**
186    * Cleans a log file and all older files from ZK. Called when we are sure that a
187    * log file is closed and has no more entries.
188    * @param key Path to the log
189    * @param id id of the peer cluster
190    * @param queueRecovered Whether this is a recovered queue
191    */
192   public void cleanOldLogs(String key, String id, boolean queueRecovered) {
193     if (queueRecovered) {
194       SortedSet<String> wals = walsByIdRecoveredQueues.get(id);
195       if (wals != null && !wals.first().equals(key)) {
196         cleanOldLogs(wals, key, id);
197       }
198     } else {
199       synchronized (this.walsById) {
200         SortedSet<String> wals = walsById.get(id);
201         if (!wals.first().equals(key)) {
202           cleanOldLogs(wals, key, id);
203         }
204       }
205     }
206  }
207 
208   private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
209     SortedSet<String> walSet = wals.headSet(key);
210     LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
211     for (String wal : walSet) {
212       this.replicationQueues.removeLog(id, wal);
213     }
214     walSet.clear();
215   }
216 
217   /**
218    * Adds a normal source per registered peer cluster and tries to process all
219    * old region server wal queues
220    */
221   protected void init() throws IOException, ReplicationException {
222     boolean replicationForBulkLoadDataEnabled =
223         conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
224           HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
225     for (String id : this.replicationPeers.getPeerIds()) {
226       addSource(id);
227       if (replicationForBulkLoadDataEnabled) {
228         // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
229         // when a peer was added before replication for bulk loaded data was enabled.
230         this.replicationQueues.addPeerToHFileRefs(id);
231       }
232     }
233     List<String> currentReplicators = this.replicationQueues.getListOfReplicators();
234     if (currentReplicators == null || currentReplicators.size() == 0) {
235       return;
236     }
237     List<String> otherRegionServers = replicationTracker.getListOfRegionServers();
238     LOG.info("Current list of replicators: " + currentReplicators + " other RSs: "
239         + otherRegionServers);
240 
241     // Look if there's anything to process after a restart
242     for (String rs : currentReplicators) {
243       if (!otherRegionServers.contains(rs)) {
244         transferQueues(rs);
245       }
246     }
247   }
248 
249   /**
250    * Add a new normal source to this region server
251    * @param id the id of the peer cluster
252    * @return the source that was created
253    * @throws IOException
254    */
255   protected ReplicationSourceInterface addSource(String id) throws IOException,
256       ReplicationException {
257     ReplicationPeerConfig peerConfig
258       = replicationPeers.getReplicationPeerConfig(id);
259     ReplicationPeer peer = replicationPeers.getPeer(id);
260     ReplicationSourceInterface src =
261         getReplicationSource(this.conf, this.fs, this, this.replicationQueues,
262           this.replicationPeers, server, id, this.clusterId, peerConfig, peer);
263     synchronized (this.walsById) {
264       this.sources.add(src);
265       this.walsById.put(id, new TreeSet<String>());
266       // Add the latest wal to that source's queue
267       if (this.latestPath != null) {
268         String name = this.latestPath.getName();
269         this.walsById.get(id).add(name);
270         try {
271           this.replicationQueues.addLog(src.getPeerClusterZnode(), name);
272         } catch (ReplicationException e) {
273           String message =
274               "Cannot add log to queue when creating a new source, queueId="
275                   + src.getPeerClusterZnode() + ", filename=" + name;
276           server.stop(message);
277           throw e;
278         }
279         src.enqueueLog(this.latestPath);
280       }
281     }
282     src.startup();
283     return src;
284   }
285 
286   /**
287    * Delete a complete queue of wals associated with a peer cluster
288    * @param peerId Id of the peer cluster queue of wals to delete
289    */
290   public void deleteSource(String peerId, boolean closeConnection) {
291     this.replicationQueues.removeQueue(peerId);
292     if (closeConnection) {
293       this.replicationPeers.peerRemoved(peerId);
294     }
295   }
296 
297   /**
298    * Terminate the replication on this region server
299    */
300   public void join() {
301     this.executor.shutdown();
302     if (this.sources.size() == 0) {
303       this.replicationQueues.removeAllQueues();
304     }
305     for (ReplicationSourceInterface source : this.sources) {
306       source.terminate("Region server is closing");
307     }
308   }
309 
310   /**
311    * Get a copy of the wals of the first source on this rs
312    * @return a sorted set of wal names
313    */
314   protected Map<String, SortedSet<String>> getWALs() {
315     return Collections.unmodifiableMap(walsById);
316   }
317 
318   /**
319    * Get a copy of the wals of the recovered sources on this rs
320    * @return a sorted set of wal names
321    */
322   protected Map<String, SortedSet<String>> getWalsByIdRecoveredQueues() {
323     return Collections.unmodifiableMap(walsByIdRecoveredQueues);
324   }
325 
326   /**
327    * Get a list of all the normal sources of this rs
328    * @return lis of all sources
329    */
330   public List<ReplicationSourceInterface> getSources() {
331     return this.sources;
332   }
333 
334   /**
335    * Get a list of all the old sources of this rs
336    * @return list of all old sources
337    */
338   public List<ReplicationSourceInterface> getOldSources() {
339     return this.oldsources;
340   }
341 
342   void preLogRoll(Path newLog) throws IOException {
343     synchronized (this.walsById) {
344       String name = newLog.getName();
345       for (ReplicationSourceInterface source : this.sources) {
346         try {
347           this.replicationQueues.addLog(source.getPeerClusterZnode(), name);
348         } catch (ReplicationException e) {
349           throw new IOException("Cannot add log to replication queue with id="
350               + source.getPeerClusterZnode() + ", filename=" + name, e);
351         }
352       }
353       for (SortedSet<String> wals : this.walsById.values()) {
354         if (this.sources.isEmpty()) {
355           // If there's no slaves, don't need to keep the old wals since
356           // we only consider the last one when a new slave comes in
357           wals.clear();
358         }
359         wals.add(name);
360       }
361     }
362 
363     this.latestPath = newLog;
364   }
365 
366   void postLogRoll(Path newLog) throws IOException {
367     // This only updates the sources we own, not the recovered ones
368     for (ReplicationSourceInterface source : this.sources) {
369       source.enqueueLog(newLog);
370     }
371   }
372 
373   /**
374    * Factory method to create a replication source
375    * @param conf the configuration to use
376    * @param fs the file system to use
377    * @param manager the manager to use
378    * @param server the server object for this region server
379    * @param peerId the id of the peer cluster
380    * @return the created source
381    * @throws IOException
382    */
383   protected ReplicationSourceInterface getReplicationSource(final Configuration conf,
384       final FileSystem fs, final ReplicationSourceManager manager,
385       final ReplicationQueues replicationQueues, final ReplicationPeers replicationPeers,
386       final Server server, final String peerId, final UUID clusterId,
387       final ReplicationPeerConfig peerConfig, final ReplicationPeer replicationPeer)
388           throws IOException {
389     RegionServerCoprocessorHost rsServerHost = null;
390     TableDescriptors tableDescriptors = null;
391     if (server instanceof HRegionServer) {
392       rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
393       tableDescriptors = ((HRegionServer) server).getTableDescriptors();
394     }
395     ReplicationSourceInterface src;
396     try {
397       @SuppressWarnings("rawtypes")
398       Class c = Class.forName(conf.get("replication.replicationsource.implementation",
399           ReplicationSource.class.getCanonicalName()));
400       src = (ReplicationSourceInterface) c.newInstance();
401     } catch (Exception e) {
402       LOG.warn("Passed replication source implementation throws errors, " +
403           "defaulting to ReplicationSource", e);
404       src = new ReplicationSource();
405     }
406 
407     ReplicationEndpoint replicationEndpoint = null;
408     try {
409       String replicationEndpointImpl = peerConfig.getReplicationEndpointImpl();
410       if (replicationEndpointImpl == null) {
411         // Default to HBase inter-cluster replication endpoint
412         replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
413       }
414       @SuppressWarnings("rawtypes")
415       Class c = Class.forName(replicationEndpointImpl);
416       replicationEndpoint = (ReplicationEndpoint) c.newInstance();
417       if(rsServerHost != null) {
418         ReplicationEndpoint newReplicationEndPoint = rsServerHost
419             .postCreateReplicationEndPoint(replicationEndpoint);
420         if(newReplicationEndPoint != null) {
421           // Override the newly created endpoint from the hook with configured end point
422           replicationEndpoint = newReplicationEndPoint;
423         }
424       }
425     } catch (Exception e) {
426       LOG.warn("Passed replication endpoint implementation throws errors", e);
427       throw new IOException(e);
428     }
429 
430     MetricsSource metrics = new MetricsSource(peerId);
431     // init replication source
432     src.init(conf, fs, manager, replicationQueues, replicationPeers, server, peerId,
433       clusterId, replicationEndpoint, metrics);
434 
435     // init replication endpoint
436     replicationEndpoint.init(new ReplicationEndpoint.Context(replicationPeer.getConfiguration(),
437       fs, peerConfig, peerId, clusterId, replicationPeer, metrics, tableDescriptors));
438 
439     return src;
440   }
441 
442   /**
443    * Transfer all the queues of the specified to this region server.
444    * First it tries to grab a lock and if it works it will move the
445    * znodes and finally will delete the old znodes.
446    *
447    * It creates one old source for any type of source of the old rs.
448    * @param rsZnode
449    */
450   private void transferQueues(String rsZnode) {
451     NodeFailoverWorker transfer =
452         new NodeFailoverWorker(rsZnode, this.replicationQueues, this.replicationPeers,
453             this.clusterId);
454     try {
455       this.executor.execute(transfer);
456     } catch (RejectedExecutionException ex) {
457       LOG.info("Cancelling the transfer of " + rsZnode + " because of " + ex.getMessage());
458     }
459   }
460 
461   /**
462    * Clear the references to the specified old source
463    * @param src source to clear
464    */
465   public void closeRecoveredQueue(ReplicationSourceInterface src) {
466     LOG.info("Done with the recovered queue " + src.getPeerClusterZnode());
467     this.oldsources.remove(src);
468     deleteSource(src.getPeerClusterZnode(), false);
469     this.walsByIdRecoveredQueues.remove(src.getPeerClusterZnode());
470   }
471 
472   /**
473    * Thie method first deletes all the recovered sources for the specified
474    * id, then deletes the normal source (deleting all related data in ZK).
475    * @param id The id of the peer cluster
476    */
477   public void removePeer(String id) {
478     LOG.info("Closing the following queue " + id + ", currently have "
479         + sources.size() + " and another "
480         + oldsources.size() + " that were recovered");
481     String terminateMessage = "Replication stream was removed by a user";
482     ReplicationSourceInterface srcToRemove = null;
483     List<ReplicationSourceInterface> oldSourcesToDelete =
484         new ArrayList<ReplicationSourceInterface>();
485     // First close all the recovered sources for this peer
486     for (ReplicationSourceInterface src : oldsources) {
487       if (id.equals(src.getPeerClusterId())) {
488         oldSourcesToDelete.add(src);
489       }
490     }
491     for (ReplicationSourceInterface src : oldSourcesToDelete) {
492       src.terminate(terminateMessage);
493       closeRecoveredQueue((src));
494     }
495     LOG.info("Number of deleted recovered sources for " + id + ": "
496         + oldSourcesToDelete.size());
497     // Now look for the one on this cluster
498     for (ReplicationSourceInterface src : this.sources) {
499       if (id.equals(src.getPeerClusterId())) {
500         srcToRemove = src;
501         break;
502       }
503     }
504     if (srcToRemove == null) {
505       LOG.error("The queue we wanted to close is missing " + id);
506       return;
507     }
508     srcToRemove.terminate(terminateMessage);
509     this.sources.remove(srcToRemove);
510     deleteSource(id, true);
511   }
512 
513   @Override
514   public void regionServerRemoved(String regionserver) {
515     transferQueues(regionserver);
516   }
517 
518   @Override
519   public void peerRemoved(String peerId) {
520     removePeer(peerId);
521   }
522 
523   @Override
524   public void peerListChanged(List<String> peerIds) {
525     for (String id : peerIds) {
526       try {
527         boolean added = this.replicationPeers.peerAdded(id);
528         if (added) {
529           addSource(id);
530         }
531       } catch (Exception e) {
532         LOG.error("Error while adding a new peer", e);
533       }
534     }
535   }
536 
537   /**
538    * Class responsible to setup new ReplicationSources to take care of the
539    * queues from dead region servers.
540    */
541   class NodeFailoverWorker extends Thread {
542 
543     private String rsZnode;
544     private final ReplicationQueues rq;
545     private final ReplicationPeers rp;
546     private final UUID clusterId;
547 
548     /**
549      *
550      * @param rsZnode
551      */
552     public NodeFailoverWorker(String rsZnode, final ReplicationQueues replicationQueues,
553         final ReplicationPeers replicationPeers, final UUID clusterId) {
554       super("Failover-for-"+rsZnode);
555       this.rsZnode = rsZnode;
556       this.rq = replicationQueues;
557       this.rp = replicationPeers;
558       this.clusterId = clusterId;
559     }
560 
561     @Override
562     public void run() {
563       if (this.rq.isThisOurZnode(rsZnode)) {
564         return;
565       }
566       // Wait a bit before transferring the queues, we may be shutting down.
567       // This sleep may not be enough in some cases.
568       try {
569         Thread.sleep(sleepBeforeFailover + (long) (rand.nextFloat() * sleepBeforeFailover));
570       } catch (InterruptedException e) {
571         LOG.warn("Interrupted while waiting before transferring a queue.");
572         Thread.currentThread().interrupt();
573       }
574       // We try to lock that rs' queue directory
575       if (server.isStopped()) {
576         LOG.info("Not transferring queue since we are shutting down");
577         return;
578       }
579       SortedMap<String, SortedSet<String>> newQueues = null;
580 
581       newQueues = this.rq.claimQueues(rsZnode);
582 
583       // Copying over the failed queue is completed.
584       if (newQueues.isEmpty()) {
585         // We either didn't get the lock or the failed region server didn't have any outstanding
586         // WALs to replicate, so we are done.
587         return;
588       }
589 
590       for (Map.Entry<String, SortedSet<String>> entry : newQueues.entrySet()) {
591         String peerId = entry.getKey();
592         try {
593           // there is not an actual peer defined corresponding to peerId for the failover.
594           ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
595           String actualPeerId = replicationQueueInfo.getPeerId();
596           ReplicationPeer peer = replicationPeers.getPeer(actualPeerId);
597           ReplicationPeerConfig peerConfig = null;
598           try {
599             peerConfig = replicationPeers.getReplicationPeerConfig(actualPeerId);
600           } catch (ReplicationException ex) {
601             LOG.warn("Received exception while getting replication peer config, skipping replay"
602                 + ex);
603           }
604           if (peer == null || peerConfig == null) {
605             LOG.warn("Skipping failover for peer:" + actualPeerId + " of node" + rsZnode);
606             continue;
607           }
608 
609           ReplicationSourceInterface src =
610               getReplicationSource(conf, fs, ReplicationSourceManager.this, this.rq, this.rp,
611                 server, peerId, this.clusterId, peerConfig, peer);
612           if (!this.rp.getPeerIds().contains((src.getPeerClusterId()))) {
613             src.terminate("Recovered queue doesn't belong to any current peer");
614             break;
615           }
616           oldsources.add(src);
617           SortedSet<String> walsSet = entry.getValue();
618           for (String wal : walsSet) {
619             src.enqueueLog(new Path(oldLogDir, wal));
620           }
621           src.startup();
622           walsByIdRecoveredQueues.put(peerId, walsSet);
623         } catch (IOException e) {
624           // TODO manage it
625           LOG.error("Failed creating a source", e);
626         }
627       }
628     }
629   }
630 
631   /**
632    * Get the directory where wals are archived
633    * @return the directory where wals are archived
634    */
635   public Path getOldLogDir() {
636     return this.oldLogDir;
637   }
638 
639   /**
640    * Get the directory where wals are stored by their RSs
641    * @return the directory where wals are stored by their RSs
642    */
643   public Path getLogDir() {
644     return this.logDir;
645   }
646 
647   /**
648    * Get the handle on the local file system
649    * @return Handle on the local file system
650    */
651   public FileSystem getFs() {
652     return this.fs;
653   }
654 
655   /**
656    * Get a string representation of all the sources' metrics
657    */
658   public String getStats() {
659     StringBuffer stats = new StringBuffer();
660     for (ReplicationSourceInterface source : sources) {
661       stats.append("Normal source for cluster " + source.getPeerClusterId() + ": ");
662       stats.append(source.getStats() + "\n");
663     }
664     for (ReplicationSourceInterface oldSource : oldsources) {
665       stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerClusterId()+": ");
666       stats.append(oldSource.getStats()+ "\n");
667     }
668     return stats.toString();
669   }
670 
671   public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)
672       throws ReplicationException {
673     for (ReplicationSourceInterface source : this.sources) {
674       source.addHFileRefs(tableName, family, pairs);
675     }
676   }
677 
678   public void cleanUpHFileRefs(String peerId, List<String> files) {
679     this.replicationQueues.removeHFileRefs(peerId, files);
680   }
681 }