View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.replication.regionserver;
20  
21  import java.util.HashMap;
22  import java.util.Map;
23  
24  import org.apache.hadoop.hbase.classification.InterfaceAudience;
25  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
26  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
27  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
28  
29  /**
30   * This class is for maintaining the various replication statistics for a source and publishing them
31   * through the metrics interfaces.
32   */
33  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
34  public class MetricsSource {
35  
36    // tracks last shipped timestamp for each wal group
37    private Map<String, Long> lastTimeStamps = new HashMap<String, Long>();
38    private long lastHFileRefsQueueSize = 0;
39    private String id;
40  
41    private final MetricsReplicationSourceSource singleSourceSource;
42    private final MetricsReplicationSourceSource globalSourceSource;
43    private Map<String, MetricsReplicationSourceSource> singleSourceSourceByTable;
44  
45    /**
46     * Constructor used to register the metrics
47     *
48     * @param id Name of the source this class is monitoring
49     */
50    public MetricsSource(String id) {
51      this.id = id;
52      singleSourceSource =
53          CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)
54              .getSource(id);
55      globalSourceSource = CompatibilitySingletonFactory
56          .getInstance(MetricsReplicationSourceFactory.class).getGlobalSource();
57      singleSourceSourceByTable = new HashMap<>();
58    }
59  
60    /**
61     * Constructor for injecting custom (or test) MetricsReplicationSourceSources
62     * @param id Name of the source this class is monitoring
63     * @param singleSourceSource Class to monitor id-scoped metrics
64     * @param globalSourceSource Class to monitor global-scoped metrics
65     */
66    public MetricsSource(String id, MetricsReplicationSourceSource singleSourceSource,
67                         MetricsReplicationSourceSource globalSourceSource,
68                         Map<String, MetricsReplicationSourceSource> singleSourceSourceByTable) {
69      this.id = id;
70      this.singleSourceSource = singleSourceSource;
71      this.globalSourceSource = globalSourceSource;
72      this.singleSourceSourceByTable = singleSourceSourceByTable;
73    }
74  
75    /**
76     * Set the age of the last edit that was shipped
77     * @param timestamp write time of the edit
78     * @param walGroup which group we are setting
79     */
80    public void setAgeOfLastShippedOp(long timestamp, String walGroup) {
81      long age = EnvironmentEdgeManager.currentTime() - timestamp;
82      singleSourceSource.setLastShippedAge(age);
83      globalSourceSource.setLastShippedAge(Math.max(age, globalSourceSource.getLastShippedAge()));
84      this.lastTimeStamps.put(walGroup, timestamp);
85    }
86  
87    /**
88     * Set the age of the last edit that was shipped group by table
89     * @param timestamp write time of the edit
90     * @param tableName String as group and tableName
91     */
92    public void setAgeOfLastShippedOpByTable(long timestamp, String tableName) {
93      long age = EnvironmentEdgeManager.currentTime() - timestamp;
94      if (!this.getSingleSourceSourceByTable().containsKey(tableName)) {
95        this.getSingleSourceSourceByTable().put(tableName,
96            CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)
97            .getSource(tableName));
98      }
99      this.singleSourceSourceByTable.get(tableName).setLastShippedAge(age);
100   }
101   /**
102    * Convenience method to use the last given timestamp to refresh the age of the last edit. Used
103    * when replication fails and need to keep that metric accurate.
104    * @param walGroupId id of the group to update
105    */
106   public void refreshAgeOfLastShippedOp(String walGroupId) {
107     Long lastTimestamp = this.lastTimeStamps.get(walGroupId);
108     if (lastTimestamp == null) {
109       this.lastTimeStamps.put(walGroupId, 0L);
110       lastTimestamp = 0L;
111     }
112     if (lastTimestamp > 0) {
113       setAgeOfLastShippedOp(lastTimestamp, walGroupId);
114     }
115   }
116 
117   /**
118    * Increment size of the log queue.
119    */
120   public void incrSizeOfLogQueue() {
121     singleSourceSource.incrSizeOfLogQueue(1);
122     globalSourceSource.incrSizeOfLogQueue(1);
123   }
124 
125   public void decrSizeOfLogQueue() {
126     singleSourceSource.decrSizeOfLogQueue(1);
127     globalSourceSource.decrSizeOfLogQueue(1);
128   }
129 
130   /**
131    * Add on the the number of log edits read
132    *
133    * @param delta the number of log edits read.
134    */
135   private void incrLogEditsRead(long delta) {
136     singleSourceSource.incrLogReadInEdits(delta);
137     globalSourceSource.incrLogReadInEdits(delta);
138   }
139 
140   /** Increment the number of log edits read by one. */
141   public void incrLogEditsRead() {
142     incrLogEditsRead(1);
143   }
144 
145   /**
146    * Add on the number of log edits filtered
147    *
148    * @param delta the number filtered.
149    */
150   public void incrLogEditsFiltered(long delta) {
151     singleSourceSource.incrLogEditsFiltered(delta);
152     globalSourceSource.incrLogEditsFiltered(delta);
153   }
154 
155   /** The number of log edits filtered out. */
156   public void incrLogEditsFiltered() {
157     incrLogEditsFiltered(1);
158   }
159 
160   /**
161    * Convience method to apply changes to metrics do to shipping a batch of logs.
162    *
163    * @param batchSize the size of the batch that was shipped to sinks.
164    */
165   public void shipBatch(long batchSize, int sizeInBytes) {
166     singleSourceSource.incrBatchesShipped(1);
167     globalSourceSource.incrBatchesShipped(1);
168 
169     singleSourceSource.incrOpsShipped(batchSize);
170     globalSourceSource.incrOpsShipped(batchSize);
171 
172     singleSourceSource.incrShippedBytes(sizeInBytes);
173     globalSourceSource.incrShippedBytes(sizeInBytes);
174   }
175 
176   /**
177    * Convience method to apply changes to metrics do to shipping a batch of logs.
178    *
179    * @param batchSize the size of the batch that was shipped to sinks.
180    * @param hfiles total number of hfiles shipped to sinks.
181    */
182   public void shipBatch(long batchSize, int sizeInBytes, long hfiles) {
183     shipBatch(batchSize, sizeInBytes);
184     singleSourceSource.incrHFilesShipped(hfiles);
185     globalSourceSource.incrHFilesShipped(hfiles);
186   }
187 
188   /** increase the byte number read by source from log file */
189   public void incrLogReadInBytes(long readInBytes) {
190     singleSourceSource.incrLogReadInBytes(readInBytes);
191     globalSourceSource.incrLogReadInBytes(readInBytes);
192   }
193 
194   /** Removes all metrics about this Source. */
195   public void clear() {
196     int lastQueueSize = singleSourceSource.getSizeOfLogQueue();
197     globalSourceSource.decrSizeOfLogQueue(lastQueueSize);
198     singleSourceSource.decrSizeOfLogQueue(lastQueueSize);
199     singleSourceSource.clear();
200     globalSourceSource.decrSizeOfHFileRefsQueue(lastHFileRefsQueueSize);
201     lastTimeStamps.clear();
202     lastHFileRefsQueueSize = 0;
203   }
204 
205   /**
206    * Get AgeOfLastShippedOp
207    * @return AgeOfLastShippedOp
208    */
209   public Long getAgeOfLastShippedOp() {
210     return singleSourceSource.getLastShippedAge();
211   }
212 
213   /**
214    * Get the sizeOfLogQueue
215    * @return sizeOfLogQueue
216    */
217   public int getSizeOfLogQueue() {
218     return singleSourceSource.getSizeOfLogQueue();
219   }
220 
221   /**
222    * Get the timeStampsOfLastShippedOp, if there are multiple groups, return the latest one
223    * @return lastTimestampForAge
224    */
225   public long getTimeStampOfLastShippedOp() {
226     long lastTimestamp = 0L;
227     for (long ts : lastTimeStamps.values()) {
228       if (ts > lastTimestamp) {
229         lastTimestamp = ts;
230       }
231     }
232     return lastTimestamp;
233   }
234 
235   /**
236    * Get the slave peer ID
237    * @return peerID
238    */
239   public String getPeerID() {
240     return id;
241   }
242 
243   public void incrSizeOfHFileRefsQueue(long size) {
244     singleSourceSource.incrSizeOfHFileRefsQueue(size);
245     globalSourceSource.incrSizeOfHFileRefsQueue(size);
246     lastHFileRefsQueueSize = size;
247   }
248 
249   public void decrSizeOfHFileRefsQueue(int size) {
250     singleSourceSource.decrSizeOfHFileRefsQueue(size);
251     globalSourceSource.decrSizeOfHFileRefsQueue(size);
252     lastHFileRefsQueueSize -= size;
253     if (lastHFileRefsQueueSize < 0) {
254       lastHFileRefsQueueSize = 0;
255     }
256   }
257 
258   public void incrUnknownFileLengthForClosedWAL() {
259     singleSourceSource.incrUnknownFileLengthForClosedWAL();
260     globalSourceSource.incrUnknownFileLengthForClosedWAL();
261   }
262 
263   public void incrUncleanlyClosedWALs() {
264     singleSourceSource.incrUncleanlyClosedWALs();
265     globalSourceSource.incrUncleanlyClosedWALs();
266   }
267 
268   public void incrBytesSkippedInUncleanlyClosedWALs(final long bytes) {
269     singleSourceSource.incrBytesSkippedInUncleanlyClosedWALs(bytes);
270     globalSourceSource.incrBytesSkippedInUncleanlyClosedWALs(bytes);
271   }
272 
273   public void incrRestartedWALReading() {
274     singleSourceSource.incrRestartedWALReading();
275     globalSourceSource.incrRestartedWALReading();
276   }
277 
278   public void incrRepeatedFileBytes(final long bytes) {
279     singleSourceSource.incrRepeatedFileBytes(bytes);
280     globalSourceSource.incrRepeatedFileBytes(bytes);
281   }
282 
283   public void incrCompletedWAL() {
284     singleSourceSource.incrCompletedWAL();
285     globalSourceSource.incrCompletedWAL();
286   }
287 
288   public void incrCompletedRecoveryQueue() {
289     singleSourceSource.incrCompletedRecoveryQueue();
290     globalSourceSource.incrCompletedRecoveryQueue();
291   }
292 
293   public void incrFailedRecoveryQueue() {
294     globalSourceSource.incrFailedRecoveryQueue();
295   }
296 
297   public Map<String, MetricsReplicationSourceSource> getSingleSourceSourceByTable() {
298     return singleSourceSourceByTable;
299   }
300 }