View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.Comparator;
23  import java.util.List;
24  import java.util.concurrent.BlockingQueue;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.Stoppable;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.hbase.io.HFileLink;
35  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
36  import org.apache.hadoop.hbase.util.StealJobQueue;
37  
38  import com.google.common.annotations.VisibleForTesting;
39  /**
40   * This Chore, every time it runs, will clear the HFiles in the hfile archive
41   * folder that are deletable for each HFile cleaner in the chain.
42   */
43  @InterfaceAudience.Private
44  public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
45  
46    public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins";
47  
48    // Configuration key for large/small throttle point
49    public final static String HFILE_DELETE_THROTTLE_THRESHOLD =
50        "hbase.regionserver.thread.hfilecleaner.throttle";
51    public final static int DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD = 64 * 1024 * 1024;// 64M
52  
53    // Configuration key for large queue initial size
54    public final static String LARGE_HFILE_QUEUE_INIT_SIZE =
55        "hbase.regionserver.hfilecleaner.large.queue.size";
56    public final static int DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE = 10240;
57  
58    // Configuration key for small queue initial size
59    public final static String SMALL_HFILE_QUEUE_INIT_SIZE =
60        "hbase.regionserver.hfilecleaner.small.queue.size";
61    public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
62  
63    private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
64  
65    StealJobQueue<HFileDeleteTask> largeFileQueue;
66    BlockingQueue<HFileDeleteTask> smallFileQueue;
67    private int throttlePoint;
68    private int largeQueueInitSize;
69    private int smallQueueInitSize;
70    private List<Thread> threads = new ArrayList<Thread>();
71    private boolean running;
72  
73    private long deletedLargeFiles = 0L;
74    private long deletedSmallFiles = 0L;
75  
76    /**
77     * @param period the period of time to sleep between each run
78     * @param stopper the stopper
79     * @param conf configuration to use
80     * @param fs handle to the FS
81     * @param directory directory to be cleaned
82     */
83    public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs,
84        Path directory) {
85      super("HFileCleaner", period, stopper, conf, fs, directory, MASTER_HFILE_CLEANER_PLUGINS);
86      throttlePoint =
87          conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
88      largeQueueInitSize =
89          conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
90      smallQueueInitSize =
91          conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
92      largeFileQueue = new StealJobQueue<HFileDeleteTask>(largeQueueInitSize, smallQueueInitSize,
93          COMPARATOR);
94      smallFileQueue = largeFileQueue.getStealFromQueue();
95      startHFileDeleteThreads();
96    }
97  
98    @Override
99    protected boolean validate(Path file) {
100     if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
101       return true;
102     }
103     return StoreFileInfo.validateStoreFileName(file.getName());
104   }
105 
106   /**
107    * Exposed for TESTING!
108    */
109   public List<BaseHFileCleanerDelegate> getDelegatesForTesting() {
110     return this.cleanersChain;
111   }
112 
113   @Override
114   public int deleteFiles(Iterable<FileStatus> filesToDelete) {
115     int deletedFiles = 0;
116     List<HFileDeleteTask> tasks = new ArrayList<HFileDeleteTask>();
117     // construct delete tasks and add into relative queue
118     for (FileStatus file : filesToDelete) {
119       HFileDeleteTask task = deleteFile(file);
120       if (task != null) {
121         tasks.add(task);
122       }
123     }
124     // wait for each submitted task to finish
125     for (HFileDeleteTask task : tasks) {
126       if (task.getResult()) {
127         deletedFiles++;
128       }
129     }
130     return deletedFiles;
131   }
132 
133   /**
134    * Construct an {@link HFileDeleteTask} for each file to delete and add into the correct queue
135    * @param file the file to delete
136    * @return HFileDeleteTask to track progress
137    */
138   private HFileDeleteTask deleteFile(FileStatus file) {
139     HFileDeleteTask task = new HFileDeleteTask(file);
140     boolean enqueued = dispatch(task);
141     return enqueued ? task : null;
142   }
143 
144   private boolean dispatch(HFileDeleteTask task) {
145     if (task.fileLength >= this.throttlePoint) {
146       if (!this.largeFileQueue.offer(task)) {
147         // should never arrive here as long as we use PriorityQueue
148         if (LOG.isTraceEnabled()) {
149           LOG.trace("Large file deletion queue is full");
150         }
151         return false;
152       }
153     } else {
154       if (!this.smallFileQueue.offer(task)) {
155         // should never arrive here as long as we use PriorityQueue
156         if (LOG.isTraceEnabled()) {
157           LOG.trace("Small file deletion queue is full");
158         }
159         return false;
160       }
161     }
162     return true;
163   }
164 
165   @Override
166   public void cleanup() {
167     super.cleanup();
168     stopHFileDeleteThreads();
169   }
170 
171   /**
172    * Start threads for hfile deletion
173    */
174   private void startHFileDeleteThreads() {
175     final String n = Thread.currentThread().getName();
176     running = true;
177     // start thread for large file deletion
178     Thread large = new Thread() {
179       @Override
180       public void run() {
181         consumerLoop(largeFileQueue);
182       }
183     };
184     large.setDaemon(true);
185     large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
186     large.start();
187     LOG.debug("Starting hfile cleaner for large files: " + large.getName());
188     threads.add(large);
189 
190     // start thread for small file deletion
191     Thread small = new Thread() {
192       @Override
193       public void run() {
194         consumerLoop(smallFileQueue);
195       }
196     };
197     small.setDaemon(true);
198     small.setName(n + "-HFileCleaner.small-" + System.currentTimeMillis());
199     small.start();
200     LOG.debug("Starting hfile cleaner for small files: " + small.getName());
201     threads.add(small);
202   }
203 
204   protected void consumerLoop(BlockingQueue<HFileDeleteTask> queue) {
205     try {
206       while (running) {
207         HFileDeleteTask task = null;
208         try {
209           task = queue.take();
210         } catch (InterruptedException e) {
211           if (LOG.isDebugEnabled()) {
212             LOG.debug("Interrupted while trying to take a task from queue", e);
213           }
214           break;
215         }
216         if (task != null) {
217           if (LOG.isDebugEnabled()) {
218             LOG.debug("Removing: " + task.filePath + " from archive");
219           }
220           boolean succeed;
221           try {
222             succeed = this.fs.delete(task.filePath, false);
223           } catch (IOException e) {
224             LOG.warn("Failed to delete file " + task.filePath, e);
225             succeed = false;
226           }
227           task.setResult(succeed);
228           if (succeed) {
229             countDeletedFiles(task.fileLength >= throttlePoint, queue == largeFileQueue);
230           }
231         }
232       }
233     } finally {
234       if (LOG.isDebugEnabled()) {
235         LOG.debug("Exit thread: " + Thread.currentThread());
236       }
237     }
238   }
239 
240   // Currently only for testing purpose
241   private void countDeletedFiles(boolean isLargeFile, boolean fromLargeQueue) {
242     if (isLargeFile) {
243       if (deletedLargeFiles == Long.MAX_VALUE) {
244         LOG.info("Deleted more than Long.MAX_VALUE large files, reset counter to 0");
245         deletedLargeFiles = 0L;
246       }
247       deletedLargeFiles++;
248     } else {
249       if (deletedSmallFiles == Long.MAX_VALUE) {
250         LOG.info("Deleted more than Long.MAX_VALUE small files, reset counter to 0");
251         deletedSmallFiles = 0L;
252       }
253       if (fromLargeQueue && LOG.isTraceEnabled()) {
254         LOG.trace("Stolen a small file deletion task in large file thread");
255       }
256       deletedSmallFiles++;
257     }
258   }
259 
260   /**
261    * Stop threads for hfile deletion
262    */
263   private void stopHFileDeleteThreads() {
264     running = false;
265     if (LOG.isDebugEnabled()) {
266       LOG.debug("Stopping file delete threads");
267     }
268     for(Thread thread: threads){
269       thread.interrupt();
270     }
271   }
272 
273   private static final Comparator<HFileDeleteTask> COMPARATOR = new Comparator<HFileDeleteTask>() {
274 
275     @Override
276     public int compare(HFileDeleteTask o1, HFileDeleteTask o2) {
277       // larger file first so reverse compare
278       int cmp = Long.compare(o2.fileLength, o1.fileLength);
279       if (cmp != 0) {
280         return cmp;
281       }
282       // just use hashCode to generate a stable result.
283       return System.identityHashCode(o1) - System.identityHashCode(o2);
284     }
285   };
286 
287   private static class HFileDeleteTask {
288     private static final long MAX_WAIT = 60 * 1000L;
289     private static final long WAIT_UNIT = 1000L;
290 
291     boolean done = false;
292     boolean result;
293     final Path filePath;
294     final long fileLength;
295 
296     public HFileDeleteTask(FileStatus file) {
297       this.filePath = file.getPath();
298       this.fileLength = file.getLen();
299     }
300 
301     public synchronized void setResult(boolean result) {
302       this.done = true;
303       this.result = result;
304       notify();
305     }
306 
307     public synchronized boolean getResult() {
308       long waitTime = 0;
309       try {
310         while (!done) {
311           wait(WAIT_UNIT);
312           waitTime += WAIT_UNIT;
313           if (done) {
314             return this.result;
315           }
316           if (waitTime > MAX_WAIT) {
317             LOG.warn("Wait more than " + MAX_WAIT + " ms for deleting " + this.filePath
318                 + ", exit...");
319             return false;
320           }
321         }
322       } catch (InterruptedException e) {
323         LOG.warn("Interrupted while waiting for result of deleting " + filePath
324             + ", will return false", e);
325         return false;
326       }
327       return this.result;
328     }
329 
330     @Override
331     public boolean equals(Object o) {
332       if (this == o) {
333         return true;
334       }
335       if (o == null || !(o instanceof HFileDeleteTask)) {
336         return false;
337       }
338       HFileDeleteTask otherTask = (HFileDeleteTask) o;
339       return this.filePath.equals(otherTask.filePath) && (this.fileLength == otherTask.fileLength);
340     }
341 
342     @Override
343     public int hashCode() {
344       return filePath.hashCode();
345     }
346   }
347 
348   @VisibleForTesting
349   public List<Thread> getCleanerThreads() {
350     return threads;
351   }
352 
353   @VisibleForTesting
354   public long getNumOfDeletedLargeFiles() {
355     return deletedLargeFiles;
356   }
357 
358   @VisibleForTesting
359   public long getNumOfDeletedSmallFiles() {
360     return deletedSmallFiles;
361   }
362 
363   @VisibleForTesting
364   public long getLargeQueueInitSize() {
365     return largeQueueInitSize;
366   }
367 
368   @VisibleForTesting
369   public long getSmallQueueInitSize() {
370     return smallQueueInitSize;
371   }
372 
373   @VisibleForTesting
374   public long getThrottlePoint() {
375     return throttlePoint;
376   }
377 
378   //@Override
379   public void onConfigurationChange(Configuration conf) {
380     StringBuilder builder = new StringBuilder();
381     builder.append("Updating configuration for HFileCleaner, previous throttle point: ")
382         .append(throttlePoint).append(", largeQueueInitSize: ").append(largeQueueInitSize)
383         .append(", smallQueueInitSize: ").append(smallQueueInitSize);
384     stopHFileDeleteThreads();
385     this.throttlePoint =
386         conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
387     this.largeQueueInitSize =
388         conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
389     this.smallQueueInitSize =
390         conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
391     // record the left over tasks
392     List<HFileDeleteTask> leftOverTasks = new ArrayList<>();
393     for (HFileDeleteTask task : largeFileQueue) {
394       leftOverTasks.add(task);
395     }
396     for (HFileDeleteTask task : smallFileQueue) {
397       leftOverTasks.add(task);
398     }
399     largeFileQueue = new StealJobQueue<HFileDeleteTask>(largeQueueInitSize, smallQueueInitSize,
400         COMPARATOR);
401     smallFileQueue = largeFileQueue.getStealFromQueue();
402     threads.clear();
403     builder.append("; new throttle point: ").append(throttlePoint).append(", largeQueueInitSize: ")
404         .append(largeQueueInitSize).append(", smallQueueInitSize: ").append(smallQueueInitSize);
405     LOG.debug(builder.toString());
406     startHFileDeleteThreads();
407     // re-dispatch the left over tasks
408     for (HFileDeleteTask task : leftOverTasks) {
409       dispatch(task);
410     }
411   }
412 }