1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Comparator;
23 import java.util.List;
24 import java.util.concurrent.BlockingQueue;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.FileStatus;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.hbase.io.HFileLink;
35 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
36 import org.apache.hadoop.hbase.util.StealJobQueue;
37
38 import com.google.common.annotations.VisibleForTesting;
39
40
41
42
43 @InterfaceAudience.Private
44 public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
45
46 public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins";
47
48
49 public final static String HFILE_DELETE_THROTTLE_THRESHOLD =
50 "hbase.regionserver.thread.hfilecleaner.throttle";
51 public final static int DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD = 64 * 1024 * 1024;
52
53
54 public final static String LARGE_HFILE_QUEUE_INIT_SIZE =
55 "hbase.regionserver.hfilecleaner.large.queue.size";
56 public final static int DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE = 10240;
57
58
59 public final static String SMALL_HFILE_QUEUE_INIT_SIZE =
60 "hbase.regionserver.hfilecleaner.small.queue.size";
61 public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
62
63 private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
64
65 StealJobQueue<HFileDeleteTask> largeFileQueue;
66 BlockingQueue<HFileDeleteTask> smallFileQueue;
67 private int throttlePoint;
68 private int largeQueueInitSize;
69 private int smallQueueInitSize;
70 private List<Thread> threads = new ArrayList<Thread>();
71 private boolean running;
72
73 private long deletedLargeFiles = 0L;
74 private long deletedSmallFiles = 0L;
75
76
77
78
79
80
81
82
83 public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs,
84 Path directory) {
85 super("HFileCleaner", period, stopper, conf, fs, directory, MASTER_HFILE_CLEANER_PLUGINS);
86 throttlePoint =
87 conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
88 largeQueueInitSize =
89 conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
90 smallQueueInitSize =
91 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
92 largeFileQueue = new StealJobQueue<HFileDeleteTask>(largeQueueInitSize, smallQueueInitSize,
93 COMPARATOR);
94 smallFileQueue = largeFileQueue.getStealFromQueue();
95 startHFileDeleteThreads();
96 }
97
98 @Override
99 protected boolean validate(Path file) {
100 if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
101 return true;
102 }
103 return StoreFileInfo.validateStoreFileName(file.getName());
104 }
105
106
107
108
109 public List<BaseHFileCleanerDelegate> getDelegatesForTesting() {
110 return this.cleanersChain;
111 }
112
113 @Override
114 public int deleteFiles(Iterable<FileStatus> filesToDelete) {
115 int deletedFiles = 0;
116 List<HFileDeleteTask> tasks = new ArrayList<HFileDeleteTask>();
117
118 for (FileStatus file : filesToDelete) {
119 HFileDeleteTask task = deleteFile(file);
120 if (task != null) {
121 tasks.add(task);
122 }
123 }
124
125 for (HFileDeleteTask task : tasks) {
126 if (task.getResult()) {
127 deletedFiles++;
128 }
129 }
130 return deletedFiles;
131 }
132
133
134
135
136
137
138 private HFileDeleteTask deleteFile(FileStatus file) {
139 HFileDeleteTask task = new HFileDeleteTask(file);
140 boolean enqueued = dispatch(task);
141 return enqueued ? task : null;
142 }
143
144 private boolean dispatch(HFileDeleteTask task) {
145 if (task.fileLength >= this.throttlePoint) {
146 if (!this.largeFileQueue.offer(task)) {
147
148 if (LOG.isTraceEnabled()) {
149 LOG.trace("Large file deletion queue is full");
150 }
151 return false;
152 }
153 } else {
154 if (!this.smallFileQueue.offer(task)) {
155
156 if (LOG.isTraceEnabled()) {
157 LOG.trace("Small file deletion queue is full");
158 }
159 return false;
160 }
161 }
162 return true;
163 }
164
165 @Override
166 public void cleanup() {
167 super.cleanup();
168 stopHFileDeleteThreads();
169 }
170
171
172
173
174 private void startHFileDeleteThreads() {
175 final String n = Thread.currentThread().getName();
176 running = true;
177
178 Thread large = new Thread() {
179 @Override
180 public void run() {
181 consumerLoop(largeFileQueue);
182 }
183 };
184 large.setDaemon(true);
185 large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
186 large.start();
187 LOG.debug("Starting hfile cleaner for large files: " + large.getName());
188 threads.add(large);
189
190
191 Thread small = new Thread() {
192 @Override
193 public void run() {
194 consumerLoop(smallFileQueue);
195 }
196 };
197 small.setDaemon(true);
198 small.setName(n + "-HFileCleaner.small-" + System.currentTimeMillis());
199 small.start();
200 LOG.debug("Starting hfile cleaner for small files: " + small.getName());
201 threads.add(small);
202 }
203
204 protected void consumerLoop(BlockingQueue<HFileDeleteTask> queue) {
205 try {
206 while (running) {
207 HFileDeleteTask task = null;
208 try {
209 task = queue.take();
210 } catch (InterruptedException e) {
211 if (LOG.isDebugEnabled()) {
212 LOG.debug("Interrupted while trying to take a task from queue", e);
213 }
214 break;
215 }
216 if (task != null) {
217 if (LOG.isDebugEnabled()) {
218 LOG.debug("Removing: " + task.filePath + " from archive");
219 }
220 boolean succeed;
221 try {
222 succeed = this.fs.delete(task.filePath, false);
223 } catch (IOException e) {
224 LOG.warn("Failed to delete file " + task.filePath, e);
225 succeed = false;
226 }
227 task.setResult(succeed);
228 if (succeed) {
229 countDeletedFiles(task.fileLength >= throttlePoint, queue == largeFileQueue);
230 }
231 }
232 }
233 } finally {
234 if (LOG.isDebugEnabled()) {
235 LOG.debug("Exit thread: " + Thread.currentThread());
236 }
237 }
238 }
239
240
241 private void countDeletedFiles(boolean isLargeFile, boolean fromLargeQueue) {
242 if (isLargeFile) {
243 if (deletedLargeFiles == Long.MAX_VALUE) {
244 LOG.info("Deleted more than Long.MAX_VALUE large files, reset counter to 0");
245 deletedLargeFiles = 0L;
246 }
247 deletedLargeFiles++;
248 } else {
249 if (deletedSmallFiles == Long.MAX_VALUE) {
250 LOG.info("Deleted more than Long.MAX_VALUE small files, reset counter to 0");
251 deletedSmallFiles = 0L;
252 }
253 if (fromLargeQueue && LOG.isTraceEnabled()) {
254 LOG.trace("Stolen a small file deletion task in large file thread");
255 }
256 deletedSmallFiles++;
257 }
258 }
259
260
261
262
263 private void stopHFileDeleteThreads() {
264 running = false;
265 if (LOG.isDebugEnabled()) {
266 LOG.debug("Stopping file delete threads");
267 }
268 for(Thread thread: threads){
269 thread.interrupt();
270 }
271 }
272
273 private static final Comparator<HFileDeleteTask> COMPARATOR = new Comparator<HFileDeleteTask>() {
274
275 @Override
276 public int compare(HFileDeleteTask o1, HFileDeleteTask o2) {
277
278 int cmp = Long.compare(o2.fileLength, o1.fileLength);
279 if (cmp != 0) {
280 return cmp;
281 }
282
283 return System.identityHashCode(o1) - System.identityHashCode(o2);
284 }
285 };
286
287 private static class HFileDeleteTask {
288 private static final long MAX_WAIT = 60 * 1000L;
289 private static final long WAIT_UNIT = 1000L;
290
291 boolean done = false;
292 boolean result;
293 final Path filePath;
294 final long fileLength;
295
296 public HFileDeleteTask(FileStatus file) {
297 this.filePath = file.getPath();
298 this.fileLength = file.getLen();
299 }
300
301 public synchronized void setResult(boolean result) {
302 this.done = true;
303 this.result = result;
304 notify();
305 }
306
307 public synchronized boolean getResult() {
308 long waitTime = 0;
309 try {
310 while (!done) {
311 wait(WAIT_UNIT);
312 waitTime += WAIT_UNIT;
313 if (done) {
314 return this.result;
315 }
316 if (waitTime > MAX_WAIT) {
317 LOG.warn("Wait more than " + MAX_WAIT + " ms for deleting " + this.filePath
318 + ", exit...");
319 return false;
320 }
321 }
322 } catch (InterruptedException e) {
323 LOG.warn("Interrupted while waiting for result of deleting " + filePath
324 + ", will return false", e);
325 return false;
326 }
327 return this.result;
328 }
329
330 @Override
331 public boolean equals(Object o) {
332 if (this == o) {
333 return true;
334 }
335 if (o == null || !(o instanceof HFileDeleteTask)) {
336 return false;
337 }
338 HFileDeleteTask otherTask = (HFileDeleteTask) o;
339 return this.filePath.equals(otherTask.filePath) && (this.fileLength == otherTask.fileLength);
340 }
341
342 @Override
343 public int hashCode() {
344 return filePath.hashCode();
345 }
346 }
347
348 @VisibleForTesting
349 public List<Thread> getCleanerThreads() {
350 return threads;
351 }
352
353 @VisibleForTesting
354 public long getNumOfDeletedLargeFiles() {
355 return deletedLargeFiles;
356 }
357
358 @VisibleForTesting
359 public long getNumOfDeletedSmallFiles() {
360 return deletedSmallFiles;
361 }
362
363 @VisibleForTesting
364 public long getLargeQueueInitSize() {
365 return largeQueueInitSize;
366 }
367
368 @VisibleForTesting
369 public long getSmallQueueInitSize() {
370 return smallQueueInitSize;
371 }
372
373 @VisibleForTesting
374 public long getThrottlePoint() {
375 return throttlePoint;
376 }
377
378
379 public void onConfigurationChange(Configuration conf) {
380 StringBuilder builder = new StringBuilder();
381 builder.append("Updating configuration for HFileCleaner, previous throttle point: ")
382 .append(throttlePoint).append(", largeQueueInitSize: ").append(largeQueueInitSize)
383 .append(", smallQueueInitSize: ").append(smallQueueInitSize);
384 stopHFileDeleteThreads();
385 this.throttlePoint =
386 conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
387 this.largeQueueInitSize =
388 conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
389 this.smallQueueInitSize =
390 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
391
392 List<HFileDeleteTask> leftOverTasks = new ArrayList<>();
393 for (HFileDeleteTask task : largeFileQueue) {
394 leftOverTasks.add(task);
395 }
396 for (HFileDeleteTask task : smallFileQueue) {
397 leftOverTasks.add(task);
398 }
399 largeFileQueue = new StealJobQueue<HFileDeleteTask>(largeQueueInitSize, smallQueueInitSize,
400 COMPARATOR);
401 smallFileQueue = largeFileQueue.getStealFromQueue();
402 threads.clear();
403 builder.append("; new throttle point: ").append(throttlePoint).append(", largeQueueInitSize: ")
404 .append(largeQueueInitSize).append(", smallQueueInitSize: ").append(smallQueueInitSize);
405 LOG.debug(builder.toString());
406 startHFileDeleteThreads();
407
408 for (HFileDeleteTask task : leftOverTasks) {
409 dispatch(task);
410 }
411 }
412 }