1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.io.PrintWriter;
25 import java.io.StringWriter;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.SortedMap;
41 import java.util.SortedSet;
42 import java.util.TreeMap;
43 import java.util.TreeSet;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.ConcurrentSkipListMap;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.FutureTask;
51 import java.util.concurrent.ScheduledThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.TimeoutException;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.concurrent.atomic.AtomicInteger;
56
57 import org.apache.commons.io.IOUtils;
58 import org.apache.commons.lang.StringUtils;
59 import org.apache.commons.logging.Log;
60 import org.apache.commons.logging.LogFactory;
61 import org.apache.hadoop.conf.Configuration;
62 import org.apache.hadoop.conf.Configured;
63 import org.apache.hadoop.fs.FSDataOutputStream;
64 import org.apache.hadoop.fs.FileStatus;
65 import org.apache.hadoop.fs.FileSystem;
66 import org.apache.hadoop.fs.Path;
67 import org.apache.hadoop.fs.permission.FsAction;
68 import org.apache.hadoop.fs.permission.FsPermission;
69 import org.apache.hadoop.hbase.Abortable;
70 import org.apache.hadoop.hbase.Cell;
71 import org.apache.hadoop.hbase.ClusterStatus;
72 import org.apache.hadoop.hbase.CoordinatedStateException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75 import org.apache.hadoop.hbase.HColumnDescriptor;
76 import org.apache.hadoop.hbase.HConstants;
77 import org.apache.hadoop.hbase.HRegionInfo;
78 import org.apache.hadoop.hbase.HRegionLocation;
79 import org.apache.hadoop.hbase.HTableDescriptor;
80 import org.apache.hadoop.hbase.KeyValue;
81 import org.apache.hadoop.hbase.MasterNotRunningException;
82 import org.apache.hadoop.hbase.MetaTableAccessor;
83 import org.apache.hadoop.hbase.RegionLocations;
84 import org.apache.hadoop.hbase.ServerName;
85 import org.apache.hadoop.hbase.TableName;
86 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
87 import org.apache.hadoop.hbase.classification.InterfaceAudience;
88 import org.apache.hadoop.hbase.classification.InterfaceStability;
89 import org.apache.hadoop.hbase.client.Admin;
90 import org.apache.hadoop.hbase.client.ClusterConnection;
91 import org.apache.hadoop.hbase.client.ConnectionFactory;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Get;
94 import org.apache.hadoop.hbase.client.HBaseAdmin;
95 import org.apache.hadoop.hbase.client.HConnectable;
96 import org.apache.hadoop.hbase.client.HConnection;
97 import org.apache.hadoop.hbase.client.HConnectionManager;
98 import org.apache.hadoop.hbase.client.MetaScanner;
99 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.RowMutations;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116 import org.apache.hadoop.hbase.security.AccessDeniedException;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
123 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
124 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
125 import org.apache.hadoop.hbase.wal.WALSplitter;
126 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
127 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
128 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
129 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
130 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
131 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
132 import org.apache.hadoop.ipc.RemoteException;
133 import org.apache.hadoop.security.UserGroupInformation;
134 import org.apache.hadoop.util.ReflectionUtils;
135 import org.apache.hadoop.util.Tool;
136 import org.apache.hadoop.util.ToolRunner;
137 import org.apache.zookeeper.KeeperException;
138
139 import com.google.common.base.Joiner;
140 import com.google.common.base.Preconditions;
141 import com.google.common.collect.ImmutableList;
142 import com.google.common.collect.Lists;
143 import com.google.common.collect.Multimap;
144 import com.google.common.collect.Ordering;
145 import com.google.common.collect.TreeMultimap;
146 import com.google.protobuf.ServiceException;
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
194 @InterfaceStability.Evolving
195 public class HBaseFsck extends Configured implements Closeable {
196 public static final long DEFAULT_TIME_LAG = 60000;
197 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
198 private static final int MAX_NUM_THREADS = 50;
199 private static boolean rsSupportsOffline = true;
200 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
201 private static final int DEFAULT_MAX_MERGE = 5;
202 private static final String TO_BE_LOADED = "to_be_loaded";
203 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
204 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
205 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200;
206 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000;
207
208
209
210
211 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80;
212
213
214
215
216 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
217 private ClusterStatus status;
218 private ClusterConnection connection;
219 private Admin admin;
220 private Table meta;
221
222 protected ExecutorService executor;
223 private long startMillis = EnvironmentEdgeManager.currentTime();
224 private HFileCorruptionChecker hfcc;
225 private int retcode = 0;
226 private Path HBCK_LOCK_PATH;
227 private FSDataOutputStream hbckOutFd;
228
229
230
231 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
232
233
234
235
236 private static boolean details = false;
237 private long timelag = DEFAULT_TIME_LAG;
238 private static boolean forceExclusive = false;
239 private static boolean disableBalancer = false;
240 private static boolean disableSplitAndMerge = false;
241 private boolean fixAssignments = false;
242 private boolean fixMeta = false;
243 private boolean checkHdfs = true;
244 private boolean fixHdfsHoles = false;
245 private boolean fixHdfsOverlaps = false;
246 private boolean fixHdfsOrphans = false;
247 private boolean fixTableOrphans = false;
248 private boolean fixVersionFile = false;
249 private boolean fixSplitParents = false;
250 private boolean removeParents = false;
251 private boolean fixReferenceFiles = false;
252 private boolean fixEmptyMetaCells = false;
253 private boolean fixTableLocks = false;
254 private boolean fixTableZNodes = false;
255 private boolean fixReplication = false;
256 private boolean fixAny = false;
257
258
259
260 private Set<TableName> tablesIncluded = new HashSet<TableName>();
261 private int maxMerge = DEFAULT_MAX_MERGE;
262 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
263 private boolean sidelineBigOverlaps = false;
264 private Path sidelineDir = null;
265
266 private boolean rerun = false;
267 private static boolean SUMMARY = false;
268 private boolean checkMetaOnly = false;
269 private boolean checkRegionBoundaries = false;
270 private boolean ignorePreCheckPermission = false;
271
272
273
274
275 final private ErrorReporter errors;
276 int fixes = 0;
277
278
279
280
281
282
283 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
284 private TreeSet<TableName> disabledTables =
285 new TreeSet<TableName>();
286
287 private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
288
289
290
291
292
293
294
295
296
297
298
299 private SortedMap<TableName, TableInfo> tablesInfo =
300 new ConcurrentSkipListMap<TableName, TableInfo>();
301
302
303
304
305 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
306
307 private Map<TableName, Set<String>> orphanTableDirs =
308 new HashMap<TableName, Set<String>>();
309
310 private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
311
312
313
314
315 private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
316 private final RetryCounterFactory lockFileRetryCounterFactory;
317
318
319
320
321
322
323
324
325
326 public HBaseFsck(Configuration conf) throws MasterNotRunningException,
327 ZooKeeperConnectionException, IOException, ClassNotFoundException {
328 this(conf, createThreadPool(conf));
329 }
330
331 private static ExecutorService createThreadPool(Configuration conf) {
332 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
333 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
334 }
335
336
337
338
339
340
341
342
343
344
345
346 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
347 ZooKeeperConnectionException, IOException, ClassNotFoundException {
348 super(conf);
349 errors = getErrorReporter(getConf());
350 this.executor = exec;
351 lockFileRetryCounterFactory = new RetryCounterFactory(
352 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
353 getConf().getInt(
354 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
355 getConf().getInt(
356 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
357 }
358
359 private class FileLockCallable implements Callable<FSDataOutputStream> {
360 RetryCounter retryCounter;
361
362 public FileLockCallable(RetryCounter retryCounter) {
363 this.retryCounter = retryCounter;
364 }
365 @Override
366 public FSDataOutputStream call() throws IOException {
367 try {
368 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
369 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
370 HConstants.DATA_FILE_UMASK_KEY);
371 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
372 fs.mkdirs(tmpDir);
373 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
374 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
375 out.writeBytes(InetAddress.getLocalHost().toString());
376 out.flush();
377 return out;
378 } catch(RemoteException e) {
379 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
380 return null;
381 } else {
382 throw e;
383 }
384 }
385 }
386
387 private FSDataOutputStream createFileWithRetries(final FileSystem fs,
388 final Path hbckLockFilePath, final FsPermission defaultPerms)
389 throws IOException {
390
391 IOException exception = null;
392 do {
393 try {
394 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
395 } catch (IOException ioe) {
396 LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
397 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
398 + retryCounter.getMaxAttempts());
399 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
400 ioe);
401 try {
402 exception = ioe;
403 retryCounter.sleepUntilNextRetry();
404 } catch (InterruptedException ie) {
405 throw (InterruptedIOException) new InterruptedIOException(
406 "Can't create lock file " + hbckLockFilePath.getName())
407 .initCause(ie);
408 }
409 }
410 } while (retryCounter.shouldRetry());
411
412 throw exception;
413 }
414 }
415
416
417
418
419
420
421
422 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
423 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
424 FileLockCallable callable = new FileLockCallable(retryCounter);
425 ExecutorService executor = Executors.newFixedThreadPool(1);
426 FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
427 executor.execute(futureTask);
428 final int timeoutInSeconds = getConf().getInt(
429 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
430 FSDataOutputStream stream = null;
431 try {
432 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
433 } catch (ExecutionException ee) {
434 LOG.warn("Encountered exception when opening lock file", ee);
435 } catch (InterruptedException ie) {
436 LOG.warn("Interrupted when opening lock file", ie);
437 Thread.currentThread().interrupt();
438 } catch (TimeoutException exception) {
439
440 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
441 futureTask.cancel(true);
442 } finally {
443 executor.shutdownNow();
444 }
445 return stream;
446 }
447
448 private void unlockHbck() {
449 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
450 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
451 do {
452 try {
453 IOUtils.closeQuietly(hbckOutFd);
454 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
455 HBCK_LOCK_PATH, true);
456 LOG.info("Finishing hbck");
457 return;
458 } catch (IOException ioe) {
459 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
460 + (retryCounter.getAttemptTimes() + 1) + " of "
461 + retryCounter.getMaxAttempts());
462 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
463 try {
464 retryCounter.sleepUntilNextRetry();
465 } catch (InterruptedException ie) {
466 Thread.currentThread().interrupt();
467 LOG.warn("Interrupted while deleting lock file" +
468 HBCK_LOCK_PATH);
469 return;
470 }
471 }
472 } while (retryCounter.shouldRetry());
473 }
474 }
475
476
477
478
479
480 public void connect() throws IOException {
481
482 if (isExclusive()) {
483
484 hbckOutFd = checkAndMarkRunningHbck();
485 if (hbckOutFd == null) {
486 setRetCode(-1);
487 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
488 "[If you are sure no other instance is running, delete the lock file " +
489 HBCK_LOCK_PATH + " and rerun the tool]");
490 throw new IOException("Duplicate hbck - Abort");
491 }
492
493
494 hbckLockCleanup.set(true);
495 }
496
497
498
499
500
501 Runtime.getRuntime().addShutdownHook(new Thread() {
502 @Override
503 public void run() {
504 IOUtils.closeQuietly(HBaseFsck.this);
505 unlockHbck();
506 }
507 });
508
509 LOG.info("Launching hbck");
510
511 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
512 admin = connection.getAdmin();
513 meta = connection.getTable(TableName.META_TABLE_NAME);
514 status = admin.getClusterStatus();
515 }
516
517
518
519
520 private void loadDeployedRegions() throws IOException, InterruptedException {
521
522 Collection<ServerName> regionServers = status.getServers();
523 errors.print("Number of live region servers: " + regionServers.size());
524 if (details) {
525 for (ServerName rsinfo: regionServers) {
526 errors.print(" " + rsinfo.getServerName());
527 }
528 }
529
530
531 Collection<ServerName> deadRegionServers = status.getDeadServerNames();
532 errors.print("Number of dead region servers: " + deadRegionServers.size());
533 if (details) {
534 for (ServerName name: deadRegionServers) {
535 errors.print(" " + name);
536 }
537 }
538
539
540 errors.print("Master: " + status.getMaster());
541
542
543 Collection<ServerName> backupMasters = status.getBackupMasters();
544 errors.print("Number of backup masters: " + backupMasters.size());
545 if (details) {
546 for (ServerName name: backupMasters) {
547 errors.print(" " + name);
548 }
549 }
550
551 errors.print("Average load: " + status.getAverageLoad());
552 errors.print("Number of requests: " + status.getRequestsCount());
553 errors.print("Number of regions: " + status.getRegionsCount());
554
555 Map<String, RegionState> rits = status.getRegionsInTransition();
556 errors.print("Number of regions in transition: " + rits.size());
557 if (details) {
558 for (RegionState state: rits.values()) {
559 errors.print(" " + state.toDescriptiveString());
560 }
561 }
562
563
564 processRegionServers(regionServers);
565 }
566
567
568
569
570 private void clearState() {
571
572 fixes = 0;
573 regionInfoMap.clear();
574 emptyRegionInfoQualifiers.clear();
575 disabledTables.clear();
576 errors.clear();
577 tablesInfo.clear();
578 orphanHdfsDirs.clear();
579 skippedRegions.clear();
580 }
581
582
583
584
585
586
587 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
588
589 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
590 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
591 LOG.info("Loading regioninfos HDFS");
592
593 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
594 int curIter = 0;
595 do {
596 clearState();
597
598 restoreHdfsIntegrity();
599 curIter++;
600 } while (fixes > 0 && curIter <= maxIterations);
601
602
603
604 if (curIter > 2) {
605 if (curIter == maxIterations) {
606 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
607 + "Tables integrity may not be fully repaired!");
608 } else {
609 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
610 }
611 }
612 }
613 }
614
615
616
617
618
619
620
621
622
623 public int onlineConsistencyRepair() throws IOException, KeeperException,
624 InterruptedException {
625 clearState();
626
627
628 loadDeployedRegions();
629
630 recordMetaRegion();
631
632 if (!checkMetaRegion()) {
633 String errorMsg = "hbase:meta table is not consistent. ";
634 if (shouldFixAssignments()) {
635 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
636 } else {
637 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
638 }
639 errors.reportError(errorMsg + " Exiting...");
640 return -2;
641 }
642
643 LOG.info("Loading regionsinfo from the hbase:meta table");
644 boolean success = loadMetaEntries();
645 if (!success) return -1;
646
647
648 reportEmptyMetaCells();
649
650
651 if (shouldFixEmptyMetaCells()) {
652 fixEmptyMetaCells();
653 }
654
655
656 if (!checkMetaOnly) {
657 reportTablesInFlux();
658 }
659
660
661 if (shouldCheckHdfs()) {
662 LOG.info("Loading region directories from HDFS");
663 loadHdfsRegionDirs();
664 LOG.info("Loading region information from HDFS");
665 loadHdfsRegionInfos();
666 }
667
668
669 loadDisabledTables();
670
671
672 fixOrphanTables();
673
674 LOG.info("Checking and fixing region consistency");
675
676 checkAndFixConsistency();
677
678
679 checkIntegrity();
680 return errors.getErrorList().size();
681 }
682
683
684
685
686
687 public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
688
689 errors.print("Version: " + status.getHBaseVersion());
690 offlineHdfsIntegrityRepair();
691
692 boolean oldBalancer = false;
693 if (shouldDisableBalancer()) {
694 oldBalancer = admin.setBalancerRunning(false, true);
695 }
696 boolean[] oldSplitAndMerge = null;
697 if (shouldDisableSplitAndMerge()) {
698 oldSplitAndMerge = admin.setSplitOrMergeEnabled(false, false,
699 Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
700 }
701
702 try {
703 onlineConsistencyRepair();
704 }
705 finally {
706
707
708
709 if (shouldDisableBalancer() && oldBalancer) {
710 admin.setBalancerRunning(oldBalancer, false);
711 }
712
713 if (shouldDisableSplitAndMerge()) {
714 if (oldSplitAndMerge != null) {
715 if (oldSplitAndMerge[0] && oldSplitAndMerge[1]) {
716 admin.setSplitOrMergeEnabled(true, false,
717 Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
718 } else if (oldSplitAndMerge[0]) {
719 admin.setSplitOrMergeEnabled(true, false, Admin.MasterSwitchType.SPLIT);
720 } else if (oldSplitAndMerge[1]) {
721 admin.setSplitOrMergeEnabled(true, false, Admin.MasterSwitchType.MERGE);
722 }
723 }
724 }
725 }
726
727 if (checkRegionBoundaries) {
728 checkRegionBoundaries();
729 }
730
731 offlineReferenceFileRepair();
732
733 checkAndFixTableLocks();
734
735
736 checkAndFixOrphanedTableZNodes();
737
738 checkAndFixReplication();
739
740
741 unlockHbck();
742
743
744 printTableSummary(tablesInfo);
745 return errors.summarize();
746 }
747
748 public static byte[] keyOnly (byte[] b) {
749 if (b == null)
750 return b;
751 int rowlength = Bytes.toShort(b, 0);
752 byte[] result = new byte[rowlength];
753 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
754 return result;
755 }
756
757 @Override
758 public void close() throws IOException {
759 IOUtils.closeQuietly(admin);
760 IOUtils.closeQuietly(meta);
761 IOUtils.closeQuietly(connection);
762 }
763
764 private static class RegionBoundariesInformation {
765 public byte [] regionName;
766 public byte [] metaFirstKey;
767 public byte [] metaLastKey;
768 public byte [] storesFirstKey;
769 public byte [] storesLastKey;
770 @Override
771 public String toString () {
772 return "regionName=" + Bytes.toStringBinary(regionName) +
773 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
774 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
775 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
776 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
777 }
778 }
779
780 public void checkRegionBoundaries() {
781 try {
782 ByteArrayComparator comparator = new ByteArrayComparator();
783 List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
784 final RegionBoundariesInformation currentRegionBoundariesInformation =
785 new RegionBoundariesInformation();
786 Path hbaseRoot = FSUtils.getRootDir(getConf());
787 for (HRegionInfo regionInfo : regions) {
788 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
789 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
790
791
792 Path path = new Path(tableDir, regionInfo.getEncodedName());
793 FileSystem fs = path.getFileSystem(getConf());
794 FileStatus[] files = fs.listStatus(path);
795
796 byte[] storeFirstKey = null;
797 byte[] storeLastKey = null;
798 for (FileStatus file : files) {
799 String fileName = file.getPath().toString();
800 fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
801 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
802 FileStatus[] storeFiles = fs.listStatus(file.getPath());
803
804 for (FileStatus storeFile : storeFiles) {
805 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
806 getConf()), getConf());
807 if ((reader.getFirstKey() != null)
808 && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
809 reader.getFirstKey()) > 0))) {
810 storeFirstKey = reader.getFirstKey();
811 }
812 if ((reader.getLastKey() != null)
813 && ((storeLastKey == null) || (comparator.compare(storeLastKey,
814 reader.getLastKey())) < 0)) {
815 storeLastKey = reader.getLastKey();
816 }
817 reader.close();
818 }
819 }
820 }
821 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
822 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
823 currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
824 currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
825 if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
826 currentRegionBoundariesInformation.metaFirstKey = null;
827 if (currentRegionBoundariesInformation.metaLastKey.length == 0)
828 currentRegionBoundariesInformation.metaLastKey = null;
829
830
831
832
833
834
835 boolean valid = true;
836
837 if ((currentRegionBoundariesInformation.storesFirstKey != null)
838 && (currentRegionBoundariesInformation.metaFirstKey != null)) {
839 valid = valid
840 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
841 currentRegionBoundariesInformation.metaFirstKey) >= 0;
842 }
843
844 if ((currentRegionBoundariesInformation.storesLastKey != null)
845 && (currentRegionBoundariesInformation.metaLastKey != null)) {
846 valid = valid
847 && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
848 currentRegionBoundariesInformation.metaLastKey) < 0;
849 }
850 if (!valid) {
851 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
852 tablesInfo.get(regionInfo.getTable()));
853 LOG.warn("Region's boundaries not aligned between stores and META for:");
854 LOG.warn(currentRegionBoundariesInformation);
855 }
856 }
857 } catch (IOException e) {
858 LOG.error(e);
859 }
860 }
861
862
863
864
865 private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
866 for (HbckInfo hi : orphanHdfsDirs) {
867 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
868 adoptHdfsOrphan(hi);
869 }
870 }
871
872
873
874
875
876
877
878
879
880
881 @SuppressWarnings("deprecation")
882 private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
883 Path p = hi.getHdfsRegionDir();
884 FileSystem fs = p.getFileSystem(getConf());
885 FileStatus[] dirs = fs.listStatus(p);
886 if (dirs == null) {
887 LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
888 p + ". This dir could probably be deleted.");
889 return ;
890 }
891
892 TableName tableName = hi.getTableName();
893 TableInfo tableInfo = tablesInfo.get(tableName);
894 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
895 HTableDescriptor template = tableInfo.getHTD();
896
897
898 Pair<byte[],byte[]> orphanRegionRange = null;
899 for (FileStatus cf : dirs) {
900 String cfName= cf.getPath().getName();
901
902 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
903
904 FileStatus[] hfiles = fs.listStatus(cf.getPath());
905 for (FileStatus hfile : hfiles) {
906 byte[] start, end;
907 HFile.Reader hf = null;
908 try {
909 CacheConfig cacheConf = new CacheConfig(getConf());
910 hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
911 hf.loadFileInfo();
912 KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
913 start = startKv.getRow();
914 KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
915 end = endKv.getRow();
916 } catch (IOException ioe) {
917 LOG.warn("Problem reading orphan file " + hfile + ", skipping");
918 continue;
919 } catch (NullPointerException ioe) {
920 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
921 continue;
922 } finally {
923 if (hf != null) {
924 hf.close();
925 }
926 }
927
928
929 if (orphanRegionRange == null) {
930
931 orphanRegionRange = new Pair<byte[], byte[]>(start, end);
932 } else {
933
934
935
936 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
937 orphanRegionRange.setFirst(start);
938 }
939 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
940 orphanRegionRange.setSecond(end);
941 }
942 }
943 }
944 }
945 if (orphanRegionRange == null) {
946 LOG.warn("No data in dir " + p + ", sidelining data");
947 fixes++;
948 sidelineRegionDir(fs, hi);
949 return;
950 }
951 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
952 Bytes.toString(orphanRegionRange.getSecond()) + ")");
953
954
955 HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(),
956 Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
957 LOG.info("Creating new region : " + hri);
958 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
959 Path target = region.getRegionFileSystem().getRegionDir();
960
961
962 mergeRegionDirs(target, hi);
963 fixes++;
964 }
965
966
967
968
969
970
971
972
973
974 private int restoreHdfsIntegrity() throws IOException, InterruptedException {
975
976 LOG.info("Loading HBase regioninfo from HDFS...");
977 loadHdfsRegionDirs();
978
979 int errs = errors.getErrorList().size();
980
981 tablesInfo = loadHdfsRegionInfos();
982 checkHdfsIntegrity(false, false);
983
984 if (errors.getErrorList().size() == errs) {
985 LOG.info("No integrity errors. We are done with this phase. Glorious.");
986 return 0;
987 }
988
989 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
990 adoptHdfsOrphans(orphanHdfsDirs);
991
992 }
993
994
995 if (shouldFixHdfsHoles()) {
996 clearState();
997 loadHdfsRegionDirs();
998 tablesInfo = loadHdfsRegionInfos();
999 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1000 }
1001
1002
1003 if (shouldFixHdfsOverlaps()) {
1004
1005 clearState();
1006 loadHdfsRegionDirs();
1007 tablesInfo = loadHdfsRegionInfos();
1008 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1009 }
1010
1011 return errors.getErrorList().size();
1012 }
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022 private void offlineReferenceFileRepair() throws IOException {
1023 Configuration conf = getConf();
1024 Path hbaseRoot = FSUtils.getRootDir(conf);
1025 FileSystem fs = hbaseRoot.getFileSystem(conf);
1026 LOG.info("Computing mapping of all store files");
1027 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1028 errors.print("");
1029 LOG.info("Validating mapping using HDFS state");
1030 for (Path path: allFiles.values()) {
1031 boolean isReference = false;
1032 try {
1033 isReference = StoreFileInfo.isReference(path);
1034 } catch (Throwable t) {
1035
1036
1037
1038
1039 }
1040 if (!isReference) continue;
1041
1042 Path referredToFile = StoreFileInfo.getReferredToFile(path);
1043 if (fs.exists(referredToFile)) continue;
1044
1045
1046 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1047 "Found lingering reference file " + path);
1048 if (!shouldFixReferenceFiles()) continue;
1049
1050
1051 boolean success = false;
1052 String pathStr = path.toString();
1053
1054
1055
1056
1057
1058 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1059 for (int i = 0; index > 0 && i < 5; i++) {
1060 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1061 }
1062 if (index > 0) {
1063 Path rootDir = getSidelineDir();
1064 Path dst = new Path(rootDir, pathStr.substring(index + 1));
1065 fs.mkdirs(dst.getParent());
1066 LOG.info("Trying to sideline reference file "
1067 + path + " to " + dst);
1068 setShouldRerun();
1069
1070 success = fs.rename(path, dst);
1071 debugLsr(dst);
1072
1073 }
1074 if (!success) {
1075 LOG.error("Failed to sideline reference file " + path);
1076 }
1077 }
1078 }
1079
1080
1081
1082
1083 private void reportEmptyMetaCells() {
1084 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1085 emptyRegionInfoQualifiers.size());
1086 if (details) {
1087 for (Result r: emptyRegionInfoQualifiers) {
1088 errors.print(" " + r);
1089 }
1090 }
1091 }
1092
1093
1094
1095
1096 private void reportTablesInFlux() {
1097 AtomicInteger numSkipped = new AtomicInteger(0);
1098 HTableDescriptor[] allTables = getTables(numSkipped);
1099 errors.print("Number of Tables: " + allTables.length);
1100 if (details) {
1101 if (numSkipped.get() > 0) {
1102 errors.detail("Number of Tables in flux: " + numSkipped.get());
1103 }
1104 for (HTableDescriptor td : allTables) {
1105 errors.detail(" Table: " + td.getTableName() + "\t" +
1106 (td.isReadOnly() ? "ro" : "rw") + "\t" +
1107 (td.isMetaRegion() ? "META" : " ") + "\t" +
1108 " families: " + td.getFamilies().size());
1109 }
1110 }
1111 }
1112
1113 public ErrorReporter getErrors() {
1114 return errors;
1115 }
1116
1117
1118
1119
1120
1121 private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1122 Path regionDir = hbi.getHdfsRegionDir();
1123 if (regionDir == null) {
1124 if (hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1125
1126 LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1127 }
1128 return;
1129 }
1130
1131 if (hbi.hdfsEntry.hri != null) {
1132
1133 return;
1134 }
1135
1136 FileSystem fs = FileSystem.get(getConf());
1137 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1138 LOG.debug("HRegionInfo read: " + hri.toString());
1139 hbi.hdfsEntry.hri = hri;
1140 }
1141
1142
1143
1144
1145
1146 public static class RegionRepairException extends IOException {
1147 private static final long serialVersionUID = 1L;
1148 final IOException ioe;
1149 public RegionRepairException(String s, IOException ioe) {
1150 super(s);
1151 this.ioe = ioe;
1152 }
1153 }
1154
1155
1156
1157
1158 private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1159 throws IOException, InterruptedException {
1160 tablesInfo.clear();
1161
1162 Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1163
1164
1165 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1166 List<Future<Void>> hbiFutures;
1167
1168 for (HbckInfo hbi : hbckInfos) {
1169 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1170 hbis.add(work);
1171 }
1172
1173
1174 hbiFutures = executor.invokeAll(hbis);
1175
1176 for(int i=0; i<hbiFutures.size(); i++) {
1177 WorkItemHdfsRegionInfo work = hbis.get(i);
1178 Future<Void> f = hbiFutures.get(i);
1179 try {
1180 f.get();
1181 } catch(ExecutionException e) {
1182 LOG.warn("Failed to read .regioninfo file for region " +
1183 work.hbi.getRegionNameAsString(), e.getCause());
1184 }
1185 }
1186
1187 Path hbaseRoot = FSUtils.getRootDir(getConf());
1188 FileSystem fs = hbaseRoot.getFileSystem(getConf());
1189
1190 for (HbckInfo hbi: hbckInfos) {
1191
1192 if (hbi.getHdfsHRI() == null) {
1193
1194 continue;
1195 }
1196
1197
1198
1199 TableName tableName = hbi.getTableName();
1200 if (tableName == null) {
1201
1202 LOG.warn("tableName was null for: " + hbi);
1203 continue;
1204 }
1205
1206 TableInfo modTInfo = tablesInfo.get(tableName);
1207 if (modTInfo == null) {
1208
1209 modTInfo = new TableInfo(tableName);
1210 tablesInfo.put(tableName, modTInfo);
1211 try {
1212 HTableDescriptor htd =
1213 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1214 modTInfo.htds.add(htd);
1215 } catch (IOException ioe) {
1216 if (!orphanTableDirs.containsKey(tableName)) {
1217 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1218
1219 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1220 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1221 Set<String> columns = new HashSet<String>();
1222 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1223 }
1224 }
1225 }
1226 if (!hbi.isSkipChecks()) {
1227 modTInfo.addRegionInfo(hbi);
1228 }
1229 }
1230
1231 loadTableInfosForTablesWithNoRegion();
1232 errors.print("");
1233
1234 return tablesInfo;
1235 }
1236
1237
1238
1239
1240
1241
1242
1243
1244 private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1245 Path regionDir = hbi.getHdfsRegionDir();
1246 FileSystem fs = regionDir.getFileSystem(getConf());
1247 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1248 for (FileStatus subdir : subDirs) {
1249 String columnfamily = subdir.getPath().getName();
1250 columns.add(columnfamily);
1251 }
1252 return columns;
1253 }
1254
1255
1256
1257
1258
1259
1260
1261
1262 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1263 Set<String> columns) throws IOException {
1264 if (columns ==null || columns.isEmpty()) return false;
1265 HTableDescriptor htd = new HTableDescriptor(tableName);
1266 for (String columnfamimly : columns) {
1267 htd.addFamily(new HColumnDescriptor(columnfamimly));
1268 }
1269 fstd.createTableDescriptor(htd, true);
1270 return true;
1271 }
1272
1273
1274
1275
1276
1277 public void fixEmptyMetaCells() throws IOException {
1278 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1279 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1280 for (Result region : emptyRegionInfoQualifiers) {
1281 deleteMetaRegion(region.getRow());
1282 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1283 }
1284 emptyRegionInfoQualifiers.clear();
1285 }
1286 }
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297 public void fixOrphanTables() throws IOException {
1298 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1299
1300 List<TableName> tmpList = new ArrayList<TableName>();
1301 tmpList.addAll(orphanTableDirs.keySet());
1302 HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1303 Iterator<Entry<TableName, Set<String>>> iter =
1304 orphanTableDirs.entrySet().iterator();
1305 int j = 0;
1306 int numFailedCase = 0;
1307 FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1308 while (iter.hasNext()) {
1309 Entry<TableName, Set<String>> entry =
1310 iter.next();
1311 TableName tableName = entry.getKey();
1312 LOG.info("Trying to fix orphan table error: " + tableName);
1313 if (j < htds.length) {
1314 if (tableName.equals(htds[j].getTableName())) {
1315 HTableDescriptor htd = htds[j];
1316 LOG.info("fixing orphan table: " + tableName + " from cache");
1317 fstd.createTableDescriptor(htd, true);
1318 j++;
1319 iter.remove();
1320 }
1321 } else {
1322 if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1323 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1324 LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1325 iter.remove();
1326 } else {
1327 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1328 numFailedCase++;
1329 }
1330 }
1331 fixes++;
1332 }
1333
1334 if (orphanTableDirs.isEmpty()) {
1335
1336
1337 setShouldRerun();
1338 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1339 } else if (numFailedCase > 0) {
1340 LOG.error("Failed to fix " + numFailedCase
1341 + " OrphanTables with default .tableinfo files");
1342 }
1343
1344 }
1345
1346 orphanTableDirs.clear();
1347
1348 }
1349
1350
1351
1352
1353
1354
1355 private HRegion createNewMeta() throws IOException {
1356 Path rootdir = FSUtils.getRootDir(getConf());
1357 Configuration c = getConf();
1358 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1359 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1360 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1361 HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1362 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1363 return meta;
1364 }
1365
1366
1367
1368
1369
1370
1371
1372 private ArrayList<Put> generatePuts(
1373 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1374 ArrayList<Put> puts = new ArrayList<Put>();
1375 boolean hasProblems = false;
1376 for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1377 TableName name = e.getKey();
1378
1379
1380 if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1381 continue;
1382 }
1383
1384 TableInfo ti = e.getValue();
1385 for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1386 .entrySet()) {
1387 Collection<HbckInfo> his = spl.getValue();
1388 int sz = his.size();
1389 if (sz != 1) {
1390
1391 LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1392 + " had " + sz + " regions instead of exactly 1." );
1393 hasProblems = true;
1394 continue;
1395 }
1396
1397
1398 HbckInfo hi = his.iterator().next();
1399 HRegionInfo hri = hi.getHdfsHRI();
1400 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1401 puts.add(p);
1402 }
1403 }
1404 return hasProblems ? null : puts;
1405 }
1406
1407
1408
1409
1410 private void suggestFixes(
1411 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1412 logParallelMerge();
1413 for (TableInfo tInfo : tablesInfo.values()) {
1414 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1415 tInfo.checkRegionChain(handler);
1416 }
1417 }
1418
1419
1420
1421
1422
1423
1424
1425
1426 public boolean rebuildMeta(boolean fix) throws IOException,
1427 InterruptedException {
1428
1429
1430
1431
1432
1433 LOG.info("Loading HBase regioninfo from HDFS...");
1434 loadHdfsRegionDirs();
1435
1436 int errs = errors.getErrorList().size();
1437 tablesInfo = loadHdfsRegionInfos();
1438 checkHdfsIntegrity(false, false);
1439
1440
1441 if (errors.getErrorList().size() != errs) {
1442
1443 while(true) {
1444 fixes = 0;
1445 suggestFixes(tablesInfo);
1446 errors.clear();
1447 loadHdfsRegionInfos();
1448 checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1449
1450 int errCount = errors.getErrorList().size();
1451
1452 if (fixes == 0) {
1453 if (errCount > 0) {
1454 return false;
1455 } else {
1456 break;
1457 }
1458 }
1459 }
1460 }
1461
1462
1463 LOG.info("HDFS regioninfo's seems good. Sidelining old hbase:meta");
1464 Path backupDir = sidelineOldMeta();
1465
1466 LOG.info("Creating new hbase:meta");
1467 HRegion meta = createNewMeta();
1468
1469
1470 List<Put> puts = generatePuts(tablesInfo);
1471 if (puts == null) {
1472 LOG.fatal("Problem encountered when creating new hbase:meta entries. " +
1473 "You may need to restore the previously sidelined hbase:meta");
1474 return false;
1475 }
1476 meta.batchMutate(puts.toArray(new Put[puts.size()]));
1477 HRegion.closeHRegion(meta);
1478 LOG.info("Success! hbase:meta table rebuilt.");
1479 LOG.info("Old hbase:meta is moved into " + backupDir);
1480 return true;
1481 }
1482
1483
1484
1485
1486 private void logParallelMerge() {
1487 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1488 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1489 " false to run serially.");
1490 } else {
1491 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
1492 " true to run in parallel.");
1493 }
1494 }
1495
1496 private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1497 boolean fixOverlaps) throws IOException {
1498 LOG.info("Checking HBase region split map from HDFS data...");
1499 logParallelMerge();
1500 for (TableInfo tInfo : tablesInfo.values()) {
1501 TableIntegrityErrorHandler handler;
1502 if (fixHoles || fixOverlaps) {
1503 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1504 fixHoles, fixOverlaps);
1505 } else {
1506 handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1507 }
1508 if (!tInfo.checkRegionChain(handler)) {
1509
1510 errors.report("Found inconsistency in table " + tInfo.getName());
1511 }
1512 }
1513 return tablesInfo;
1514 }
1515
1516 private Path getSidelineDir() throws IOException {
1517 if (sidelineDir == null) {
1518 Path hbaseDir = FSUtils.getRootDir(getConf());
1519 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1520 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1521 + startMillis);
1522 }
1523 return sidelineDir;
1524 }
1525
1526
1527
1528
1529 Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1530 return sidelineRegionDir(fs, null, hi);
1531 }
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541 Path sidelineRegionDir(FileSystem fs,
1542 String parentDir, HbckInfo hi) throws IOException {
1543 TableName tableName = hi.getTableName();
1544 Path regionDir = hi.getHdfsRegionDir();
1545
1546 if (!fs.exists(regionDir)) {
1547 LOG.warn("No previous " + regionDir + " exists. Continuing.");
1548 return null;
1549 }
1550
1551 Path rootDir = getSidelineDir();
1552 if (parentDir != null) {
1553 rootDir = new Path(rootDir, parentDir);
1554 }
1555 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1556 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1557 fs.mkdirs(sidelineRegionDir);
1558 boolean success = false;
1559 FileStatus[] cfs = fs.listStatus(regionDir);
1560 if (cfs == null) {
1561 LOG.info("Region dir is empty: " + regionDir);
1562 } else {
1563 for (FileStatus cf : cfs) {
1564 Path src = cf.getPath();
1565 Path dst = new Path(sidelineRegionDir, src.getName());
1566 if (fs.isFile(src)) {
1567
1568 success = fs.rename(src, dst);
1569 if (!success) {
1570 String msg = "Unable to rename file " + src + " to " + dst;
1571 LOG.error(msg);
1572 throw new IOException(msg);
1573 }
1574 continue;
1575 }
1576
1577
1578 fs.mkdirs(dst);
1579
1580 LOG.info("Sidelining files from " + src + " into containing region " + dst);
1581
1582
1583
1584
1585 FileStatus[] hfiles = fs.listStatus(src);
1586 if (hfiles != null && hfiles.length > 0) {
1587 for (FileStatus hfile : hfiles) {
1588 success = fs.rename(hfile.getPath(), dst);
1589 if (!success) {
1590 String msg = "Unable to rename file " + src + " to " + dst;
1591 LOG.error(msg);
1592 throw new IOException(msg);
1593 }
1594 }
1595 }
1596 LOG.debug("Sideline directory contents:");
1597 debugLsr(sidelineRegionDir);
1598 }
1599 }
1600
1601 LOG.info("Removing old region dir: " + regionDir);
1602 success = fs.delete(regionDir, true);
1603 if (!success) {
1604 String msg = "Unable to delete dir " + regionDir;
1605 LOG.error(msg);
1606 throw new IOException(msg);
1607 }
1608 return sidelineRegionDir;
1609 }
1610
1611
1612
1613
1614 void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1615 Path backupHbaseDir) throws IOException {
1616 Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1617 if (fs.exists(tableDir)) {
1618 Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1619 fs.mkdirs(backupTableDir.getParent());
1620 boolean success = fs.rename(tableDir, backupTableDir);
1621 if (!success) {
1622 throw new IOException("Failed to move " + tableName + " from "
1623 + tableDir + " to " + backupTableDir);
1624 }
1625 } else {
1626 LOG.info("No previous " + tableName + " exists. Continuing.");
1627 }
1628 }
1629
1630
1631
1632
1633 Path sidelineOldMeta() throws IOException {
1634
1635 Path hbaseDir = FSUtils.getRootDir(getConf());
1636 FileSystem fs = hbaseDir.getFileSystem(getConf());
1637 Path backupDir = getSidelineDir();
1638 fs.mkdirs(backupDir);
1639
1640 try {
1641 sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1642 } catch (IOException e) {
1643 LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore "
1644 + "try to rename hbase:meta in " + backupDir.getName() + " to "
1645 + hbaseDir.getName() + ".", e);
1646 throw e;
1647 }
1648 return backupDir;
1649 }
1650
1651
1652
1653
1654
1655
1656 private void loadDisabledTables()
1657 throws ZooKeeperConnectionException, IOException {
1658 HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1659 @Override
1660 public Void connect(HConnection connection) throws IOException {
1661 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1662 try {
1663 for (TableName tableName :
1664 ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1665 disabledTables.add(tableName);
1666 }
1667 } catch (KeeperException ke) {
1668 throw new IOException(ke);
1669 } catch (InterruptedException e) {
1670 throw new InterruptedIOException();
1671 } finally {
1672 zkw.close();
1673 }
1674 return null;
1675 }
1676 });
1677 }
1678
1679
1680
1681
1682 private boolean isTableDisabled(HRegionInfo regionInfo) {
1683 return disabledTables.contains(regionInfo.getTable());
1684 }
1685
1686
1687
1688
1689
1690 public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1691 Path rootDir = FSUtils.getRootDir(getConf());
1692 FileSystem fs = rootDir.getFileSystem(getConf());
1693
1694
1695 List<FileStatus> tableDirs = Lists.newArrayList();
1696
1697 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1698
1699 List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1700 for (Path path : paths) {
1701 TableName tableName = FSUtils.getTableName(path);
1702 if ((!checkMetaOnly &&
1703 isTableIncluded(tableName)) ||
1704 tableName.equals(TableName.META_TABLE_NAME)) {
1705 tableDirs.add(fs.getFileStatus(path));
1706 }
1707 }
1708
1709
1710 if (!foundVersionFile) {
1711 errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1712 "Version file does not exist in root dir " + rootDir);
1713 if (shouldFixVersionFile()) {
1714 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1715 + " file.");
1716 setShouldRerun();
1717 FSUtils.setVersion(fs, rootDir, getConf().getInt(
1718 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1719 HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1720 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1721 }
1722 }
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749 private boolean recordMetaRegion() throws IOException {
1750 RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1751 HConstants.EMPTY_START_ROW, false, false);
1752 if (rl == null) {
1753 errors.reportError(ERROR_CODE.NULL_META_REGION,
1754 "META region or some of its attributes are null.");
1755 return false;
1756 }
1757 for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1758
1759 if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1760 metaLocation.getHostname() == null) {
1761 errors.reportError(ERROR_CODE.NULL_META_REGION,
1762 "META region or some of its attributes are null.");
1763 return false;
1764 }
1765 ServerName sn = metaLocation.getServerName();
1766 MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1767 HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1768 if (hbckInfo == null) {
1769 regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1770 } else {
1771 hbckInfo.metaEntry = m;
1772 }
1773 }
1774 return true;
1775 }
1776
1777 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1778 return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1779 @Override
1780 public void abort(String why, Throwable e) {
1781 LOG.error(why, e);
1782 System.exit(1);
1783 }
1784
1785 @Override
1786 public boolean isAborted() {
1787 return false;
1788 }
1789
1790 });
1791 }
1792
1793 private ServerName getMetaRegionServerName(int replicaId)
1794 throws IOException, KeeperException {
1795 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1796 ServerName sn = null;
1797 try {
1798 sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1799 } finally {
1800 zkw.close();
1801 }
1802 return sn;
1803 }
1804
1805
1806
1807
1808
1809
1810 void processRegionServers(Collection<ServerName> regionServerList)
1811 throws IOException, InterruptedException {
1812
1813 List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1814 List<Future<Void>> workFutures;
1815
1816
1817 for (ServerName rsinfo: regionServerList) {
1818 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1819 }
1820
1821 workFutures = executor.invokeAll(workItems);
1822
1823 for(int i=0; i<workFutures.size(); i++) {
1824 WorkItemRegion item = workItems.get(i);
1825 Future<Void> f = workFutures.get(i);
1826 try {
1827 f.get();
1828 } catch(ExecutionException e) {
1829 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1830 e.getCause());
1831 }
1832 }
1833 }
1834
1835
1836
1837
1838 private void checkAndFixConsistency()
1839 throws IOException, KeeperException, InterruptedException {
1840
1841
1842 List<CheckRegionConsistencyWorkItem> workItems =
1843 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1844 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1845 if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1846 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1847 }
1848 }
1849 checkRegionConsistencyConcurrently(workItems);
1850
1851 boolean prevHdfsCheck = shouldCheckHdfs();
1852 setCheckHdfs(false);
1853
1854
1855 List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1856 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1857 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1858 if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1859 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1860 }
1861 }
1862 checkRegionConsistencyConcurrently(replicaWorkItems);
1863 setCheckHdfs(prevHdfsCheck);
1864
1865
1866
1867
1868
1869 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1870 int numOfSkippedRegions = skippedRegions.size();
1871 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1872 throw new IOException(numOfSkippedRegions
1873 + " region(s) could not be checked or repaired. See logs for detail.");
1874 }
1875 }
1876
1877
1878
1879
1880 private void checkRegionConsistencyConcurrently(
1881 final List<CheckRegionConsistencyWorkItem> workItems)
1882 throws IOException, KeeperException, InterruptedException {
1883 if (workItems.isEmpty()) {
1884 return;
1885 }
1886
1887 List<Future<Void>> workFutures = executor.invokeAll(workItems);
1888 for(Future<Void> f: workFutures) {
1889 try {
1890 f.get();
1891 } catch(ExecutionException e1) {
1892 LOG.warn("Could not check region consistency " , e1.getCause());
1893 if (e1.getCause() instanceof IOException) {
1894 throw (IOException)e1.getCause();
1895 } else if (e1.getCause() instanceof KeeperException) {
1896 throw (KeeperException)e1.getCause();
1897 } else if (e1.getCause() instanceof InterruptedException) {
1898 throw (InterruptedException)e1.getCause();
1899 } else {
1900 throw new IOException(e1.getCause());
1901 }
1902 }
1903 }
1904 }
1905
1906 class CheckRegionConsistencyWorkItem implements Callable<Void> {
1907 private final String key;
1908 private final HbckInfo hbi;
1909
1910 CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1911 this.key = key;
1912 this.hbi = hbi;
1913 }
1914
1915 @Override
1916 public synchronized Void call() throws Exception {
1917 try {
1918 checkRegionConsistency(key, hbi);
1919 } catch (Exception e) {
1920
1921
1922 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1923 + "'.", e);
1924 if (hbi.getHdfsHRI().isMetaRegion()) {
1925 throw e;
1926 }
1927 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1928 addSkippedRegion(hbi);
1929 }
1930 return null;
1931 }
1932 }
1933
1934 private void addSkippedRegion(final HbckInfo hbi) {
1935 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1936 if (skippedRegionNames == null) {
1937 skippedRegionNames = new HashSet<String>();
1938 }
1939 skippedRegionNames.add(hbi.getRegionNameAsString());
1940 skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1941 }
1942
1943 private void preCheckPermission() throws IOException, AccessDeniedException {
1944 if (shouldIgnorePreCheckPermission()) {
1945 return;
1946 }
1947
1948 Path hbaseDir = FSUtils.getRootDir(getConf());
1949 FileSystem fs = hbaseDir.getFileSystem(getConf());
1950 UserProvider userProvider = UserProvider.instantiate(getConf());
1951 UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1952 FileStatus[] files = fs.listStatus(hbaseDir);
1953 for (FileStatus file : files) {
1954 try {
1955 FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1956 } catch (AccessDeniedException ace) {
1957 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1958 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1959 + " does not have write perms to " + file.getPath()
1960 + ". Please rerun hbck as hdfs user " + file.getOwner());
1961 throw ace;
1962 }
1963 }
1964 }
1965
1966
1967
1968
1969 private void deleteMetaRegion(HbckInfo hi) throws IOException {
1970 deleteMetaRegion(hi.metaEntry.getRegionName());
1971 }
1972
1973
1974
1975
1976 private void deleteMetaRegion(byte[] metaKey) throws IOException {
1977 Delete d = new Delete(metaKey);
1978 meta.delete(d);
1979 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1980 }
1981
1982
1983
1984
1985 private void resetSplitParent(HbckInfo hi) throws IOException {
1986 RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1987 Delete d = new Delete(hi.metaEntry.getRegionName());
1988 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1989 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1990 mutations.add(d);
1991
1992 HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1993 hri.setOffline(false);
1994 hri.setSplit(false);
1995 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1996 mutations.add(p);
1997
1998 meta.mutateRow(mutations);
1999 LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2000 }
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010 private void offline(byte[] regionName) throws IOException {
2011 String regionString = Bytes.toStringBinary(regionName);
2012 if (!rsSupportsOffline) {
2013 LOG.warn("Using unassign region " + regionString
2014 + " instead of using offline method, you should"
2015 + " restart HMaster after these repairs");
2016 admin.unassign(regionName, true);
2017 return;
2018 }
2019
2020
2021 try {
2022 LOG.info("Offlining region " + regionString);
2023 admin.offline(regionName);
2024 } catch (IOException ioe) {
2025 String notFoundMsg = "java.lang.NoSuchMethodException: " +
2026 "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2027 if (ioe.getMessage().contains(notFoundMsg)) {
2028 LOG.warn("Using unassign region " + regionString
2029 + " instead of using offline method, you should"
2030 + " restart HMaster after these repairs");
2031 rsSupportsOffline = false;
2032 admin.unassign(regionName, true);
2033 return;
2034 }
2035 throw ioe;
2036 }
2037 }
2038
2039 private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2040 undeployRegionsForHbi(hi);
2041
2042 if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2043 return;
2044 }
2045 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2046 for (int i = 1; i < numReplicas; i++) {
2047 if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2048 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2049 hi.getPrimaryHRIForDeployedReplica(), i);
2050 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2051 if (h != null) {
2052 undeployRegionsForHbi(h);
2053
2054
2055 h.setSkipChecks(true);
2056 }
2057 }
2058 }
2059
2060 private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2061 for (OnlineEntry rse : hi.deployedEntries) {
2062 LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa);
2063 try {
2064 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2065 offline(rse.hri.getRegionName());
2066 } catch (IOException ioe) {
2067 LOG.warn("Got exception when attempting to offline region "
2068 + Bytes.toString(rse.hri.getRegionName()), ioe);
2069 }
2070 }
2071 }
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085 private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2086 if (hi.metaEntry == null && hi.hdfsEntry == null) {
2087 undeployRegions(hi);
2088 return;
2089 }
2090
2091
2092 Get get = new Get(hi.getRegionName());
2093 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2094 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2095 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2096
2097 if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2098 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2099 for (int i = 0; i < numReplicas; i++) {
2100 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2101 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2102 }
2103 }
2104 Result r = meta.get(get);
2105 RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2106 if (rl == null) {
2107 LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2108 " since meta does not have handle to reach it");
2109 return;
2110 }
2111 for (HRegionLocation h : rl.getRegionLocations()) {
2112 ServerName serverName = h.getServerName();
2113 if (serverName == null) {
2114 errors.reportError("Unable to close region "
2115 + hi.getRegionNameAsString() + " because meta does not "
2116 + "have handle to reach it.");
2117 continue;
2118 }
2119 HRegionInfo hri = h.getRegionInfo();
2120 if (hri == null) {
2121 LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2122 + " because hbase:meta had invalid or missing "
2123 + HConstants.CATALOG_FAMILY_STR + ":"
2124 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2125 + " qualifier value.");
2126 continue;
2127 }
2128
2129 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2130 }
2131 }
2132
2133 private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2134 KeeperException, InterruptedException {
2135
2136 if (shouldFixAssignments()) {
2137 errors.print(msg);
2138 undeployRegions(hbi);
2139 setShouldRerun();
2140 HRegionInfo hri = hbi.getHdfsHRI();
2141 if (hri == null) {
2142 hri = hbi.metaEntry;
2143 }
2144 HBaseFsckRepair.fixUnassigned(admin, hri);
2145 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2146
2147
2148 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2149 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2150 for (int i = 1; i < replicationCount; i++) {
2151 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2152 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2153 if (h != null) {
2154 undeployRegions(h);
2155
2156
2157 h.setSkipChecks(true);
2158 }
2159 HBaseFsckRepair.fixUnassigned(admin, hri);
2160 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2161 }
2162
2163 }
2164 }
2165
2166
2167
2168
2169 private void checkRegionConsistency(final String key, final HbckInfo hbi)
2170 throws IOException, KeeperException, InterruptedException {
2171
2172 if (hbi.isSkipChecks()) return;
2173 String descriptiveName = hbi.toString();
2174 boolean inMeta = hbi.metaEntry != null;
2175
2176 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2177 boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2178 boolean isDeployed = !hbi.deployedOn.isEmpty();
2179 boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2180 boolean deploymentMatchesMeta =
2181 hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2182 hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2183 boolean splitParent =
2184 (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2185 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2186 boolean recentlyModified = inHdfs &&
2187 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2188
2189
2190 if (hbi.containsOnlyHdfsEdits()) {
2191 return;
2192 }
2193 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2194 return;
2195 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2196 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2197 "tabled that is not deployed");
2198 return;
2199 } else if (recentlyModified) {
2200 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2201 return;
2202 }
2203
2204 else if (!inMeta && !inHdfs && !isDeployed) {
2205
2206 assert false : "Entry for region with no data";
2207 } else if (!inMeta && !inHdfs && isDeployed) {
2208 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2209 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2210 "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2211 if (shouldFixAssignments()) {
2212 undeployRegions(hbi);
2213 }
2214
2215 } else if (!inMeta && inHdfs && !isDeployed) {
2216 if (hbi.isMerged()) {
2217
2218
2219 hbi.setSkipChecks(true);
2220 LOG.info("Region " + descriptiveName
2221 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2222 return;
2223 }
2224 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2225 + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2226 "or deployed on any region server");
2227
2228 if (shouldFixMeta()) {
2229 if (!hbi.isHdfsRegioninfoPresent()) {
2230 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2231 + " in table integrity repair phase if -fixHdfsOrphans was" +
2232 " used.");
2233 return;
2234 }
2235
2236 HRegionInfo hri = hbi.getHdfsHRI();
2237 TableInfo tableInfo = tablesInfo.get(hri.getTable());
2238
2239 for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2240 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2241 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2242 hri.getEndKey()) >= 0)
2243 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2244 if(region.isSplit() || region.isOffline()) continue;
2245 Path regionDir = hbi.getHdfsRegionDir();
2246 FileSystem fs = regionDir.getFileSystem(getConf());
2247 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2248 for (Path familyDir : familyDirs) {
2249 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2250 for (Path referenceFilePath : referenceFilePaths) {
2251 Path parentRegionDir =
2252 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2253 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2254 LOG.warn(hri + " start and stop keys are in the range of " + region
2255 + ". The region might not be cleaned up from hdfs when region " + region
2256 + " split failed. Hence deleting from hdfs.");
2257 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2258 regionDir.getParent(), hri);
2259 return;
2260 }
2261 }
2262 }
2263 }
2264 }
2265
2266 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2267 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2268 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2269 admin.getClusterStatus().getServers(), numReplicas);
2270
2271 tryAssignmentRepair(hbi, "Trying to reassign region...");
2272 }
2273
2274 } else if (!inMeta && inHdfs && isDeployed) {
2275 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2276 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2277 debugLsr(hbi.getHdfsRegionDir());
2278 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2279
2280
2281
2282
2283 if (shouldFixAssignments()) {
2284 undeployRegionsForHbi(hbi);
2285 }
2286 }
2287 if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2288 if (!hbi.isHdfsRegioninfoPresent()) {
2289 LOG.error("This should have been repaired in table integrity repair phase");
2290 return;
2291 }
2292
2293 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2294 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2295 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2296 admin.getClusterStatus().getServers(), numReplicas);
2297 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2298 }
2299
2300
2301 } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2302
2303
2304 if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2305
2306 HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2307 HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2308 if (infoA != null && infoB != null) {
2309
2310 hbi.setSkipChecks(true);
2311 return;
2312 }
2313 }
2314 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2315 + descriptiveName + " is a split parent in META, in HDFS, "
2316 + "and not deployed on any region server. This could be transient, "
2317 + "consider to run the catalog janitor first!");
2318 if (shouldFixSplitParents()) {
2319 setShouldRerun();
2320 resetSplitParent(hbi);
2321 }
2322 } else if (inMeta && !inHdfs && !isDeployed) {
2323 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2324 + descriptiveName + " found in META, but not in HDFS "
2325 + "or deployed on any region server.");
2326 if (shouldFixMeta()) {
2327 deleteMetaRegion(hbi);
2328 }
2329 } else if (inMeta && !inHdfs && isDeployed) {
2330 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2331 + " found in META, but not in HDFS, " +
2332 "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2333
2334
2335
2336 if (shouldFixAssignments()) {
2337 errors.print("Trying to fix unassigned region...");
2338 undeployRegions(hbi);
2339 }
2340 if (shouldFixMeta()) {
2341
2342 deleteMetaRegion(hbi);
2343 }
2344 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2345 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2346 + " not deployed on any region server.");
2347 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2348 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2349 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2350 "Region " + descriptiveName + " should not be deployed according " +
2351 "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2352 if (shouldFixAssignments()) {
2353 errors.print("Trying to close the region " + descriptiveName);
2354 setShouldRerun();
2355 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2356 }
2357 } else if (inMeta && inHdfs && isMultiplyDeployed) {
2358 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2359 + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2360 + " but is multiply assigned to region servers " +
2361 Joiner.on(", ").join(hbi.deployedOn));
2362
2363 if (shouldFixAssignments()) {
2364 errors.print("Trying to fix assignment error...");
2365 setShouldRerun();
2366 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2367 }
2368 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2369 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2370 + descriptiveName + " listed in hbase:meta on region server " +
2371 hbi.metaEntry.regionServer + " but found on region server " +
2372 hbi.deployedOn.get(0));
2373
2374 if (shouldFixAssignments()) {
2375 errors.print("Trying to fix assignment error...");
2376 setShouldRerun();
2377 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2378 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2379 }
2380 } else {
2381 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2382 " is in an unforeseen state:" +
2383 " inMeta=" + inMeta +
2384 " inHdfs=" + inHdfs +
2385 " isDeployed=" + isDeployed +
2386 " isMultiplyDeployed=" + isMultiplyDeployed +
2387 " deploymentMatchesMeta=" + deploymentMatchesMeta +
2388 " shouldBeDeployed=" + shouldBeDeployed);
2389 }
2390 }
2391
2392
2393
2394
2395
2396
2397
2398 SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2399 tablesInfo = new TreeMap<TableName,TableInfo> ();
2400 LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2401 for (HbckInfo hbi : regionInfoMap.values()) {
2402
2403 if (hbi.metaEntry == null) {
2404
2405 Path p = hbi.getHdfsRegionDir();
2406 if (p == null) {
2407 errors.report("No regioninfo in Meta or HDFS. " + hbi);
2408 }
2409
2410
2411 continue;
2412 }
2413 if (hbi.metaEntry.regionServer == null) {
2414 errors.detail("Skipping region because no region server: " + hbi);
2415 continue;
2416 }
2417 if (hbi.metaEntry.isOffline()) {
2418 errors.detail("Skipping region because it is offline: " + hbi);
2419 continue;
2420 }
2421 if (hbi.containsOnlyHdfsEdits()) {
2422 errors.detail("Skipping region because it only contains edits" + hbi);
2423 continue;
2424 }
2425
2426
2427
2428
2429
2430
2431 if (hbi.deployedOn.size() == 0) continue;
2432
2433
2434 TableName tableName = hbi.metaEntry.getTable();
2435 TableInfo modTInfo = tablesInfo.get(tableName);
2436 if (modTInfo == null) {
2437 modTInfo = new TableInfo(tableName);
2438 }
2439 for (ServerName server : hbi.deployedOn) {
2440 modTInfo.addServer(server);
2441 }
2442
2443 if (!hbi.isSkipChecks()) {
2444 modTInfo.addRegionInfo(hbi);
2445 }
2446
2447 tablesInfo.put(tableName, modTInfo);
2448 }
2449
2450 loadTableInfosForTablesWithNoRegion();
2451
2452 logParallelMerge();
2453 for (TableInfo tInfo : tablesInfo.values()) {
2454 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2455 if (!tInfo.checkRegionChain(handler)) {
2456 errors.report("Found inconsistency in table " + tInfo.getName());
2457 }
2458 }
2459 return tablesInfo;
2460 }
2461
2462
2463
2464
2465 private void loadTableInfosForTablesWithNoRegion() throws IOException {
2466 Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2467 for (HTableDescriptor htd : allTables.values()) {
2468 if (checkMetaOnly && !htd.isMetaTable()) {
2469 continue;
2470 }
2471
2472 TableName tableName = htd.getTableName();
2473 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2474 TableInfo tableInfo = new TableInfo(tableName);
2475 tableInfo.htds.add(htd);
2476 tablesInfo.put(htd.getTableName(), tableInfo);
2477 }
2478 }
2479 }
2480
2481
2482
2483
2484
2485 public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2486 int fileMoves = 0;
2487 String thread = Thread.currentThread().getName();
2488 LOG.debug("[" + thread + "] Contained region dir after close and pause");
2489 debugLsr(contained.getHdfsRegionDir());
2490
2491
2492 FileSystem fs = targetRegionDir.getFileSystem(getConf());
2493 FileStatus[] dirs = null;
2494 try {
2495 dirs = fs.listStatus(contained.getHdfsRegionDir());
2496 } catch (FileNotFoundException fnfe) {
2497
2498
2499 if (!fs.exists(contained.getHdfsRegionDir())) {
2500 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2501 + " is missing. Assuming already sidelined or moved.");
2502 } else {
2503 sidelineRegionDir(fs, contained);
2504 }
2505 return fileMoves;
2506 }
2507
2508 if (dirs == null) {
2509 if (!fs.exists(contained.getHdfsRegionDir())) {
2510 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2511 + " already sidelined.");
2512 } else {
2513 sidelineRegionDir(fs, contained);
2514 }
2515 return fileMoves;
2516 }
2517
2518 for (FileStatus cf : dirs) {
2519 Path src = cf.getPath();
2520 Path dst = new Path(targetRegionDir, src.getName());
2521
2522 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2523
2524 continue;
2525 }
2526
2527 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2528
2529 continue;
2530 }
2531
2532 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2533
2534
2535
2536
2537 for (FileStatus hfile : fs.listStatus(src)) {
2538 boolean success = fs.rename(hfile.getPath(), dst);
2539 if (success) {
2540 fileMoves++;
2541 }
2542 }
2543 LOG.debug("[" + thread + "] Sideline directory contents:");
2544 debugLsr(targetRegionDir);
2545 }
2546
2547
2548 sidelineRegionDir(fs, contained);
2549 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2550 getSidelineDir());
2551 debugLsr(contained.getHdfsRegionDir());
2552
2553 return fileMoves;
2554 }
2555
2556
2557 static class WorkItemOverlapMerge implements Callable<Void> {
2558 private TableIntegrityErrorHandler handler;
2559 Collection<HbckInfo> overlapgroup;
2560
2561 WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2562 this.handler = handler;
2563 this.overlapgroup = overlapgroup;
2564 }
2565
2566 @Override
2567 public Void call() throws Exception {
2568 handler.handleOverlapGroup(overlapgroup);
2569 return null;
2570 }
2571 };
2572
2573
2574
2575
2576
2577 public class TableInfo {
2578 TableName tableName;
2579 TreeSet <ServerName> deployedOn;
2580
2581
2582 final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2583
2584
2585 final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2586
2587
2588 final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2589
2590
2591 final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2592
2593
2594 final Multimap<byte[], HbckInfo> overlapGroups =
2595 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2596
2597
2598 private ImmutableList<HRegionInfo> regionsFromMeta = null;
2599
2600 TableInfo(TableName name) {
2601 this.tableName = name;
2602 deployedOn = new TreeSet <ServerName>();
2603 }
2604
2605
2606
2607
2608 private HTableDescriptor getHTD() {
2609 if (htds.size() == 1) {
2610 return (HTableDescriptor)htds.toArray()[0];
2611 } else {
2612 LOG.error("None/Multiple table descriptors found for table '"
2613 + tableName + "' regions: " + htds);
2614 }
2615 return null;
2616 }
2617
2618 public void addRegionInfo(HbckInfo hir) {
2619 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2620
2621
2622 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2623 return;
2624 }
2625
2626
2627 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2628 errors.reportError(
2629 ERROR_CODE.REGION_CYCLE,
2630 String.format("The endkey for this region comes before the "
2631 + "startkey, startkey=%s, endkey=%s",
2632 Bytes.toStringBinary(hir.getStartKey()),
2633 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2634 backwards.add(hir);
2635 return;
2636 }
2637
2638
2639
2640 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2641 }
2642
2643 public void addServer(ServerName server) {
2644 this.deployedOn.add(server);
2645 }
2646
2647 public TableName getName() {
2648 return tableName;
2649 }
2650
2651 public int getNumRegions() {
2652 return sc.getStarts().size() + backwards.size();
2653 }
2654
2655 public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2656
2657 if (regionsFromMeta == null) {
2658 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2659 for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2660 if (tableName.equals(h.getTableName())) {
2661 if (h.metaEntry != null) {
2662 regions.add((HRegionInfo) h.metaEntry);
2663 }
2664 }
2665 }
2666 regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2667 }
2668
2669 return regionsFromMeta;
2670 }
2671
2672
2673 private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2674 ErrorReporter errors;
2675
2676 IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2677 this.errors = errors;
2678 setTableInfo(ti);
2679 }
2680
2681 @Override
2682 public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2683 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2684 "First region should start with an empty key. You need to "
2685 + " create a new region and regioninfo in HDFS to plug the hole.",
2686 getTableInfo(), hi);
2687 }
2688
2689 @Override
2690 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2691 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2692 "Last region should end with an empty key. You need to "
2693 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2694 }
2695
2696 @Override
2697 public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2698 errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2699 "Region has the same start and end key.", getTableInfo(), hi);
2700 }
2701
2702 @Override
2703 public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2704 byte[] key = r1.getStartKey();
2705
2706 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2707 "Multiple regions have the same startkey: "
2708 + Bytes.toStringBinary(key), getTableInfo(), r1);
2709 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2710 "Multiple regions have the same startkey: "
2711 + Bytes.toStringBinary(key), getTableInfo(), r2);
2712 }
2713
2714 @Override
2715 public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
2716 byte[] key = r1.getStartKey();
2717
2718 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2719 "Multiple regions have the same regionID: "
2720 + Bytes.toStringBinary(key), getTableInfo(), r1);
2721 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2722 "Multiple regions have the same regionID: "
2723 + Bytes.toStringBinary(key), getTableInfo(), r2);
2724 }
2725
2726 @Override
2727 public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2728 errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2729 "There is an overlap in the region chain.",
2730 getTableInfo(), hi1, hi2);
2731 }
2732
2733 @Override
2734 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2735 errors.reportError(
2736 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2737 "There is a hole in the region chain between "
2738 + Bytes.toStringBinary(holeStart) + " and "
2739 + Bytes.toStringBinary(holeStop)
2740 + ". You need to create a new .regioninfo and region "
2741 + "dir in hdfs to plug the hole.");
2742 }
2743 };
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757 private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2758 Configuration conf;
2759
2760 boolean fixOverlaps = true;
2761
2762 HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2763 boolean fixHoles, boolean fixOverlaps) {
2764 super(ti, errors);
2765 this.conf = conf;
2766 this.fixOverlaps = fixOverlaps;
2767
2768 }
2769
2770
2771
2772
2773
2774
2775 @Override
2776 public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2777 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2778 "First region should start with an empty key. Creating a new " +
2779 "region and regioninfo in HDFS to plug the hole.",
2780 getTableInfo(), next);
2781 HTableDescriptor htd = getTableInfo().getHTD();
2782
2783 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2784 HConstants.EMPTY_START_ROW, next.getStartKey());
2785
2786
2787 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2788 LOG.info("Table region start key was not empty. Created new empty region: "
2789 + newRegion + " " +region);
2790 fixes++;
2791 }
2792
2793 @Override
2794 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2795 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2796 "Last region should end with an empty key. Creating a new "
2797 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2798 HTableDescriptor htd = getTableInfo().getHTD();
2799
2800 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2801 HConstants.EMPTY_START_ROW);
2802
2803 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2804 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
2805 + " " + region);
2806 fixes++;
2807 }
2808
2809
2810
2811
2812
2813 @Override
2814 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2815 errors.reportError(
2816 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2817 "There is a hole in the region chain between "
2818 + Bytes.toStringBinary(holeStartKey) + " and "
2819 + Bytes.toStringBinary(holeStopKey)
2820 + ". Creating a new regioninfo and region "
2821 + "dir in hdfs to plug the hole.");
2822 HTableDescriptor htd = getTableInfo().getHTD();
2823 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2824 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2825 LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2826 fixes++;
2827 }
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840 @Override
2841 public void handleOverlapGroup(Collection<HbckInfo> overlap)
2842 throws IOException {
2843 Preconditions.checkNotNull(overlap);
2844 Preconditions.checkArgument(overlap.size() >0);
2845
2846 if (!this.fixOverlaps) {
2847 LOG.warn("Not attempting to repair overlaps.");
2848 return;
2849 }
2850
2851 if (overlap.size() > maxMerge) {
2852 LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2853 "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2854 if (sidelineBigOverlaps) {
2855
2856 sidelineBigOverlaps(overlap);
2857 }
2858 return;
2859 }
2860 if (shouldRemoveParents()) {
2861 removeParentsAndFixSplits(overlap);
2862 }
2863 mergeOverlaps(overlap);
2864 }
2865
2866 void removeParentsAndFixSplits(Collection<HbckInfo> overlap) throws IOException {
2867 Pair<byte[], byte[]> range = null;
2868 HbckInfo parent = null;
2869 HbckInfo daughterA = null;
2870 HbckInfo daughterB = null;
2871 Collection<HbckInfo> daughters = new ArrayList<HbckInfo>(overlap);
2872
2873 String thread = Thread.currentThread().getName();
2874 LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
2875
2876
2877 if (overlap.size() > 3) {
2878 LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
2879 return;
2880 }
2881
2882 for (HbckInfo hi : overlap) {
2883 if (range == null) {
2884 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2885 } else {
2886 if (RegionSplitCalculator.BYTES_COMPARATOR
2887 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2888 range.setFirst(hi.getStartKey());
2889 }
2890 if (RegionSplitCalculator.BYTES_COMPARATOR
2891 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2892 range.setSecond(hi.getEndKey());
2893 }
2894 }
2895 }
2896
2897 LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
2898 + Bytes.toStringBinary(range.getSecond()) + "]");
2899
2900
2901 for (HbckInfo hi : overlap) {
2902 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
2903 && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2904 LOG.info("This is a parent for this group: " + hi.toString());
2905 parent = hi;
2906 }
2907 }
2908
2909
2910 if (parent != null) {
2911 daughters.remove(parent);
2912 }
2913
2914
2915
2916 for (HbckInfo hi : daughters) {
2917 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
2918 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2919 daughterA = hi;
2920 }
2921 }
2922 if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2923 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2924 daughterB = hi;
2925 }
2926 }
2927 }
2928
2929
2930 if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() || parent == null)
2931 return;
2932
2933 FileSystem fs = FileSystem.get(conf);
2934 LOG.info("Found parent: " + parent.getRegionNameAsString());
2935 LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
2936 LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
2937 LOG.info("Trying to fix parent in overlap by removing the parent.");
2938 try {
2939 closeRegion(parent);
2940 } catch (IOException ioe) {
2941 LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
2942 return;
2943 } catch (InterruptedException ie) {
2944 LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
2945 return;
2946 }
2947
2948 try {
2949 offline(parent.getRegionName());
2950 } catch (IOException ioe) {
2951 LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
2952 + ". Just continuing with regular merge... ", ioe);
2953 return;
2954 }
2955
2956 try {
2957 HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
2958 } catch (IOException ioe) {
2959 LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
2960 + ". Just continuing with regular merge... ", ioe);
2961 return;
2962 }
2963
2964 sidelineRegionDir(fs, parent);
2965 LOG.info("[" + thread + "] Sidelined parent region dir "+ parent.getHdfsRegionDir() + " into " +
2966 getSidelineDir());
2967 debugLsr(parent.getHdfsRegionDir());
2968
2969
2970 overlap.remove(parent);
2971 overlap.remove(daughterA);
2972 overlap.remove(daughterB);
2973
2974 LOG.info("Done fixing split.");
2975
2976 }
2977
2978 void mergeOverlaps(Collection<HbckInfo> overlap)
2979 throws IOException {
2980 String thread = Thread.currentThread().getName();
2981 LOG.info("== [" + thread + "] Merging regions into one region: "
2982 + Joiner.on(",").join(overlap));
2983
2984 Pair<byte[], byte[]> range = null;
2985 for (HbckInfo hi : overlap) {
2986 if (range == null) {
2987 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2988 } else {
2989 if (RegionSplitCalculator.BYTES_COMPARATOR
2990 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2991 range.setFirst(hi.getStartKey());
2992 }
2993 if (RegionSplitCalculator.BYTES_COMPARATOR
2994 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2995 range.setSecond(hi.getEndKey());
2996 }
2997 }
2998
2999 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
3000 LOG.debug("[" + thread + "] Contained region dir before close");
3001 debugLsr(hi.getHdfsRegionDir());
3002 try {
3003 LOG.info("[" + thread + "] Closing region: " + hi);
3004 closeRegion(hi);
3005 } catch (IOException ioe) {
3006 LOG.warn("[" + thread + "] Was unable to close region " + hi
3007 + ". Just continuing... ", ioe);
3008 } catch (InterruptedException e) {
3009 LOG.warn("[" + thread + "] Was unable to close region " + hi
3010 + ". Just continuing... ", e);
3011 }
3012
3013 try {
3014 LOG.info("[" + thread + "] Offlining region: " + hi);
3015 offline(hi.getRegionName());
3016 } catch (IOException ioe) {
3017 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
3018 + ". Just continuing... ", ioe);
3019 }
3020 }
3021
3022
3023 HTableDescriptor htd = getTableInfo().getHTD();
3024
3025 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
3026 range.getSecond());
3027 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
3028 LOG.info("[" + thread + "] Created new empty container region: " +
3029 newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
3030 debugLsr(region.getRegionFileSystem().getRegionDir());
3031
3032
3033 boolean didFix= false;
3034 Path target = region.getRegionFileSystem().getRegionDir();
3035 for (HbckInfo contained : overlap) {
3036 LOG.info("[" + thread + "] Merging " + contained + " into " + target );
3037 int merges = mergeRegionDirs(target, contained);
3038 if (merges > 0) {
3039 didFix = true;
3040 }
3041 }
3042 if (didFix) {
3043 fixes++;
3044 }
3045 }
3046
3047
3048
3049
3050
3051
3052
3053
3054 void sidelineBigOverlaps(
3055 Collection<HbckInfo> bigOverlap) throws IOException {
3056 int overlapsToSideline = bigOverlap.size() - maxMerge;
3057 if (overlapsToSideline > maxOverlapsToSideline) {
3058 overlapsToSideline = maxOverlapsToSideline;
3059 }
3060 List<HbckInfo> regionsToSideline =
3061 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
3062 FileSystem fs = FileSystem.get(conf);
3063 for (HbckInfo regionToSideline: regionsToSideline) {
3064 try {
3065 LOG.info("Closing region: " + regionToSideline);
3066 closeRegion(regionToSideline);
3067 } catch (IOException ioe) {
3068 LOG.warn("Was unable to close region " + regionToSideline
3069 + ". Just continuing... ", ioe);
3070 } catch (InterruptedException e) {
3071 LOG.warn("Was unable to close region " + regionToSideline
3072 + ". Just continuing... ", e);
3073 }
3074
3075 try {
3076 LOG.info("Offlining region: " + regionToSideline);
3077 offline(regionToSideline.getRegionName());
3078 } catch (IOException ioe) {
3079 LOG.warn("Unable to offline region from master: " + regionToSideline
3080 + ". Just continuing... ", ioe);
3081 }
3082
3083 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
3084 Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
3085 if (sidelineRegionDir != null) {
3086 sidelinedRegions.put(sidelineRegionDir, regionToSideline);
3087 LOG.info("After sidelined big overlapped region: "
3088 + regionToSideline.getRegionNameAsString()
3089 + " to " + sidelineRegionDir.toString());
3090 fixes++;
3091 }
3092 }
3093 }
3094 }
3095
3096
3097
3098
3099
3100
3101
3102 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3103
3104
3105
3106 if (disabledTables.contains(this.tableName)) {
3107 return true;
3108 }
3109 int originalErrorsCount = errors.getErrorList().size();
3110 Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3111 SortedSet<byte[]> splits = sc.getSplits();
3112
3113 byte[] prevKey = null;
3114 byte[] problemKey = null;
3115
3116 if (splits.size() == 0) {
3117
3118 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3119 }
3120
3121 for (byte[] key : splits) {
3122 Collection<HbckInfo> ranges = regions.get(key);
3123 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3124 for (HbckInfo rng : ranges) {
3125 handler.handleRegionStartKeyNotEmpty(rng);
3126 }
3127 }
3128
3129
3130 for (HbckInfo rng : ranges) {
3131
3132 byte[] endKey = rng.getEndKey();
3133 endKey = (endKey.length == 0) ? null : endKey;
3134 if (Bytes.equals(rng.getStartKey(),endKey)) {
3135 handler.handleDegenerateRegion(rng);
3136 }
3137 }
3138
3139 if (ranges.size() == 1) {
3140
3141 if (problemKey != null) {
3142 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3143 }
3144 problemKey = null;
3145 } else if (ranges.size() > 1) {
3146
3147
3148 if (problemKey == null) {
3149
3150 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3151 problemKey = key;
3152 }
3153 overlapGroups.putAll(problemKey, ranges);
3154
3155
3156 ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3157
3158 for (HbckInfo r1 : ranges) {
3159 if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3160 subRange.remove(r1);
3161 for (HbckInfo r2 : subRange) {
3162 if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3163
3164 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3165 handler.handleDuplicateStartKeys(r1,r2);
3166 } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 &&
3167 r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) {
3168 LOG.info("this is a split, log to splits");
3169 handler.handleSplit(r1, r2);
3170 } else {
3171
3172 handler.handleOverlapInRegionChain(r1, r2);
3173 }
3174 }
3175 }
3176
3177 } else if (ranges.size() == 0) {
3178 if (problemKey != null) {
3179 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3180 }
3181 problemKey = null;
3182
3183 byte[] holeStopKey = sc.getSplits().higher(key);
3184
3185 if (holeStopKey != null) {
3186
3187 handler.handleHoleInRegionChain(key, holeStopKey);
3188 }
3189 }
3190 prevKey = key;
3191 }
3192
3193
3194
3195 if (prevKey != null) {
3196 handler.handleRegionEndKeyNotEmpty(prevKey);
3197 }
3198
3199
3200 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3201 boolean ok = handleOverlapsParallel(handler, prevKey);
3202 if (!ok) {
3203 return false;
3204 }
3205 } else {
3206 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3207 handler.handleOverlapGroup(overlap);
3208 }
3209 }
3210
3211 if (details) {
3212
3213 errors.print("---- Table '" + this.tableName
3214 + "': region split map");
3215 dump(splits, regions);
3216 errors.print("---- Table '" + this.tableName
3217 + "': overlap groups");
3218 dumpOverlapProblems(overlapGroups);
3219 errors.print("There are " + overlapGroups.keySet().size()
3220 + " overlap groups with " + overlapGroups.size()
3221 + " overlapping regions");
3222 }
3223 if (!sidelinedRegions.isEmpty()) {
3224 LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3225 errors.print("---- Table '" + this.tableName
3226 + "': sidelined big overlapped regions");
3227 dumpSidelinedRegions(sidelinedRegions);
3228 }
3229 return errors.getErrorList().size() == originalErrorsCount;
3230 }
3231
3232 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3233 throws IOException {
3234
3235
3236 List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3237 List<Future<Void>> rets;
3238 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3239
3240 merges.add(new WorkItemOverlapMerge(overlap, handler));
3241 }
3242 try {
3243 rets = executor.invokeAll(merges);
3244 } catch (InterruptedException e) {
3245 LOG.error("Overlap merges were interrupted", e);
3246 return false;
3247 }
3248 for(int i=0; i<merges.size(); i++) {
3249 WorkItemOverlapMerge work = merges.get(i);
3250 Future<Void> f = rets.get(i);
3251 try {
3252 f.get();
3253 } catch(ExecutionException e) {
3254 LOG.warn("Failed to merge overlap group" + work, e.getCause());
3255 } catch (InterruptedException e) {
3256 LOG.error("Waiting for overlap merges was interrupted", e);
3257 return false;
3258 }
3259 }
3260 return true;
3261 }
3262
3263
3264
3265
3266
3267
3268
3269 void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3270
3271 StringBuilder sb = new StringBuilder();
3272 for (byte[] k : splits) {
3273 sb.setLength(0);
3274 sb.append(Bytes.toStringBinary(k) + ":\t");
3275 for (HbckInfo r : regions.get(k)) {
3276 sb.append("[ "+ r.toString() + ", "
3277 + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3278 }
3279 errors.print(sb.toString());
3280 }
3281 }
3282 }
3283
3284 public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3285
3286
3287 for (byte[] k : regions.keySet()) {
3288 errors.print(Bytes.toStringBinary(k) + ":");
3289 for (HbckInfo r : regions.get(k)) {
3290 errors.print("[ " + r.toString() + ", "
3291 + Bytes.toStringBinary(r.getEndKey()) + "]");
3292 }
3293 errors.print("----");
3294 }
3295 }
3296
3297 public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3298 for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3299 TableName tableName = entry.getValue().getTableName();
3300 Path path = entry.getKey();
3301 errors.print("This sidelined region dir should be bulk loaded: "
3302 + path.toString());
3303 errors.print("Bulk load command looks like: "
3304 + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3305 + path.toUri().getPath() + " "+ tableName);
3306 }
3307 }
3308
3309 public Multimap<byte[], HbckInfo> getOverlapGroups(
3310 TableName table) {
3311 TableInfo ti = tablesInfo.get(table);
3312 return ti.overlapGroups;
3313 }
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324 HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3325 List<TableName> tableNames = new ArrayList<TableName>();
3326 long now = EnvironmentEdgeManager.currentTime();
3327
3328 for (HbckInfo hbi : regionInfoMap.values()) {
3329 MetaEntry info = hbi.metaEntry;
3330
3331
3332
3333 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3334 if (info.modTime + timelag < now) {
3335 tableNames.add(info.getTable());
3336 } else {
3337 numSkipped.incrementAndGet();
3338 }
3339 }
3340 }
3341 return getHTableDescriptors(tableNames);
3342 }
3343
3344 HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3345 HTableDescriptor[] htd = new HTableDescriptor[0];
3346 Admin admin = null;
3347 try {
3348 LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3349 admin = new HBaseAdmin(getConf());
3350 htd = admin.getTableDescriptorsByTableName(tableNames);
3351 } catch (IOException e) {
3352 LOG.debug("Exception getting table descriptors", e);
3353 } finally {
3354 if (admin != null) {
3355 try {
3356 admin.close();
3357 } catch (IOException e) {
3358 LOG.debug("Exception closing HBaseAdmin", e);
3359 }
3360 }
3361 }
3362 return htd;
3363 }
3364
3365
3366
3367
3368
3369
3370 private synchronized HbckInfo getOrCreateInfo(String name) {
3371 HbckInfo hbi = regionInfoMap.get(name);
3372 if (hbi == null) {
3373 hbi = new HbckInfo(null);
3374 regionInfoMap.put(name, hbi);
3375 }
3376 return hbi;
3377 }
3378
3379 private void checkAndFixTableLocks() throws IOException {
3380 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3381
3382 try {
3383 TableLockChecker checker = new TableLockChecker(zkw, errors);
3384 checker.checkTableLocks();
3385
3386 if (this.fixTableLocks) {
3387 checker.fixExpiredTableLocks();
3388 }
3389 } finally {
3390 zkw.close();
3391 }
3392 }
3393
3394 private void checkAndFixReplication() throws IOException {
3395 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3396 try {
3397 ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3398 checker.checkUnDeletedQueues();
3399
3400 if (checker.hasUnDeletedQueues() && this.fixReplication) {
3401 checker.fixUnDeletedQueues();
3402 setShouldRerun();
3403 }
3404 } finally {
3405 zkw.close();
3406 }
3407 }
3408
3409
3410
3411
3412
3413
3414
3415 private void checkAndFixOrphanedTableZNodes()
3416 throws IOException, KeeperException, InterruptedException {
3417 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3418
3419 try {
3420 Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3421 String msg;
3422 TableInfo tableInfo;
3423
3424 for (TableName tableName : enablingTables) {
3425
3426 tableInfo = tablesInfo.get(tableName);
3427 if (tableInfo != null) {
3428
3429 continue;
3430 }
3431
3432 msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3433 LOG.warn(msg);
3434 orphanedTableZNodes.add(tableName);
3435 errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3436 }
3437
3438 if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3439 ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3440
3441 for (TableName tableName : orphanedTableZNodes) {
3442 try {
3443
3444
3445
3446
3447 zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3448 } catch (CoordinatedStateException e) {
3449
3450 LOG.error(
3451 "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3452 e);
3453 }
3454 }
3455 }
3456 } finally {
3457 zkw.close();
3458 }
3459 }
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3471 Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3472 for (HbckInfo value : regionInfoMap.values()) {
3473 if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3474 metaRegions.put(value.getReplicaId(), value);
3475 }
3476 }
3477 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3478 .getRegionReplication();
3479 boolean noProblem = true;
3480
3481
3482 for (int i = 0; i < metaReplication; i++) {
3483 HbckInfo metaHbckInfo = metaRegions.remove(i);
3484 List<ServerName> servers = new ArrayList<ServerName>();
3485 if (metaHbckInfo != null) {
3486 servers = metaHbckInfo.deployedOn;
3487 }
3488 if (servers.size() != 1) {
3489 noProblem = false;
3490 if (servers.size() == 0) {
3491 assignMetaReplica(i);
3492 } else if (servers.size() > 1) {
3493 errors
3494 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3495 metaHbckInfo.getReplicaId() + " is found on more than one region.");
3496 if (shouldFixAssignments()) {
3497 errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3498 metaHbckInfo.getReplicaId() +"..");
3499 setShouldRerun();
3500
3501 HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3502 }
3503 }
3504 }
3505 }
3506
3507 for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3508 noProblem = false;
3509 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3510 "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3511 ", deployed " + metaRegions.size());
3512 if (shouldFixAssignments()) {
3513 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3514 " of hbase:meta..");
3515 setShouldRerun();
3516 unassignMetaReplica(entry.getValue());
3517 }
3518 }
3519
3520
3521 return noProblem;
3522 }
3523
3524 private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3525 KeeperException {
3526 undeployRegions(hi);
3527 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3528 try {
3529 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3530 } finally {
3531 zkw.close();
3532 }
3533 }
3534
3535 private void assignMetaReplica(int replicaId)
3536 throws IOException, KeeperException, InterruptedException {
3537 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3538 replicaId +" is not found on any region.");
3539 if (shouldFixAssignments()) {
3540 errors.print("Trying to fix a problem with hbase:meta..");
3541 setShouldRerun();
3542
3543 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3544 HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3545 HBaseFsckRepair.fixUnassigned(admin, h);
3546 HBaseFsckRepair.waitUntilAssigned(admin, h);
3547 }
3548 }
3549
3550
3551
3552
3553
3554 boolean loadMetaEntries() throws IOException {
3555 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3556 int countRecord = 1;
3557
3558
3559 final Comparator<Cell> comp = new Comparator<Cell>() {
3560 @Override
3561 public int compare(Cell k1, Cell k2) {
3562 return Long.compare(k1.getTimestamp(), k2.getTimestamp());
3563 }
3564 };
3565
3566 @Override
3567 public boolean processRow(Result result) throws IOException {
3568 try {
3569
3570
3571 long ts = Collections.max(result.listCells(), comp).getTimestamp();
3572 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3573 if (rl == null) {
3574 emptyRegionInfoQualifiers.add(result);
3575 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3576 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3577 return true;
3578 }
3579 ServerName sn = null;
3580 if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3581 rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3582 emptyRegionInfoQualifiers.add(result);
3583 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3584 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3585 return true;
3586 }
3587 HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3588 if (!(isTableIncluded(hri.getTable())
3589 || hri.isMetaRegion())) {
3590 return true;
3591 }
3592 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3593 for (HRegionLocation h : rl.getRegionLocations()) {
3594 if (h == null || h.getRegionInfo() == null) {
3595 continue;
3596 }
3597 sn = h.getServerName();
3598 hri = h.getRegionInfo();
3599
3600 MetaEntry m = null;
3601 if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3602 m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3603 } else {
3604 m = new MetaEntry(hri, sn, ts, null, null);
3605 }
3606 HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3607 if (previous == null) {
3608 regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3609 } else if (previous.metaEntry == null) {
3610 previous.metaEntry = m;
3611 } else {
3612 throw new IOException("Two entries in hbase:meta are same " + previous);
3613 }
3614 }
3615 PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3616 for (HRegionInfo mergeRegion : new HRegionInfo[] {
3617 mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3618 if (mergeRegion != null) {
3619
3620 HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3621 hbInfo.setMerged(true);
3622 }
3623 }
3624
3625
3626 if (countRecord % 100 == 0) {
3627 errors.progress();
3628 }
3629 countRecord++;
3630 return true;
3631 } catch (RuntimeException e) {
3632 LOG.error("Result=" + result);
3633 throw e;
3634 }
3635 }
3636 };
3637 if (!checkMetaOnly) {
3638
3639 MetaScanner.metaScan(connection, visitor);
3640 }
3641
3642 errors.print("");
3643 return true;
3644 }
3645
3646
3647
3648
3649 static class MetaEntry extends HRegionInfo {
3650 ServerName regionServer;
3651 long modTime;
3652 HRegionInfo splitA, splitB;
3653
3654 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3655 this(rinfo, regionServer, modTime, null, null);
3656 }
3657
3658 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3659 HRegionInfo splitA, HRegionInfo splitB) {
3660 super(rinfo);
3661 this.regionServer = regionServer;
3662 this.modTime = modTime;
3663 this.splitA = splitA;
3664 this.splitB = splitB;
3665 }
3666
3667 @Override
3668 public boolean equals(Object o) {
3669 boolean superEq = super.equals(o);
3670 if (!superEq) {
3671 return superEq;
3672 }
3673
3674 MetaEntry me = (MetaEntry) o;
3675 if (!regionServer.equals(me.regionServer)) {
3676 return false;
3677 }
3678 return (modTime == me.modTime);
3679 }
3680
3681 @Override
3682 public int hashCode() {
3683 int hash = Arrays.hashCode(getRegionName());
3684 hash ^= getRegionId();
3685 hash ^= Arrays.hashCode(getStartKey());
3686 hash ^= Arrays.hashCode(getEndKey());
3687 hash ^= Boolean.valueOf(isOffline()).hashCode();
3688 hash ^= getTable().hashCode();
3689 if (regionServer != null) {
3690 hash ^= regionServer.hashCode();
3691 }
3692 hash ^= modTime;
3693 return hash;
3694 }
3695 }
3696
3697
3698
3699
3700 static class HdfsEntry {
3701 HRegionInfo hri;
3702 Path hdfsRegionDir = null;
3703 long hdfsRegionDirModTime = 0;
3704 boolean hdfsRegioninfoFilePresent = false;
3705 boolean hdfsOnlyEdits = false;
3706 }
3707
3708
3709
3710
3711 static class OnlineEntry {
3712 HRegionInfo hri;
3713 ServerName hsa;
3714
3715 @Override
3716 public String toString() {
3717 return hsa.toString() + ";" + hri.getRegionNameAsString();
3718 }
3719 }
3720
3721
3722
3723
3724
3725 public static class HbckInfo implements KeyRange {
3726 private MetaEntry metaEntry = null;
3727 private HdfsEntry hdfsEntry = null;
3728 private List<OnlineEntry> deployedEntries = Lists.newArrayList();
3729 private List<ServerName> deployedOn = Lists.newArrayList();
3730 private boolean skipChecks = false;
3731 private boolean isMerged = false;
3732 private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3733 private HRegionInfo primaryHRIForDeployedReplica = null;
3734
3735 HbckInfo(MetaEntry metaEntry) {
3736 this.metaEntry = metaEntry;
3737 }
3738
3739 public synchronized int getReplicaId() {
3740 return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3741 }
3742
3743 public synchronized void addServer(HRegionInfo hri, ServerName server) {
3744 OnlineEntry rse = new OnlineEntry() ;
3745 rse.hri = hri;
3746 rse.hsa = server;
3747 this.deployedEntries.add(rse);
3748 this.deployedOn.add(server);
3749
3750 this.deployedReplicaId = hri.getReplicaId();
3751 this.primaryHRIForDeployedReplica =
3752 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3753 }
3754
3755 @Override
3756 public synchronized String toString() {
3757 StringBuilder sb = new StringBuilder();
3758 sb.append("{ meta => ");
3759 sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3760 sb.append( ", hdfs => " + getHdfsRegionDir());
3761 sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3762 sb.append( ", replicaId => " + getReplicaId());
3763 sb.append(" }");
3764 return sb.toString();
3765 }
3766
3767 @Override
3768 public byte[] getStartKey() {
3769 if (this.metaEntry != null) {
3770 return this.metaEntry.getStartKey();
3771 } else if (this.hdfsEntry != null) {
3772 return this.hdfsEntry.hri.getStartKey();
3773 } else {
3774 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3775 return null;
3776 }
3777 }
3778
3779 @Override
3780 public byte[] getEndKey() {
3781 if (this.metaEntry != null) {
3782 return this.metaEntry.getEndKey();
3783 } else if (this.hdfsEntry != null) {
3784 return this.hdfsEntry.hri.getEndKey();
3785 } else {
3786 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3787 return null;
3788 }
3789 }
3790
3791 public TableName getTableName() {
3792 if (this.metaEntry != null) {
3793 return this.metaEntry.getTable();
3794 } else if (this.hdfsEntry != null) {
3795
3796
3797 Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3798 return FSUtils.getTableName(tableDir);
3799 } else {
3800
3801 for (OnlineEntry e : deployedEntries) {
3802 return e.hri.getTable();
3803 }
3804 return null;
3805 }
3806 }
3807
3808 public String getRegionNameAsString() {
3809 if (metaEntry != null) {
3810 return metaEntry.getRegionNameAsString();
3811 } else if (hdfsEntry != null) {
3812 if (hdfsEntry.hri != null) {
3813 return hdfsEntry.hri.getRegionNameAsString();
3814 }
3815 } else {
3816
3817 for (OnlineEntry e : deployedEntries) {
3818 return e.hri.getRegionNameAsString();
3819 }
3820 }
3821 return null;
3822 }
3823
3824 public byte[] getRegionName() {
3825 if (metaEntry != null) {
3826 return metaEntry.getRegionName();
3827 } else if (hdfsEntry != null) {
3828 return hdfsEntry.hri.getRegionName();
3829 } else {
3830
3831 for (OnlineEntry e : deployedEntries) {
3832 return e.hri.getRegionName();
3833 }
3834 return null;
3835 }
3836 }
3837
3838 public HRegionInfo getPrimaryHRIForDeployedReplica() {
3839 return primaryHRIForDeployedReplica;
3840 }
3841
3842 Path getHdfsRegionDir() {
3843 if (hdfsEntry == null) {
3844 return null;
3845 }
3846 return hdfsEntry.hdfsRegionDir;
3847 }
3848
3849 boolean containsOnlyHdfsEdits() {
3850 if (hdfsEntry == null) {
3851 return false;
3852 }
3853 return hdfsEntry.hdfsOnlyEdits;
3854 }
3855
3856 boolean isHdfsRegioninfoPresent() {
3857 if (hdfsEntry == null) {
3858 return false;
3859 }
3860 return hdfsEntry.hdfsRegioninfoFilePresent;
3861 }
3862
3863 long getModTime() {
3864 if (hdfsEntry == null) {
3865 return 0;
3866 }
3867 return hdfsEntry.hdfsRegionDirModTime;
3868 }
3869
3870 HRegionInfo getHdfsHRI() {
3871 if (hdfsEntry == null) {
3872 return null;
3873 }
3874 return hdfsEntry.hri;
3875 }
3876
3877 public void setSkipChecks(boolean skipChecks) {
3878 this.skipChecks = skipChecks;
3879 }
3880
3881 public boolean isSkipChecks() {
3882 return skipChecks;
3883 }
3884
3885 public void setMerged(boolean isMerged) {
3886 this.isMerged = isMerged;
3887 }
3888
3889 public boolean isMerged() {
3890 return this.isMerged;
3891 }
3892 }
3893
3894 final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3895 @Override
3896 public int compare(HbckInfo l, HbckInfo r) {
3897 if (l == r) {
3898
3899 return 0;
3900 }
3901
3902 int tableCompare = l.getTableName().compareTo(r.getTableName());
3903 if (tableCompare != 0) {
3904 return tableCompare;
3905 }
3906
3907 int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3908 l.getStartKey(), r.getStartKey());
3909 if (startComparison != 0) {
3910 return startComparison;
3911 }
3912
3913
3914 byte[] endKey = r.getEndKey();
3915 endKey = (endKey.length == 0) ? null : endKey;
3916 byte[] endKey2 = l.getEndKey();
3917 endKey2 = (endKey2.length == 0) ? null : endKey2;
3918 int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3919 endKey2, endKey);
3920
3921 if (endComparison != 0) {
3922 return endComparison;
3923 }
3924
3925
3926
3927 if (l.hdfsEntry == null && r.hdfsEntry == null) {
3928 return 0;
3929 }
3930 if (l.hdfsEntry == null && r.hdfsEntry != null) {
3931 return 1;
3932 }
3933
3934 if (r.hdfsEntry == null) {
3935 return -1;
3936 }
3937
3938 return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3939 }
3940 };
3941
3942
3943
3944
3945 private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3946 StringBuilder sb = new StringBuilder();
3947 int numOfSkippedRegions;
3948 errors.print("Summary:");
3949 for (TableInfo tInfo : tablesInfo.values()) {
3950 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3951 skippedRegions.get(tInfo.getName()).size() : 0;
3952
3953 if (errors.tableHasErrors(tInfo)) {
3954 errors.print("Table " + tInfo.getName() + " is inconsistent.");
3955 } else if (numOfSkippedRegions > 0){
3956 errors.print("Table " + tInfo.getName() + " is okay (with "
3957 + numOfSkippedRegions + " skipped regions).");
3958 }
3959 else {
3960 errors.print("Table " + tInfo.getName() + " is okay.");
3961 }
3962 errors.print(" Number of regions: " + tInfo.getNumRegions());
3963 if (numOfSkippedRegions > 0) {
3964 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3965 System.out.println(" Number of skipped regions: " + numOfSkippedRegions);
3966 System.out.println(" List of skipped regions:");
3967 for(String sr : skippedRegionStrings) {
3968 System.out.println(" " + sr);
3969 }
3970 }
3971 sb.setLength(0);
3972 sb.append(" Deployed on: ");
3973 for (ServerName server : tInfo.deployedOn) {
3974 sb.append(" " + server.toString());
3975 }
3976 errors.print(sb.toString());
3977 }
3978 }
3979
3980 static ErrorReporter getErrorReporter(
3981 final Configuration conf) throws ClassNotFoundException {
3982 Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3983 return ReflectionUtils.newInstance(reporter, conf);
3984 }
3985
3986 public interface ErrorReporter {
3987 enum ERROR_CODE {
3988 UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3989 NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3990 MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3991 FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3992 HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3993 ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3994 WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR,
3995 UNDELETED_REPLICATION_QUEUE, DUPE_ENDKEYS
3996 }
3997 void clear();
3998 void report(String message);
3999 void reportError(String message);
4000 void reportError(ERROR_CODE errorCode, String message);
4001 void reportError(ERROR_CODE errorCode, String message, TableInfo table);
4002 void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
4003 void reportError(
4004 ERROR_CODE errorCode,
4005 String message,
4006 TableInfo table,
4007 HbckInfo info1,
4008 HbckInfo info2
4009 );
4010 int summarize();
4011 void detail(String details);
4012 ArrayList<ERROR_CODE> getErrorList();
4013 void progress();
4014 void print(String message);
4015 void resetErrors();
4016 boolean tableHasErrors(TableInfo table);
4017 }
4018
4019 static class PrintingErrorReporter implements ErrorReporter {
4020 public int errorCount = 0;
4021 private int showProgress;
4022
4023 private static final int progressThreshold = 100;
4024
4025 Set<TableInfo> errorTables = new HashSet<TableInfo>();
4026
4027
4028 private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
4029
4030 @Override
4031 public void clear() {
4032 errorTables.clear();
4033 errorList.clear();
4034 errorCount = 0;
4035 }
4036
4037 @Override
4038 public synchronized void reportError(ERROR_CODE errorCode, String message) {
4039 if (errorCode == ERROR_CODE.WRONG_USAGE) {
4040 System.err.println(message);
4041 return;
4042 }
4043
4044 errorList.add(errorCode);
4045 if (!getSUMMARY()) {
4046 System.out.println("ERROR: " + message);
4047 }
4048 errorCount++;
4049 showProgress = 0;
4050 }
4051
4052 @Override
4053 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
4054 errorTables.add(table);
4055 reportError(errorCode, message);
4056 }
4057
4058 @Override
4059 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4060 HbckInfo info) {
4061 errorTables.add(table);
4062 String reference = "(region " + info.getRegionNameAsString() + ")";
4063 reportError(errorCode, reference + " " + message);
4064 }
4065
4066 @Override
4067 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4068 HbckInfo info1, HbckInfo info2) {
4069 errorTables.add(table);
4070 String reference = "(regions " + info1.getRegionNameAsString()
4071 + " and " + info2.getRegionNameAsString() + ")";
4072 reportError(errorCode, reference + " " + message);
4073 }
4074
4075 @Override
4076 public synchronized void reportError(String message) {
4077 reportError(ERROR_CODE.UNKNOWN, message);
4078 }
4079
4080
4081
4082
4083
4084
4085 @Override
4086 public synchronized void report(String message) {
4087 if (!getSUMMARY()) {
4088 System.out.println("ERROR: " + message);
4089 }
4090 showProgress = 0;
4091 }
4092
4093 @Override
4094 public synchronized int summarize() {
4095 System.out.println(Integer.toString(errorCount) +
4096 " inconsistencies detected.");
4097 if (errorCount == 0) {
4098 System.out.println("Status: OK");
4099 return 0;
4100 } else {
4101 System.out.println("Status: INCONSISTENT");
4102 return -1;
4103 }
4104 }
4105
4106 @Override
4107 public ArrayList<ERROR_CODE> getErrorList() {
4108 return errorList;
4109 }
4110
4111 @Override
4112 public synchronized void print(String message) {
4113 if (!getSUMMARY()) {
4114 System.out.println(message);
4115 }
4116 }
4117
4118 private synchronized static boolean getSUMMARY() {
4119 return SUMMARY;
4120 }
4121
4122 @Override
4123 public boolean tableHasErrors(TableInfo table) {
4124 return errorTables.contains(table);
4125 }
4126
4127 @Override
4128 public void resetErrors() {
4129 errorCount = 0;
4130 }
4131
4132 @Override
4133 public synchronized void detail(String message) {
4134 if (details) {
4135 System.out.println(message);
4136 }
4137 showProgress = 0;
4138 }
4139
4140 @Override
4141 public synchronized void progress() {
4142 if (showProgress++ == progressThreshold) {
4143 if (!getSUMMARY()) {
4144 System.out.print(".");
4145 }
4146 showProgress = 0;
4147 }
4148 }
4149 }
4150
4151
4152
4153
4154 static class WorkItemRegion implements Callable<Void> {
4155 private HBaseFsck hbck;
4156 private ServerName rsinfo;
4157 private ErrorReporter errors;
4158 private HConnection connection;
4159
4160 WorkItemRegion(HBaseFsck hbck, ServerName info,
4161 ErrorReporter errors, HConnection connection) {
4162 this.hbck = hbck;
4163 this.rsinfo = info;
4164 this.errors = errors;
4165 this.connection = connection;
4166 }
4167
4168 @Override
4169 public synchronized Void call() throws IOException {
4170 errors.progress();
4171 try {
4172 BlockingInterface server = connection.getAdmin(rsinfo);
4173
4174
4175 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4176 regions = filterRegions(regions);
4177
4178 if (details) {
4179 errors.detail("RegionServer: " + rsinfo.getServerName() +
4180 " number of regions: " + regions.size());
4181 for (HRegionInfo rinfo: regions) {
4182 errors.detail(" " + rinfo.getRegionNameAsString() +
4183 " id: " + rinfo.getRegionId() +
4184 " encoded_name: " + rinfo.getEncodedName() +
4185 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4186 " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4187 }
4188 }
4189
4190
4191 for (HRegionInfo r:regions) {
4192 HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4193 hbi.addServer(r, rsinfo);
4194 }
4195 } catch (IOException e) {
4196 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4197 " Unable to fetch region information. " + e);
4198 throw e;
4199 }
4200 return null;
4201 }
4202
4203 private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4204 List<HRegionInfo> ret = Lists.newArrayList();
4205 for (HRegionInfo hri : regions) {
4206 if (hri.isMetaTable() || (!hbck.checkMetaOnly
4207 && hbck.isTableIncluded(hri.getTable()))) {
4208 ret.add(hri);
4209 }
4210 }
4211 return ret;
4212 }
4213 }
4214
4215
4216
4217
4218
4219 static class WorkItemHdfsDir implements Callable<Void> {
4220 private HBaseFsck hbck;
4221 private FileStatus tableDir;
4222 private ErrorReporter errors;
4223 private FileSystem fs;
4224
4225 WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4226 FileStatus status) {
4227 this.hbck = hbck;
4228 this.fs = fs;
4229 this.tableDir = status;
4230 this.errors = errors;
4231 }
4232
4233 @Override
4234 public synchronized Void call() throws IOException {
4235 try {
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292 static class WorkItemHdfsRegionInfo implements Callable<Void> {
4293 private HbckInfo hbi;
4294 private HBaseFsck hbck;
4295 private ErrorReporter errors;
4296
4297 WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4298 this.hbi = hbi;
4299 this.hbck = hbck;
4300 this.errors = errors;
4301 }
4302
4303 @Override
4304 public synchronized Void call() throws IOException {
4305
4306 if (hbi.getHdfsHRI() == null) {
4307 try {
4308 errors.progress();
4309 hbck.loadHdfsRegioninfo(hbi);
4310 } catch (IOException ioe) {
4311 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4312 + hbi.getTableName() + " in hdfs dir "
4313 + hbi.getHdfsRegionDir()
4314 + "! It may be an invalid format or version file. Treating as "
4315 + "an orphaned regiondir.";
4316 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4317 try {
4318 hbck.debugLsr(hbi.getHdfsRegionDir());
4319 } catch (IOException ioe2) {
4320 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4321 throw ioe2;
4322 }
4323 hbck.orphanHdfsDirs.add(hbi);
4324 throw ioe;
4325 }
4326 }
4327 return null;
4328 }
4329 };
4330
4331
4332
4333
4334
4335 public static void setDisplayFullReport() {
4336 details = true;
4337 }
4338
4339
4340
4341
4342 public static void setForceExclusive() {
4343 forceExclusive = true;
4344 }
4345
4346
4347
4348
4349 public boolean isExclusive() {
4350 return fixAny || forceExclusive;
4351 }
4352
4353
4354
4355
4356 public static void setDisableBalancer() {
4357 disableBalancer = true;
4358 }
4359
4360
4361
4362
4363 public static void setDisableSplitAndMerge() {
4364 disableSplitAndMerge = true;
4365 }
4366
4367
4368
4369
4370
4371
4372 public boolean shouldDisableBalancer() {
4373 return fixAny || disableBalancer;
4374 }
4375
4376
4377
4378
4379
4380
4381 public boolean shouldDisableSplitAndMerge() {
4382 return fixAny || disableSplitAndMerge;
4383 }
4384
4385
4386
4387
4388
4389 synchronized static void setSummary() {
4390 SUMMARY = true;
4391 }
4392
4393
4394
4395
4396
4397 void setCheckMetaOnly() {
4398 checkMetaOnly = true;
4399 }
4400
4401
4402
4403
4404 void setRegionBoundariesCheck() {
4405 checkRegionBoundaries = true;
4406 }
4407
4408
4409
4410
4411
4412 public void setFixTableLocks(boolean shouldFix) {
4413 fixTableLocks = shouldFix;
4414 fixAny |= shouldFix;
4415 }
4416
4417
4418
4419
4420 public void setFixReplication(boolean shouldFix) {
4421 fixReplication = shouldFix;
4422 fixAny |= shouldFix;
4423 }
4424
4425
4426
4427
4428
4429 public void setFixTableZNodes(boolean shouldFix) {
4430 fixTableZNodes = shouldFix;
4431 fixAny |= shouldFix;
4432 }
4433
4434
4435
4436
4437
4438
4439
4440 void setShouldRerun() {
4441 rerun = true;
4442 }
4443
4444 boolean shouldRerun() {
4445 return rerun;
4446 }
4447
4448
4449
4450
4451
4452 public void setFixAssignments(boolean shouldFix) {
4453 fixAssignments = shouldFix;
4454 fixAny |= shouldFix;
4455 }
4456
4457 boolean shouldFixAssignments() {
4458 return fixAssignments;
4459 }
4460
4461 public void setFixMeta(boolean shouldFix) {
4462 fixMeta = shouldFix;
4463 fixAny |= shouldFix;
4464 }
4465
4466 boolean shouldFixMeta() {
4467 return fixMeta;
4468 }
4469
4470 public void setFixEmptyMetaCells(boolean shouldFix) {
4471 fixEmptyMetaCells = shouldFix;
4472 fixAny |= shouldFix;
4473 }
4474
4475 boolean shouldFixEmptyMetaCells() {
4476 return fixEmptyMetaCells;
4477 }
4478
4479 public void setCheckHdfs(boolean checking) {
4480 checkHdfs = checking;
4481 }
4482
4483 boolean shouldCheckHdfs() {
4484 return checkHdfs;
4485 }
4486
4487 public void setFixHdfsHoles(boolean shouldFix) {
4488 fixHdfsHoles = shouldFix;
4489 fixAny |= shouldFix;
4490 }
4491
4492 boolean shouldFixHdfsHoles() {
4493 return fixHdfsHoles;
4494 }
4495
4496 public void setFixTableOrphans(boolean shouldFix) {
4497 fixTableOrphans = shouldFix;
4498 fixAny |= shouldFix;
4499 }
4500
4501 boolean shouldFixTableOrphans() {
4502 return fixTableOrphans;
4503 }
4504
4505 public void setFixHdfsOverlaps(boolean shouldFix) {
4506 fixHdfsOverlaps = shouldFix;
4507 fixAny |= shouldFix;
4508 }
4509
4510 boolean shouldFixHdfsOverlaps() {
4511 return fixHdfsOverlaps;
4512 }
4513
4514 public void setFixHdfsOrphans(boolean shouldFix) {
4515 fixHdfsOrphans = shouldFix;
4516 fixAny |= shouldFix;
4517 }
4518
4519 boolean shouldFixHdfsOrphans() {
4520 return fixHdfsOrphans;
4521 }
4522
4523 public void setFixVersionFile(boolean shouldFix) {
4524 fixVersionFile = shouldFix;
4525 fixAny |= shouldFix;
4526 }
4527
4528 public boolean shouldFixVersionFile() {
4529 return fixVersionFile;
4530 }
4531
4532 public void setSidelineBigOverlaps(boolean sbo) {
4533 this.sidelineBigOverlaps = sbo;
4534 }
4535
4536 public boolean shouldSidelineBigOverlaps() {
4537 return sidelineBigOverlaps;
4538 }
4539
4540 public void setFixSplitParents(boolean shouldFix) {
4541 fixSplitParents = shouldFix;
4542 fixAny |= shouldFix;
4543 }
4544
4545 public void setRemoveParents(boolean shouldFix) {
4546 removeParents = shouldFix;
4547 fixAny |= shouldFix;
4548 }
4549
4550 boolean shouldFixSplitParents() {
4551 return fixSplitParents;
4552 }
4553
4554 boolean shouldRemoveParents() {
4555 return removeParents;
4556 }
4557
4558 public void setFixReferenceFiles(boolean shouldFix) {
4559 fixReferenceFiles = shouldFix;
4560 fixAny |= shouldFix;
4561 }
4562
4563 boolean shouldFixReferenceFiles() {
4564 return fixReferenceFiles;
4565 }
4566
4567 public boolean shouldIgnorePreCheckPermission() {
4568 return !fixAny || ignorePreCheckPermission;
4569 }
4570
4571 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4572 this.ignorePreCheckPermission = ignorePreCheckPermission;
4573 }
4574
4575
4576
4577
4578 public void setMaxMerge(int mm) {
4579 this.maxMerge = mm;
4580 }
4581
4582 public int getMaxMerge() {
4583 return maxMerge;
4584 }
4585
4586 public void setMaxOverlapsToSideline(int mo) {
4587 this.maxOverlapsToSideline = mo;
4588 }
4589
4590 public int getMaxOverlapsToSideline() {
4591 return maxOverlapsToSideline;
4592 }
4593
4594
4595
4596
4597
4598 boolean isTableIncluded(TableName table) {
4599 return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4600 }
4601
4602 public void includeTable(TableName table) {
4603 tablesIncluded.add(table);
4604 }
4605
4606 Set<TableName> getIncludedTables() {
4607 return new HashSet<TableName>(tablesIncluded);
4608 }
4609
4610
4611
4612
4613
4614
4615 public void setTimeLag(long seconds) {
4616 timelag = seconds * 1000;
4617 }
4618
4619
4620
4621
4622
4623 public void setSidelineDir(String sidelineDir) {
4624 this.sidelineDir = new Path(sidelineDir);
4625 }
4626
4627 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4628 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4629 }
4630
4631 public HFileCorruptionChecker getHFilecorruptionChecker() {
4632 return hfcc;
4633 }
4634
4635 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4636 this.hfcc = hfcc;
4637 }
4638
4639 public void setRetCode(int code) {
4640 this.retcode = code;
4641 }
4642
4643 public int getRetCode() {
4644 return retcode;
4645 }
4646
4647 protected HBaseFsck printUsageAndExit() {
4648 StringWriter sw = new StringWriter(2048);
4649 PrintWriter out = new PrintWriter(sw);
4650 out.println("Usage: fsck [opts] {only tables}");
4651 out.println(" where [opts] are:");
4652 out.println(" -help Display help options (this)");
4653 out.println(" -details Display full report of all regions.");
4654 out.println(" -timelag <timeInSeconds> Process only regions that " +
4655 " have not experienced any metadata updates in the last " +
4656 " <timeInSeconds> seconds.");
4657 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4658 " before checking if the fix worked if run with -fix");
4659 out.println(" -summary Print only summary of the tables and status.");
4660 out.println(" -metaonly Only check the state of the hbase:meta table.");
4661 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4662 out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
4663 out.println(" -exclusive Abort if another hbck is exclusive or fixing.");
4664 out.println(" -disableBalancer Disable the load balancer.");
4665
4666 out.println("");
4667 out.println(" Metadata Repair options: (expert features, use with caution!)");
4668 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity");
4669 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");
4670 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");
4671 out.println(" -noHdfsChecking Don't load/check region info from HDFS."
4672 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4673 out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
4674 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");
4675 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4676 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
4677 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
4678 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4679 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
4680 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4681 out.println(" -fixSplitParents Try to force offline split parents to be online.");
4682 out.println(" -removeParents Try to offline and sideline lingering parents and keep daughter regions.");
4683 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
4684 out.println(" -fixReferenceFiles Try to offline lingering reference store files");
4685 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"
4686 + " (empty REGIONINFO_QUALIFIER rows)");
4687
4688 out.println("");
4689 out.println(" Datafile Repair options: (expert features, use with caution!)");
4690 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");
4691 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");
4692
4693 out.println("");
4694 out.println(" Metadata Repair shortcuts");
4695 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4696 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4697 "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4698 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4699
4700 out.println("");
4701 out.println(" Table lock options");
4702 out.println(" -fixTableLocks Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4703
4704 out.println("");
4705 out.println(" Table Znode options");
4706 out.println(" -fixOrphanedTableZnodes Set table state in ZNode to disabled if table does not exists");
4707
4708 out.println("");
4709 out.println(" Replication options");
4710 out.println(" -fixReplication Deletes replication queues for removed peers");
4711
4712 out.flush();
4713 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4714
4715 setRetCode(-2);
4716 return this;
4717 }
4718
4719
4720
4721
4722
4723
4724
4725 public static void main(String[] args) throws Exception {
4726
4727 Configuration conf = HBaseConfiguration.create();
4728 Path hbasedir = FSUtils.getRootDir(conf);
4729 URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4730 FSUtils.setFsDefault(conf, new Path(defaultFs));
4731 int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4732 System.exit(ret);
4733 }
4734
4735
4736
4737
4738 static class HBaseFsckTool extends Configured implements Tool {
4739 HBaseFsckTool(Configuration conf) { super(conf); }
4740 @Override
4741 public int run(String[] args) throws Exception {
4742 HBaseFsck hbck = new HBaseFsck(getConf());
4743 hbck.exec(hbck.executor, args);
4744 hbck.close();
4745 return hbck.getRetCode();
4746 }
4747 };
4748
4749
4750 public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4751 ServiceException, InterruptedException {
4752 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4753
4754 boolean checkCorruptHFiles = false;
4755 boolean sidelineCorruptHFiles = false;
4756
4757
4758 for (int i = 0; i < args.length; i++) {
4759 String cmd = args[i];
4760 if (cmd.equals("-help") || cmd.equals("-h")) {
4761 return printUsageAndExit();
4762 } else if (cmd.equals("-details")) {
4763 setDisplayFullReport();
4764 } else if (cmd.equals("-exclusive")) {
4765 setForceExclusive();
4766 } else if (cmd.equals("-disableBalancer")) {
4767 setDisableBalancer();
4768 } else if (cmd.equals("-disableSplitAndMerge")) {
4769 setDisableSplitAndMerge();
4770 } else if (cmd.equals("-timelag")) {
4771 if (i == args.length - 1) {
4772 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4773 return printUsageAndExit();
4774 }
4775 try {
4776 long timelag = Long.parseLong(args[i+1]);
4777 setTimeLag(timelag);
4778 } catch (NumberFormatException e) {
4779 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4780 return printUsageAndExit();
4781 }
4782 i++;
4783 } else if (cmd.equals("-sleepBeforeRerun")) {
4784 if (i == args.length - 1) {
4785 errors.reportError(ERROR_CODE.WRONG_USAGE,
4786 "HBaseFsck: -sleepBeforeRerun needs a value.");
4787 return printUsageAndExit();
4788 }
4789 try {
4790 sleepBeforeRerun = Long.parseLong(args[i+1]);
4791 } catch (NumberFormatException e) {
4792 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4793 return printUsageAndExit();
4794 }
4795 i++;
4796 } else if (cmd.equals("-sidelineDir")) {
4797 if (i == args.length - 1) {
4798 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4799 return printUsageAndExit();
4800 }
4801 i++;
4802 setSidelineDir(args[i]);
4803 } else if (cmd.equals("-fix")) {
4804 errors.reportError(ERROR_CODE.WRONG_USAGE,
4805 "This option is deprecated, please use -fixAssignments instead.");
4806 setFixAssignments(true);
4807 } else if (cmd.equals("-fixAssignments")) {
4808 setFixAssignments(true);
4809 } else if (cmd.equals("-fixMeta")) {
4810 setFixMeta(true);
4811 } else if (cmd.equals("-noHdfsChecking")) {
4812 setCheckHdfs(false);
4813 } else if (cmd.equals("-fixHdfsHoles")) {
4814 setFixHdfsHoles(true);
4815 } else if (cmd.equals("-fixHdfsOrphans")) {
4816 setFixHdfsOrphans(true);
4817 } else if (cmd.equals("-fixTableOrphans")) {
4818 setFixTableOrphans(true);
4819 } else if (cmd.equals("-fixHdfsOverlaps")) {
4820 setFixHdfsOverlaps(true);
4821 } else if (cmd.equals("-fixVersionFile")) {
4822 setFixVersionFile(true);
4823 } else if (cmd.equals("-sidelineBigOverlaps")) {
4824 setSidelineBigOverlaps(true);
4825 } else if (cmd.equals("-fixSplitParents")) {
4826 setFixSplitParents(true);
4827 } else if (cmd.equals("-removeParents")) {
4828 setRemoveParents(true);
4829 } else if (cmd.equals("-ignorePreCheckPermission")) {
4830 setIgnorePreCheckPermission(true);
4831 } else if (cmd.equals("-checkCorruptHFiles")) {
4832 checkCorruptHFiles = true;
4833 } else if (cmd.equals("-sidelineCorruptHFiles")) {
4834 sidelineCorruptHFiles = true;
4835 } else if (cmd.equals("-fixReferenceFiles")) {
4836 setFixReferenceFiles(true);
4837 } else if (cmd.equals("-fixEmptyMetaCells")) {
4838 setFixEmptyMetaCells(true);
4839 } else if (cmd.equals("-repair")) {
4840
4841
4842 setFixHdfsHoles(true);
4843 setFixHdfsOrphans(true);
4844 setFixMeta(true);
4845 setFixAssignments(true);
4846 setFixHdfsOverlaps(true);
4847 setFixVersionFile(true);
4848 setSidelineBigOverlaps(true);
4849 setFixSplitParents(false);
4850 setCheckHdfs(true);
4851 setFixReferenceFiles(true);
4852 setFixTableLocks(true);
4853 setFixTableZNodes(true);
4854 } else if (cmd.equals("-repairHoles")) {
4855
4856 setFixHdfsHoles(true);
4857 setFixHdfsOrphans(false);
4858 setFixMeta(true);
4859 setFixAssignments(true);
4860 setFixHdfsOverlaps(false);
4861 setSidelineBigOverlaps(false);
4862 setFixSplitParents(false);
4863 setCheckHdfs(true);
4864 } else if (cmd.equals("-maxOverlapsToSideline")) {
4865 if (i == args.length - 1) {
4866 errors.reportError(ERROR_CODE.WRONG_USAGE,
4867 "-maxOverlapsToSideline needs a numeric value argument.");
4868 return printUsageAndExit();
4869 }
4870 try {
4871 int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4872 setMaxOverlapsToSideline(maxOverlapsToSideline);
4873 } catch (NumberFormatException e) {
4874 errors.reportError(ERROR_CODE.WRONG_USAGE,
4875 "-maxOverlapsToSideline needs a numeric value argument.");
4876 return printUsageAndExit();
4877 }
4878 i++;
4879 } else if (cmd.equals("-maxMerge")) {
4880 if (i == args.length - 1) {
4881 errors.reportError(ERROR_CODE.WRONG_USAGE,
4882 "-maxMerge needs a numeric value argument.");
4883 return printUsageAndExit();
4884 }
4885 try {
4886 int maxMerge = Integer.parseInt(args[i+1]);
4887 setMaxMerge(maxMerge);
4888 } catch (NumberFormatException e) {
4889 errors.reportError(ERROR_CODE.WRONG_USAGE,
4890 "-maxMerge needs a numeric value argument.");
4891 return printUsageAndExit();
4892 }
4893 i++;
4894 } else if (cmd.equals("-summary")) {
4895 setSummary();
4896 } else if (cmd.equals("-metaonly")) {
4897 setCheckMetaOnly();
4898 } else if (cmd.equals("-boundaries")) {
4899 setRegionBoundariesCheck();
4900 } else if (cmd.equals("-fixTableLocks")) {
4901 setFixTableLocks(true);
4902 } else if (cmd.equals("-fixReplication")) {
4903 setFixReplication(true);
4904 } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4905 setFixTableZNodes(true);
4906 } else if (cmd.startsWith("-")) {
4907 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4908 return printUsageAndExit();
4909 } else {
4910 includeTable(TableName.valueOf(cmd));
4911 errors.print("Allow checking/fixes for table: " + cmd);
4912 }
4913 }
4914
4915 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4916
4917
4918 try {
4919 preCheckPermission();
4920 } catch (AccessDeniedException ace) {
4921 Runtime.getRuntime().exit(-1);
4922 } catch (IOException ioe) {
4923 Runtime.getRuntime().exit(-1);
4924 }
4925
4926
4927 connect();
4928
4929 try {
4930
4931 if (checkCorruptHFiles || sidelineCorruptHFiles) {
4932 LOG.info("Checking all hfiles for corruption");
4933 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4934 setHFileCorruptionChecker(hfcc);
4935 Collection<TableName> tables = getIncludedTables();
4936 Collection<Path> tableDirs = new ArrayList<Path>();
4937 Path rootdir = FSUtils.getRootDir(getConf());
4938 if (tables.size() > 0) {
4939 for (TableName t : tables) {
4940 tableDirs.add(FSUtils.getTableDir(rootdir, t));
4941 }
4942 } else {
4943 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4944 }
4945 hfcc.checkTables(tableDirs);
4946 hfcc.report(errors);
4947 }
4948
4949
4950 int code = onlineHbck();
4951 setRetCode(code);
4952
4953
4954
4955
4956 if (shouldRerun()) {
4957 try {
4958 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4959 Thread.sleep(sleepBeforeRerun);
4960 } catch (InterruptedException ie) {
4961 LOG.warn("Interrupted while sleeping");
4962 return this;
4963 }
4964
4965 setFixAssignments(false);
4966 setFixMeta(false);
4967 setFixHdfsHoles(false);
4968 setFixHdfsOverlaps(false);
4969 setFixVersionFile(false);
4970 setFixTableOrphans(false);
4971 errors.resetErrors();
4972 code = onlineHbck();
4973 setRetCode(code);
4974 }
4975 } finally {
4976 IOUtils.closeQuietly(this);
4977 }
4978 return this;
4979 }
4980
4981
4982
4983
4984 void debugLsr(Path p) throws IOException {
4985 debugLsr(getConf(), p, errors);
4986 }
4987
4988
4989
4990
4991 public static void debugLsr(Configuration conf,
4992 Path p) throws IOException {
4993 debugLsr(conf, p, new PrintingErrorReporter());
4994 }
4995
4996
4997
4998
4999 public static void debugLsr(Configuration conf,
5000 Path p, ErrorReporter errors) throws IOException {
5001 if (!LOG.isDebugEnabled() || p == null) {
5002 return;
5003 }
5004 FileSystem fs = p.getFileSystem(conf);
5005
5006 if (!fs.exists(p)) {
5007
5008 return;
5009 }
5010 errors.print(p.toString());
5011
5012 if (fs.isFile(p)) {
5013 return;
5014 }
5015
5016 if (fs.getFileStatus(p).isDirectory()) {
5017 FileStatus[] fss= fs.listStatus(p);
5018 for (FileStatus status : fss) {
5019 debugLsr(conf, status.getPath(), errors);
5020 }
5021 }
5022 }
5023 }