1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.io.PrintWriter;
25 import java.io.StringWriter;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.SortedMap;
41 import java.util.SortedSet;
42 import java.util.TreeMap;
43 import java.util.TreeSet;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.ConcurrentSkipListMap;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.FutureTask;
51 import java.util.concurrent.ScheduledThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.TimeoutException;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.concurrent.atomic.AtomicInteger;
56
57 import org.apache.commons.io.IOUtils;
58 import org.apache.commons.lang.StringUtils;
59 import org.apache.commons.logging.Log;
60 import org.apache.commons.logging.LogFactory;
61 import org.apache.hadoop.conf.Configuration;
62 import org.apache.hadoop.conf.Configured;
63 import org.apache.hadoop.fs.FSDataOutputStream;
64 import org.apache.hadoop.fs.FileStatus;
65 import org.apache.hadoop.fs.FileSystem;
66 import org.apache.hadoop.fs.Path;
67 import org.apache.hadoop.fs.permission.FsAction;
68 import org.apache.hadoop.fs.permission.FsPermission;
69 import org.apache.hadoop.hbase.Abortable;
70 import org.apache.hadoop.hbase.Cell;
71 import org.apache.hadoop.hbase.ClusterStatus;
72 import org.apache.hadoop.hbase.CoordinatedStateException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75 import org.apache.hadoop.hbase.HColumnDescriptor;
76 import org.apache.hadoop.hbase.HConstants;
77 import org.apache.hadoop.hbase.HRegionInfo;
78 import org.apache.hadoop.hbase.HRegionLocation;
79 import org.apache.hadoop.hbase.HTableDescriptor;
80 import org.apache.hadoop.hbase.KeyValue;
81 import org.apache.hadoop.hbase.MasterNotRunningException;
82 import org.apache.hadoop.hbase.MetaTableAccessor;
83 import org.apache.hadoop.hbase.RegionLocations;
84 import org.apache.hadoop.hbase.ServerName;
85 import org.apache.hadoop.hbase.TableName;
86 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
87 import org.apache.hadoop.hbase.classification.InterfaceAudience;
88 import org.apache.hadoop.hbase.classification.InterfaceStability;
89 import org.apache.hadoop.hbase.client.Admin;
90 import org.apache.hadoop.hbase.client.ClusterConnection;
91 import org.apache.hadoop.hbase.client.ConnectionFactory;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Get;
94 import org.apache.hadoop.hbase.client.HBaseAdmin;
95 import org.apache.hadoop.hbase.client.HConnectable;
96 import org.apache.hadoop.hbase.client.HConnection;
97 import org.apache.hadoop.hbase.client.HConnectionManager;
98 import org.apache.hadoop.hbase.client.MetaScanner;
99 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.RowMutations;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116 import org.apache.hadoop.hbase.security.AccessDeniedException;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
123 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
124 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
125 import org.apache.hadoop.hbase.wal.WALSplitter;
126 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
127 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
128 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
129 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
130 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
131 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
132 import org.apache.hadoop.ipc.RemoteException;
133 import org.apache.hadoop.security.UserGroupInformation;
134 import org.apache.hadoop.util.ReflectionUtils;
135 import org.apache.hadoop.util.Tool;
136 import org.apache.hadoop.util.ToolRunner;
137 import org.apache.zookeeper.KeeperException;
138
139 import com.google.common.base.Joiner;
140 import com.google.common.base.Preconditions;
141 import com.google.common.collect.ImmutableList;
142 import com.google.common.collect.Lists;
143 import com.google.common.collect.Multimap;
144 import com.google.common.collect.Ordering;
145 import com.google.common.collect.TreeMultimap;
146 import com.google.protobuf.ServiceException;
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
194 @InterfaceStability.Evolving
195 public class HBaseFsck extends Configured implements Closeable {
196 public static final long DEFAULT_TIME_LAG = 60000;
197 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
198 private static final int MAX_NUM_THREADS = 50;
199 private static boolean rsSupportsOffline = true;
200 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
201 private static final int DEFAULT_MAX_MERGE = 5;
202 private static final String TO_BE_LOADED = "to_be_loaded";
203 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
204 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
205 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200;
206 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000;
207
208
209
210
211 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80;
212
213
214
215
216 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
217 private ClusterStatus status;
218 private ClusterConnection connection;
219 private Admin admin;
220 private Table meta;
221
222 protected ExecutorService executor;
223 private long startMillis = EnvironmentEdgeManager.currentTime();
224 private HFileCorruptionChecker hfcc;
225 private int retcode = 0;
226 private Path HBCK_LOCK_PATH;
227 private FSDataOutputStream hbckOutFd;
228
229
230
231 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
232
233
234
235
236 private static boolean details = false;
237 private long timelag = DEFAULT_TIME_LAG;
238 private static boolean forceExclusive = false;
239 private static boolean disableBalancer = false;
240 private static boolean disableSplitAndMerge = false;
241 private boolean fixAssignments = false;
242 private boolean fixMeta = false;
243 private boolean checkHdfs = true;
244 private boolean fixHdfsHoles = false;
245 private boolean fixHdfsOverlaps = false;
246 private boolean fixHdfsOrphans = false;
247 private boolean fixTableOrphans = false;
248 private boolean fixVersionFile = false;
249 private boolean fixSplitParents = false;
250 private boolean removeParents = false;
251 private boolean fixReferenceFiles = false;
252 private boolean fixEmptyMetaCells = false;
253 private boolean fixTableLocks = false;
254 private boolean fixTableZNodes = false;
255 private boolean fixReplication = false;
256 private boolean fixAny = false;
257
258
259
260 private Set<TableName> tablesIncluded = new HashSet<TableName>();
261 private int maxMerge = DEFAULT_MAX_MERGE;
262 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
263 private boolean sidelineBigOverlaps = false;
264 private Path sidelineDir = null;
265
266 private boolean rerun = false;
267 private static boolean SUMMARY = false;
268 private boolean checkMetaOnly = false;
269 private boolean checkRegionBoundaries = false;
270 private boolean ignorePreCheckPermission = false;
271
272
273
274
275 final private ErrorReporter errors;
276 int fixes = 0;
277
278
279
280
281
282
283 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
284 private TreeSet<TableName> disabledTables =
285 new TreeSet<TableName>();
286
287 private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
288
289
290
291
292
293
294
295
296
297
298
299 private SortedMap<TableName, TableInfo> tablesInfo =
300 new ConcurrentSkipListMap<TableName, TableInfo>();
301
302
303
304
305 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
306
307 private Map<TableName, Set<String>> orphanTableDirs =
308 new HashMap<TableName, Set<String>>();
309
310 private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
311
312
313
314
315 private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
316 private final RetryCounterFactory lockFileRetryCounterFactory;
317
318
319
320
321
322
323
324
325
326 public HBaseFsck(Configuration conf) throws MasterNotRunningException,
327 ZooKeeperConnectionException, IOException, ClassNotFoundException {
328 this(conf, createThreadPool(conf));
329 }
330
331 private static ExecutorService createThreadPool(Configuration conf) {
332 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
333 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
334 }
335
336
337
338
339
340
341
342
343
344
345
346 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
347 ZooKeeperConnectionException, IOException, ClassNotFoundException {
348 super(conf);
349 errors = getErrorReporter(getConf());
350 this.executor = exec;
351 lockFileRetryCounterFactory = new RetryCounterFactory(
352 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
353 getConf().getInt(
354 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
355 getConf().getInt(
356 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
357 }
358
359 private class FileLockCallable implements Callable<FSDataOutputStream> {
360 RetryCounter retryCounter;
361
362 public FileLockCallable(RetryCounter retryCounter) {
363 this.retryCounter = retryCounter;
364 }
365 @Override
366 public FSDataOutputStream call() throws IOException {
367 try {
368 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
369 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
370 HConstants.DATA_FILE_UMASK_KEY);
371 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
372 fs.mkdirs(tmpDir);
373 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
374 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
375 out.writeBytes(InetAddress.getLocalHost().toString());
376 out.flush();
377 return out;
378 } catch(RemoteException e) {
379 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
380 return null;
381 } else {
382 throw e;
383 }
384 }
385 }
386
387 private FSDataOutputStream createFileWithRetries(final FileSystem fs,
388 final Path hbckLockFilePath, final FsPermission defaultPerms)
389 throws IOException {
390
391 IOException exception = null;
392 do {
393 try {
394 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
395 } catch (IOException ioe) {
396 LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
397 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
398 + retryCounter.getMaxAttempts());
399 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
400 ioe);
401 try {
402 exception = ioe;
403 retryCounter.sleepUntilNextRetry();
404 } catch (InterruptedException ie) {
405 throw (InterruptedIOException) new InterruptedIOException(
406 "Can't create lock file " + hbckLockFilePath.getName())
407 .initCause(ie);
408 }
409 }
410 } while (retryCounter.shouldRetry());
411
412 throw exception;
413 }
414 }
415
416
417
418
419
420
421
422 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
423 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
424 FileLockCallable callable = new FileLockCallable(retryCounter);
425 ExecutorService executor = Executors.newFixedThreadPool(1);
426 FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
427 executor.execute(futureTask);
428 final int timeoutInSeconds = getConf().getInt(
429 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
430 FSDataOutputStream stream = null;
431 try {
432 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
433 } catch (ExecutionException ee) {
434 LOG.warn("Encountered exception when opening lock file", ee);
435 } catch (InterruptedException ie) {
436 LOG.warn("Interrupted when opening lock file", ie);
437 Thread.currentThread().interrupt();
438 } catch (TimeoutException exception) {
439
440 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
441 futureTask.cancel(true);
442 } finally {
443 executor.shutdownNow();
444 }
445 return stream;
446 }
447
448 private void unlockHbck() {
449 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
450 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
451 do {
452 try {
453 IOUtils.closeQuietly(hbckOutFd);
454 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
455 HBCK_LOCK_PATH, true);
456 LOG.info("Finishing hbck");
457 return;
458 } catch (IOException ioe) {
459 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
460 + (retryCounter.getAttemptTimes() + 1) + " of "
461 + retryCounter.getMaxAttempts());
462 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
463 try {
464 retryCounter.sleepUntilNextRetry();
465 } catch (InterruptedException ie) {
466 Thread.currentThread().interrupt();
467 LOG.warn("Interrupted while deleting lock file" +
468 HBCK_LOCK_PATH);
469 return;
470 }
471 }
472 } while (retryCounter.shouldRetry());
473 }
474 }
475
476
477
478
479
480 public void connect() throws IOException {
481
482 if (isExclusive()) {
483
484 hbckOutFd = checkAndMarkRunningHbck();
485 if (hbckOutFd == null) {
486 setRetCode(-1);
487 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
488 "[If you are sure no other instance is running, delete the lock file " +
489 HBCK_LOCK_PATH + " and rerun the tool]");
490 throw new IOException("Duplicate hbck - Abort");
491 }
492
493
494 hbckLockCleanup.set(true);
495 }
496
497
498
499
500
501 Runtime.getRuntime().addShutdownHook(new Thread() {
502 @Override
503 public void run() {
504 IOUtils.closeQuietly(HBaseFsck.this);
505 unlockHbck();
506 }
507 });
508
509 LOG.info("Launching hbck");
510
511 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
512 admin = connection.getAdmin();
513 meta = connection.getTable(TableName.META_TABLE_NAME);
514 status = admin.getClusterStatus();
515 }
516
517
518
519
520 private void loadDeployedRegions() throws IOException, InterruptedException {
521
522 Collection<ServerName> regionServers = status.getServers();
523 errors.print("Number of live region servers: " + regionServers.size());
524 if (details) {
525 for (ServerName rsinfo: regionServers) {
526 errors.print(" " + rsinfo.getServerName());
527 }
528 }
529
530
531 Collection<ServerName> deadRegionServers = status.getDeadServerNames();
532 errors.print("Number of dead region servers: " + deadRegionServers.size());
533 if (details) {
534 for (ServerName name: deadRegionServers) {
535 errors.print(" " + name);
536 }
537 }
538
539
540 errors.print("Master: " + status.getMaster());
541
542
543 Collection<ServerName> backupMasters = status.getBackupMasters();
544 errors.print("Number of backup masters: " + backupMasters.size());
545 if (details) {
546 for (ServerName name: backupMasters) {
547 errors.print(" " + name);
548 }
549 }
550
551 errors.print("Average load: " + status.getAverageLoad());
552 errors.print("Number of requests: " + status.getRequestsCount());
553 errors.print("Number of regions: " + status.getRegionsCount());
554
555 Map<String, RegionState> rits = status.getRegionsInTransition();
556 errors.print("Number of regions in transition: " + rits.size());
557 if (details) {
558 for (RegionState state: rits.values()) {
559 errors.print(" " + state.toDescriptiveString());
560 }
561 }
562
563
564 processRegionServers(regionServers);
565 }
566
567
568
569
570 private void clearState() {
571
572 fixes = 0;
573 regionInfoMap.clear();
574 emptyRegionInfoQualifiers.clear();
575 disabledTables.clear();
576 errors.clear();
577 tablesInfo.clear();
578 orphanHdfsDirs.clear();
579 skippedRegions.clear();
580 }
581
582
583
584
585
586
587 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
588
589 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
590 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
591 LOG.info("Loading regioninfos HDFS");
592
593 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
594 int curIter = 0;
595 do {
596 clearState();
597
598 restoreHdfsIntegrity();
599 curIter++;
600 } while (fixes > 0 && curIter <= maxIterations);
601
602
603
604 if (curIter > 2) {
605 if (curIter == maxIterations) {
606 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
607 + "Tables integrity may not be fully repaired!");
608 } else {
609 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
610 }
611 }
612 }
613 }
614
615
616
617
618
619
620
621
622
623 public int onlineConsistencyRepair() throws IOException, KeeperException,
624 InterruptedException {
625 clearState();
626
627
628 loadDeployedRegions();
629
630 recordMetaRegion();
631
632 if (!checkMetaRegion()) {
633 String errorMsg = "hbase:meta table is not consistent. ";
634 if (shouldFixAssignments()) {
635 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
636 } else {
637 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
638 }
639 errors.reportError(errorMsg + " Exiting...");
640 return -2;
641 }
642
643 LOG.info("Loading regionsinfo from the hbase:meta table");
644 boolean success = loadMetaEntries();
645 if (!success) return -1;
646
647
648 reportEmptyMetaCells();
649
650
651 if (shouldFixEmptyMetaCells()) {
652 fixEmptyMetaCells();
653 }
654
655
656 if (!checkMetaOnly) {
657 reportTablesInFlux();
658 }
659
660
661 if (shouldCheckHdfs()) {
662 LOG.info("Loading region directories from HDFS");
663 loadHdfsRegionDirs();
664 LOG.info("Loading region information from HDFS");
665 loadHdfsRegionInfos();
666 }
667
668
669 loadDisabledTables();
670
671
672 fixOrphanTables();
673
674 LOG.info("Checking and fixing region consistency");
675
676 checkAndFixConsistency();
677
678
679 checkIntegrity();
680 return errors.getErrorList().size();
681 }
682
683
684
685
686
687 public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
688
689 errors.print("Version: " + status.getHBaseVersion());
690 offlineHdfsIntegrityRepair();
691
692 boolean oldBalancer = false;
693 if (shouldDisableBalancer()) {
694 oldBalancer = admin.setBalancerRunning(false, true);
695 }
696 boolean[] oldSplitAndMerge = null;
697 if (shouldDisableSplitAndMerge()) {
698 oldSplitAndMerge = admin.setSplitOrMergeEnabled(false, false,
699 Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
700 }
701
702 try {
703 onlineConsistencyRepair();
704 }
705 finally {
706
707
708
709 if (shouldDisableBalancer() && oldBalancer) {
710 admin.setBalancerRunning(oldBalancer, false);
711 }
712
713 if (shouldDisableSplitAndMerge()) {
714 if (oldSplitAndMerge != null) {
715 if (oldSplitAndMerge[0] && oldSplitAndMerge[1]) {
716 admin.setSplitOrMergeEnabled(true, false,
717 Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
718 } else if (oldSplitAndMerge[0]) {
719 admin.setSplitOrMergeEnabled(true, false, Admin.MasterSwitchType.SPLIT);
720 } else if (oldSplitAndMerge[1]) {
721 admin.setSplitOrMergeEnabled(true, false, Admin.MasterSwitchType.MERGE);
722 }
723 }
724 }
725 }
726
727 if (checkRegionBoundaries) {
728 checkRegionBoundaries();
729 }
730
731 offlineReferenceFileRepair();
732
733 checkAndFixTableLocks();
734
735
736 checkAndFixOrphanedTableZNodes();
737
738 checkAndFixReplication();
739
740
741 unlockHbck();
742
743
744 printTableSummary(tablesInfo);
745 return errors.summarize();
746 }
747
748 public static byte[] keyOnly (byte[] b) {
749 if (b == null)
750 return b;
751 int rowlength = Bytes.toShort(b, 0);
752 byte[] result = new byte[rowlength];
753 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
754 return result;
755 }
756
757 @Override
758 public void close() throws IOException {
759 IOUtils.closeQuietly(admin);
760 IOUtils.closeQuietly(meta);
761 IOUtils.closeQuietly(connection);
762 }
763
764 private static class RegionBoundariesInformation {
765 public byte [] regionName;
766 public byte [] metaFirstKey;
767 public byte [] metaLastKey;
768 public byte [] storesFirstKey;
769 public byte [] storesLastKey;
770 @Override
771 public String toString () {
772 return "regionName=" + Bytes.toStringBinary(regionName) +
773 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
774 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
775 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
776 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
777 }
778 }
779
780 public void checkRegionBoundaries() {
781 try {
782 ByteArrayComparator comparator = new ByteArrayComparator();
783 List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
784 final RegionBoundariesInformation currentRegionBoundariesInformation =
785 new RegionBoundariesInformation();
786 Path hbaseRoot = FSUtils.getRootDir(getConf());
787 for (HRegionInfo regionInfo : regions) {
788 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
789 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
790
791
792 Path path = new Path(tableDir, regionInfo.getEncodedName());
793 FileSystem fs = path.getFileSystem(getConf());
794 FileStatus[] files = fs.listStatus(path);
795
796 byte[] storeFirstKey = null;
797 byte[] storeLastKey = null;
798 for (FileStatus file : files) {
799 String fileName = file.getPath().toString();
800 fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
801 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
802 FileStatus[] storeFiles = fs.listStatus(file.getPath());
803
804 for (FileStatus storeFile : storeFiles) {
805 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
806 getConf()), getConf());
807 if (reader.getFirstKey() != null) {
808 byte[] firstKey = keyOnly(reader.getFirstKey());
809 if (storeFirstKey == null || comparator.compare(storeFirstKey, firstKey) > 0) {
810 storeFirstKey = firstKey;
811 }
812 }
813 if (reader.getLastKey() != null) {
814 byte[] lastKey = keyOnly(reader.getLastKey());
815 if (storeLastKey == null || comparator.compare(storeLastKey, lastKey) < 0) {
816 storeLastKey = lastKey;
817 }
818 }
819 reader.close();
820 }
821 }
822 }
823 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
824 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
825 currentRegionBoundariesInformation.storesFirstKey = storeFirstKey;
826 currentRegionBoundariesInformation.storesLastKey = storeLastKey;
827 if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
828 currentRegionBoundariesInformation.metaFirstKey = null;
829 if (currentRegionBoundariesInformation.metaLastKey.length == 0)
830 currentRegionBoundariesInformation.metaLastKey = null;
831
832
833
834
835
836
837 boolean valid = true;
838
839 if ((currentRegionBoundariesInformation.storesFirstKey != null)
840 && (currentRegionBoundariesInformation.metaFirstKey != null)) {
841 valid = valid
842 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
843 currentRegionBoundariesInformation.metaFirstKey) >= 0;
844 }
845
846 if ((currentRegionBoundariesInformation.storesLastKey != null)
847 && (currentRegionBoundariesInformation.metaLastKey != null)) {
848 valid = valid
849 && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
850 currentRegionBoundariesInformation.metaLastKey) < 0;
851 }
852 if (!valid) {
853 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
854 tablesInfo.get(regionInfo.getTable()));
855 LOG.warn("Region's boundaries not aligned between stores and META for:");
856 LOG.warn(currentRegionBoundariesInformation);
857 }
858 }
859 } catch (IOException e) {
860 LOG.error(e);
861 }
862 }
863
864
865
866
867 private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
868 for (HbckInfo hi : orphanHdfsDirs) {
869 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
870 adoptHdfsOrphan(hi);
871 }
872 }
873
874
875
876
877
878
879
880
881
882
883 @SuppressWarnings("deprecation")
884 private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
885 Path p = hi.getHdfsRegionDir();
886 FileSystem fs = p.getFileSystem(getConf());
887 FileStatus[] dirs = fs.listStatus(p);
888 if (dirs == null) {
889 LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
890 p + ". This dir could probably be deleted.");
891 return ;
892 }
893
894 TableName tableName = hi.getTableName();
895 TableInfo tableInfo = tablesInfo.get(tableName);
896 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
897 HTableDescriptor template = tableInfo.getHTD();
898
899
900 Pair<byte[],byte[]> orphanRegionRange = null;
901 for (FileStatus cf : dirs) {
902 String cfName= cf.getPath().getName();
903
904 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
905
906 FileStatus[] hfiles = fs.listStatus(cf.getPath());
907 for (FileStatus hfile : hfiles) {
908 byte[] start, end;
909 HFile.Reader hf = null;
910 try {
911 CacheConfig cacheConf = new CacheConfig(getConf());
912 hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
913 hf.loadFileInfo();
914 KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
915 start = startKv.getRow();
916 KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
917 end = endKv.getRow();
918 } catch (IOException ioe) {
919 LOG.warn("Problem reading orphan file " + hfile + ", skipping");
920 continue;
921 } catch (NullPointerException ioe) {
922 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
923 continue;
924 } finally {
925 if (hf != null) {
926 hf.close();
927 }
928 }
929
930
931 if (orphanRegionRange == null) {
932
933 orphanRegionRange = new Pair<byte[], byte[]>(start, end);
934 } else {
935
936
937
938 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
939 orphanRegionRange.setFirst(start);
940 }
941 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
942 orphanRegionRange.setSecond(end);
943 }
944 }
945 }
946 }
947 if (orphanRegionRange == null) {
948 LOG.warn("No data in dir " + p + ", sidelining data");
949 fixes++;
950 sidelineRegionDir(fs, hi);
951 return;
952 }
953 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
954 Bytes.toString(orphanRegionRange.getSecond()) + ")");
955
956
957 HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(),
958 Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
959 LOG.info("Creating new region : " + hri);
960 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
961 Path target = region.getRegionFileSystem().getRegionDir();
962
963
964 mergeRegionDirs(target, hi);
965 fixes++;
966 }
967
968
969
970
971
972
973
974
975
976 private int restoreHdfsIntegrity() throws IOException, InterruptedException {
977
978 LOG.info("Loading HBase regioninfo from HDFS...");
979 loadHdfsRegionDirs();
980
981 int errs = errors.getErrorList().size();
982
983 tablesInfo = loadHdfsRegionInfos();
984 checkHdfsIntegrity(false, false);
985
986 if (errors.getErrorList().size() == errs) {
987 LOG.info("No integrity errors. We are done with this phase. Glorious.");
988 return 0;
989 }
990
991 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
992 adoptHdfsOrphans(orphanHdfsDirs);
993
994 }
995
996
997 if (shouldFixHdfsHoles()) {
998 clearState();
999 loadHdfsRegionDirs();
1000 tablesInfo = loadHdfsRegionInfos();
1001 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1002 }
1003
1004
1005 if (shouldFixHdfsOverlaps()) {
1006
1007 clearState();
1008 loadHdfsRegionDirs();
1009 tablesInfo = loadHdfsRegionInfos();
1010 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1011 }
1012
1013 return errors.getErrorList().size();
1014 }
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024 private void offlineReferenceFileRepair() throws IOException {
1025 Configuration conf = getConf();
1026 Path hbaseRoot = FSUtils.getRootDir(conf);
1027 FileSystem fs = hbaseRoot.getFileSystem(conf);
1028 LOG.info("Computing mapping of all store files");
1029 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1030 errors.print("");
1031 LOG.info("Validating mapping using HDFS state");
1032 for (Path path: allFiles.values()) {
1033 boolean isReference = false;
1034 try {
1035 isReference = StoreFileInfo.isReference(path);
1036 } catch (Throwable t) {
1037
1038
1039
1040
1041 }
1042 if (!isReference) continue;
1043
1044 Path referredToFile = StoreFileInfo.getReferredToFile(path);
1045 if (fs.exists(referredToFile)) continue;
1046
1047
1048 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1049 "Found lingering reference file " + path);
1050 if (!shouldFixReferenceFiles()) continue;
1051
1052
1053 boolean success = false;
1054 String pathStr = path.toString();
1055
1056
1057
1058
1059
1060 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1061 for (int i = 0; index > 0 && i < 5; i++) {
1062 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1063 }
1064 if (index > 0) {
1065 Path rootDir = getSidelineDir();
1066 Path dst = new Path(rootDir, pathStr.substring(index + 1));
1067 fs.mkdirs(dst.getParent());
1068 LOG.info("Trying to sideline reference file "
1069 + path + " to " + dst);
1070 setShouldRerun();
1071
1072 success = fs.rename(path, dst);
1073 debugLsr(dst);
1074
1075 }
1076 if (!success) {
1077 LOG.error("Failed to sideline reference file " + path);
1078 }
1079 }
1080 }
1081
1082
1083
1084
1085 private void reportEmptyMetaCells() {
1086 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1087 emptyRegionInfoQualifiers.size());
1088 if (details) {
1089 for (Result r: emptyRegionInfoQualifiers) {
1090 errors.print(" " + r);
1091 }
1092 }
1093 }
1094
1095
1096
1097
1098 private void reportTablesInFlux() {
1099 AtomicInteger numSkipped = new AtomicInteger(0);
1100 HTableDescriptor[] allTables = getTables(numSkipped);
1101 errors.print("Number of Tables: " + allTables.length);
1102 if (details) {
1103 if (numSkipped.get() > 0) {
1104 errors.detail("Number of Tables in flux: " + numSkipped.get());
1105 }
1106 for (HTableDescriptor td : allTables) {
1107 errors.detail(" Table: " + td.getTableName() + "\t" +
1108 (td.isReadOnly() ? "ro" : "rw") + "\t" +
1109 (td.isMetaRegion() ? "META" : " ") + "\t" +
1110 " families: " + td.getFamilies().size());
1111 }
1112 }
1113 }
1114
1115 public ErrorReporter getErrors() {
1116 return errors;
1117 }
1118
1119
1120
1121
1122
1123 private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1124 Path regionDir = hbi.getHdfsRegionDir();
1125 if (regionDir == null) {
1126 if (hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1127
1128 LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1129 }
1130 return;
1131 }
1132
1133 if (hbi.hdfsEntry.hri != null) {
1134
1135 return;
1136 }
1137
1138 FileSystem fs = FileSystem.get(getConf());
1139 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1140 LOG.debug("HRegionInfo read: " + hri.toString());
1141 hbi.hdfsEntry.hri = hri;
1142 }
1143
1144
1145
1146
1147
1148 public static class RegionRepairException extends IOException {
1149 private static final long serialVersionUID = 1L;
1150 final IOException ioe;
1151 public RegionRepairException(String s, IOException ioe) {
1152 super(s);
1153 this.ioe = ioe;
1154 }
1155 }
1156
1157
1158
1159
1160 private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1161 throws IOException, InterruptedException {
1162 tablesInfo.clear();
1163
1164 Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1165
1166
1167 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1168 List<Future<Void>> hbiFutures;
1169
1170 for (HbckInfo hbi : hbckInfos) {
1171 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1172 hbis.add(work);
1173 }
1174
1175
1176 hbiFutures = executor.invokeAll(hbis);
1177
1178 for(int i=0; i<hbiFutures.size(); i++) {
1179 WorkItemHdfsRegionInfo work = hbis.get(i);
1180 Future<Void> f = hbiFutures.get(i);
1181 try {
1182 f.get();
1183 } catch(ExecutionException e) {
1184 LOG.warn("Failed to read .regioninfo file for region " +
1185 work.hbi.getRegionNameAsString(), e.getCause());
1186 }
1187 }
1188
1189 Path hbaseRoot = FSUtils.getRootDir(getConf());
1190 FileSystem fs = hbaseRoot.getFileSystem(getConf());
1191
1192 for (HbckInfo hbi: hbckInfos) {
1193
1194 if (hbi.getHdfsHRI() == null) {
1195
1196 continue;
1197 }
1198
1199
1200
1201 TableName tableName = hbi.getTableName();
1202 if (tableName == null) {
1203
1204 LOG.warn("tableName was null for: " + hbi);
1205 continue;
1206 }
1207
1208 TableInfo modTInfo = tablesInfo.get(tableName);
1209 if (modTInfo == null) {
1210
1211 modTInfo = new TableInfo(tableName);
1212 tablesInfo.put(tableName, modTInfo);
1213 try {
1214 HTableDescriptor htd =
1215 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1216 modTInfo.htds.add(htd);
1217 } catch (IOException ioe) {
1218 if (!orphanTableDirs.containsKey(tableName)) {
1219 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1220
1221 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1222 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1223 Set<String> columns = new HashSet<String>();
1224 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1225 }
1226 }
1227 }
1228 if (!hbi.isSkipChecks()) {
1229 modTInfo.addRegionInfo(hbi);
1230 }
1231 }
1232
1233 loadTableInfosForTablesWithNoRegion();
1234 errors.print("");
1235
1236 return tablesInfo;
1237 }
1238
1239
1240
1241
1242
1243
1244
1245
1246 private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1247 Path regionDir = hbi.getHdfsRegionDir();
1248 FileSystem fs = regionDir.getFileSystem(getConf());
1249 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1250 for (FileStatus subdir : subDirs) {
1251 String columnfamily = subdir.getPath().getName();
1252 columns.add(columnfamily);
1253 }
1254 return columns;
1255 }
1256
1257
1258
1259
1260
1261
1262
1263
1264 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1265 Set<String> columns) throws IOException {
1266 if (columns ==null || columns.isEmpty()) return false;
1267 HTableDescriptor htd = new HTableDescriptor(tableName);
1268 for (String columnfamimly : columns) {
1269 htd.addFamily(new HColumnDescriptor(columnfamimly));
1270 }
1271 fstd.createTableDescriptor(htd, true);
1272 return true;
1273 }
1274
1275
1276
1277
1278
1279 public void fixEmptyMetaCells() throws IOException {
1280 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1281 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1282 for (Result region : emptyRegionInfoQualifiers) {
1283 deleteMetaRegion(region.getRow());
1284 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1285 }
1286 emptyRegionInfoQualifiers.clear();
1287 }
1288 }
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299 public void fixOrphanTables() throws IOException {
1300 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1301
1302 List<TableName> tmpList = new ArrayList<TableName>();
1303 tmpList.addAll(orphanTableDirs.keySet());
1304 HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1305 Iterator<Entry<TableName, Set<String>>> iter =
1306 orphanTableDirs.entrySet().iterator();
1307 int j = 0;
1308 int numFailedCase = 0;
1309 FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1310 while (iter.hasNext()) {
1311 Entry<TableName, Set<String>> entry =
1312 iter.next();
1313 TableName tableName = entry.getKey();
1314 LOG.info("Trying to fix orphan table error: " + tableName);
1315 if (j < htds.length) {
1316 if (tableName.equals(htds[j].getTableName())) {
1317 HTableDescriptor htd = htds[j];
1318 LOG.info("fixing orphan table: " + tableName + " from cache");
1319 fstd.createTableDescriptor(htd, true);
1320 j++;
1321 iter.remove();
1322 }
1323 } else {
1324 if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1325 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1326 LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1327 iter.remove();
1328 } else {
1329 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1330 numFailedCase++;
1331 }
1332 }
1333 fixes++;
1334 }
1335
1336 if (orphanTableDirs.isEmpty()) {
1337
1338
1339 setShouldRerun();
1340 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1341 } else if (numFailedCase > 0) {
1342 LOG.error("Failed to fix " + numFailedCase
1343 + " OrphanTables with default .tableinfo files");
1344 }
1345
1346 }
1347
1348 orphanTableDirs.clear();
1349
1350 }
1351
1352
1353
1354
1355
1356
1357 private HRegion createNewMeta() throws IOException {
1358 Path rootdir = FSUtils.getRootDir(getConf());
1359 Configuration c = getConf();
1360 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1361 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1362 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1363 HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1364 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1365 return meta;
1366 }
1367
1368
1369
1370
1371
1372
1373
1374 private ArrayList<Put> generatePuts(
1375 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1376 ArrayList<Put> puts = new ArrayList<Put>();
1377 boolean hasProblems = false;
1378 for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1379 TableName name = e.getKey();
1380
1381
1382 if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1383 continue;
1384 }
1385
1386 TableInfo ti = e.getValue();
1387 for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1388 .entrySet()) {
1389 Collection<HbckInfo> his = spl.getValue();
1390 int sz = his.size();
1391 if (sz != 1) {
1392
1393 LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1394 + " had " + sz + " regions instead of exactly 1." );
1395 hasProblems = true;
1396 continue;
1397 }
1398
1399
1400 HbckInfo hi = his.iterator().next();
1401 HRegionInfo hri = hi.getHdfsHRI();
1402 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1403 puts.add(p);
1404 }
1405 }
1406 return hasProblems ? null : puts;
1407 }
1408
1409
1410
1411
1412 private void suggestFixes(
1413 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1414 logParallelMerge();
1415 for (TableInfo tInfo : tablesInfo.values()) {
1416 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1417 tInfo.checkRegionChain(handler);
1418 }
1419 }
1420
1421
1422
1423
1424
1425
1426
1427
1428 public boolean rebuildMeta(boolean fix) throws IOException,
1429 InterruptedException {
1430
1431
1432
1433
1434
1435 LOG.info("Loading HBase regioninfo from HDFS...");
1436 loadHdfsRegionDirs();
1437
1438 int errs = errors.getErrorList().size();
1439 tablesInfo = loadHdfsRegionInfos();
1440 checkHdfsIntegrity(false, false);
1441
1442
1443 if (errors.getErrorList().size() != errs) {
1444
1445 while(true) {
1446 fixes = 0;
1447 suggestFixes(tablesInfo);
1448 errors.clear();
1449 loadHdfsRegionInfos();
1450 checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1451
1452 int errCount = errors.getErrorList().size();
1453
1454 if (fixes == 0) {
1455 if (errCount > 0) {
1456 return false;
1457 } else {
1458 break;
1459 }
1460 }
1461 }
1462 }
1463
1464
1465 LOG.info("HDFS regioninfo's seems good. Sidelining old hbase:meta");
1466 Path backupDir = sidelineOldMeta();
1467
1468 LOG.info("Creating new hbase:meta");
1469 HRegion meta = createNewMeta();
1470
1471
1472 List<Put> puts = generatePuts(tablesInfo);
1473 if (puts == null) {
1474 LOG.fatal("Problem encountered when creating new hbase:meta entries. " +
1475 "You may need to restore the previously sidelined hbase:meta");
1476 return false;
1477 }
1478 meta.batchMutate(puts.toArray(new Put[puts.size()]));
1479 HRegion.closeHRegion(meta);
1480 LOG.info("Success! hbase:meta table rebuilt.");
1481 LOG.info("Old hbase:meta is moved into " + backupDir);
1482 return true;
1483 }
1484
1485
1486
1487
1488 private void logParallelMerge() {
1489 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1490 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1491 " false to run serially.");
1492 } else {
1493 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
1494 " true to run in parallel.");
1495 }
1496 }
1497
1498 private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1499 boolean fixOverlaps) throws IOException {
1500 LOG.info("Checking HBase region split map from HDFS data...");
1501 logParallelMerge();
1502 for (TableInfo tInfo : tablesInfo.values()) {
1503 TableIntegrityErrorHandler handler;
1504 if (fixHoles || fixOverlaps) {
1505 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1506 fixHoles, fixOverlaps);
1507 } else {
1508 handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1509 }
1510 if (!tInfo.checkRegionChain(handler)) {
1511
1512 errors.report("Found inconsistency in table " + tInfo.getName());
1513 }
1514 }
1515 return tablesInfo;
1516 }
1517
1518 private Path getSidelineDir() throws IOException {
1519 if (sidelineDir == null) {
1520 Path hbaseDir = FSUtils.getRootDir(getConf());
1521 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1522 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1523 + startMillis);
1524 }
1525 return sidelineDir;
1526 }
1527
1528
1529
1530
1531 Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1532 return sidelineRegionDir(fs, null, hi);
1533 }
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543 Path sidelineRegionDir(FileSystem fs,
1544 String parentDir, HbckInfo hi) throws IOException {
1545 TableName tableName = hi.getTableName();
1546 Path regionDir = hi.getHdfsRegionDir();
1547
1548 if (!fs.exists(regionDir)) {
1549 LOG.warn("No previous " + regionDir + " exists. Continuing.");
1550 return null;
1551 }
1552
1553 Path rootDir = getSidelineDir();
1554 if (parentDir != null) {
1555 rootDir = new Path(rootDir, parentDir);
1556 }
1557 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1558 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1559 fs.mkdirs(sidelineRegionDir);
1560 boolean success = false;
1561 FileStatus[] cfs = fs.listStatus(regionDir);
1562 if (cfs == null) {
1563 LOG.info("Region dir is empty: " + regionDir);
1564 } else {
1565 for (FileStatus cf : cfs) {
1566 Path src = cf.getPath();
1567 Path dst = new Path(sidelineRegionDir, src.getName());
1568 if (fs.isFile(src)) {
1569
1570 success = fs.rename(src, dst);
1571 if (!success) {
1572 String msg = "Unable to rename file " + src + " to " + dst;
1573 LOG.error(msg);
1574 throw new IOException(msg);
1575 }
1576 continue;
1577 }
1578
1579
1580 fs.mkdirs(dst);
1581
1582 LOG.info("Sidelining files from " + src + " into containing region " + dst);
1583
1584
1585
1586
1587 FileStatus[] hfiles = fs.listStatus(src);
1588 if (hfiles != null && hfiles.length > 0) {
1589 for (FileStatus hfile : hfiles) {
1590 success = fs.rename(hfile.getPath(), dst);
1591 if (!success) {
1592 String msg = "Unable to rename file " + src + " to " + dst;
1593 LOG.error(msg);
1594 throw new IOException(msg);
1595 }
1596 }
1597 }
1598 LOG.debug("Sideline directory contents:");
1599 debugLsr(sidelineRegionDir);
1600 }
1601 }
1602
1603 LOG.info("Removing old region dir: " + regionDir);
1604 success = fs.delete(regionDir, true);
1605 if (!success) {
1606 String msg = "Unable to delete dir " + regionDir;
1607 LOG.error(msg);
1608 throw new IOException(msg);
1609 }
1610 return sidelineRegionDir;
1611 }
1612
1613
1614
1615
1616 void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1617 Path backupHbaseDir) throws IOException {
1618 Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1619 if (fs.exists(tableDir)) {
1620 Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1621 fs.mkdirs(backupTableDir.getParent());
1622 boolean success = fs.rename(tableDir, backupTableDir);
1623 if (!success) {
1624 throw new IOException("Failed to move " + tableName + " from "
1625 + tableDir + " to " + backupTableDir);
1626 }
1627 } else {
1628 LOG.info("No previous " + tableName + " exists. Continuing.");
1629 }
1630 }
1631
1632
1633
1634
1635 Path sidelineOldMeta() throws IOException {
1636
1637 Path hbaseDir = FSUtils.getRootDir(getConf());
1638 FileSystem fs = hbaseDir.getFileSystem(getConf());
1639 Path backupDir = getSidelineDir();
1640 fs.mkdirs(backupDir);
1641
1642 try {
1643 sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1644 } catch (IOException e) {
1645 LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore "
1646 + "try to rename hbase:meta in " + backupDir.getName() + " to "
1647 + hbaseDir.getName() + ".", e);
1648 throw e;
1649 }
1650 return backupDir;
1651 }
1652
1653
1654
1655
1656
1657
1658 private void loadDisabledTables()
1659 throws ZooKeeperConnectionException, IOException {
1660 HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1661 @Override
1662 public Void connect(HConnection connection) throws IOException {
1663 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1664 try {
1665 for (TableName tableName :
1666 ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1667 disabledTables.add(tableName);
1668 }
1669 } catch (KeeperException ke) {
1670 throw new IOException(ke);
1671 } catch (InterruptedException e) {
1672 throw new InterruptedIOException();
1673 } finally {
1674 zkw.close();
1675 }
1676 return null;
1677 }
1678 });
1679 }
1680
1681
1682
1683
1684 private boolean isTableDisabled(HRegionInfo regionInfo) {
1685 return disabledTables.contains(regionInfo.getTable());
1686 }
1687
1688
1689
1690
1691
1692 public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1693 Path rootDir = FSUtils.getRootDir(getConf());
1694 FileSystem fs = rootDir.getFileSystem(getConf());
1695
1696
1697 List<FileStatus> tableDirs = Lists.newArrayList();
1698
1699 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1700
1701 List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1702 for (Path path : paths) {
1703 TableName tableName = FSUtils.getTableName(path);
1704 if ((!checkMetaOnly &&
1705 isTableIncluded(tableName)) ||
1706 tableName.equals(TableName.META_TABLE_NAME)) {
1707 tableDirs.add(fs.getFileStatus(path));
1708 }
1709 }
1710
1711
1712 if (!foundVersionFile) {
1713 errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1714 "Version file does not exist in root dir " + rootDir);
1715 if (shouldFixVersionFile()) {
1716 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1717 + " file.");
1718 setShouldRerun();
1719 FSUtils.setVersion(fs, rootDir, getConf().getInt(
1720 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1721 HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1722 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1723 }
1724 }
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751 private boolean recordMetaRegion() throws IOException {
1752 RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1753 HConstants.EMPTY_START_ROW, false, false);
1754 if (rl == null) {
1755 errors.reportError(ERROR_CODE.NULL_META_REGION,
1756 "META region or some of its attributes are null.");
1757 return false;
1758 }
1759 for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1760
1761 if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1762 metaLocation.getHostname() == null) {
1763 errors.reportError(ERROR_CODE.NULL_META_REGION,
1764 "META region or some of its attributes are null.");
1765 return false;
1766 }
1767 ServerName sn = metaLocation.getServerName();
1768 MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1769 HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1770 if (hbckInfo == null) {
1771 regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1772 } else {
1773 hbckInfo.metaEntry = m;
1774 }
1775 }
1776 return true;
1777 }
1778
1779 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1780 return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1781 @Override
1782 public void abort(String why, Throwable e) {
1783 LOG.error(why, e);
1784 System.exit(1);
1785 }
1786
1787 @Override
1788 public boolean isAborted() {
1789 return false;
1790 }
1791
1792 });
1793 }
1794
1795 private ServerName getMetaRegionServerName(int replicaId)
1796 throws IOException, KeeperException {
1797 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1798 ServerName sn = null;
1799 try {
1800 sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1801 } finally {
1802 zkw.close();
1803 }
1804 return sn;
1805 }
1806
1807
1808
1809
1810
1811
1812 void processRegionServers(Collection<ServerName> regionServerList)
1813 throws IOException, InterruptedException {
1814
1815 List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1816 List<Future<Void>> workFutures;
1817
1818
1819 for (ServerName rsinfo: regionServerList) {
1820 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1821 }
1822
1823 workFutures = executor.invokeAll(workItems);
1824
1825 for(int i=0; i<workFutures.size(); i++) {
1826 WorkItemRegion item = workItems.get(i);
1827 Future<Void> f = workFutures.get(i);
1828 try {
1829 f.get();
1830 } catch(ExecutionException e) {
1831 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1832 e.getCause());
1833 }
1834 }
1835 }
1836
1837
1838
1839
1840 private void checkAndFixConsistency()
1841 throws IOException, KeeperException, InterruptedException {
1842
1843
1844 List<CheckRegionConsistencyWorkItem> workItems =
1845 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1846 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1847 if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1848 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1849 }
1850 }
1851 checkRegionConsistencyConcurrently(workItems);
1852
1853 boolean prevHdfsCheck = shouldCheckHdfs();
1854 setCheckHdfs(false);
1855
1856
1857 List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1858 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1859 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1860 if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1861 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1862 }
1863 }
1864 checkRegionConsistencyConcurrently(replicaWorkItems);
1865 setCheckHdfs(prevHdfsCheck);
1866
1867
1868
1869
1870
1871 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1872 int numOfSkippedRegions = skippedRegions.size();
1873 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1874 throw new IOException(numOfSkippedRegions
1875 + " region(s) could not be checked or repaired. See logs for detail.");
1876 }
1877 }
1878
1879
1880
1881
1882 private void checkRegionConsistencyConcurrently(
1883 final List<CheckRegionConsistencyWorkItem> workItems)
1884 throws IOException, KeeperException, InterruptedException {
1885 if (workItems.isEmpty()) {
1886 return;
1887 }
1888
1889 List<Future<Void>> workFutures = executor.invokeAll(workItems);
1890 for(Future<Void> f: workFutures) {
1891 try {
1892 f.get();
1893 } catch(ExecutionException e1) {
1894 LOG.warn("Could not check region consistency " , e1.getCause());
1895 if (e1.getCause() instanceof IOException) {
1896 throw (IOException)e1.getCause();
1897 } else if (e1.getCause() instanceof KeeperException) {
1898 throw (KeeperException)e1.getCause();
1899 } else if (e1.getCause() instanceof InterruptedException) {
1900 throw (InterruptedException)e1.getCause();
1901 } else {
1902 throw new IOException(e1.getCause());
1903 }
1904 }
1905 }
1906 }
1907
1908 class CheckRegionConsistencyWorkItem implements Callable<Void> {
1909 private final String key;
1910 private final HbckInfo hbi;
1911
1912 CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1913 this.key = key;
1914 this.hbi = hbi;
1915 }
1916
1917 @Override
1918 public synchronized Void call() throws Exception {
1919 try {
1920 checkRegionConsistency(key, hbi);
1921 } catch (Exception e) {
1922
1923
1924 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1925 + "'.", e);
1926 if (hbi.getHdfsHRI().isMetaRegion()) {
1927 throw e;
1928 }
1929 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1930 addSkippedRegion(hbi);
1931 }
1932 return null;
1933 }
1934 }
1935
1936 private void addSkippedRegion(final HbckInfo hbi) {
1937 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1938 if (skippedRegionNames == null) {
1939 skippedRegionNames = new HashSet<String>();
1940 }
1941 skippedRegionNames.add(hbi.getRegionNameAsString());
1942 skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1943 }
1944
1945 private void preCheckPermission() throws IOException, AccessDeniedException {
1946 if (shouldIgnorePreCheckPermission()) {
1947 return;
1948 }
1949
1950 Path hbaseDir = FSUtils.getRootDir(getConf());
1951 FileSystem fs = hbaseDir.getFileSystem(getConf());
1952 UserProvider userProvider = UserProvider.instantiate(getConf());
1953 UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1954 FileStatus[] files = fs.listStatus(hbaseDir);
1955 for (FileStatus file : files) {
1956 try {
1957 FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1958 } catch (AccessDeniedException ace) {
1959 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1960 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1961 + " does not have write perms to " + file.getPath()
1962 + ". Please rerun hbck as hdfs user " + file.getOwner());
1963 throw ace;
1964 }
1965 }
1966 }
1967
1968
1969
1970
1971 private void deleteMetaRegion(HbckInfo hi) throws IOException {
1972 deleteMetaRegion(hi.metaEntry.getRegionName());
1973 }
1974
1975
1976
1977
1978 private void deleteMetaRegion(byte[] metaKey) throws IOException {
1979 Delete d = new Delete(metaKey);
1980 meta.delete(d);
1981 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1982 }
1983
1984
1985
1986
1987 private void resetSplitParent(HbckInfo hi) throws IOException {
1988 RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1989 Delete d = new Delete(hi.metaEntry.getRegionName());
1990 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1991 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1992 mutations.add(d);
1993
1994 HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1995 hri.setOffline(false);
1996 hri.setSplit(false);
1997 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1998 mutations.add(p);
1999
2000 meta.mutateRow(mutations);
2001 LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2002 }
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012 private void offline(byte[] regionName) throws IOException {
2013 String regionString = Bytes.toStringBinary(regionName);
2014 if (!rsSupportsOffline) {
2015 LOG.warn("Using unassign region " + regionString
2016 + " instead of using offline method, you should"
2017 + " restart HMaster after these repairs");
2018 admin.unassign(regionName, true);
2019 return;
2020 }
2021
2022
2023 try {
2024 LOG.info("Offlining region " + regionString);
2025 admin.offline(regionName);
2026 } catch (IOException ioe) {
2027 String notFoundMsg = "java.lang.NoSuchMethodException: " +
2028 "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2029 if (ioe.getMessage().contains(notFoundMsg)) {
2030 LOG.warn("Using unassign region " + regionString
2031 + " instead of using offline method, you should"
2032 + " restart HMaster after these repairs");
2033 rsSupportsOffline = false;
2034 admin.unassign(regionName, true);
2035 return;
2036 }
2037 throw ioe;
2038 }
2039 }
2040
2041 private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2042 undeployRegionsForHbi(hi);
2043
2044 if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2045 return;
2046 }
2047 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2048 for (int i = 1; i < numReplicas; i++) {
2049 if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2050 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2051 hi.getPrimaryHRIForDeployedReplica(), i);
2052 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2053 if (h != null) {
2054 undeployRegionsForHbi(h);
2055
2056
2057 h.setSkipChecks(true);
2058 }
2059 }
2060 }
2061
2062 private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2063 for (OnlineEntry rse : hi.deployedEntries) {
2064 LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa);
2065 try {
2066 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2067 offline(rse.hri.getRegionName());
2068 } catch (IOException ioe) {
2069 LOG.warn("Got exception when attempting to offline region "
2070 + Bytes.toString(rse.hri.getRegionName()), ioe);
2071 }
2072 }
2073 }
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087 private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2088 if (hi.metaEntry == null && hi.hdfsEntry == null) {
2089 undeployRegions(hi);
2090 return;
2091 }
2092
2093
2094 Get get = new Get(hi.getRegionName());
2095 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2096 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2097 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2098
2099 if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2100 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2101 for (int i = 0; i < numReplicas; i++) {
2102 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2103 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2104 }
2105 }
2106 Result r = meta.get(get);
2107 RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2108 if (rl == null) {
2109 LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2110 " since meta does not have handle to reach it");
2111 return;
2112 }
2113 for (HRegionLocation h : rl.getRegionLocations()) {
2114 ServerName serverName = h.getServerName();
2115 if (serverName == null) {
2116 errors.reportError("Unable to close region "
2117 + hi.getRegionNameAsString() + " because meta does not "
2118 + "have handle to reach it.");
2119 continue;
2120 }
2121 HRegionInfo hri = h.getRegionInfo();
2122 if (hri == null) {
2123 LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2124 + " because hbase:meta had invalid or missing "
2125 + HConstants.CATALOG_FAMILY_STR + ":"
2126 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2127 + " qualifier value.");
2128 continue;
2129 }
2130
2131 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2132 }
2133 }
2134
2135 private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2136 KeeperException, InterruptedException {
2137
2138 if (shouldFixAssignments()) {
2139 errors.print(msg);
2140 undeployRegions(hbi);
2141 setShouldRerun();
2142 HRegionInfo hri = hbi.getHdfsHRI();
2143 if (hri == null) {
2144 hri = hbi.metaEntry;
2145 }
2146 HBaseFsckRepair.fixUnassigned(admin, hri);
2147 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2148
2149
2150 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2151 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2152 for (int i = 1; i < replicationCount; i++) {
2153 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2154 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2155 if (h != null) {
2156 undeployRegions(h);
2157
2158
2159 h.setSkipChecks(true);
2160 }
2161 HBaseFsckRepair.fixUnassigned(admin, hri);
2162 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2163 }
2164
2165 }
2166 }
2167
2168
2169
2170
2171 private void checkRegionConsistency(final String key, final HbckInfo hbi)
2172 throws IOException, KeeperException, InterruptedException {
2173
2174 if (hbi.isSkipChecks()) return;
2175 String descriptiveName = hbi.toString();
2176 boolean inMeta = hbi.metaEntry != null;
2177
2178 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2179 boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2180 boolean isDeployed = !hbi.deployedOn.isEmpty();
2181 boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2182 boolean deploymentMatchesMeta =
2183 hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2184 hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2185 boolean splitParent =
2186 (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2187 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2188 boolean recentlyModified = inHdfs &&
2189 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2190
2191
2192 if (hbi.containsOnlyHdfsEdits()) {
2193 return;
2194 }
2195 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2196 return;
2197 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2198 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2199 "tabled that is not deployed");
2200 return;
2201 } else if (recentlyModified) {
2202 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2203 return;
2204 }
2205
2206 else if (!inMeta && !inHdfs && !isDeployed) {
2207
2208 assert false : "Entry for region with no data";
2209 } else if (!inMeta && !inHdfs && isDeployed) {
2210 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2211 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2212 "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2213 if (shouldFixAssignments()) {
2214 undeployRegions(hbi);
2215 }
2216
2217 } else if (!inMeta && inHdfs && !isDeployed) {
2218 if (hbi.isMerged()) {
2219
2220
2221 hbi.setSkipChecks(true);
2222 LOG.info("Region " + descriptiveName
2223 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2224 return;
2225 }
2226 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2227 + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2228 "or deployed on any region server");
2229
2230 if (shouldFixMeta()) {
2231 if (!hbi.isHdfsRegioninfoPresent()) {
2232 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2233 + " in table integrity repair phase if -fixHdfsOrphans was" +
2234 " used.");
2235 return;
2236 }
2237
2238 HRegionInfo hri = hbi.getHdfsHRI();
2239 TableInfo tableInfo = tablesInfo.get(hri.getTable());
2240
2241 for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2242 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2243 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2244 hri.getEndKey()) >= 0)
2245 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2246 if(region.isSplit() || region.isOffline()) continue;
2247 Path regionDir = hbi.getHdfsRegionDir();
2248 FileSystem fs = regionDir.getFileSystem(getConf());
2249 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2250 for (Path familyDir : familyDirs) {
2251 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2252 for (Path referenceFilePath : referenceFilePaths) {
2253 Path parentRegionDir =
2254 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2255 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2256 LOG.warn(hri + " start and stop keys are in the range of " + region
2257 + ". The region might not be cleaned up from hdfs when region " + region
2258 + " split failed. Hence deleting from hdfs.");
2259 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2260 regionDir.getParent(), hri);
2261 return;
2262 }
2263 }
2264 }
2265 }
2266 }
2267
2268 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2269 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2270 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2271 admin.getClusterStatus().getServers(), numReplicas);
2272
2273 tryAssignmentRepair(hbi, "Trying to reassign region...");
2274 }
2275
2276 } else if (!inMeta && inHdfs && isDeployed) {
2277 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2278 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2279 debugLsr(hbi.getHdfsRegionDir());
2280 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2281
2282
2283
2284
2285 if (shouldFixAssignments()) {
2286 undeployRegionsForHbi(hbi);
2287 }
2288 }
2289 if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2290 if (!hbi.isHdfsRegioninfoPresent()) {
2291 LOG.error("This should have been repaired in table integrity repair phase");
2292 return;
2293 }
2294
2295 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2296 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2297 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2298 admin.getClusterStatus().getServers(), numReplicas);
2299 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2300 }
2301
2302
2303 } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2304
2305
2306 if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2307
2308 HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2309 HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2310 if (infoA != null && infoB != null) {
2311
2312 hbi.setSkipChecks(true);
2313 return;
2314 }
2315 }
2316
2317
2318
2319 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2320 LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "
2321 + "and not deployed on any region server. This may be transient.");
2322 hbi.setSkipChecks(true);
2323 return;
2324 }
2325
2326 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2327 + descriptiveName + " is a split parent in META, in HDFS, "
2328 + "and not deployed on any region server. This could be transient, "
2329 + "consider to run the catalog janitor first!");
2330 if (shouldFixSplitParents()) {
2331 setShouldRerun();
2332 resetSplitParent(hbi);
2333 }
2334 } else if (inMeta && !inHdfs && !isDeployed) {
2335 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2336 + descriptiveName + " found in META, but not in HDFS "
2337 + "or deployed on any region server.");
2338 if (shouldFixMeta()) {
2339 deleteMetaRegion(hbi);
2340 }
2341 } else if (inMeta && !inHdfs && isDeployed) {
2342 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2343 + " found in META, but not in HDFS, " +
2344 "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2345
2346
2347
2348 if (shouldFixAssignments()) {
2349 errors.print("Trying to fix unassigned region...");
2350 undeployRegions(hbi);
2351 }
2352 if (shouldFixMeta()) {
2353
2354 deleteMetaRegion(hbi);
2355 }
2356 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2357 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2358 + " not deployed on any region server.");
2359 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2360 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2361 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2362 "Region " + descriptiveName + " should not be deployed according " +
2363 "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2364 if (shouldFixAssignments()) {
2365 errors.print("Trying to close the region " + descriptiveName);
2366 setShouldRerun();
2367 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2368 }
2369 } else if (inMeta && inHdfs && isMultiplyDeployed) {
2370 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2371 + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2372 + " but is multiply assigned to region servers " +
2373 Joiner.on(", ").join(hbi.deployedOn));
2374
2375 if (shouldFixAssignments()) {
2376 errors.print("Trying to fix assignment error...");
2377 setShouldRerun();
2378 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2379 }
2380 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2381 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2382 + descriptiveName + " listed in hbase:meta on region server " +
2383 hbi.metaEntry.regionServer + " but found on region server " +
2384 hbi.deployedOn.get(0));
2385
2386 if (shouldFixAssignments()) {
2387 errors.print("Trying to fix assignment error...");
2388 setShouldRerun();
2389 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2390 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2391 }
2392 } else {
2393 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2394 " is in an unforeseen state:" +
2395 " inMeta=" + inMeta +
2396 " inHdfs=" + inHdfs +
2397 " isDeployed=" + isDeployed +
2398 " isMultiplyDeployed=" + isMultiplyDeployed +
2399 " deploymentMatchesMeta=" + deploymentMatchesMeta +
2400 " shouldBeDeployed=" + shouldBeDeployed);
2401 }
2402 }
2403
2404
2405
2406
2407
2408
2409
2410 SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2411 tablesInfo = new TreeMap<TableName,TableInfo> ();
2412 LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2413 for (HbckInfo hbi : regionInfoMap.values()) {
2414
2415 if (hbi.metaEntry == null) {
2416
2417 Path p = hbi.getHdfsRegionDir();
2418 if (p == null) {
2419 errors.report("No regioninfo in Meta or HDFS. " + hbi);
2420 }
2421
2422
2423 continue;
2424 }
2425 if (hbi.metaEntry.regionServer == null) {
2426 errors.detail("Skipping region because no region server: " + hbi);
2427 continue;
2428 }
2429 if (hbi.metaEntry.isOffline()) {
2430 errors.detail("Skipping region because it is offline: " + hbi);
2431 continue;
2432 }
2433 if (hbi.containsOnlyHdfsEdits()) {
2434 errors.detail("Skipping region because it only contains edits" + hbi);
2435 continue;
2436 }
2437
2438
2439
2440
2441
2442
2443 if (hbi.deployedOn.size() == 0) continue;
2444
2445
2446 TableName tableName = hbi.metaEntry.getTable();
2447 TableInfo modTInfo = tablesInfo.get(tableName);
2448 if (modTInfo == null) {
2449 modTInfo = new TableInfo(tableName);
2450 }
2451 for (ServerName server : hbi.deployedOn) {
2452 modTInfo.addServer(server);
2453 }
2454
2455 if (!hbi.isSkipChecks()) {
2456 modTInfo.addRegionInfo(hbi);
2457 }
2458
2459 tablesInfo.put(tableName, modTInfo);
2460 }
2461
2462 loadTableInfosForTablesWithNoRegion();
2463
2464 logParallelMerge();
2465 for (TableInfo tInfo : tablesInfo.values()) {
2466 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2467 if (!tInfo.checkRegionChain(handler)) {
2468 errors.report("Found inconsistency in table " + tInfo.getName());
2469 }
2470 }
2471 return tablesInfo;
2472 }
2473
2474
2475
2476
2477 private void loadTableInfosForTablesWithNoRegion() throws IOException {
2478 Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2479 for (HTableDescriptor htd : allTables.values()) {
2480 if (checkMetaOnly && !htd.isMetaTable()) {
2481 continue;
2482 }
2483
2484 TableName tableName = htd.getTableName();
2485 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2486 TableInfo tableInfo = new TableInfo(tableName);
2487 tableInfo.htds.add(htd);
2488 tablesInfo.put(htd.getTableName(), tableInfo);
2489 }
2490 }
2491 }
2492
2493
2494
2495
2496
2497 public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2498 int fileMoves = 0;
2499 String thread = Thread.currentThread().getName();
2500 LOG.debug("[" + thread + "] Contained region dir after close and pause");
2501 debugLsr(contained.getHdfsRegionDir());
2502
2503
2504 FileSystem fs = targetRegionDir.getFileSystem(getConf());
2505 FileStatus[] dirs = null;
2506 try {
2507 dirs = fs.listStatus(contained.getHdfsRegionDir());
2508 } catch (FileNotFoundException fnfe) {
2509
2510
2511 if (!fs.exists(contained.getHdfsRegionDir())) {
2512 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2513 + " is missing. Assuming already sidelined or moved.");
2514 } else {
2515 sidelineRegionDir(fs, contained);
2516 }
2517 return fileMoves;
2518 }
2519
2520 if (dirs == null) {
2521 if (!fs.exists(contained.getHdfsRegionDir())) {
2522 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2523 + " already sidelined.");
2524 } else {
2525 sidelineRegionDir(fs, contained);
2526 }
2527 return fileMoves;
2528 }
2529
2530 for (FileStatus cf : dirs) {
2531 Path src = cf.getPath();
2532 Path dst = new Path(targetRegionDir, src.getName());
2533
2534 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2535
2536 continue;
2537 }
2538
2539 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2540
2541 continue;
2542 }
2543
2544 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2545
2546
2547
2548
2549 for (FileStatus hfile : fs.listStatus(src)) {
2550 boolean success = fs.rename(hfile.getPath(), dst);
2551 if (success) {
2552 fileMoves++;
2553 }
2554 }
2555 LOG.debug("[" + thread + "] Sideline directory contents:");
2556 debugLsr(targetRegionDir);
2557 }
2558
2559
2560 sidelineRegionDir(fs, contained);
2561 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2562 getSidelineDir());
2563 debugLsr(contained.getHdfsRegionDir());
2564
2565 return fileMoves;
2566 }
2567
2568
2569 static class WorkItemOverlapMerge implements Callable<Void> {
2570 private TableIntegrityErrorHandler handler;
2571 Collection<HbckInfo> overlapgroup;
2572
2573 WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2574 this.handler = handler;
2575 this.overlapgroup = overlapgroup;
2576 }
2577
2578 @Override
2579 public Void call() throws Exception {
2580 handler.handleOverlapGroup(overlapgroup);
2581 return null;
2582 }
2583 };
2584
2585
2586
2587
2588
2589 public class TableInfo {
2590 TableName tableName;
2591 TreeSet <ServerName> deployedOn;
2592
2593
2594 final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2595
2596
2597 final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2598
2599
2600 final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2601
2602
2603 final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2604
2605
2606 final Multimap<byte[], HbckInfo> overlapGroups =
2607 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2608
2609
2610 private ImmutableList<HRegionInfo> regionsFromMeta = null;
2611
2612 TableInfo(TableName name) {
2613 this.tableName = name;
2614 deployedOn = new TreeSet <ServerName>();
2615 }
2616
2617
2618
2619
2620 private HTableDescriptor getHTD() {
2621 if (htds.size() == 1) {
2622 return (HTableDescriptor)htds.toArray()[0];
2623 } else {
2624 LOG.error("None/Multiple table descriptors found for table '"
2625 + tableName + "' regions: " + htds);
2626 }
2627 return null;
2628 }
2629
2630 public void addRegionInfo(HbckInfo hir) {
2631 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2632
2633
2634 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2635 return;
2636 }
2637
2638
2639 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2640 errors.reportError(
2641 ERROR_CODE.REGION_CYCLE,
2642 String.format("The endkey for this region comes before the "
2643 + "startkey, startkey=%s, endkey=%s",
2644 Bytes.toStringBinary(hir.getStartKey()),
2645 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2646 backwards.add(hir);
2647 return;
2648 }
2649
2650
2651
2652 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2653 }
2654
2655 public void addServer(ServerName server) {
2656 this.deployedOn.add(server);
2657 }
2658
2659 public TableName getName() {
2660 return tableName;
2661 }
2662
2663 public int getNumRegions() {
2664 return sc.getStarts().size() + backwards.size();
2665 }
2666
2667 public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2668
2669 if (regionsFromMeta == null) {
2670 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2671 for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2672 if (tableName.equals(h.getTableName())) {
2673 if (h.metaEntry != null) {
2674 regions.add((HRegionInfo) h.metaEntry);
2675 }
2676 }
2677 }
2678 regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2679 }
2680
2681 return regionsFromMeta;
2682 }
2683
2684
2685 private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2686 ErrorReporter errors;
2687
2688 IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2689 this.errors = errors;
2690 setTableInfo(ti);
2691 }
2692
2693 @Override
2694 public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2695 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2696 "First region should start with an empty key. You need to "
2697 + " create a new region and regioninfo in HDFS to plug the hole.",
2698 getTableInfo(), hi);
2699 }
2700
2701 @Override
2702 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2703 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2704 "Last region should end with an empty key. You need to "
2705 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2706 }
2707
2708 @Override
2709 public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2710 errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2711 "Region has the same start and end key.", getTableInfo(), hi);
2712 }
2713
2714 @Override
2715 public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2716 byte[] key = r1.getStartKey();
2717
2718 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2719 "Multiple regions have the same startkey: "
2720 + Bytes.toStringBinary(key), getTableInfo(), r1);
2721 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2722 "Multiple regions have the same startkey: "
2723 + Bytes.toStringBinary(key), getTableInfo(), r2);
2724 }
2725
2726 @Override
2727 public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
2728 byte[] key = r1.getStartKey();
2729
2730 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2731 "Multiple regions have the same regionID: "
2732 + Bytes.toStringBinary(key), getTableInfo(), r1);
2733 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2734 "Multiple regions have the same regionID: "
2735 + Bytes.toStringBinary(key), getTableInfo(), r2);
2736 }
2737
2738 @Override
2739 public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2740 errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2741 "There is an overlap in the region chain.",
2742 getTableInfo(), hi1, hi2);
2743 }
2744
2745 @Override
2746 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2747 errors.reportError(
2748 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2749 "There is a hole in the region chain between "
2750 + Bytes.toStringBinary(holeStart) + " and "
2751 + Bytes.toStringBinary(holeStop)
2752 + ". You need to create a new .regioninfo and region "
2753 + "dir in hdfs to plug the hole.");
2754 }
2755 };
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769 private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2770 Configuration conf;
2771
2772 boolean fixOverlaps = true;
2773
2774 HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2775 boolean fixHoles, boolean fixOverlaps) {
2776 super(ti, errors);
2777 this.conf = conf;
2778 this.fixOverlaps = fixOverlaps;
2779
2780 }
2781
2782
2783
2784
2785
2786
2787 @Override
2788 public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2789 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2790 "First region should start with an empty key. Creating a new " +
2791 "region and regioninfo in HDFS to plug the hole.",
2792 getTableInfo(), next);
2793 HTableDescriptor htd = getTableInfo().getHTD();
2794
2795 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2796 HConstants.EMPTY_START_ROW, next.getStartKey());
2797
2798
2799 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2800 LOG.info("Table region start key was not empty. Created new empty region: "
2801 + newRegion + " " +region);
2802 fixes++;
2803 }
2804
2805 @Override
2806 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2807 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2808 "Last region should end with an empty key. Creating a new "
2809 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2810 HTableDescriptor htd = getTableInfo().getHTD();
2811
2812 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2813 HConstants.EMPTY_START_ROW);
2814
2815 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2816 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
2817 + " " + region);
2818 fixes++;
2819 }
2820
2821
2822
2823
2824
2825 @Override
2826 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2827 errors.reportError(
2828 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2829 "There is a hole in the region chain between "
2830 + Bytes.toStringBinary(holeStartKey) + " and "
2831 + Bytes.toStringBinary(holeStopKey)
2832 + ". Creating a new regioninfo and region "
2833 + "dir in hdfs to plug the hole.");
2834 HTableDescriptor htd = getTableInfo().getHTD();
2835 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2836 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2837 LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2838 fixes++;
2839 }
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852 @Override
2853 public void handleOverlapGroup(Collection<HbckInfo> overlap)
2854 throws IOException {
2855 Preconditions.checkNotNull(overlap);
2856 Preconditions.checkArgument(overlap.size() >0);
2857
2858 if (!this.fixOverlaps) {
2859 LOG.warn("Not attempting to repair overlaps.");
2860 return;
2861 }
2862
2863 if (overlap.size() > maxMerge) {
2864 LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2865 "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2866 if (sidelineBigOverlaps) {
2867
2868 sidelineBigOverlaps(overlap);
2869 }
2870 return;
2871 }
2872 if (shouldRemoveParents()) {
2873 removeParentsAndFixSplits(overlap);
2874 }
2875 mergeOverlaps(overlap);
2876 }
2877
2878 void removeParentsAndFixSplits(Collection<HbckInfo> overlap) throws IOException {
2879 Pair<byte[], byte[]> range = null;
2880 HbckInfo parent = null;
2881 HbckInfo daughterA = null;
2882 HbckInfo daughterB = null;
2883 Collection<HbckInfo> daughters = new ArrayList<HbckInfo>(overlap);
2884
2885 String thread = Thread.currentThread().getName();
2886 LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
2887
2888
2889 if (overlap.size() > 3) {
2890 LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
2891 return;
2892 }
2893
2894 for (HbckInfo hi : overlap) {
2895 if (range == null) {
2896 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2897 } else {
2898 if (RegionSplitCalculator.BYTES_COMPARATOR
2899 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2900 range.setFirst(hi.getStartKey());
2901 }
2902 if (RegionSplitCalculator.BYTES_COMPARATOR
2903 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2904 range.setSecond(hi.getEndKey());
2905 }
2906 }
2907 }
2908
2909 LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
2910 + Bytes.toStringBinary(range.getSecond()) + "]");
2911
2912
2913 for (HbckInfo hi : overlap) {
2914 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
2915 && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2916 LOG.info("This is a parent for this group: " + hi.toString());
2917 parent = hi;
2918 }
2919 }
2920
2921
2922 if (parent != null) {
2923 daughters.remove(parent);
2924 }
2925
2926
2927
2928 for (HbckInfo hi : daughters) {
2929 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
2930 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2931 daughterA = hi;
2932 }
2933 }
2934 if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2935 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2936 daughterB = hi;
2937 }
2938 }
2939 }
2940
2941
2942 if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() || parent == null)
2943 return;
2944
2945 FileSystem fs = FileSystem.get(conf);
2946 LOG.info("Found parent: " + parent.getRegionNameAsString());
2947 LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
2948 LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
2949 LOG.info("Trying to fix parent in overlap by removing the parent.");
2950 try {
2951 closeRegion(parent);
2952 } catch (IOException ioe) {
2953 LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
2954 return;
2955 } catch (InterruptedException ie) {
2956 LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
2957 return;
2958 }
2959
2960 try {
2961 offline(parent.getRegionName());
2962 } catch (IOException ioe) {
2963 LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
2964 + ". Just continuing with regular merge... ", ioe);
2965 return;
2966 }
2967
2968 try {
2969 HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
2970 } catch (IOException ioe) {
2971 LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
2972 + ". Just continuing with regular merge... ", ioe);
2973 return;
2974 }
2975
2976 sidelineRegionDir(fs, parent);
2977 LOG.info("[" + thread + "] Sidelined parent region dir "+ parent.getHdfsRegionDir() + " into " +
2978 getSidelineDir());
2979 debugLsr(parent.getHdfsRegionDir());
2980
2981
2982 overlap.remove(parent);
2983 overlap.remove(daughterA);
2984 overlap.remove(daughterB);
2985
2986 LOG.info("Done fixing split.");
2987
2988 }
2989
2990 void mergeOverlaps(Collection<HbckInfo> overlap)
2991 throws IOException {
2992 String thread = Thread.currentThread().getName();
2993 LOG.info("== [" + thread + "] Merging regions into one region: "
2994 + Joiner.on(",").join(overlap));
2995
2996 Pair<byte[], byte[]> range = null;
2997 for (HbckInfo hi : overlap) {
2998 if (range == null) {
2999 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
3000 } else {
3001 if (RegionSplitCalculator.BYTES_COMPARATOR
3002 .compare(hi.getStartKey(), range.getFirst()) < 0) {
3003 range.setFirst(hi.getStartKey());
3004 }
3005 if ((RegionSplitCalculator.BYTES_COMPARATOR
3006 .compare(range.getSecond(), HConstants.EMPTY_END_ROW) == 0)
3007 || (RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(),
3008 HConstants.EMPTY_END_ROW) == 0)) {
3009 range.setSecond(HConstants.EMPTY_END_ROW);
3010 } else if (RegionSplitCalculator.BYTES_COMPARATOR
3011 .compare(hi.getEndKey(), range.getSecond()) > 0) {
3012 range.setSecond(hi.getEndKey());
3013 }
3014 }
3015
3016 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
3017 LOG.debug("[" + thread + "] Contained region dir before close");
3018 debugLsr(hi.getHdfsRegionDir());
3019 try {
3020 LOG.info("[" + thread + "] Closing region: " + hi);
3021 closeRegion(hi);
3022 } catch (IOException ioe) {
3023 LOG.warn("[" + thread + "] Was unable to close region " + hi
3024 + ". Just continuing... ", ioe);
3025 } catch (InterruptedException e) {
3026 LOG.warn("[" + thread + "] Was unable to close region " + hi
3027 + ". Just continuing... ", e);
3028 }
3029
3030 try {
3031 LOG.info("[" + thread + "] Offlining region: " + hi);
3032 offline(hi.getRegionName());
3033 } catch (IOException ioe) {
3034 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
3035 + ". Just continuing... ", ioe);
3036 }
3037 }
3038
3039
3040 HTableDescriptor htd = getTableInfo().getHTD();
3041
3042 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
3043 range.getSecond());
3044 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
3045 LOG.info("[" + thread + "] Created new empty container region: " +
3046 newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
3047 debugLsr(region.getRegionFileSystem().getRegionDir());
3048
3049
3050 boolean didFix= false;
3051 Path target = region.getRegionFileSystem().getRegionDir();
3052 for (HbckInfo contained : overlap) {
3053 LOG.info("[" + thread + "] Merging " + contained + " into " + target );
3054 int merges = mergeRegionDirs(target, contained);
3055 if (merges > 0) {
3056 didFix = true;
3057 }
3058 }
3059 if (didFix) {
3060 fixes++;
3061 }
3062 }
3063
3064
3065
3066
3067
3068
3069
3070
3071 void sidelineBigOverlaps(
3072 Collection<HbckInfo> bigOverlap) throws IOException {
3073 int overlapsToSideline = bigOverlap.size() - maxMerge;
3074 if (overlapsToSideline > maxOverlapsToSideline) {
3075 overlapsToSideline = maxOverlapsToSideline;
3076 }
3077 List<HbckInfo> regionsToSideline =
3078 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
3079 FileSystem fs = FileSystem.get(conf);
3080 for (HbckInfo regionToSideline: regionsToSideline) {
3081 try {
3082 LOG.info("Closing region: " + regionToSideline);
3083 closeRegion(regionToSideline);
3084 } catch (IOException ioe) {
3085 LOG.warn("Was unable to close region " + regionToSideline
3086 + ". Just continuing... ", ioe);
3087 } catch (InterruptedException e) {
3088 LOG.warn("Was unable to close region " + regionToSideline
3089 + ". Just continuing... ", e);
3090 }
3091
3092 try {
3093 LOG.info("Offlining region: " + regionToSideline);
3094 offline(regionToSideline.getRegionName());
3095 } catch (IOException ioe) {
3096 LOG.warn("Unable to offline region from master: " + regionToSideline
3097 + ". Just continuing... ", ioe);
3098 }
3099
3100 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
3101 Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
3102 if (sidelineRegionDir != null) {
3103 sidelinedRegions.put(sidelineRegionDir, regionToSideline);
3104 LOG.info("After sidelined big overlapped region: "
3105 + regionToSideline.getRegionNameAsString()
3106 + " to " + sidelineRegionDir.toString());
3107 fixes++;
3108 }
3109 }
3110 }
3111 }
3112
3113
3114
3115
3116
3117
3118
3119 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3120
3121
3122
3123 if (disabledTables.contains(this.tableName)) {
3124 return true;
3125 }
3126 int originalErrorsCount = errors.getErrorList().size();
3127 Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3128 SortedSet<byte[]> splits = sc.getSplits();
3129
3130 byte[] prevKey = null;
3131 byte[] problemKey = null;
3132
3133 if (splits.size() == 0) {
3134
3135 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3136 }
3137
3138 for (byte[] key : splits) {
3139 Collection<HbckInfo> ranges = regions.get(key);
3140 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3141 for (HbckInfo rng : ranges) {
3142 handler.handleRegionStartKeyNotEmpty(rng);
3143 }
3144 }
3145
3146
3147 for (HbckInfo rng : ranges) {
3148
3149 byte[] endKey = rng.getEndKey();
3150 endKey = (endKey.length == 0) ? null : endKey;
3151 if (Bytes.equals(rng.getStartKey(),endKey)) {
3152 handler.handleDegenerateRegion(rng);
3153 }
3154 }
3155
3156 if (ranges.size() == 1) {
3157
3158 if (problemKey != null) {
3159 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3160 }
3161 problemKey = null;
3162 } else if (ranges.size() > 1) {
3163
3164
3165 if (problemKey == null) {
3166
3167 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3168 problemKey = key;
3169 }
3170 overlapGroups.putAll(problemKey, ranges);
3171
3172
3173 ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3174
3175 for (HbckInfo r1 : ranges) {
3176 if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3177 subRange.remove(r1);
3178 for (HbckInfo r2 : subRange) {
3179 if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3180
3181 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3182 handler.handleDuplicateStartKeys(r1,r2);
3183 } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 &&
3184 r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) {
3185 LOG.info("this is a split, log to splits");
3186 handler.handleSplit(r1, r2);
3187 } else {
3188
3189 handler.handleOverlapInRegionChain(r1, r2);
3190 }
3191 }
3192 }
3193
3194 } else if (ranges.size() == 0) {
3195 if (problemKey != null) {
3196 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3197 }
3198 problemKey = null;
3199
3200 byte[] holeStopKey = sc.getSplits().higher(key);
3201
3202 if (holeStopKey != null) {
3203
3204 handler.handleHoleInRegionChain(key, holeStopKey);
3205 }
3206 }
3207 prevKey = key;
3208 }
3209
3210
3211
3212 if (prevKey != null) {
3213 handler.handleRegionEndKeyNotEmpty(prevKey);
3214 }
3215
3216
3217 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3218 boolean ok = handleOverlapsParallel(handler, prevKey);
3219 if (!ok) {
3220 return false;
3221 }
3222 } else {
3223 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3224 handler.handleOverlapGroup(overlap);
3225 }
3226 }
3227
3228 if (details) {
3229
3230 errors.print("---- Table '" + this.tableName
3231 + "': region split map");
3232 dump(splits, regions);
3233 errors.print("---- Table '" + this.tableName
3234 + "': overlap groups");
3235 dumpOverlapProblems(overlapGroups);
3236 errors.print("There are " + overlapGroups.keySet().size()
3237 + " overlap groups with " + overlapGroups.size()
3238 + " overlapping regions");
3239 }
3240 if (!sidelinedRegions.isEmpty()) {
3241 LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3242 errors.print("---- Table '" + this.tableName
3243 + "': sidelined big overlapped regions");
3244 dumpSidelinedRegions(sidelinedRegions);
3245 }
3246 return errors.getErrorList().size() == originalErrorsCount;
3247 }
3248
3249 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3250 throws IOException {
3251
3252
3253 List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3254 List<Future<Void>> rets;
3255 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3256
3257 merges.add(new WorkItemOverlapMerge(overlap, handler));
3258 }
3259 try {
3260 rets = executor.invokeAll(merges);
3261 } catch (InterruptedException e) {
3262 LOG.error("Overlap merges were interrupted", e);
3263 return false;
3264 }
3265 for(int i=0; i<merges.size(); i++) {
3266 WorkItemOverlapMerge work = merges.get(i);
3267 Future<Void> f = rets.get(i);
3268 try {
3269 f.get();
3270 } catch(ExecutionException e) {
3271 LOG.warn("Failed to merge overlap group" + work, e.getCause());
3272 } catch (InterruptedException e) {
3273 LOG.error("Waiting for overlap merges was interrupted", e);
3274 return false;
3275 }
3276 }
3277 return true;
3278 }
3279
3280
3281
3282
3283
3284
3285
3286 void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3287
3288 StringBuilder sb = new StringBuilder();
3289 for (byte[] k : splits) {
3290 sb.setLength(0);
3291 sb.append(Bytes.toStringBinary(k) + ":\t");
3292 for (HbckInfo r : regions.get(k)) {
3293 sb.append("[ "+ r.toString() + ", "
3294 + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3295 }
3296 errors.print(sb.toString());
3297 }
3298 }
3299 }
3300
3301 public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3302
3303
3304 for (byte[] k : regions.keySet()) {
3305 errors.print(Bytes.toStringBinary(k) + ":");
3306 for (HbckInfo r : regions.get(k)) {
3307 errors.print("[ " + r.toString() + ", "
3308 + Bytes.toStringBinary(r.getEndKey()) + "]");
3309 }
3310 errors.print("----");
3311 }
3312 }
3313
3314 public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3315 for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3316 TableName tableName = entry.getValue().getTableName();
3317 Path path = entry.getKey();
3318 errors.print("This sidelined region dir should be bulk loaded: "
3319 + path.toString());
3320 errors.print("Bulk load command looks like: "
3321 + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3322 + path.toUri().getPath() + " "+ tableName);
3323 }
3324 }
3325
3326 public Multimap<byte[], HbckInfo> getOverlapGroups(
3327 TableName table) {
3328 TableInfo ti = tablesInfo.get(table);
3329 return ti.overlapGroups;
3330 }
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341 HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3342 List<TableName> tableNames = new ArrayList<TableName>();
3343 long now = EnvironmentEdgeManager.currentTime();
3344
3345 for (HbckInfo hbi : regionInfoMap.values()) {
3346 MetaEntry info = hbi.metaEntry;
3347
3348
3349
3350 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3351 if (info.modTime + timelag < now) {
3352 tableNames.add(info.getTable());
3353 } else {
3354 numSkipped.incrementAndGet();
3355 }
3356 }
3357 }
3358 return getHTableDescriptors(tableNames);
3359 }
3360
3361 HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3362 HTableDescriptor[] htd = new HTableDescriptor[0];
3363 Admin admin = null;
3364 try {
3365 LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3366 admin = new HBaseAdmin(getConf());
3367 htd = admin.getTableDescriptorsByTableName(tableNames);
3368 } catch (IOException e) {
3369 LOG.debug("Exception getting table descriptors", e);
3370 } finally {
3371 if (admin != null) {
3372 try {
3373 admin.close();
3374 } catch (IOException e) {
3375 LOG.debug("Exception closing HBaseAdmin", e);
3376 }
3377 }
3378 }
3379 return htd;
3380 }
3381
3382
3383
3384
3385
3386
3387 private synchronized HbckInfo getOrCreateInfo(String name) {
3388 HbckInfo hbi = regionInfoMap.get(name);
3389 if (hbi == null) {
3390 hbi = new HbckInfo(null);
3391 regionInfoMap.put(name, hbi);
3392 }
3393 return hbi;
3394 }
3395
3396 private void checkAndFixTableLocks() throws IOException {
3397 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3398
3399 try {
3400 TableLockChecker checker = new TableLockChecker(zkw, errors);
3401 checker.checkTableLocks();
3402
3403 if (this.fixTableLocks) {
3404 checker.fixExpiredTableLocks();
3405 }
3406 } finally {
3407 zkw.close();
3408 }
3409 }
3410
3411 private void checkAndFixReplication() throws IOException {
3412 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3413 try {
3414 ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3415 checker.checkUnDeletedQueues();
3416
3417 if (checker.hasUnDeletedQueues() && this.fixReplication) {
3418 checker.fixUnDeletedQueues();
3419 setShouldRerun();
3420 }
3421 } finally {
3422 zkw.close();
3423 }
3424 }
3425
3426
3427
3428
3429
3430
3431
3432 private void checkAndFixOrphanedTableZNodes()
3433 throws IOException, KeeperException, InterruptedException {
3434 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3435
3436 try {
3437 Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3438 String msg;
3439 TableInfo tableInfo;
3440
3441 for (TableName tableName : enablingTables) {
3442
3443 tableInfo = tablesInfo.get(tableName);
3444 if (tableInfo != null) {
3445
3446 continue;
3447 }
3448
3449 msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3450 LOG.warn(msg);
3451 orphanedTableZNodes.add(tableName);
3452 errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3453 }
3454
3455 if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3456 ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3457
3458 for (TableName tableName : orphanedTableZNodes) {
3459 try {
3460
3461
3462
3463
3464 zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3465 } catch (CoordinatedStateException e) {
3466
3467 LOG.error(
3468 "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3469 e);
3470 }
3471 }
3472 }
3473 } finally {
3474 zkw.close();
3475 }
3476 }
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3488 Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3489 for (HbckInfo value : regionInfoMap.values()) {
3490 if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3491 metaRegions.put(value.getReplicaId(), value);
3492 }
3493 }
3494 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3495 .getRegionReplication();
3496 boolean noProblem = true;
3497
3498
3499 for (int i = 0; i < metaReplication; i++) {
3500 HbckInfo metaHbckInfo = metaRegions.remove(i);
3501 List<ServerName> servers = new ArrayList<ServerName>();
3502 if (metaHbckInfo != null) {
3503 servers = metaHbckInfo.deployedOn;
3504 }
3505 if (servers.size() != 1) {
3506 noProblem = false;
3507 if (servers.size() == 0) {
3508 assignMetaReplica(i);
3509 } else if (servers.size() > 1) {
3510 errors
3511 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3512 metaHbckInfo.getReplicaId() + " is found on more than one region.");
3513 if (shouldFixAssignments()) {
3514 errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3515 metaHbckInfo.getReplicaId() +"..");
3516 setShouldRerun();
3517
3518 HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3519 }
3520 }
3521 }
3522 }
3523
3524 for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3525 noProblem = false;
3526 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3527 "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3528 ", deployed " + metaRegions.size());
3529 if (shouldFixAssignments()) {
3530 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3531 " of hbase:meta..");
3532 setShouldRerun();
3533 unassignMetaReplica(entry.getValue());
3534 }
3535 }
3536
3537
3538 return noProblem;
3539 }
3540
3541 private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3542 KeeperException {
3543 undeployRegions(hi);
3544 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3545 try {
3546 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3547 } finally {
3548 zkw.close();
3549 }
3550 }
3551
3552 private void assignMetaReplica(int replicaId)
3553 throws IOException, KeeperException, InterruptedException {
3554 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3555 replicaId +" is not found on any region.");
3556 if (shouldFixAssignments()) {
3557 errors.print("Trying to fix a problem with hbase:meta..");
3558 setShouldRerun();
3559
3560 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3561 HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3562 HBaseFsckRepair.fixUnassigned(admin, h);
3563 HBaseFsckRepair.waitUntilAssigned(admin, h);
3564 }
3565 }
3566
3567
3568
3569
3570
3571 boolean loadMetaEntries() throws IOException {
3572 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3573 int countRecord = 1;
3574
3575
3576 final Comparator<Cell> comp = new Comparator<Cell>() {
3577 @Override
3578 public int compare(Cell k1, Cell k2) {
3579 return Long.compare(k1.getTimestamp(), k2.getTimestamp());
3580 }
3581 };
3582
3583 @Override
3584 public boolean processRow(Result result) throws IOException {
3585 try {
3586
3587
3588 long ts = Collections.max(result.listCells(), comp).getTimestamp();
3589 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3590 if (rl == null) {
3591 emptyRegionInfoQualifiers.add(result);
3592 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3593 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3594 return true;
3595 }
3596 ServerName sn = null;
3597 if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3598 rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3599 emptyRegionInfoQualifiers.add(result);
3600 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3601 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3602 return true;
3603 }
3604 HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3605 if (!(isTableIncluded(hri.getTable())
3606 || hri.isMetaRegion())) {
3607 return true;
3608 }
3609 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3610 for (HRegionLocation h : rl.getRegionLocations()) {
3611 if (h == null || h.getRegionInfo() == null) {
3612 continue;
3613 }
3614 sn = h.getServerName();
3615 hri = h.getRegionInfo();
3616
3617 MetaEntry m = null;
3618 if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3619 m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3620 } else {
3621 m = new MetaEntry(hri, sn, ts, null, null);
3622 }
3623 HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3624 if (previous == null) {
3625 regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3626 } else if (previous.metaEntry == null) {
3627 previous.metaEntry = m;
3628 } else {
3629 throw new IOException("Two entries in hbase:meta are same " + previous);
3630 }
3631 }
3632 PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3633 for (HRegionInfo mergeRegion : new HRegionInfo[] {
3634 mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3635 if (mergeRegion != null) {
3636
3637 HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3638 hbInfo.setMerged(true);
3639 }
3640 }
3641
3642
3643 if (countRecord % 100 == 0) {
3644 errors.progress();
3645 }
3646 countRecord++;
3647 return true;
3648 } catch (RuntimeException e) {
3649 LOG.error("Result=" + result);
3650 throw e;
3651 }
3652 }
3653 };
3654 if (!checkMetaOnly) {
3655
3656 MetaScanner.metaScan(connection, visitor);
3657 }
3658
3659 errors.print("");
3660 return true;
3661 }
3662
3663
3664
3665
3666 static class MetaEntry extends HRegionInfo {
3667 ServerName regionServer;
3668 long modTime;
3669 HRegionInfo splitA, splitB;
3670
3671 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3672 this(rinfo, regionServer, modTime, null, null);
3673 }
3674
3675 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3676 HRegionInfo splitA, HRegionInfo splitB) {
3677 super(rinfo);
3678 this.regionServer = regionServer;
3679 this.modTime = modTime;
3680 this.splitA = splitA;
3681 this.splitB = splitB;
3682 }
3683
3684 @Override
3685 public boolean equals(Object o) {
3686 boolean superEq = super.equals(o);
3687 if (!superEq) {
3688 return superEq;
3689 }
3690
3691 MetaEntry me = (MetaEntry) o;
3692 if (!regionServer.equals(me.regionServer)) {
3693 return false;
3694 }
3695 return (modTime == me.modTime);
3696 }
3697
3698 @Override
3699 public int hashCode() {
3700 int hash = Arrays.hashCode(getRegionName());
3701 hash ^= getRegionId();
3702 hash ^= Arrays.hashCode(getStartKey());
3703 hash ^= Arrays.hashCode(getEndKey());
3704 hash ^= Boolean.valueOf(isOffline()).hashCode();
3705 hash ^= getTable().hashCode();
3706 if (regionServer != null) {
3707 hash ^= regionServer.hashCode();
3708 }
3709 hash ^= modTime;
3710 return hash;
3711 }
3712 }
3713
3714
3715
3716
3717 static class HdfsEntry {
3718 HRegionInfo hri;
3719 Path hdfsRegionDir = null;
3720 long hdfsRegionDirModTime = 0;
3721 boolean hdfsRegioninfoFilePresent = false;
3722 boolean hdfsOnlyEdits = false;
3723 }
3724
3725
3726
3727
3728 static class OnlineEntry {
3729 HRegionInfo hri;
3730 ServerName hsa;
3731
3732 @Override
3733 public String toString() {
3734 return hsa.toString() + ";" + hri.getRegionNameAsString();
3735 }
3736 }
3737
3738
3739
3740
3741
3742 public static class HbckInfo implements KeyRange {
3743 private MetaEntry metaEntry = null;
3744 private HdfsEntry hdfsEntry = null;
3745 private List<OnlineEntry> deployedEntries = Lists.newArrayList();
3746 private List<ServerName> deployedOn = Lists.newArrayList();
3747 private boolean skipChecks = false;
3748 private boolean isMerged = false;
3749 private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3750 private HRegionInfo primaryHRIForDeployedReplica = null;
3751
3752 HbckInfo(MetaEntry metaEntry) {
3753 this.metaEntry = metaEntry;
3754 }
3755
3756 public synchronized int getReplicaId() {
3757 return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3758 }
3759
3760 public synchronized void addServer(HRegionInfo hri, ServerName server) {
3761 OnlineEntry rse = new OnlineEntry() ;
3762 rse.hri = hri;
3763 rse.hsa = server;
3764 this.deployedEntries.add(rse);
3765 this.deployedOn.add(server);
3766
3767 this.deployedReplicaId = hri.getReplicaId();
3768 this.primaryHRIForDeployedReplica =
3769 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3770 }
3771
3772 @Override
3773 public synchronized String toString() {
3774 StringBuilder sb = new StringBuilder();
3775 sb.append("{ meta => ");
3776 sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3777 sb.append( ", hdfs => " + getHdfsRegionDir());
3778 sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3779 sb.append( ", replicaId => " + getReplicaId());
3780 sb.append(" }");
3781 return sb.toString();
3782 }
3783
3784 @Override
3785 public byte[] getStartKey() {
3786 if (this.metaEntry != null) {
3787 return this.metaEntry.getStartKey();
3788 } else if (this.hdfsEntry != null) {
3789 return this.hdfsEntry.hri.getStartKey();
3790 } else {
3791 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3792 return null;
3793 }
3794 }
3795
3796 @Override
3797 public byte[] getEndKey() {
3798 if (this.metaEntry != null) {
3799 return this.metaEntry.getEndKey();
3800 } else if (this.hdfsEntry != null) {
3801 return this.hdfsEntry.hri.getEndKey();
3802 } else {
3803 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3804 return null;
3805 }
3806 }
3807
3808 public TableName getTableName() {
3809 if (this.metaEntry != null) {
3810 return this.metaEntry.getTable();
3811 } else if (this.hdfsEntry != null) {
3812
3813
3814 Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3815 return FSUtils.getTableName(tableDir);
3816 } else {
3817
3818 for (OnlineEntry e : deployedEntries) {
3819 return e.hri.getTable();
3820 }
3821 return null;
3822 }
3823 }
3824
3825 public String getRegionNameAsString() {
3826 if (metaEntry != null) {
3827 return metaEntry.getRegionNameAsString();
3828 } else if (hdfsEntry != null) {
3829 if (hdfsEntry.hri != null) {
3830 return hdfsEntry.hri.getRegionNameAsString();
3831 }
3832 } else {
3833
3834 for (OnlineEntry e : deployedEntries) {
3835 return e.hri.getRegionNameAsString();
3836 }
3837 }
3838 return null;
3839 }
3840
3841 public byte[] getRegionName() {
3842 if (metaEntry != null) {
3843 return metaEntry.getRegionName();
3844 } else if (hdfsEntry != null) {
3845 return hdfsEntry.hri.getRegionName();
3846 } else {
3847
3848 for (OnlineEntry e : deployedEntries) {
3849 return e.hri.getRegionName();
3850 }
3851 return null;
3852 }
3853 }
3854
3855 public HRegionInfo getPrimaryHRIForDeployedReplica() {
3856 return primaryHRIForDeployedReplica;
3857 }
3858
3859 Path getHdfsRegionDir() {
3860 if (hdfsEntry == null) {
3861 return null;
3862 }
3863 return hdfsEntry.hdfsRegionDir;
3864 }
3865
3866 boolean containsOnlyHdfsEdits() {
3867 if (hdfsEntry == null) {
3868 return false;
3869 }
3870 return hdfsEntry.hdfsOnlyEdits;
3871 }
3872
3873 boolean isHdfsRegioninfoPresent() {
3874 if (hdfsEntry == null) {
3875 return false;
3876 }
3877 return hdfsEntry.hdfsRegioninfoFilePresent;
3878 }
3879
3880 long getModTime() {
3881 if (hdfsEntry == null) {
3882 return 0;
3883 }
3884 return hdfsEntry.hdfsRegionDirModTime;
3885 }
3886
3887 HRegionInfo getHdfsHRI() {
3888 if (hdfsEntry == null) {
3889 return null;
3890 }
3891 return hdfsEntry.hri;
3892 }
3893
3894 public void setSkipChecks(boolean skipChecks) {
3895 this.skipChecks = skipChecks;
3896 }
3897
3898 public boolean isSkipChecks() {
3899 return skipChecks;
3900 }
3901
3902 public void setMerged(boolean isMerged) {
3903 this.isMerged = isMerged;
3904 }
3905
3906 public boolean isMerged() {
3907 return this.isMerged;
3908 }
3909 }
3910
3911 final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3912 @Override
3913 public int compare(HbckInfo l, HbckInfo r) {
3914 if (l == r) {
3915
3916 return 0;
3917 }
3918
3919 int tableCompare = l.getTableName().compareTo(r.getTableName());
3920 if (tableCompare != 0) {
3921 return tableCompare;
3922 }
3923
3924 int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3925 l.getStartKey(), r.getStartKey());
3926 if (startComparison != 0) {
3927 return startComparison;
3928 }
3929
3930
3931 byte[] endKey = r.getEndKey();
3932 endKey = (endKey.length == 0) ? null : endKey;
3933 byte[] endKey2 = l.getEndKey();
3934 endKey2 = (endKey2.length == 0) ? null : endKey2;
3935 int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3936 endKey2, endKey);
3937
3938 if (endComparison != 0) {
3939 return endComparison;
3940 }
3941
3942
3943
3944 if (l.hdfsEntry == null && r.hdfsEntry == null) {
3945 return 0;
3946 }
3947 if (l.hdfsEntry == null && r.hdfsEntry != null) {
3948 return 1;
3949 }
3950
3951 if (r.hdfsEntry == null) {
3952 return -1;
3953 }
3954
3955 return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3956 }
3957 };
3958
3959
3960
3961
3962 private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3963 StringBuilder sb = new StringBuilder();
3964 int numOfSkippedRegions;
3965 errors.print("Summary:");
3966 for (TableInfo tInfo : tablesInfo.values()) {
3967 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3968 skippedRegions.get(tInfo.getName()).size() : 0;
3969
3970 if (errors.tableHasErrors(tInfo)) {
3971 errors.print("Table " + tInfo.getName() + " is inconsistent.");
3972 } else if (numOfSkippedRegions > 0){
3973 errors.print("Table " + tInfo.getName() + " is okay (with "
3974 + numOfSkippedRegions + " skipped regions).");
3975 }
3976 else {
3977 errors.print("Table " + tInfo.getName() + " is okay.");
3978 }
3979 errors.print(" Number of regions: " + tInfo.getNumRegions());
3980 if (numOfSkippedRegions > 0) {
3981 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3982 System.out.println(" Number of skipped regions: " + numOfSkippedRegions);
3983 System.out.println(" List of skipped regions:");
3984 for(String sr : skippedRegionStrings) {
3985 System.out.println(" " + sr);
3986 }
3987 }
3988 sb.setLength(0);
3989 sb.append(" Deployed on: ");
3990 for (ServerName server : tInfo.deployedOn) {
3991 sb.append(" " + server.toString());
3992 }
3993 errors.print(sb.toString());
3994 }
3995 }
3996
3997 static ErrorReporter getErrorReporter(
3998 final Configuration conf) throws ClassNotFoundException {
3999 Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
4000 return ReflectionUtils.newInstance(reporter, conf);
4001 }
4002
4003 public interface ErrorReporter {
4004 enum ERROR_CODE {
4005 UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
4006 NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
4007 MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
4008 FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
4009 HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
4010 ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
4011 WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR,
4012 UNDELETED_REPLICATION_QUEUE, DUPE_ENDKEYS
4013 }
4014 void clear();
4015 void report(String message);
4016 void reportError(String message);
4017 void reportError(ERROR_CODE errorCode, String message);
4018 void reportError(ERROR_CODE errorCode, String message, TableInfo table);
4019 void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
4020 void reportError(
4021 ERROR_CODE errorCode,
4022 String message,
4023 TableInfo table,
4024 HbckInfo info1,
4025 HbckInfo info2
4026 );
4027 int summarize();
4028 void detail(String details);
4029 ArrayList<ERROR_CODE> getErrorList();
4030 void progress();
4031 void print(String message);
4032 void resetErrors();
4033 boolean tableHasErrors(TableInfo table);
4034 }
4035
4036 static class PrintingErrorReporter implements ErrorReporter {
4037 public int errorCount = 0;
4038 private int showProgress;
4039
4040 private static final int progressThreshold = 100;
4041
4042 Set<TableInfo> errorTables = new HashSet<TableInfo>();
4043
4044
4045 private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
4046
4047 @Override
4048 public void clear() {
4049 errorTables.clear();
4050 errorList.clear();
4051 errorCount = 0;
4052 }
4053
4054 @Override
4055 public synchronized void reportError(ERROR_CODE errorCode, String message) {
4056 if (errorCode == ERROR_CODE.WRONG_USAGE) {
4057 System.err.println(message);
4058 return;
4059 }
4060
4061 errorList.add(errorCode);
4062 if (!getSUMMARY()) {
4063 System.out.println("ERROR: " + message);
4064 }
4065 errorCount++;
4066 showProgress = 0;
4067 }
4068
4069 @Override
4070 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
4071 errorTables.add(table);
4072 reportError(errorCode, message);
4073 }
4074
4075 @Override
4076 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4077 HbckInfo info) {
4078 errorTables.add(table);
4079 String reference = "(region " + info.getRegionNameAsString() + ")";
4080 reportError(errorCode, reference + " " + message);
4081 }
4082
4083 @Override
4084 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4085 HbckInfo info1, HbckInfo info2) {
4086 errorTables.add(table);
4087 String reference = "(regions " + info1.getRegionNameAsString()
4088 + " and " + info2.getRegionNameAsString() + ")";
4089 reportError(errorCode, reference + " " + message);
4090 }
4091
4092 @Override
4093 public synchronized void reportError(String message) {
4094 reportError(ERROR_CODE.UNKNOWN, message);
4095 }
4096
4097
4098
4099
4100
4101
4102 @Override
4103 public synchronized void report(String message) {
4104 if (!getSUMMARY()) {
4105 System.out.println("ERROR: " + message);
4106 }
4107 showProgress = 0;
4108 }
4109
4110 @Override
4111 public synchronized int summarize() {
4112 System.out.println(Integer.toString(errorCount) +
4113 " inconsistencies detected.");
4114 if (errorCount == 0) {
4115 System.out.println("Status: OK");
4116 return 0;
4117 } else {
4118 System.out.println("Status: INCONSISTENT");
4119 return -1;
4120 }
4121 }
4122
4123 @Override
4124 public ArrayList<ERROR_CODE> getErrorList() {
4125 return errorList;
4126 }
4127
4128 @Override
4129 public synchronized void print(String message) {
4130 if (!getSUMMARY()) {
4131 System.out.println(message);
4132 }
4133 }
4134
4135 private synchronized static boolean getSUMMARY() {
4136 return SUMMARY;
4137 }
4138
4139 @Override
4140 public boolean tableHasErrors(TableInfo table) {
4141 return errorTables.contains(table);
4142 }
4143
4144 @Override
4145 public void resetErrors() {
4146 errorCount = 0;
4147 }
4148
4149 @Override
4150 public synchronized void detail(String message) {
4151 if (details) {
4152 System.out.println(message);
4153 }
4154 showProgress = 0;
4155 }
4156
4157 @Override
4158 public synchronized void progress() {
4159 if (showProgress++ == progressThreshold) {
4160 if (!getSUMMARY()) {
4161 System.out.print(".");
4162 }
4163 showProgress = 0;
4164 }
4165 }
4166 }
4167
4168
4169
4170
4171 static class WorkItemRegion implements Callable<Void> {
4172 private HBaseFsck hbck;
4173 private ServerName rsinfo;
4174 private ErrorReporter errors;
4175 private HConnection connection;
4176
4177 WorkItemRegion(HBaseFsck hbck, ServerName info,
4178 ErrorReporter errors, HConnection connection) {
4179 this.hbck = hbck;
4180 this.rsinfo = info;
4181 this.errors = errors;
4182 this.connection = connection;
4183 }
4184
4185 @Override
4186 public synchronized Void call() throws IOException {
4187 errors.progress();
4188 try {
4189 BlockingInterface server = connection.getAdmin(rsinfo);
4190
4191
4192 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4193 regions = filterRegions(regions);
4194
4195 if (details) {
4196 errors.detail("RegionServer: " + rsinfo.getServerName() +
4197 " number of regions: " + regions.size());
4198 for (HRegionInfo rinfo: regions) {
4199 errors.detail(" " + rinfo.getRegionNameAsString() +
4200 " id: " + rinfo.getRegionId() +
4201 " encoded_name: " + rinfo.getEncodedName() +
4202 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4203 " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4204 }
4205 }
4206
4207
4208 for (HRegionInfo r:regions) {
4209 HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4210 hbi.addServer(r, rsinfo);
4211 }
4212 } catch (IOException e) {
4213 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4214 " Unable to fetch region information. " + e);
4215 throw e;
4216 }
4217 return null;
4218 }
4219
4220 private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4221 List<HRegionInfo> ret = Lists.newArrayList();
4222 for (HRegionInfo hri : regions) {
4223 if (hri.isMetaTable() || (!hbck.checkMetaOnly
4224 && hbck.isTableIncluded(hri.getTable()))) {
4225 ret.add(hri);
4226 }
4227 }
4228 return ret;
4229 }
4230 }
4231
4232
4233
4234
4235
4236 static class WorkItemHdfsDir implements Callable<Void> {
4237 private HBaseFsck hbck;
4238 private FileStatus tableDir;
4239 private ErrorReporter errors;
4240 private FileSystem fs;
4241
4242 WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4243 FileStatus status) {
4244 this.hbck = hbck;
4245 this.fs = fs;
4246 this.tableDir = status;
4247 this.errors = errors;
4248 }
4249
4250 @Override
4251 public synchronized Void call() throws IOException {
4252 try {
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309 static class WorkItemHdfsRegionInfo implements Callable<Void> {
4310 private HbckInfo hbi;
4311 private HBaseFsck hbck;
4312 private ErrorReporter errors;
4313
4314 WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4315 this.hbi = hbi;
4316 this.hbck = hbck;
4317 this.errors = errors;
4318 }
4319
4320 @Override
4321 public synchronized Void call() throws IOException {
4322
4323 if (hbi.getHdfsHRI() == null) {
4324 try {
4325 errors.progress();
4326 hbck.loadHdfsRegioninfo(hbi);
4327 } catch (IOException ioe) {
4328 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4329 + hbi.getTableName() + " in hdfs dir "
4330 + hbi.getHdfsRegionDir()
4331 + "! It may be an invalid format or version file. Treating as "
4332 + "an orphaned regiondir.";
4333 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4334 try {
4335 hbck.debugLsr(hbi.getHdfsRegionDir());
4336 } catch (IOException ioe2) {
4337 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4338 throw ioe2;
4339 }
4340 hbck.orphanHdfsDirs.add(hbi);
4341 throw ioe;
4342 }
4343 }
4344 return null;
4345 }
4346 };
4347
4348
4349
4350
4351
4352 public static void setDisplayFullReport() {
4353 details = true;
4354 }
4355
4356
4357
4358
4359 public static void setForceExclusive() {
4360 forceExclusive = true;
4361 }
4362
4363
4364
4365
4366 public boolean isExclusive() {
4367 return fixAny || forceExclusive;
4368 }
4369
4370
4371
4372
4373 public static void setDisableBalancer() {
4374 disableBalancer = true;
4375 }
4376
4377
4378
4379
4380 public static void setDisableSplitAndMerge() {
4381 disableSplitAndMerge = true;
4382 }
4383
4384
4385
4386
4387
4388
4389 public boolean shouldDisableBalancer() {
4390 return fixAny || disableBalancer;
4391 }
4392
4393
4394
4395
4396
4397
4398 public boolean shouldDisableSplitAndMerge() {
4399 return fixAny || disableSplitAndMerge;
4400 }
4401
4402
4403
4404
4405
4406 synchronized static void setSummary() {
4407 SUMMARY = true;
4408 }
4409
4410
4411
4412
4413
4414 void setCheckMetaOnly() {
4415 checkMetaOnly = true;
4416 }
4417
4418
4419
4420
4421 void setRegionBoundariesCheck() {
4422 checkRegionBoundaries = true;
4423 }
4424
4425
4426
4427
4428
4429 public void setFixTableLocks(boolean shouldFix) {
4430 fixTableLocks = shouldFix;
4431 fixAny |= shouldFix;
4432 }
4433
4434
4435
4436
4437 public void setFixReplication(boolean shouldFix) {
4438 fixReplication = shouldFix;
4439 fixAny |= shouldFix;
4440 }
4441
4442
4443
4444
4445
4446 public void setFixTableZNodes(boolean shouldFix) {
4447 fixTableZNodes = shouldFix;
4448 fixAny |= shouldFix;
4449 }
4450
4451
4452
4453
4454
4455
4456
4457 void setShouldRerun() {
4458 rerun = true;
4459 }
4460
4461 boolean shouldRerun() {
4462 return rerun;
4463 }
4464
4465
4466
4467
4468
4469 public void setFixAssignments(boolean shouldFix) {
4470 fixAssignments = shouldFix;
4471 fixAny |= shouldFix;
4472 }
4473
4474 boolean shouldFixAssignments() {
4475 return fixAssignments;
4476 }
4477
4478 public void setFixMeta(boolean shouldFix) {
4479 fixMeta = shouldFix;
4480 fixAny |= shouldFix;
4481 }
4482
4483 boolean shouldFixMeta() {
4484 return fixMeta;
4485 }
4486
4487 public void setFixEmptyMetaCells(boolean shouldFix) {
4488 fixEmptyMetaCells = shouldFix;
4489 fixAny |= shouldFix;
4490 }
4491
4492 boolean shouldFixEmptyMetaCells() {
4493 return fixEmptyMetaCells;
4494 }
4495
4496 public void setCheckHdfs(boolean checking) {
4497 checkHdfs = checking;
4498 }
4499
4500 boolean shouldCheckHdfs() {
4501 return checkHdfs;
4502 }
4503
4504 public void setFixHdfsHoles(boolean shouldFix) {
4505 fixHdfsHoles = shouldFix;
4506 fixAny |= shouldFix;
4507 }
4508
4509 boolean shouldFixHdfsHoles() {
4510 return fixHdfsHoles;
4511 }
4512
4513 public void setFixTableOrphans(boolean shouldFix) {
4514 fixTableOrphans = shouldFix;
4515 fixAny |= shouldFix;
4516 }
4517
4518 boolean shouldFixTableOrphans() {
4519 return fixTableOrphans;
4520 }
4521
4522 public void setFixHdfsOverlaps(boolean shouldFix) {
4523 fixHdfsOverlaps = shouldFix;
4524 fixAny |= shouldFix;
4525 }
4526
4527 boolean shouldFixHdfsOverlaps() {
4528 return fixHdfsOverlaps;
4529 }
4530
4531 public void setFixHdfsOrphans(boolean shouldFix) {
4532 fixHdfsOrphans = shouldFix;
4533 fixAny |= shouldFix;
4534 }
4535
4536 boolean shouldFixHdfsOrphans() {
4537 return fixHdfsOrphans;
4538 }
4539
4540 public void setFixVersionFile(boolean shouldFix) {
4541 fixVersionFile = shouldFix;
4542 fixAny |= shouldFix;
4543 }
4544
4545 public boolean shouldFixVersionFile() {
4546 return fixVersionFile;
4547 }
4548
4549 public void setSidelineBigOverlaps(boolean sbo) {
4550 this.sidelineBigOverlaps = sbo;
4551 }
4552
4553 public boolean shouldSidelineBigOverlaps() {
4554 return sidelineBigOverlaps;
4555 }
4556
4557 public void setFixSplitParents(boolean shouldFix) {
4558 fixSplitParents = shouldFix;
4559 fixAny |= shouldFix;
4560 }
4561
4562 public void setRemoveParents(boolean shouldFix) {
4563 removeParents = shouldFix;
4564 fixAny |= shouldFix;
4565 }
4566
4567 boolean shouldFixSplitParents() {
4568 return fixSplitParents;
4569 }
4570
4571 boolean shouldRemoveParents() {
4572 return removeParents;
4573 }
4574
4575 public void setFixReferenceFiles(boolean shouldFix) {
4576 fixReferenceFiles = shouldFix;
4577 fixAny |= shouldFix;
4578 }
4579
4580 boolean shouldFixReferenceFiles() {
4581 return fixReferenceFiles;
4582 }
4583
4584 public boolean shouldIgnorePreCheckPermission() {
4585 return !fixAny || ignorePreCheckPermission;
4586 }
4587
4588 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4589 this.ignorePreCheckPermission = ignorePreCheckPermission;
4590 }
4591
4592
4593
4594
4595 public void setMaxMerge(int mm) {
4596 this.maxMerge = mm;
4597 }
4598
4599 public int getMaxMerge() {
4600 return maxMerge;
4601 }
4602
4603 public void setMaxOverlapsToSideline(int mo) {
4604 this.maxOverlapsToSideline = mo;
4605 }
4606
4607 public int getMaxOverlapsToSideline() {
4608 return maxOverlapsToSideline;
4609 }
4610
4611
4612
4613
4614
4615 boolean isTableIncluded(TableName table) {
4616 return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4617 }
4618
4619 public void includeTable(TableName table) {
4620 tablesIncluded.add(table);
4621 }
4622
4623 Set<TableName> getIncludedTables() {
4624 return new HashSet<TableName>(tablesIncluded);
4625 }
4626
4627
4628
4629
4630
4631
4632 public void setTimeLag(long seconds) {
4633 timelag = seconds * 1000;
4634 }
4635
4636
4637
4638
4639
4640 public void setSidelineDir(String sidelineDir) {
4641 this.sidelineDir = new Path(sidelineDir);
4642 }
4643
4644 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4645 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4646 }
4647
4648 public HFileCorruptionChecker getHFilecorruptionChecker() {
4649 return hfcc;
4650 }
4651
4652 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4653 this.hfcc = hfcc;
4654 }
4655
4656 public void setRetCode(int code) {
4657 this.retcode = code;
4658 }
4659
4660 public int getRetCode() {
4661 return retcode;
4662 }
4663
4664 protected HBaseFsck printUsageAndExit() {
4665 StringWriter sw = new StringWriter(2048);
4666 PrintWriter out = new PrintWriter(sw);
4667 out.println("Usage: fsck [opts] {only tables}");
4668 out.println(" where [opts] are:");
4669 out.println(" -help Display help options (this)");
4670 out.println(" -details Display full report of all regions.");
4671 out.println(" -timelag <timeInSeconds> Process only regions that " +
4672 " have not experienced any metadata updates in the last " +
4673 " <timeInSeconds> seconds.");
4674 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4675 " before checking if the fix worked if run with -fix");
4676 out.println(" -summary Print only summary of the tables and status.");
4677 out.println(" -metaonly Only check the state of the hbase:meta table.");
4678 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4679 out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
4680 out.println(" -exclusive Abort if another hbck is exclusive or fixing.");
4681 out.println(" -disableBalancer Disable the load balancer.");
4682
4683 out.println("");
4684 out.println(" Metadata Repair options: (expert features, use with caution!)");
4685 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity");
4686 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");
4687 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");
4688 out.println(" -noHdfsChecking Don't load/check region info from HDFS."
4689 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4690 out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
4691 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");
4692 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4693 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
4694 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
4695 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4696 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
4697 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4698 out.println(" -fixSplitParents Try to force offline split parents to be online.");
4699 out.println(" -removeParents Try to offline and sideline lingering parents and keep daughter regions.");
4700 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
4701 out.println(" -fixReferenceFiles Try to offline lingering reference store files");
4702 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"
4703 + " (empty REGIONINFO_QUALIFIER rows)");
4704
4705 out.println("");
4706 out.println(" Datafile Repair options: (expert features, use with caution!)");
4707 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");
4708 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");
4709
4710 out.println("");
4711 out.println(" Metadata Repair shortcuts");
4712 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4713 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4714 "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4715 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4716
4717 out.println("");
4718 out.println(" Table lock options");
4719 out.println(" -fixTableLocks Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4720
4721 out.println("");
4722 out.println(" Table Znode options");
4723 out.println(" -fixOrphanedTableZnodes Set table state in ZNode to disabled if table does not exists");
4724
4725 out.println("");
4726 out.println(" Replication options");
4727 out.println(" -fixReplication Deletes replication queues for removed peers");
4728
4729 out.flush();
4730 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4731
4732 setRetCode(-2);
4733 return this;
4734 }
4735
4736
4737
4738
4739
4740
4741
4742 public static void main(String[] args) throws Exception {
4743
4744 Configuration conf = HBaseConfiguration.create();
4745 Path hbasedir = FSUtils.getRootDir(conf);
4746 URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4747 FSUtils.setFsDefault(conf, new Path(defaultFs));
4748 int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4749 System.exit(ret);
4750 }
4751
4752
4753
4754
4755 static class HBaseFsckTool extends Configured implements Tool {
4756 HBaseFsckTool(Configuration conf) { super(conf); }
4757 @Override
4758 public int run(String[] args) throws Exception {
4759 HBaseFsck hbck = new HBaseFsck(getConf());
4760 hbck.exec(hbck.executor, args);
4761 hbck.close();
4762 return hbck.getRetCode();
4763 }
4764 };
4765
4766
4767 public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4768 ServiceException, InterruptedException {
4769 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4770
4771 boolean checkCorruptHFiles = false;
4772 boolean sidelineCorruptHFiles = false;
4773
4774
4775 for (int i = 0; i < args.length; i++) {
4776 String cmd = args[i];
4777 if (cmd.equals("-help") || cmd.equals("-h")) {
4778 return printUsageAndExit();
4779 } else if (cmd.equals("-details")) {
4780 setDisplayFullReport();
4781 } else if (cmd.equals("-exclusive")) {
4782 setForceExclusive();
4783 } else if (cmd.equals("-disableBalancer")) {
4784 setDisableBalancer();
4785 } else if (cmd.equals("-disableSplitAndMerge")) {
4786 setDisableSplitAndMerge();
4787 } else if (cmd.equals("-timelag")) {
4788 if (i == args.length - 1) {
4789 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4790 return printUsageAndExit();
4791 }
4792 try {
4793 long timelag = Long.parseLong(args[i+1]);
4794 setTimeLag(timelag);
4795 } catch (NumberFormatException e) {
4796 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4797 return printUsageAndExit();
4798 }
4799 i++;
4800 } else if (cmd.equals("-sleepBeforeRerun")) {
4801 if (i == args.length - 1) {
4802 errors.reportError(ERROR_CODE.WRONG_USAGE,
4803 "HBaseFsck: -sleepBeforeRerun needs a value.");
4804 return printUsageAndExit();
4805 }
4806 try {
4807 sleepBeforeRerun = Long.parseLong(args[i+1]);
4808 } catch (NumberFormatException e) {
4809 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4810 return printUsageAndExit();
4811 }
4812 i++;
4813 } else if (cmd.equals("-sidelineDir")) {
4814 if (i == args.length - 1) {
4815 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4816 return printUsageAndExit();
4817 }
4818 i++;
4819 setSidelineDir(args[i]);
4820 } else if (cmd.equals("-fix")) {
4821 errors.reportError(ERROR_CODE.WRONG_USAGE,
4822 "This option is deprecated, please use -fixAssignments instead.");
4823 setFixAssignments(true);
4824 } else if (cmd.equals("-fixAssignments")) {
4825 setFixAssignments(true);
4826 } else if (cmd.equals("-fixMeta")) {
4827 setFixMeta(true);
4828 } else if (cmd.equals("-noHdfsChecking")) {
4829 setCheckHdfs(false);
4830 } else if (cmd.equals("-fixHdfsHoles")) {
4831 setFixHdfsHoles(true);
4832 } else if (cmd.equals("-fixHdfsOrphans")) {
4833 setFixHdfsOrphans(true);
4834 } else if (cmd.equals("-fixTableOrphans")) {
4835 setFixTableOrphans(true);
4836 } else if (cmd.equals("-fixHdfsOverlaps")) {
4837 setFixHdfsOverlaps(true);
4838 } else if (cmd.equals("-fixVersionFile")) {
4839 setFixVersionFile(true);
4840 } else if (cmd.equals("-sidelineBigOverlaps")) {
4841 setSidelineBigOverlaps(true);
4842 } else if (cmd.equals("-fixSplitParents")) {
4843 setFixSplitParents(true);
4844 } else if (cmd.equals("-removeParents")) {
4845 setRemoveParents(true);
4846 } else if (cmd.equals("-ignorePreCheckPermission")) {
4847 setIgnorePreCheckPermission(true);
4848 } else if (cmd.equals("-checkCorruptHFiles")) {
4849 checkCorruptHFiles = true;
4850 } else if (cmd.equals("-sidelineCorruptHFiles")) {
4851 sidelineCorruptHFiles = true;
4852 } else if (cmd.equals("-fixReferenceFiles")) {
4853 setFixReferenceFiles(true);
4854 } else if (cmd.equals("-fixEmptyMetaCells")) {
4855 setFixEmptyMetaCells(true);
4856 } else if (cmd.equals("-repair")) {
4857
4858
4859 setFixHdfsHoles(true);
4860 setFixHdfsOrphans(true);
4861 setFixMeta(true);
4862 setFixAssignments(true);
4863 setFixHdfsOverlaps(true);
4864 setFixVersionFile(true);
4865 setSidelineBigOverlaps(true);
4866 setFixSplitParents(false);
4867 setCheckHdfs(true);
4868 setFixReferenceFiles(true);
4869 setFixTableLocks(true);
4870 setFixTableZNodes(true);
4871 } else if (cmd.equals("-repairHoles")) {
4872
4873 setFixHdfsHoles(true);
4874 setFixHdfsOrphans(false);
4875 setFixMeta(true);
4876 setFixAssignments(true);
4877 setFixHdfsOverlaps(false);
4878 setSidelineBigOverlaps(false);
4879 setFixSplitParents(false);
4880 setCheckHdfs(true);
4881 } else if (cmd.equals("-maxOverlapsToSideline")) {
4882 if (i == args.length - 1) {
4883 errors.reportError(ERROR_CODE.WRONG_USAGE,
4884 "-maxOverlapsToSideline needs a numeric value argument.");
4885 return printUsageAndExit();
4886 }
4887 try {
4888 int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4889 setMaxOverlapsToSideline(maxOverlapsToSideline);
4890 } catch (NumberFormatException e) {
4891 errors.reportError(ERROR_CODE.WRONG_USAGE,
4892 "-maxOverlapsToSideline needs a numeric value argument.");
4893 return printUsageAndExit();
4894 }
4895 i++;
4896 } else if (cmd.equals("-maxMerge")) {
4897 if (i == args.length - 1) {
4898 errors.reportError(ERROR_CODE.WRONG_USAGE,
4899 "-maxMerge needs a numeric value argument.");
4900 return printUsageAndExit();
4901 }
4902 try {
4903 int maxMerge = Integer.parseInt(args[i+1]);
4904 setMaxMerge(maxMerge);
4905 } catch (NumberFormatException e) {
4906 errors.reportError(ERROR_CODE.WRONG_USAGE,
4907 "-maxMerge needs a numeric value argument.");
4908 return printUsageAndExit();
4909 }
4910 i++;
4911 } else if (cmd.equals("-summary")) {
4912 setSummary();
4913 } else if (cmd.equals("-metaonly")) {
4914 setCheckMetaOnly();
4915 } else if (cmd.equals("-boundaries")) {
4916 setRegionBoundariesCheck();
4917 } else if (cmd.equals("-fixTableLocks")) {
4918 setFixTableLocks(true);
4919 } else if (cmd.equals("-fixReplication")) {
4920 setFixReplication(true);
4921 } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4922 setFixTableZNodes(true);
4923 } else if (cmd.startsWith("-")) {
4924 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4925 return printUsageAndExit();
4926 } else {
4927 includeTable(TableName.valueOf(cmd));
4928 errors.print("Allow checking/fixes for table: " + cmd);
4929 }
4930 }
4931
4932 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4933
4934
4935 try {
4936 preCheckPermission();
4937 } catch (AccessDeniedException ace) {
4938 Runtime.getRuntime().exit(-1);
4939 } catch (IOException ioe) {
4940 Runtime.getRuntime().exit(-1);
4941 }
4942
4943
4944 connect();
4945
4946 try {
4947
4948 if (checkCorruptHFiles || sidelineCorruptHFiles) {
4949 LOG.info("Checking all hfiles for corruption");
4950 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4951 setHFileCorruptionChecker(hfcc);
4952 Collection<TableName> tables = getIncludedTables();
4953 Collection<Path> tableDirs = new ArrayList<Path>();
4954 Path rootdir = FSUtils.getRootDir(getConf());
4955 if (tables.size() > 0) {
4956 for (TableName t : tables) {
4957 tableDirs.add(FSUtils.getTableDir(rootdir, t));
4958 }
4959 } else {
4960 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4961 }
4962 hfcc.checkTables(tableDirs);
4963 hfcc.report(errors);
4964 }
4965
4966
4967 int code = onlineHbck();
4968 setRetCode(code);
4969
4970
4971
4972
4973 if (shouldRerun()) {
4974 try {
4975 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4976 Thread.sleep(sleepBeforeRerun);
4977 } catch (InterruptedException ie) {
4978 LOG.warn("Interrupted while sleeping");
4979 return this;
4980 }
4981
4982 setFixAssignments(false);
4983 setFixMeta(false);
4984 setFixHdfsHoles(false);
4985 setFixHdfsOverlaps(false);
4986 setFixVersionFile(false);
4987 setFixTableOrphans(false);
4988 errors.resetErrors();
4989 code = onlineHbck();
4990 setRetCode(code);
4991 }
4992 } finally {
4993 IOUtils.closeQuietly(this);
4994 }
4995 return this;
4996 }
4997
4998
4999
5000
5001 void debugLsr(Path p) throws IOException {
5002 debugLsr(getConf(), p, errors);
5003 }
5004
5005
5006
5007
5008 public static void debugLsr(Configuration conf,
5009 Path p) throws IOException {
5010 debugLsr(conf, p, new PrintingErrorReporter());
5011 }
5012
5013
5014
5015
5016 public static void debugLsr(Configuration conf,
5017 Path p, ErrorReporter errors) throws IOException {
5018 if (!LOG.isDebugEnabled() || p == null) {
5019 return;
5020 }
5021 FileSystem fs = p.getFileSystem(conf);
5022
5023 if (!fs.exists(p)) {
5024
5025 return;
5026 }
5027 errors.print(p.toString());
5028
5029 if (fs.isFile(p)) {
5030 return;
5031 }
5032
5033 if (fs.getFileStatus(p).isDirectory()) {
5034 FileStatus[] fss= fs.listStatus(p);
5035 for (FileStatus status : fss) {
5036 debugLsr(conf, status.getPath(), errors);
5037 }
5038 }
5039 }
5040 }