1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.io.PrintWriter;
25 import java.io.StringWriter;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.SortedMap;
41 import java.util.SortedSet;
42 import java.util.TreeMap;
43 import java.util.TreeSet;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.ConcurrentSkipListMap;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.FutureTask;
51 import java.util.concurrent.ScheduledThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.TimeoutException;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.concurrent.atomic.AtomicInteger;
56
57 import com.google.common.base.Joiner;
58 import com.google.common.base.Preconditions;
59 import com.google.common.collect.ImmutableList;
60 import com.google.common.collect.Lists;
61 import com.google.common.collect.Multimap;
62 import com.google.common.collect.Ordering;
63 import com.google.common.collect.TreeMultimap;
64 import com.google.protobuf.ServiceException;
65
66 import org.apache.commons.lang.StringUtils;
67 import org.apache.commons.logging.Log;
68 import org.apache.commons.logging.LogFactory;
69 import org.apache.hadoop.hbase.classification.InterfaceAudience;
70 import org.apache.hadoop.hbase.classification.InterfaceStability;
71 import org.apache.hadoop.conf.Configuration;
72 import org.apache.hadoop.conf.Configured;
73 import org.apache.hadoop.fs.FSDataOutputStream;
74 import org.apache.hadoop.fs.FileStatus;
75 import org.apache.hadoop.fs.FileSystem;
76 import org.apache.hadoop.fs.Path;
77 import org.apache.hadoop.fs.permission.FsAction;
78 import org.apache.hadoop.fs.permission.FsPermission;
79 import org.apache.hadoop.hbase.Abortable;
80 import org.apache.hadoop.hbase.Cell;
81 import org.apache.hadoop.hbase.ClusterStatus;
82 import org.apache.hadoop.hbase.CoordinatedStateException;
83 import org.apache.hadoop.hbase.HBaseConfiguration;
84 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
85 import org.apache.hadoop.hbase.HColumnDescriptor;
86 import org.apache.hadoop.hbase.HConstants;
87 import org.apache.hadoop.hbase.HRegionInfo;
88 import org.apache.hadoop.hbase.HRegionLocation;
89 import org.apache.hadoop.hbase.HTableDescriptor;
90 import org.apache.hadoop.hbase.KeyValue;
91 import org.apache.hadoop.hbase.MasterNotRunningException;
92 import org.apache.hadoop.hbase.RegionLocations;
93 import org.apache.hadoop.hbase.ServerName;
94 import org.apache.hadoop.hbase.TableName;
95 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
96 import org.apache.hadoop.hbase.MetaTableAccessor;
97 import org.apache.hadoop.hbase.classification.InterfaceAudience;
98 import org.apache.hadoop.hbase.classification.InterfaceStability;
99 import org.apache.hadoop.hbase.client.Admin;
100 import org.apache.hadoop.hbase.client.ClusterConnection;
101 import org.apache.hadoop.hbase.client.ConnectionFactory;
102 import org.apache.hadoop.hbase.client.Delete;
103 import org.apache.hadoop.hbase.client.Get;
104 import org.apache.hadoop.hbase.client.HBaseAdmin;
105 import org.apache.hadoop.hbase.client.HConnectable;
106 import org.apache.hadoop.hbase.client.HConnection;
107 import org.apache.hadoop.hbase.client.HConnectionManager;
108 import org.apache.hadoop.hbase.client.MetaScanner;
109 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
110 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
111 import org.apache.hadoop.hbase.client.Put;
112 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
113 import org.apache.hadoop.hbase.client.Result;
114 import org.apache.hadoop.hbase.client.RowMutations;
115 import org.apache.hadoop.hbase.client.Table;
116 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
117 import org.apache.hadoop.hbase.io.hfile.HFile;
118 import org.apache.hadoop.hbase.master.MasterFileSystem;
119 import org.apache.hadoop.hbase.master.RegionState;
120 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
121 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
122 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
123 import org.apache.hadoop.hbase.regionserver.HRegion;
124 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
125 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
126 import org.apache.hadoop.hbase.security.AccessDeniedException;
127 import org.apache.hadoop.hbase.security.UserProvider;
128 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
129 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
130 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
131 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
132 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
133 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
134 import org.apache.hadoop.hbase.wal.WALSplitter;
135 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
136 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
137 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
138 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
139 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
140 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
141 import org.apache.hadoop.io.IOUtils;
142 import org.apache.hadoop.ipc.RemoteException;
143 import org.apache.hadoop.security.UserGroupInformation;
144 import org.apache.hadoop.util.ReflectionUtils;
145 import org.apache.hadoop.util.Tool;
146 import org.apache.hadoop.util.ToolRunner;
147 import org.apache.zookeeper.KeeperException;
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
195 @InterfaceStability.Evolving
196 public class HBaseFsck extends Configured implements Closeable {
197 public static final long DEFAULT_TIME_LAG = 60000;
198 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
199 private static final int MAX_NUM_THREADS = 50;
200 private static boolean rsSupportsOffline = true;
201 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
202 private static final int DEFAULT_MAX_MERGE = 5;
203 private static final String TO_BE_LOADED = "to_be_loaded";
204 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
205 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
206 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200;
207 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000;
208
209
210
211
212 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80;
213
214
215
216
217 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
218 private ClusterStatus status;
219 private ClusterConnection connection;
220 private Admin admin;
221 private Table meta;
222
223 protected ExecutorService executor;
224 private long startMillis = EnvironmentEdgeManager.currentTime();
225 private HFileCorruptionChecker hfcc;
226 private int retcode = 0;
227 private Path HBCK_LOCK_PATH;
228 private FSDataOutputStream hbckOutFd;
229
230
231
232 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
233
234
235
236
237 private static boolean details = false;
238 private long timelag = DEFAULT_TIME_LAG;
239 private boolean fixAssignments = false;
240 private boolean fixMeta = false;
241 private boolean checkHdfs = true;
242 private boolean fixHdfsHoles = false;
243 private boolean fixHdfsOverlaps = false;
244 private boolean fixHdfsOrphans = false;
245 private boolean fixTableOrphans = false;
246 private boolean fixVersionFile = false;
247 private boolean fixSplitParents = false;
248 private boolean fixReferenceFiles = false;
249 private boolean fixEmptyMetaCells = false;
250 private boolean fixTableLocks = false;
251 private boolean fixTableZNodes = false;
252 private boolean fixAny = false;
253
254
255
256 private Set<TableName> tablesIncluded = new HashSet<TableName>();
257 private int maxMerge = DEFAULT_MAX_MERGE;
258 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
259 private boolean sidelineBigOverlaps = false;
260 private Path sidelineDir = null;
261
262 private boolean rerun = false;
263 private static boolean summary = false;
264 private boolean checkMetaOnly = false;
265 private boolean checkRegionBoundaries = false;
266 private boolean ignorePreCheckPermission = false;
267
268
269
270
271 final private ErrorReporter errors;
272 int fixes = 0;
273
274
275
276
277
278
279 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
280 private TreeSet<TableName> disabledTables =
281 new TreeSet<TableName>();
282
283 private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
284
285
286
287
288
289
290
291
292
293
294
295 private SortedMap<TableName, TableInfo> tablesInfo =
296 new ConcurrentSkipListMap<TableName, TableInfo>();
297
298
299
300
301 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
302
303 private Map<TableName, Set<String>> orphanTableDirs =
304 new HashMap<TableName, Set<String>>();
305
306 private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
307
308
309
310
311 private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
312 private final RetryCounterFactory lockFileRetryCounterFactory;
313
314
315
316
317
318
319
320
321
322 public HBaseFsck(Configuration conf) throws MasterNotRunningException,
323 ZooKeeperConnectionException, IOException, ClassNotFoundException {
324 this(conf, createThreadPool(conf));
325 }
326
327 private static ExecutorService createThreadPool(Configuration conf) {
328 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
329 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
330 }
331
332
333
334
335
336
337
338
339
340
341
342 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
343 ZooKeeperConnectionException, IOException, ClassNotFoundException {
344 super(conf);
345 errors = getErrorReporter(getConf());
346 this.executor = exec;
347 lockFileRetryCounterFactory = new RetryCounterFactory(
348 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
349 getConf().getInt(
350 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
351 getConf().getInt(
352 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
353 }
354
355 private class FileLockCallable implements Callable<FSDataOutputStream> {
356 RetryCounter retryCounter;
357
358 public FileLockCallable(RetryCounter retryCounter) {
359 this.retryCounter = retryCounter;
360 }
361 @Override
362 public FSDataOutputStream call() throws IOException {
363 try {
364 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
365 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
366 HConstants.DATA_FILE_UMASK_KEY);
367 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
368 fs.mkdirs(tmpDir);
369 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
370 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
371 out.writeBytes(InetAddress.getLocalHost().toString());
372 out.flush();
373 return out;
374 } catch(RemoteException e) {
375 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
376 return null;
377 } else {
378 throw e;
379 }
380 }
381 }
382
383 private FSDataOutputStream createFileWithRetries(final FileSystem fs,
384 final Path hbckLockFilePath, final FsPermission defaultPerms)
385 throws IOException {
386
387 IOException exception = null;
388 do {
389 try {
390 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
391 } catch (IOException ioe) {
392 LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
393 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
394 + retryCounter.getMaxAttempts());
395 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
396 ioe);
397 try {
398 exception = ioe;
399 retryCounter.sleepUntilNextRetry();
400 } catch (InterruptedException ie) {
401 throw (InterruptedIOException) new InterruptedIOException(
402 "Can't create lock file " + hbckLockFilePath.getName())
403 .initCause(ie);
404 }
405 }
406 } while (retryCounter.shouldRetry());
407
408 throw exception;
409 }
410 }
411
412
413
414
415
416
417
418 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
419 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
420 FileLockCallable callable = new FileLockCallable(retryCounter);
421 ExecutorService executor = Executors.newFixedThreadPool(1);
422 FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
423 executor.execute(futureTask);
424 final int timeoutInSeconds = getConf().getInt(
425 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
426 FSDataOutputStream stream = null;
427 try {
428 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
429 } catch (ExecutionException ee) {
430 LOG.warn("Encountered exception when opening lock file", ee);
431 } catch (InterruptedException ie) {
432 LOG.warn("Interrupted when opening lock file", ie);
433 Thread.currentThread().interrupt();
434 } catch (TimeoutException exception) {
435
436 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
437 futureTask.cancel(true);
438 } finally {
439 executor.shutdownNow();
440 }
441 return stream;
442 }
443
444 private void unlockHbck() {
445 if (hbckLockCleanup.compareAndSet(true, false)) {
446 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
447 do {
448 try {
449 IOUtils.closeStream(hbckOutFd);
450 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
451 HBCK_LOCK_PATH, true);
452 LOG.info("Finishing hbck");
453 return;
454 } catch (IOException ioe) {
455 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
456 + (retryCounter.getAttemptTimes() + 1) + " of "
457 + retryCounter.getMaxAttempts());
458 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
459 try {
460 retryCounter.sleepUntilNextRetry();
461 } catch (InterruptedException ie) {
462 Thread.currentThread().interrupt();
463 LOG.warn("Interrupted while deleting lock file" +
464 HBCK_LOCK_PATH);
465 return;
466 }
467 }
468 } while (retryCounter.shouldRetry());
469 }
470 }
471
472
473
474
475
476 public void connect() throws IOException {
477
478
479 hbckOutFd = checkAndMarkRunningHbck();
480 if (hbckOutFd == null) {
481 setRetCode(-1);
482 LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
483 " no other instance is running, delete the lock file " +
484 HBCK_LOCK_PATH + " and rerun the tool]");
485 throw new IOException("Duplicate hbck - Abort");
486 }
487
488
489 hbckLockCleanup.set(true);
490
491
492
493
494 Runtime.getRuntime().addShutdownHook(new Thread() {
495 @Override
496 public void run() {
497 IOUtils.closeStream(HBaseFsck.this);
498 unlockHbck();
499 }
500 });
501
502 LOG.info("Launching hbck");
503
504 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
505 admin = connection.getAdmin();
506 meta = connection.getTable(TableName.META_TABLE_NAME);
507 status = admin.getClusterStatus();
508 }
509
510
511
512
513 private void loadDeployedRegions() throws IOException, InterruptedException {
514
515 Collection<ServerName> regionServers = status.getServers();
516 errors.print("Number of live region servers: " + regionServers.size());
517 if (details) {
518 for (ServerName rsinfo: regionServers) {
519 errors.print(" " + rsinfo.getServerName());
520 }
521 }
522
523
524 Collection<ServerName> deadRegionServers = status.getDeadServerNames();
525 errors.print("Number of dead region servers: " + deadRegionServers.size());
526 if (details) {
527 for (ServerName name: deadRegionServers) {
528 errors.print(" " + name);
529 }
530 }
531
532
533 errors.print("Master: " + status.getMaster());
534
535
536 Collection<ServerName> backupMasters = status.getBackupMasters();
537 errors.print("Number of backup masters: " + backupMasters.size());
538 if (details) {
539 for (ServerName name: backupMasters) {
540 errors.print(" " + name);
541 }
542 }
543
544 errors.print("Average load: " + status.getAverageLoad());
545 errors.print("Number of requests: " + status.getRequestsCount());
546 errors.print("Number of regions: " + status.getRegionsCount());
547
548 Map<String, RegionState> rits = status.getRegionsInTransition();
549 errors.print("Number of regions in transition: " + rits.size());
550 if (details) {
551 for (RegionState state: rits.values()) {
552 errors.print(" " + state.toDescriptiveString());
553 }
554 }
555
556
557 processRegionServers(regionServers);
558 }
559
560
561
562
563 private void clearState() {
564
565 fixes = 0;
566 regionInfoMap.clear();
567 emptyRegionInfoQualifiers.clear();
568 disabledTables.clear();
569 errors.clear();
570 tablesInfo.clear();
571 orphanHdfsDirs.clear();
572 skippedRegions.clear();
573 }
574
575
576
577
578
579
580 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
581
582 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
583 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
584 LOG.info("Loading regioninfos HDFS");
585
586 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
587 int curIter = 0;
588 do {
589 clearState();
590
591 restoreHdfsIntegrity();
592 curIter++;
593 } while (fixes > 0 && curIter <= maxIterations);
594
595
596
597 if (curIter > 2) {
598 if (curIter == maxIterations) {
599 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
600 + "Tables integrity may not be fully repaired!");
601 } else {
602 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
603 }
604 }
605 }
606 }
607
608
609
610
611
612
613
614
615
616 public int onlineConsistencyRepair() throws IOException, KeeperException,
617 InterruptedException {
618 clearState();
619
620
621 loadDeployedRegions();
622
623 recordMetaRegion();
624
625 if (!checkMetaRegion()) {
626 String errorMsg = "hbase:meta table is not consistent. ";
627 if (shouldFixAssignments()) {
628 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
629 } else {
630 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
631 }
632 errors.reportError(errorMsg + " Exiting...");
633 return -2;
634 }
635
636 LOG.info("Loading regionsinfo from the hbase:meta table");
637 boolean success = loadMetaEntries();
638 if (!success) return -1;
639
640
641 reportEmptyMetaCells();
642
643
644 if (shouldFixEmptyMetaCells()) {
645 fixEmptyMetaCells();
646 }
647
648
649 if (!checkMetaOnly) {
650 reportTablesInFlux();
651 }
652
653
654 if (shouldCheckHdfs()) {
655 LOG.info("Loading region directories from HDFS");
656 loadHdfsRegionDirs();
657 LOG.info("Loading region information from HDFS");
658 loadHdfsRegionInfos();
659 }
660
661
662 loadDisabledTables();
663
664
665 fixOrphanTables();
666
667 LOG.info("Checking and fixing region consistency");
668
669
670 checkAndFixConsistency();
671
672
673 checkIntegrity();
674 return errors.getErrorList().size();
675 }
676
677
678
679
680
681 public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
682
683 errors.print("Version: " + status.getHBaseVersion());
684 offlineHdfsIntegrityRepair();
685
686
687 boolean oldBalancer = admin.setBalancerRunning(false, true);
688 try {
689 onlineConsistencyRepair();
690 }
691 finally {
692 admin.setBalancerRunning(oldBalancer, false);
693 }
694
695 if (checkRegionBoundaries) {
696 checkRegionBoundaries();
697 }
698
699 offlineReferenceFileRepair();
700
701 checkAndFixTableLocks();
702
703
704 checkAndFixOrphanedTableZNodes();
705
706
707 unlockHbck();
708
709
710 printTableSummary(tablesInfo);
711 return errors.summarize();
712 }
713
714 public static byte[] keyOnly (byte[] b) {
715 if (b == null)
716 return b;
717 int rowlength = Bytes.toShort(b, 0);
718 byte[] result = new byte[rowlength];
719 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
720 return result;
721 }
722
723 @Override
724 public void close() throws IOException {
725 IOUtils.cleanup(null, admin, meta, connection);
726 }
727
728 private static class RegionBoundariesInformation {
729 public byte [] regionName;
730 public byte [] metaFirstKey;
731 public byte [] metaLastKey;
732 public byte [] storesFirstKey;
733 public byte [] storesLastKey;
734 @Override
735 public String toString () {
736 return "regionName=" + Bytes.toStringBinary(regionName) +
737 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
738 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
739 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
740 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
741 }
742 }
743
744 public void checkRegionBoundaries() {
745 try {
746 ByteArrayComparator comparator = new ByteArrayComparator();
747 List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
748 final RegionBoundariesInformation currentRegionBoundariesInformation =
749 new RegionBoundariesInformation();
750 Path hbaseRoot = FSUtils.getRootDir(getConf());
751 for (HRegionInfo regionInfo : regions) {
752 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
753 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
754
755
756 Path path = new Path(tableDir, regionInfo.getEncodedName());
757 FileSystem fs = path.getFileSystem(getConf());
758 FileStatus[] files = fs.listStatus(path);
759
760 byte[] storeFirstKey = null;
761 byte[] storeLastKey = null;
762 for (FileStatus file : files) {
763 String fileName = file.getPath().toString();
764 fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
765 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
766 FileStatus[] storeFiles = fs.listStatus(file.getPath());
767
768 for (FileStatus storeFile : storeFiles) {
769 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
770 getConf()), getConf());
771 if ((reader.getFirstKey() != null)
772 && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
773 reader.getFirstKey()) > 0))) {
774 storeFirstKey = reader.getFirstKey();
775 }
776 if ((reader.getLastKey() != null)
777 && ((storeLastKey == null) || (comparator.compare(storeLastKey,
778 reader.getLastKey())) < 0)) {
779 storeLastKey = reader.getLastKey();
780 }
781 reader.close();
782 }
783 }
784 }
785 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
786 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
787 currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
788 currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
789 if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
790 currentRegionBoundariesInformation.metaFirstKey = null;
791 if (currentRegionBoundariesInformation.metaLastKey.length == 0)
792 currentRegionBoundariesInformation.metaLastKey = null;
793
794
795
796
797
798
799 boolean valid = true;
800
801 if ((currentRegionBoundariesInformation.storesFirstKey != null)
802 && (currentRegionBoundariesInformation.metaFirstKey != null)) {
803 valid = valid
804 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
805 currentRegionBoundariesInformation.metaFirstKey) >= 0;
806 }
807
808 if ((currentRegionBoundariesInformation.storesLastKey != null)
809 && (currentRegionBoundariesInformation.metaLastKey != null)) {
810 valid = valid
811 && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
812 currentRegionBoundariesInformation.metaLastKey) < 0;
813 }
814 if (!valid) {
815 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
816 tablesInfo.get(regionInfo.getTable()));
817 LOG.warn("Region's boundaries not alligned between stores and META for:");
818 LOG.warn(currentRegionBoundariesInformation);
819 }
820 }
821 } catch (IOException e) {
822 LOG.error(e);
823 }
824 }
825
826
827
828
829 private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
830 for (HbckInfo hi : orphanHdfsDirs) {
831 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
832 adoptHdfsOrphan(hi);
833 }
834 }
835
836
837
838
839
840
841
842
843
844
845 @SuppressWarnings("deprecation")
846 private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
847 Path p = hi.getHdfsRegionDir();
848 FileSystem fs = p.getFileSystem(getConf());
849 FileStatus[] dirs = fs.listStatus(p);
850 if (dirs == null) {
851 LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
852 p + ". This dir could probably be deleted.");
853 return ;
854 }
855
856 TableName tableName = hi.getTableName();
857 TableInfo tableInfo = tablesInfo.get(tableName);
858 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
859 HTableDescriptor template = tableInfo.getHTD();
860
861
862 Pair<byte[],byte[]> orphanRegionRange = null;
863 for (FileStatus cf : dirs) {
864 String cfName= cf.getPath().getName();
865
866 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
867
868 FileStatus[] hfiles = fs.listStatus(cf.getPath());
869 for (FileStatus hfile : hfiles) {
870 byte[] start, end;
871 HFile.Reader hf = null;
872 try {
873 CacheConfig cacheConf = new CacheConfig(getConf());
874 hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
875 hf.loadFileInfo();
876 KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
877 start = startKv.getRow();
878 KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
879 end = endKv.getRow();
880 } catch (IOException ioe) {
881 LOG.warn("Problem reading orphan file " + hfile + ", skipping");
882 continue;
883 } catch (NullPointerException ioe) {
884 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
885 continue;
886 } finally {
887 if (hf != null) {
888 hf.close();
889 }
890 }
891
892
893 if (orphanRegionRange == null) {
894
895 orphanRegionRange = new Pair<byte[], byte[]>(start, end);
896 } else {
897
898
899
900 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
901 orphanRegionRange.setFirst(start);
902 }
903 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
904 orphanRegionRange.setSecond(end);
905 }
906 }
907 }
908 }
909 if (orphanRegionRange == null) {
910 LOG.warn("No data in dir " + p + ", sidelining data");
911 fixes++;
912 sidelineRegionDir(fs, hi);
913 return;
914 }
915 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
916 Bytes.toString(orphanRegionRange.getSecond()) + ")");
917
918
919 HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
920 LOG.info("Creating new region : " + hri);
921 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
922 Path target = region.getRegionFileSystem().getRegionDir();
923
924
925 mergeRegionDirs(target, hi);
926 fixes++;
927 }
928
929
930
931
932
933
934
935
936
937 private int restoreHdfsIntegrity() throws IOException, InterruptedException {
938
939 LOG.info("Loading HBase regioninfo from HDFS...");
940 loadHdfsRegionDirs();
941
942 int errs = errors.getErrorList().size();
943
944 tablesInfo = loadHdfsRegionInfos();
945 checkHdfsIntegrity(false, false);
946
947 if (errors.getErrorList().size() == errs) {
948 LOG.info("No integrity errors. We are done with this phase. Glorious.");
949 return 0;
950 }
951
952 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
953 adoptHdfsOrphans(orphanHdfsDirs);
954
955 }
956
957
958 if (shouldFixHdfsHoles()) {
959 clearState();
960 loadHdfsRegionDirs();
961 tablesInfo = loadHdfsRegionInfos();
962 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
963 }
964
965
966 if (shouldFixHdfsOverlaps()) {
967
968 clearState();
969 loadHdfsRegionDirs();
970 tablesInfo = loadHdfsRegionInfos();
971 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
972 }
973
974 return errors.getErrorList().size();
975 }
976
977
978
979
980
981
982
983
984
985 private void offlineReferenceFileRepair() throws IOException {
986 Configuration conf = getConf();
987 Path hbaseRoot = FSUtils.getRootDir(conf);
988 FileSystem fs = hbaseRoot.getFileSystem(conf);
989 LOG.info("Computing mapping of all store files");
990 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
991 errors.print("");
992 LOG.info("Validating mapping using HDFS state");
993 for (Path path: allFiles.values()) {
994 boolean isReference = false;
995 try {
996 isReference = StoreFileInfo.isReference(path);
997 } catch (Throwable t) {
998
999
1000
1001
1002 }
1003 if (!isReference) continue;
1004
1005 Path referredToFile = StoreFileInfo.getReferredToFile(path);
1006 if (fs.exists(referredToFile)) continue;
1007
1008
1009 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1010 "Found lingering reference file " + path);
1011 if (!shouldFixReferenceFiles()) continue;
1012
1013
1014 boolean success = false;
1015 String pathStr = path.toString();
1016
1017
1018
1019
1020
1021 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1022 for (int i = 0; index > 0 && i < 5; i++) {
1023 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1024 }
1025 if (index > 0) {
1026 Path rootDir = getSidelineDir();
1027 Path dst = new Path(rootDir, pathStr.substring(index + 1));
1028 fs.mkdirs(dst.getParent());
1029 LOG.info("Trying to sildeline reference file "
1030 + path + " to " + dst);
1031 setShouldRerun();
1032
1033 success = fs.rename(path, dst);
1034 }
1035 if (!success) {
1036 LOG.error("Failed to sideline reference file " + path);
1037 }
1038 }
1039 }
1040
1041
1042
1043
1044 private void reportEmptyMetaCells() {
1045 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1046 emptyRegionInfoQualifiers.size());
1047 if (details) {
1048 for (Result r: emptyRegionInfoQualifiers) {
1049 errors.print(" " + r);
1050 }
1051 }
1052 }
1053
1054
1055
1056
1057 private void reportTablesInFlux() {
1058 AtomicInteger numSkipped = new AtomicInteger(0);
1059 HTableDescriptor[] allTables = getTables(numSkipped);
1060 errors.print("Number of Tables: " + allTables.length);
1061 if (details) {
1062 if (numSkipped.get() > 0) {
1063 errors.detail("Number of Tables in flux: " + numSkipped.get());
1064 }
1065 for (HTableDescriptor td : allTables) {
1066 errors.detail(" Table: " + td.getTableName() + "\t" +
1067 (td.isReadOnly() ? "ro" : "rw") + "\t" +
1068 (td.isMetaRegion() ? "META" : " ") + "\t" +
1069 " families: " + td.getFamilies().size());
1070 }
1071 }
1072 }
1073
1074 public ErrorReporter getErrors() {
1075 return errors;
1076 }
1077
1078
1079
1080
1081
1082 private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1083 Path regionDir = hbi.getHdfsRegionDir();
1084 if (regionDir == null) {
1085 LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1086 return;
1087 }
1088
1089 if (hbi.hdfsEntry.hri != null) {
1090
1091 return;
1092 }
1093
1094 FileSystem fs = FileSystem.get(getConf());
1095 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1096 LOG.debug("HRegionInfo read: " + hri.toString());
1097 hbi.hdfsEntry.hri = hri;
1098 }
1099
1100
1101
1102
1103
1104 public static class RegionRepairException extends IOException {
1105 private static final long serialVersionUID = 1L;
1106 final IOException ioe;
1107 public RegionRepairException(String s, IOException ioe) {
1108 super(s);
1109 this.ioe = ioe;
1110 }
1111 }
1112
1113
1114
1115
1116 private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1117 throws IOException, InterruptedException {
1118 tablesInfo.clear();
1119
1120 Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1121
1122
1123 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1124 List<Future<Void>> hbiFutures;
1125
1126 for (HbckInfo hbi : hbckInfos) {
1127 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1128 hbis.add(work);
1129 }
1130
1131
1132 hbiFutures = executor.invokeAll(hbis);
1133
1134 for(int i=0; i<hbiFutures.size(); i++) {
1135 WorkItemHdfsRegionInfo work = hbis.get(i);
1136 Future<Void> f = hbiFutures.get(i);
1137 try {
1138 f.get();
1139 } catch(ExecutionException e) {
1140 LOG.warn("Failed to read .regioninfo file for region " +
1141 work.hbi.getRegionNameAsString(), e.getCause());
1142 }
1143 }
1144
1145 Path hbaseRoot = FSUtils.getRootDir(getConf());
1146 FileSystem fs = hbaseRoot.getFileSystem(getConf());
1147
1148 for (HbckInfo hbi: hbckInfos) {
1149
1150 if (hbi.getHdfsHRI() == null) {
1151
1152 continue;
1153 }
1154
1155
1156
1157 TableName tableName = hbi.getTableName();
1158 if (tableName == null) {
1159
1160 LOG.warn("tableName was null for: " + hbi);
1161 continue;
1162 }
1163
1164 TableInfo modTInfo = tablesInfo.get(tableName);
1165 if (modTInfo == null) {
1166
1167 modTInfo = new TableInfo(tableName);
1168 tablesInfo.put(tableName, modTInfo);
1169 try {
1170 HTableDescriptor htd =
1171 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1172 modTInfo.htds.add(htd);
1173 } catch (IOException ioe) {
1174 if (!orphanTableDirs.containsKey(tableName)) {
1175 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1176
1177 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1178 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1179 Set<String> columns = new HashSet<String>();
1180 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1181 }
1182 }
1183 }
1184 if (!hbi.isSkipChecks()) {
1185 modTInfo.addRegionInfo(hbi);
1186 }
1187 }
1188
1189 loadTableInfosForTablesWithNoRegion();
1190 errors.print("");
1191
1192 return tablesInfo;
1193 }
1194
1195
1196
1197
1198
1199
1200
1201
1202 private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1203 Path regionDir = hbi.getHdfsRegionDir();
1204 FileSystem fs = regionDir.getFileSystem(getConf());
1205 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1206 for (FileStatus subdir : subDirs) {
1207 String columnfamily = subdir.getPath().getName();
1208 columns.add(columnfamily);
1209 }
1210 return columns;
1211 }
1212
1213
1214
1215
1216
1217
1218
1219
1220 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1221 Set<String> columns) throws IOException {
1222 if (columns ==null || columns.isEmpty()) return false;
1223 HTableDescriptor htd = new HTableDescriptor(tableName);
1224 for (String columnfamimly : columns) {
1225 htd.addFamily(new HColumnDescriptor(columnfamimly));
1226 }
1227 fstd.createTableDescriptor(htd, true);
1228 return true;
1229 }
1230
1231
1232
1233
1234
1235 public void fixEmptyMetaCells() throws IOException {
1236 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1237 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1238 for (Result region : emptyRegionInfoQualifiers) {
1239 deleteMetaRegion(region.getRow());
1240 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1241 }
1242 emptyRegionInfoQualifiers.clear();
1243 }
1244 }
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255 public void fixOrphanTables() throws IOException {
1256 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1257
1258 List<TableName> tmpList = new ArrayList<TableName>();
1259 tmpList.addAll(orphanTableDirs.keySet());
1260 HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1261 Iterator<Entry<TableName, Set<String>>> iter =
1262 orphanTableDirs.entrySet().iterator();
1263 int j = 0;
1264 int numFailedCase = 0;
1265 FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1266 while (iter.hasNext()) {
1267 Entry<TableName, Set<String>> entry =
1268 iter.next();
1269 TableName tableName = entry.getKey();
1270 LOG.info("Trying to fix orphan table error: " + tableName);
1271 if (j < htds.length) {
1272 if (tableName.equals(htds[j].getTableName())) {
1273 HTableDescriptor htd = htds[j];
1274 LOG.info("fixing orphan table: " + tableName + " from cache");
1275 fstd.createTableDescriptor(htd, true);
1276 j++;
1277 iter.remove();
1278 }
1279 } else {
1280 if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1281 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1282 LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1283 iter.remove();
1284 } else {
1285 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1286 numFailedCase++;
1287 }
1288 }
1289 fixes++;
1290 }
1291
1292 if (orphanTableDirs.isEmpty()) {
1293
1294
1295 setShouldRerun();
1296 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1297 } else if (numFailedCase > 0) {
1298 LOG.error("Failed to fix " + numFailedCase
1299 + " OrphanTables with default .tableinfo files");
1300 }
1301
1302 }
1303
1304 orphanTableDirs.clear();
1305
1306 }
1307
1308
1309
1310
1311
1312
1313 private HRegion createNewMeta() throws IOException {
1314 Path rootdir = FSUtils.getRootDir(getConf());
1315 Configuration c = getConf();
1316 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1317 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1318 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1319 HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1320 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1321 return meta;
1322 }
1323
1324
1325
1326
1327
1328
1329
1330 private ArrayList<Put> generatePuts(
1331 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1332 ArrayList<Put> puts = new ArrayList<Put>();
1333 boolean hasProblems = false;
1334 for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1335 TableName name = e.getKey();
1336
1337
1338 if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1339 continue;
1340 }
1341
1342 TableInfo ti = e.getValue();
1343 for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1344 .entrySet()) {
1345 Collection<HbckInfo> his = spl.getValue();
1346 int sz = his.size();
1347 if (sz != 1) {
1348
1349 LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1350 + " had " + sz + " regions instead of exactly 1." );
1351 hasProblems = true;
1352 continue;
1353 }
1354
1355
1356 HbckInfo hi = his.iterator().next();
1357 HRegionInfo hri = hi.getHdfsHRI();
1358 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1359 puts.add(p);
1360 }
1361 }
1362 return hasProblems ? null : puts;
1363 }
1364
1365
1366
1367
1368 private void suggestFixes(
1369 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1370 logParallelMerge();
1371 for (TableInfo tInfo : tablesInfo.values()) {
1372 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1373 tInfo.checkRegionChain(handler);
1374 }
1375 }
1376
1377
1378
1379
1380
1381
1382
1383
1384 public boolean rebuildMeta(boolean fix) throws IOException,
1385 InterruptedException {
1386
1387
1388
1389
1390
1391 LOG.info("Loading HBase regioninfo from HDFS...");
1392 loadHdfsRegionDirs();
1393
1394 int errs = errors.getErrorList().size();
1395 tablesInfo = loadHdfsRegionInfos();
1396 checkHdfsIntegrity(false, false);
1397
1398
1399 if (errors.getErrorList().size() != errs) {
1400
1401 while(true) {
1402 fixes = 0;
1403 suggestFixes(tablesInfo);
1404 errors.clear();
1405 loadHdfsRegionInfos();
1406 checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1407
1408 int errCount = errors.getErrorList().size();
1409
1410 if (fixes == 0) {
1411 if (errCount > 0) {
1412 return false;
1413 } else {
1414 break;
1415 }
1416 }
1417 }
1418 }
1419
1420
1421 LOG.info("HDFS regioninfo's seems good. Sidelining old hbase:meta");
1422 Path backupDir = sidelineOldMeta();
1423
1424 LOG.info("Creating new hbase:meta");
1425 HRegion meta = createNewMeta();
1426
1427
1428 List<Put> puts = generatePuts(tablesInfo);
1429 if (puts == null) {
1430 LOG.fatal("Problem encountered when creating new hbase:meta entries. " +
1431 "You may need to restore the previously sidelined hbase:meta");
1432 return false;
1433 }
1434 meta.batchMutate(puts.toArray(new Put[puts.size()]));
1435 HRegion.closeHRegion(meta);
1436 LOG.info("Success! hbase:meta table rebuilt.");
1437 LOG.info("Old hbase:meta is moved into " + backupDir);
1438 return true;
1439 }
1440
1441
1442
1443
1444 private void logParallelMerge() {
1445 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1446 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1447 " false to run serially.");
1448 } else {
1449 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
1450 " true to run in parallel.");
1451 }
1452 }
1453
1454 private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1455 boolean fixOverlaps) throws IOException {
1456 LOG.info("Checking HBase region split map from HDFS data...");
1457 logParallelMerge();
1458 for (TableInfo tInfo : tablesInfo.values()) {
1459 TableIntegrityErrorHandler handler;
1460 if (fixHoles || fixOverlaps) {
1461 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1462 fixHoles, fixOverlaps);
1463 } else {
1464 handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1465 }
1466 if (!tInfo.checkRegionChain(handler)) {
1467
1468 errors.report("Found inconsistency in table " + tInfo.getName());
1469 }
1470 }
1471 return tablesInfo;
1472 }
1473
1474 private Path getSidelineDir() throws IOException {
1475 if (sidelineDir == null) {
1476 Path hbaseDir = FSUtils.getRootDir(getConf());
1477 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1478 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1479 + startMillis);
1480 }
1481 return sidelineDir;
1482 }
1483
1484
1485
1486
1487 Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1488 return sidelineRegionDir(fs, null, hi);
1489 }
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499 Path sidelineRegionDir(FileSystem fs,
1500 String parentDir, HbckInfo hi) throws IOException {
1501 TableName tableName = hi.getTableName();
1502 Path regionDir = hi.getHdfsRegionDir();
1503
1504 if (!fs.exists(regionDir)) {
1505 LOG.warn("No previous " + regionDir + " exists. Continuing.");
1506 return null;
1507 }
1508
1509 Path rootDir = getSidelineDir();
1510 if (parentDir != null) {
1511 rootDir = new Path(rootDir, parentDir);
1512 }
1513 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1514 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1515 fs.mkdirs(sidelineRegionDir);
1516 boolean success = false;
1517 FileStatus[] cfs = fs.listStatus(regionDir);
1518 if (cfs == null) {
1519 LOG.info("Region dir is empty: " + regionDir);
1520 } else {
1521 for (FileStatus cf : cfs) {
1522 Path src = cf.getPath();
1523 Path dst = new Path(sidelineRegionDir, src.getName());
1524 if (fs.isFile(src)) {
1525
1526 success = fs.rename(src, dst);
1527 if (!success) {
1528 String msg = "Unable to rename file " + src + " to " + dst;
1529 LOG.error(msg);
1530 throw new IOException(msg);
1531 }
1532 continue;
1533 }
1534
1535
1536 fs.mkdirs(dst);
1537
1538 LOG.info("Sidelining files from " + src + " into containing region " + dst);
1539
1540
1541
1542
1543 FileStatus[] hfiles = fs.listStatus(src);
1544 if (hfiles != null && hfiles.length > 0) {
1545 for (FileStatus hfile : hfiles) {
1546 success = fs.rename(hfile.getPath(), dst);
1547 if (!success) {
1548 String msg = "Unable to rename file " + src + " to " + dst;
1549 LOG.error(msg);
1550 throw new IOException(msg);
1551 }
1552 }
1553 }
1554 LOG.debug("Sideline directory contents:");
1555 debugLsr(sidelineRegionDir);
1556 }
1557 }
1558
1559 LOG.info("Removing old region dir: " + regionDir);
1560 success = fs.delete(regionDir, true);
1561 if (!success) {
1562 String msg = "Unable to delete dir " + regionDir;
1563 LOG.error(msg);
1564 throw new IOException(msg);
1565 }
1566 return sidelineRegionDir;
1567 }
1568
1569
1570
1571
1572 void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1573 Path backupHbaseDir) throws IOException {
1574 Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1575 if (fs.exists(tableDir)) {
1576 Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1577 fs.mkdirs(backupTableDir.getParent());
1578 boolean success = fs.rename(tableDir, backupTableDir);
1579 if (!success) {
1580 throw new IOException("Failed to move " + tableName + " from "
1581 + tableDir + " to " + backupTableDir);
1582 }
1583 } else {
1584 LOG.info("No previous " + tableName + " exists. Continuing.");
1585 }
1586 }
1587
1588
1589
1590
1591 Path sidelineOldMeta() throws IOException {
1592
1593 Path hbaseDir = FSUtils.getRootDir(getConf());
1594 FileSystem fs = hbaseDir.getFileSystem(getConf());
1595 Path backupDir = getSidelineDir();
1596 fs.mkdirs(backupDir);
1597
1598 try {
1599 sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1600 } catch (IOException e) {
1601 LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore "
1602 + "try to rename hbase:meta in " + backupDir.getName() + " to "
1603 + hbaseDir.getName() + ".", e);
1604 throw e;
1605 }
1606 return backupDir;
1607 }
1608
1609
1610
1611
1612
1613
1614 private void loadDisabledTables()
1615 throws ZooKeeperConnectionException, IOException {
1616 HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1617 @Override
1618 public Void connect(HConnection connection) throws IOException {
1619 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1620 try {
1621 for (TableName tableName :
1622 ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1623 disabledTables.add(tableName);
1624 }
1625 } catch (KeeperException ke) {
1626 throw new IOException(ke);
1627 } catch (InterruptedException e) {
1628 throw new InterruptedIOException();
1629 } finally {
1630 zkw.close();
1631 }
1632 return null;
1633 }
1634 });
1635 }
1636
1637
1638
1639
1640 private boolean isTableDisabled(HRegionInfo regionInfo) {
1641 return disabledTables.contains(regionInfo.getTable());
1642 }
1643
1644
1645
1646
1647
1648 public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1649 Path rootDir = FSUtils.getRootDir(getConf());
1650 FileSystem fs = rootDir.getFileSystem(getConf());
1651
1652
1653 List<FileStatus> tableDirs = Lists.newArrayList();
1654
1655 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1656
1657 List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1658 for (Path path : paths) {
1659 TableName tableName = FSUtils.getTableName(path);
1660 if ((!checkMetaOnly &&
1661 isTableIncluded(tableName)) ||
1662 tableName.equals(TableName.META_TABLE_NAME)) {
1663 tableDirs.add(fs.getFileStatus(path));
1664 }
1665 }
1666
1667
1668 if (!foundVersionFile) {
1669 errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1670 "Version file does not exist in root dir " + rootDir);
1671 if (shouldFixVersionFile()) {
1672 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1673 + " file.");
1674 setShouldRerun();
1675 FSUtils.setVersion(fs, rootDir, getConf().getInt(
1676 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1677 HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1678 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1679 }
1680 }
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707 private boolean recordMetaRegion() throws IOException {
1708 RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1709 HConstants.EMPTY_START_ROW, false, false);
1710 if (rl == null) {
1711 errors.reportError(ERROR_CODE.NULL_META_REGION,
1712 "META region or some of its attributes are null.");
1713 return false;
1714 }
1715 for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1716
1717 if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1718 metaLocation.getHostname() == null) {
1719 errors.reportError(ERROR_CODE.NULL_META_REGION,
1720 "META region or some of its attributes are null.");
1721 return false;
1722 }
1723 ServerName sn = metaLocation.getServerName();
1724 MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1725 HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1726 if (hbckInfo == null) {
1727 regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1728 } else {
1729 hbckInfo.metaEntry = m;
1730 }
1731 }
1732 return true;
1733 }
1734
1735 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1736 return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1737 @Override
1738 public void abort(String why, Throwable e) {
1739 LOG.error(why, e);
1740 System.exit(1);
1741 }
1742
1743 @Override
1744 public boolean isAborted() {
1745 return false;
1746 }
1747
1748 });
1749 }
1750
1751 private ServerName getMetaRegionServerName(int replicaId)
1752 throws IOException, KeeperException {
1753 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1754 ServerName sn = null;
1755 try {
1756 sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1757 } finally {
1758 zkw.close();
1759 }
1760 return sn;
1761 }
1762
1763
1764
1765
1766
1767
1768 void processRegionServers(Collection<ServerName> regionServerList)
1769 throws IOException, InterruptedException {
1770
1771 List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1772 List<Future<Void>> workFutures;
1773
1774
1775 for (ServerName rsinfo: regionServerList) {
1776 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1777 }
1778
1779 workFutures = executor.invokeAll(workItems);
1780
1781 for(int i=0; i<workFutures.size(); i++) {
1782 WorkItemRegion item = workItems.get(i);
1783 Future<Void> f = workFutures.get(i);
1784 try {
1785 f.get();
1786 } catch(ExecutionException e) {
1787 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1788 e.getCause());
1789 }
1790 }
1791 }
1792
1793
1794
1795
1796 private void checkAndFixConsistency()
1797 throws IOException, KeeperException, InterruptedException {
1798
1799
1800 List<CheckRegionConsistencyWorkItem> workItems =
1801 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1802 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1803 if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1804 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1805 }
1806 }
1807 checkRegionConsistencyConcurrently(workItems);
1808
1809 boolean prevHdfsCheck = shouldCheckHdfs();
1810 setCheckHdfs(false);
1811
1812
1813 List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1814 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1815 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1816 if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1817 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1818 }
1819 }
1820 checkRegionConsistencyConcurrently(replicaWorkItems);
1821 setCheckHdfs(prevHdfsCheck);
1822
1823
1824
1825
1826
1827 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1828 int numOfSkippedRegions = skippedRegions.size();
1829 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1830 throw new IOException(numOfSkippedRegions
1831 + " region(s) could not be checked or repaired. See logs for detail.");
1832 }
1833 }
1834
1835
1836
1837
1838 private void checkRegionConsistencyConcurrently(
1839 final List<CheckRegionConsistencyWorkItem> workItems)
1840 throws IOException, KeeperException, InterruptedException {
1841 if (workItems.isEmpty()) {
1842 return;
1843 }
1844
1845 List<Future<Void>> workFutures = executor.invokeAll(workItems);
1846 for(Future<Void> f: workFutures) {
1847 try {
1848 f.get();
1849 } catch(ExecutionException e1) {
1850 LOG.warn("Could not check region consistency " , e1.getCause());
1851 if (e1.getCause() instanceof IOException) {
1852 throw (IOException)e1.getCause();
1853 } else if (e1.getCause() instanceof KeeperException) {
1854 throw (KeeperException)e1.getCause();
1855 } else if (e1.getCause() instanceof InterruptedException) {
1856 throw (InterruptedException)e1.getCause();
1857 } else {
1858 throw new IOException(e1.getCause());
1859 }
1860 }
1861 }
1862 }
1863
1864 class CheckRegionConsistencyWorkItem implements Callable<Void> {
1865 private final String key;
1866 private final HbckInfo hbi;
1867
1868 CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1869 this.key = key;
1870 this.hbi = hbi;
1871 }
1872
1873 @Override
1874 public synchronized Void call() throws Exception {
1875 try {
1876 checkRegionConsistency(key, hbi);
1877 } catch (Exception e) {
1878
1879
1880 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1881 + "'.", e);
1882 if (hbi.getHdfsHRI().isMetaRegion()) {
1883 throw e;
1884 }
1885 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1886 addSkippedRegion(hbi);
1887 }
1888 return null;
1889 }
1890 }
1891
1892 private void addSkippedRegion(final HbckInfo hbi) {
1893 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1894 if (skippedRegionNames == null) {
1895 skippedRegionNames = new HashSet<String>();
1896 }
1897 skippedRegionNames.add(hbi.getRegionNameAsString());
1898 skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1899 }
1900
1901 private void preCheckPermission() throws IOException, AccessDeniedException {
1902 if (shouldIgnorePreCheckPermission()) {
1903 return;
1904 }
1905
1906 Path hbaseDir = FSUtils.getRootDir(getConf());
1907 FileSystem fs = hbaseDir.getFileSystem(getConf());
1908 UserProvider userProvider = UserProvider.instantiate(getConf());
1909 UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1910 FileStatus[] files = fs.listStatus(hbaseDir);
1911 for (FileStatus file : files) {
1912 try {
1913 FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1914 } catch (AccessDeniedException ace) {
1915 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1916 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1917 + " does not have write perms to " + file.getPath()
1918 + ". Please rerun hbck as hdfs user " + file.getOwner());
1919 throw ace;
1920 }
1921 }
1922 }
1923
1924
1925
1926
1927 private void deleteMetaRegion(HbckInfo hi) throws IOException {
1928 deleteMetaRegion(hi.metaEntry.getRegionName());
1929 }
1930
1931
1932
1933
1934 private void deleteMetaRegion(byte[] metaKey) throws IOException {
1935 Delete d = new Delete(metaKey);
1936 meta.delete(d);
1937 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1938 }
1939
1940
1941
1942
1943 private void resetSplitParent(HbckInfo hi) throws IOException {
1944 RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1945 Delete d = new Delete(hi.metaEntry.getRegionName());
1946 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1947 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1948 mutations.add(d);
1949
1950 HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1951 hri.setOffline(false);
1952 hri.setSplit(false);
1953 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1954 mutations.add(p);
1955
1956 meta.mutateRow(mutations);
1957 LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1958 }
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968 private void offline(byte[] regionName) throws IOException {
1969 String regionString = Bytes.toStringBinary(regionName);
1970 if (!rsSupportsOffline) {
1971 LOG.warn("Using unassign region " + regionString
1972 + " instead of using offline method, you should"
1973 + " restart HMaster after these repairs");
1974 admin.unassign(regionName, true);
1975 return;
1976 }
1977
1978
1979 try {
1980 LOG.info("Offlining region " + regionString);
1981 admin.offline(regionName);
1982 } catch (IOException ioe) {
1983 String notFoundMsg = "java.lang.NoSuchMethodException: " +
1984 "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1985 if (ioe.getMessage().contains(notFoundMsg)) {
1986 LOG.warn("Using unassign region " + regionString
1987 + " instead of using offline method, you should"
1988 + " restart HMaster after these repairs");
1989 rsSupportsOffline = false;
1990 admin.unassign(regionName, true);
1991 return;
1992 }
1993 throw ioe;
1994 }
1995 }
1996
1997 private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1998 undeployRegionsForHbi(hi);
1999
2000 if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2001 return;
2002 }
2003 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2004 for (int i = 1; i < numReplicas; i++) {
2005 if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2006 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2007 hi.getPrimaryHRIForDeployedReplica(), i);
2008 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2009 if (h != null) {
2010 undeployRegionsForHbi(h);
2011
2012
2013 h.setSkipChecks(true);
2014 }
2015 }
2016 }
2017
2018 private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2019 for (OnlineEntry rse : hi.deployedEntries) {
2020 LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa);
2021 try {
2022 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2023 offline(rse.hri.getRegionName());
2024 } catch (IOException ioe) {
2025 LOG.warn("Got exception when attempting to offline region "
2026 + Bytes.toString(rse.hri.getRegionName()), ioe);
2027 }
2028 }
2029 }
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043 private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2044 if (hi.metaEntry == null && hi.hdfsEntry == null) {
2045 undeployRegions(hi);
2046 return;
2047 }
2048
2049
2050 Get get = new Get(hi.getRegionName());
2051 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2052 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2053 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2054
2055 if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2056 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2057 for (int i = 0; i < numReplicas; i++) {
2058 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2059 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2060 }
2061 }
2062 Result r = meta.get(get);
2063 RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2064 if (rl == null) {
2065 LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2066 " since meta does not have handle to reach it");
2067 return;
2068 }
2069 for (HRegionLocation h : rl.getRegionLocations()) {
2070 ServerName serverName = h.getServerName();
2071 if (serverName == null) {
2072 errors.reportError("Unable to close region "
2073 + hi.getRegionNameAsString() + " because meta does not "
2074 + "have handle to reach it.");
2075 continue;
2076 }
2077 HRegionInfo hri = h.getRegionInfo();
2078 if (hri == null) {
2079 LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2080 + " because hbase:meta had invalid or missing "
2081 + HConstants.CATALOG_FAMILY_STR + ":"
2082 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2083 + " qualifier value.");
2084 continue;
2085 }
2086
2087 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2088 }
2089 }
2090
2091 private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2092 KeeperException, InterruptedException {
2093
2094 if (shouldFixAssignments()) {
2095 errors.print(msg);
2096 undeployRegions(hbi);
2097 setShouldRerun();
2098 HRegionInfo hri = hbi.getHdfsHRI();
2099 if (hri == null) {
2100 hri = hbi.metaEntry;
2101 }
2102 HBaseFsckRepair.fixUnassigned(admin, hri);
2103 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2104
2105
2106 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2107 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2108 for (int i = 1; i < replicationCount; i++) {
2109 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2110 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2111 if (h != null) {
2112 undeployRegions(h);
2113
2114
2115 h.setSkipChecks(true);
2116 }
2117 HBaseFsckRepair.fixUnassigned(admin, hri);
2118 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2119 }
2120
2121 }
2122 }
2123
2124
2125
2126
2127 private void checkRegionConsistency(final String key, final HbckInfo hbi)
2128 throws IOException, KeeperException, InterruptedException {
2129
2130 if (hbi.isSkipChecks()) return;
2131 String descriptiveName = hbi.toString();
2132 boolean inMeta = hbi.metaEntry != null;
2133
2134 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2135 boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2136 boolean isDeployed = !hbi.deployedOn.isEmpty();
2137 boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2138 boolean deploymentMatchesMeta =
2139 hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2140 hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2141 boolean splitParent =
2142 (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2143 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2144 boolean recentlyModified = inHdfs &&
2145 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2146
2147
2148 if (hbi.containsOnlyHdfsEdits()) {
2149 return;
2150 }
2151 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2152 return;
2153 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2154 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2155 "tabled that is not deployed");
2156 return;
2157 } else if (recentlyModified) {
2158 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2159 return;
2160 }
2161
2162 else if (!inMeta && !inHdfs && !isDeployed) {
2163
2164 assert false : "Entry for region with no data";
2165 } else if (!inMeta && !inHdfs && isDeployed) {
2166 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2167 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2168 "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2169 if (shouldFixAssignments()) {
2170 undeployRegions(hbi);
2171 }
2172
2173 } else if (!inMeta && inHdfs && !isDeployed) {
2174 if (hbi.isMerged()) {
2175
2176
2177 hbi.setSkipChecks(true);
2178 LOG.info("Region " + descriptiveName
2179 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2180 return;
2181 }
2182 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2183 + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2184 "or deployed on any region server");
2185
2186 if (shouldFixMeta()) {
2187 if (!hbi.isHdfsRegioninfoPresent()) {
2188 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2189 + " in table integrity repair phase if -fixHdfsOrphans was" +
2190 " used.");
2191 return;
2192 }
2193
2194 HRegionInfo hri = hbi.getHdfsHRI();
2195 TableInfo tableInfo = tablesInfo.get(hri.getTable());
2196
2197 for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2198 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2199 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2200 hri.getEndKey()) >= 0)
2201 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2202 if(region.isSplit() || region.isOffline()) continue;
2203 Path regionDir = hbi.getHdfsRegionDir();
2204 FileSystem fs = regionDir.getFileSystem(getConf());
2205 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2206 for (Path familyDir : familyDirs) {
2207 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2208 for (Path referenceFilePath : referenceFilePaths) {
2209 Path parentRegionDir =
2210 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2211 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2212 LOG.warn(hri + " start and stop keys are in the range of " + region
2213 + ". The region might not be cleaned up from hdfs when region " + region
2214 + " split failed. Hence deleting from hdfs.");
2215 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2216 regionDir.getParent(), hri);
2217 return;
2218 }
2219 }
2220 }
2221 }
2222 }
2223
2224 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2225 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2226 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2227 admin.getClusterStatus().getServers(), numReplicas);
2228
2229 tryAssignmentRepair(hbi, "Trying to reassign region...");
2230 }
2231
2232 } else if (!inMeta && inHdfs && isDeployed) {
2233 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2234 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2235 debugLsr(hbi.getHdfsRegionDir());
2236 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2237
2238
2239
2240
2241 if (shouldFixAssignments()) {
2242 undeployRegionsForHbi(hbi);
2243 }
2244 }
2245 if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2246 if (!hbi.isHdfsRegioninfoPresent()) {
2247 LOG.error("This should have been repaired in table integrity repair phase");
2248 return;
2249 }
2250
2251 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2252 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2253 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2254 admin.getClusterStatus().getServers(), numReplicas);
2255 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2256 }
2257
2258
2259 } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2260
2261
2262 if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2263
2264 HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2265 HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2266 if (infoA != null && infoB != null) {
2267
2268 hbi.setSkipChecks(true);
2269 return;
2270 }
2271 }
2272 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2273 + descriptiveName + " is a split parent in META, in HDFS, "
2274 + "and not deployed on any region server. This could be transient.");
2275 if (shouldFixSplitParents()) {
2276 setShouldRerun();
2277 resetSplitParent(hbi);
2278 }
2279 } else if (inMeta && !inHdfs && !isDeployed) {
2280 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2281 + descriptiveName + " found in META, but not in HDFS "
2282 + "or deployed on any region server.");
2283 if (shouldFixMeta()) {
2284 deleteMetaRegion(hbi);
2285 }
2286 } else if (inMeta && !inHdfs && isDeployed) {
2287 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2288 + " found in META, but not in HDFS, " +
2289 "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2290
2291
2292
2293 if (shouldFixAssignments()) {
2294 errors.print("Trying to fix unassigned region...");
2295 undeployRegions(hbi);
2296 }
2297 if (shouldFixMeta()) {
2298
2299 deleteMetaRegion(hbi);
2300 }
2301 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2302 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2303 + " not deployed on any region server.");
2304 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2305 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2306 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2307 "Region " + descriptiveName + " should not be deployed according " +
2308 "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2309 if (shouldFixAssignments()) {
2310 errors.print("Trying to close the region " + descriptiveName);
2311 setShouldRerun();
2312 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2313 }
2314 } else if (inMeta && inHdfs && isMultiplyDeployed) {
2315 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2316 + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2317 + " but is multiply assigned to region servers " +
2318 Joiner.on(", ").join(hbi.deployedOn));
2319
2320 if (shouldFixAssignments()) {
2321 errors.print("Trying to fix assignment error...");
2322 setShouldRerun();
2323 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2324 }
2325 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2326 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2327 + descriptiveName + " listed in hbase:meta on region server " +
2328 hbi.metaEntry.regionServer + " but found on region server " +
2329 hbi.deployedOn.get(0));
2330
2331 if (shouldFixAssignments()) {
2332 errors.print("Trying to fix assignment error...");
2333 setShouldRerun();
2334 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2335 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2336 }
2337 } else {
2338 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2339 " is in an unforeseen state:" +
2340 " inMeta=" + inMeta +
2341 " inHdfs=" + inHdfs +
2342 " isDeployed=" + isDeployed +
2343 " isMultiplyDeployed=" + isMultiplyDeployed +
2344 " deploymentMatchesMeta=" + deploymentMatchesMeta +
2345 " shouldBeDeployed=" + shouldBeDeployed);
2346 }
2347 }
2348
2349
2350
2351
2352
2353
2354
2355 SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2356 tablesInfo = new TreeMap<TableName,TableInfo> ();
2357 LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2358 for (HbckInfo hbi : regionInfoMap.values()) {
2359
2360 if (hbi.metaEntry == null) {
2361
2362 Path p = hbi.getHdfsRegionDir();
2363 if (p == null) {
2364 errors.report("No regioninfo in Meta or HDFS. " + hbi);
2365 }
2366
2367
2368 continue;
2369 }
2370 if (hbi.metaEntry.regionServer == null) {
2371 errors.detail("Skipping region because no region server: " + hbi);
2372 continue;
2373 }
2374 if (hbi.metaEntry.isOffline()) {
2375 errors.detail("Skipping region because it is offline: " + hbi);
2376 continue;
2377 }
2378 if (hbi.containsOnlyHdfsEdits()) {
2379 errors.detail("Skipping region because it only contains edits" + hbi);
2380 continue;
2381 }
2382
2383
2384
2385
2386
2387
2388 if (hbi.deployedOn.size() == 0) continue;
2389
2390
2391 TableName tableName = hbi.metaEntry.getTable();
2392 TableInfo modTInfo = tablesInfo.get(tableName);
2393 if (modTInfo == null) {
2394 modTInfo = new TableInfo(tableName);
2395 }
2396 for (ServerName server : hbi.deployedOn) {
2397 modTInfo.addServer(server);
2398 }
2399
2400 if (!hbi.isSkipChecks()) {
2401 modTInfo.addRegionInfo(hbi);
2402 }
2403
2404 tablesInfo.put(tableName, modTInfo);
2405 }
2406
2407 loadTableInfosForTablesWithNoRegion();
2408
2409 logParallelMerge();
2410 for (TableInfo tInfo : tablesInfo.values()) {
2411 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2412 if (!tInfo.checkRegionChain(handler)) {
2413 errors.report("Found inconsistency in table " + tInfo.getName());
2414 }
2415 }
2416 return tablesInfo;
2417 }
2418
2419
2420
2421
2422 private void loadTableInfosForTablesWithNoRegion() throws IOException {
2423 Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2424 for (HTableDescriptor htd : allTables.values()) {
2425 if (checkMetaOnly && !htd.isMetaTable()) {
2426 continue;
2427 }
2428
2429 TableName tableName = htd.getTableName();
2430 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2431 TableInfo tableInfo = new TableInfo(tableName);
2432 tableInfo.htds.add(htd);
2433 tablesInfo.put(htd.getTableName(), tableInfo);
2434 }
2435 }
2436 }
2437
2438
2439
2440
2441
2442 public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2443 int fileMoves = 0;
2444 String thread = Thread.currentThread().getName();
2445 LOG.debug("[" + thread + "] Contained region dir after close and pause");
2446 debugLsr(contained.getHdfsRegionDir());
2447
2448
2449 FileSystem fs = targetRegionDir.getFileSystem(getConf());
2450 FileStatus[] dirs = null;
2451 try {
2452 dirs = fs.listStatus(contained.getHdfsRegionDir());
2453 } catch (FileNotFoundException fnfe) {
2454
2455
2456 if (!fs.exists(contained.getHdfsRegionDir())) {
2457 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2458 + " is missing. Assuming already sidelined or moved.");
2459 } else {
2460 sidelineRegionDir(fs, contained);
2461 }
2462 return fileMoves;
2463 }
2464
2465 if (dirs == null) {
2466 if (!fs.exists(contained.getHdfsRegionDir())) {
2467 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2468 + " already sidelined.");
2469 } else {
2470 sidelineRegionDir(fs, contained);
2471 }
2472 return fileMoves;
2473 }
2474
2475 for (FileStatus cf : dirs) {
2476 Path src = cf.getPath();
2477 Path dst = new Path(targetRegionDir, src.getName());
2478
2479 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2480
2481 continue;
2482 }
2483
2484 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2485
2486 continue;
2487 }
2488
2489 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2490
2491
2492
2493
2494 for (FileStatus hfile : fs.listStatus(src)) {
2495 boolean success = fs.rename(hfile.getPath(), dst);
2496 if (success) {
2497 fileMoves++;
2498 }
2499 }
2500 LOG.debug("[" + thread + "] Sideline directory contents:");
2501 debugLsr(targetRegionDir);
2502 }
2503
2504
2505 sidelineRegionDir(fs, contained);
2506 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2507 getSidelineDir());
2508 debugLsr(contained.getHdfsRegionDir());
2509
2510 return fileMoves;
2511 }
2512
2513
2514 static class WorkItemOverlapMerge implements Callable<Void> {
2515 private TableIntegrityErrorHandler handler;
2516 Collection<HbckInfo> overlapgroup;
2517
2518 WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2519 this.handler = handler;
2520 this.overlapgroup = overlapgroup;
2521 }
2522
2523 @Override
2524 public Void call() throws Exception {
2525 handler.handleOverlapGroup(overlapgroup);
2526 return null;
2527 }
2528 };
2529
2530
2531
2532
2533
2534 public class TableInfo {
2535 TableName tableName;
2536 TreeSet <ServerName> deployedOn;
2537
2538
2539 final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2540
2541
2542 final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2543
2544
2545 final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2546
2547
2548 final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2549
2550
2551 final Multimap<byte[], HbckInfo> overlapGroups =
2552 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2553
2554
2555 private ImmutableList<HRegionInfo> regionsFromMeta = null;
2556
2557 TableInfo(TableName name) {
2558 this.tableName = name;
2559 deployedOn = new TreeSet <ServerName>();
2560 }
2561
2562
2563
2564
2565 private HTableDescriptor getHTD() {
2566 if (htds.size() == 1) {
2567 return (HTableDescriptor)htds.toArray()[0];
2568 } else {
2569 LOG.error("None/Multiple table descriptors found for table '"
2570 + tableName + "' regions: " + htds);
2571 }
2572 return null;
2573 }
2574
2575 public void addRegionInfo(HbckInfo hir) {
2576 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2577
2578
2579 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2580 return;
2581 }
2582
2583
2584 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2585 errors.reportError(
2586 ERROR_CODE.REGION_CYCLE,
2587 String.format("The endkey for this region comes before the "
2588 + "startkey, startkey=%s, endkey=%s",
2589 Bytes.toStringBinary(hir.getStartKey()),
2590 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2591 backwards.add(hir);
2592 return;
2593 }
2594
2595
2596
2597 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2598 }
2599
2600 public void addServer(ServerName server) {
2601 this.deployedOn.add(server);
2602 }
2603
2604 public TableName getName() {
2605 return tableName;
2606 }
2607
2608 public int getNumRegions() {
2609 return sc.getStarts().size() + backwards.size();
2610 }
2611
2612 public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2613
2614 if (regionsFromMeta == null) {
2615 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2616 for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2617 if (tableName.equals(h.getTableName())) {
2618 if (h.metaEntry != null) {
2619 regions.add((HRegionInfo) h.metaEntry);
2620 }
2621 }
2622 }
2623 regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2624 }
2625
2626 return regionsFromMeta;
2627 }
2628
2629
2630 private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2631 ErrorReporter errors;
2632
2633 IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2634 this.errors = errors;
2635 setTableInfo(ti);
2636 }
2637
2638 @Override
2639 public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2640 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2641 "First region should start with an empty key. You need to "
2642 + " create a new region and regioninfo in HDFS to plug the hole.",
2643 getTableInfo(), hi);
2644 }
2645
2646 @Override
2647 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2648 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2649 "Last region should end with an empty key. You need to "
2650 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2651 }
2652
2653 @Override
2654 public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2655 errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2656 "Region has the same start and end key.", getTableInfo(), hi);
2657 }
2658
2659 @Override
2660 public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2661 byte[] key = r1.getStartKey();
2662
2663 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2664 "Multiple regions have the same startkey: "
2665 + Bytes.toStringBinary(key), getTableInfo(), r1);
2666 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2667 "Multiple regions have the same startkey: "
2668 + Bytes.toStringBinary(key), getTableInfo(), r2);
2669 }
2670
2671 @Override
2672 public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2673 errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2674 "There is an overlap in the region chain.",
2675 getTableInfo(), hi1, hi2);
2676 }
2677
2678 @Override
2679 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2680 errors.reportError(
2681 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2682 "There is a hole in the region chain between "
2683 + Bytes.toStringBinary(holeStart) + " and "
2684 + Bytes.toStringBinary(holeStop)
2685 + ". You need to create a new .regioninfo and region "
2686 + "dir in hdfs to plug the hole.");
2687 }
2688 };
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702 private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2703 Configuration conf;
2704
2705 boolean fixOverlaps = true;
2706
2707 HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2708 boolean fixHoles, boolean fixOverlaps) {
2709 super(ti, errors);
2710 this.conf = conf;
2711 this.fixOverlaps = fixOverlaps;
2712
2713 }
2714
2715
2716
2717
2718
2719
2720 @Override
2721 public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2722 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2723 "First region should start with an empty key. Creating a new " +
2724 "region and regioninfo in HDFS to plug the hole.",
2725 getTableInfo(), next);
2726 HTableDescriptor htd = getTableInfo().getHTD();
2727
2728 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2729 HConstants.EMPTY_START_ROW, next.getStartKey());
2730
2731
2732 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2733 LOG.info("Table region start key was not empty. Created new empty region: "
2734 + newRegion + " " +region);
2735 fixes++;
2736 }
2737
2738 @Override
2739 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2740 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2741 "Last region should end with an empty key. Creating a new "
2742 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2743 HTableDescriptor htd = getTableInfo().getHTD();
2744
2745 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2746 HConstants.EMPTY_START_ROW);
2747
2748 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2749 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
2750 + " " + region);
2751 fixes++;
2752 }
2753
2754
2755
2756
2757
2758 @Override
2759 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2760 errors.reportError(
2761 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2762 "There is a hole in the region chain between "
2763 + Bytes.toStringBinary(holeStartKey) + " and "
2764 + Bytes.toStringBinary(holeStopKey)
2765 + ". Creating a new regioninfo and region "
2766 + "dir in hdfs to plug the hole.");
2767 HTableDescriptor htd = getTableInfo().getHTD();
2768 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2769 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2770 LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2771 fixes++;
2772 }
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785 @Override
2786 public void handleOverlapGroup(Collection<HbckInfo> overlap)
2787 throws IOException {
2788 Preconditions.checkNotNull(overlap);
2789 Preconditions.checkArgument(overlap.size() >0);
2790
2791 if (!this.fixOverlaps) {
2792 LOG.warn("Not attempting to repair overlaps.");
2793 return;
2794 }
2795
2796 if (overlap.size() > maxMerge) {
2797 LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2798 "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2799 if (sidelineBigOverlaps) {
2800
2801 sidelineBigOverlaps(overlap);
2802 }
2803 return;
2804 }
2805
2806 mergeOverlaps(overlap);
2807 }
2808
2809 void mergeOverlaps(Collection<HbckInfo> overlap)
2810 throws IOException {
2811 String thread = Thread.currentThread().getName();
2812 LOG.info("== [" + thread + "] Merging regions into one region: "
2813 + Joiner.on(",").join(overlap));
2814
2815 Pair<byte[], byte[]> range = null;
2816 for (HbckInfo hi : overlap) {
2817 if (range == null) {
2818 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2819 } else {
2820 if (RegionSplitCalculator.BYTES_COMPARATOR
2821 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2822 range.setFirst(hi.getStartKey());
2823 }
2824 if (RegionSplitCalculator.BYTES_COMPARATOR
2825 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2826 range.setSecond(hi.getEndKey());
2827 }
2828 }
2829
2830 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
2831 LOG.debug("[" + thread + "] Contained region dir before close");
2832 debugLsr(hi.getHdfsRegionDir());
2833 try {
2834 LOG.info("[" + thread + "] Closing region: " + hi);
2835 closeRegion(hi);
2836 } catch (IOException ioe) {
2837 LOG.warn("[" + thread + "] Was unable to close region " + hi
2838 + ". Just continuing... ", ioe);
2839 } catch (InterruptedException e) {
2840 LOG.warn("[" + thread + "] Was unable to close region " + hi
2841 + ". Just continuing... ", e);
2842 }
2843
2844 try {
2845 LOG.info("[" + thread + "] Offlining region: " + hi);
2846 offline(hi.getRegionName());
2847 } catch (IOException ioe) {
2848 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2849 + ". Just continuing... ", ioe);
2850 }
2851 }
2852
2853
2854 HTableDescriptor htd = getTableInfo().getHTD();
2855
2856 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2857 range.getSecond());
2858 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2859 LOG.info("[" + thread + "] Created new empty container region: " +
2860 newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2861 debugLsr(region.getRegionFileSystem().getRegionDir());
2862
2863
2864 boolean didFix= false;
2865 Path target = region.getRegionFileSystem().getRegionDir();
2866 for (HbckInfo contained : overlap) {
2867 LOG.info("[" + thread + "] Merging " + contained + " into " + target );
2868 int merges = mergeRegionDirs(target, contained);
2869 if (merges > 0) {
2870 didFix = true;
2871 }
2872 }
2873 if (didFix) {
2874 fixes++;
2875 }
2876 }
2877
2878
2879
2880
2881
2882
2883
2884
2885 void sidelineBigOverlaps(
2886 Collection<HbckInfo> bigOverlap) throws IOException {
2887 int overlapsToSideline = bigOverlap.size() - maxMerge;
2888 if (overlapsToSideline > maxOverlapsToSideline) {
2889 overlapsToSideline = maxOverlapsToSideline;
2890 }
2891 List<HbckInfo> regionsToSideline =
2892 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2893 FileSystem fs = FileSystem.get(conf);
2894 for (HbckInfo regionToSideline: regionsToSideline) {
2895 try {
2896 LOG.info("Closing region: " + regionToSideline);
2897 closeRegion(regionToSideline);
2898 } catch (IOException ioe) {
2899 LOG.warn("Was unable to close region " + regionToSideline
2900 + ". Just continuing... ", ioe);
2901 } catch (InterruptedException e) {
2902 LOG.warn("Was unable to close region " + regionToSideline
2903 + ". Just continuing... ", e);
2904 }
2905
2906 try {
2907 LOG.info("Offlining region: " + regionToSideline);
2908 offline(regionToSideline.getRegionName());
2909 } catch (IOException ioe) {
2910 LOG.warn("Unable to offline region from master: " + regionToSideline
2911 + ". Just continuing... ", ioe);
2912 }
2913
2914 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2915 Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2916 if (sidelineRegionDir != null) {
2917 sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2918 LOG.info("After sidelined big overlapped region: "
2919 + regionToSideline.getRegionNameAsString()
2920 + " to " + sidelineRegionDir.toString());
2921 fixes++;
2922 }
2923 }
2924 }
2925 }
2926
2927
2928
2929
2930
2931
2932
2933 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2934
2935
2936
2937 if (disabledTables.contains(this.tableName)) {
2938 return true;
2939 }
2940 int originalErrorsCount = errors.getErrorList().size();
2941 Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2942 SortedSet<byte[]> splits = sc.getSplits();
2943
2944 byte[] prevKey = null;
2945 byte[] problemKey = null;
2946
2947 if (splits.size() == 0) {
2948
2949 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2950 }
2951
2952 for (byte[] key : splits) {
2953 Collection<HbckInfo> ranges = regions.get(key);
2954 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2955 for (HbckInfo rng : ranges) {
2956 handler.handleRegionStartKeyNotEmpty(rng);
2957 }
2958 }
2959
2960
2961 for (HbckInfo rng : ranges) {
2962
2963 byte[] endKey = rng.getEndKey();
2964 endKey = (endKey.length == 0) ? null : endKey;
2965 if (Bytes.equals(rng.getStartKey(),endKey)) {
2966 handler.handleDegenerateRegion(rng);
2967 }
2968 }
2969
2970 if (ranges.size() == 1) {
2971
2972 if (problemKey != null) {
2973 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2974 }
2975 problemKey = null;
2976 } else if (ranges.size() > 1) {
2977
2978
2979 if (problemKey == null) {
2980
2981 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2982 problemKey = key;
2983 }
2984 overlapGroups.putAll(problemKey, ranges);
2985
2986
2987 ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2988
2989 for (HbckInfo r1 : ranges) {
2990 if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2991 subRange.remove(r1);
2992 for (HbckInfo r2 : subRange) {
2993 if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2994 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2995 handler.handleDuplicateStartKeys(r1,r2);
2996 } else {
2997
2998 handler.handleOverlapInRegionChain(r1, r2);
2999 }
3000 }
3001 }
3002
3003 } else if (ranges.size() == 0) {
3004 if (problemKey != null) {
3005 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3006 }
3007 problemKey = null;
3008
3009 byte[] holeStopKey = sc.getSplits().higher(key);
3010
3011 if (holeStopKey != null) {
3012
3013 handler.handleHoleInRegionChain(key, holeStopKey);
3014 }
3015 }
3016 prevKey = key;
3017 }
3018
3019
3020
3021 if (prevKey != null) {
3022 handler.handleRegionEndKeyNotEmpty(prevKey);
3023 }
3024
3025
3026 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3027 boolean ok = handleOverlapsParallel(handler, prevKey);
3028 if (!ok) {
3029 return false;
3030 }
3031 } else {
3032 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3033 handler.handleOverlapGroup(overlap);
3034 }
3035 }
3036
3037 if (details) {
3038
3039 errors.print("---- Table '" + this.tableName
3040 + "': region split map");
3041 dump(splits, regions);
3042 errors.print("---- Table '" + this.tableName
3043 + "': overlap groups");
3044 dumpOverlapProblems(overlapGroups);
3045 errors.print("There are " + overlapGroups.keySet().size()
3046 + " overlap groups with " + overlapGroups.size()
3047 + " overlapping regions");
3048 }
3049 if (!sidelinedRegions.isEmpty()) {
3050 LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3051 errors.print("---- Table '" + this.tableName
3052 + "': sidelined big overlapped regions");
3053 dumpSidelinedRegions(sidelinedRegions);
3054 }
3055 return errors.getErrorList().size() == originalErrorsCount;
3056 }
3057
3058 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3059 throws IOException {
3060
3061
3062 List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3063 List<Future<Void>> rets;
3064 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3065
3066 merges.add(new WorkItemOverlapMerge(overlap, handler));
3067 }
3068 try {
3069 rets = executor.invokeAll(merges);
3070 } catch (InterruptedException e) {
3071 LOG.error("Overlap merges were interrupted", e);
3072 return false;
3073 }
3074 for(int i=0; i<merges.size(); i++) {
3075 WorkItemOverlapMerge work = merges.get(i);
3076 Future<Void> f = rets.get(i);
3077 try {
3078 f.get();
3079 } catch(ExecutionException e) {
3080 LOG.warn("Failed to merge overlap group" + work, e.getCause());
3081 } catch (InterruptedException e) {
3082 LOG.error("Waiting for overlap merges was interrupted", e);
3083 return false;
3084 }
3085 }
3086 return true;
3087 }
3088
3089
3090
3091
3092
3093
3094
3095 void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3096
3097 StringBuilder sb = new StringBuilder();
3098 for (byte[] k : splits) {
3099 sb.setLength(0);
3100 sb.append(Bytes.toStringBinary(k) + ":\t");
3101 for (HbckInfo r : regions.get(k)) {
3102 sb.append("[ "+ r.toString() + ", "
3103 + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3104 }
3105 errors.print(sb.toString());
3106 }
3107 }
3108 }
3109
3110 public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3111
3112
3113 for (byte[] k : regions.keySet()) {
3114 errors.print(Bytes.toStringBinary(k) + ":");
3115 for (HbckInfo r : regions.get(k)) {
3116 errors.print("[ " + r.toString() + ", "
3117 + Bytes.toStringBinary(r.getEndKey()) + "]");
3118 }
3119 errors.print("----");
3120 }
3121 }
3122
3123 public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3124 for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3125 TableName tableName = entry.getValue().getTableName();
3126 Path path = entry.getKey();
3127 errors.print("This sidelined region dir should be bulk loaded: "
3128 + path.toString());
3129 errors.print("Bulk load command looks like: "
3130 + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3131 + path.toUri().getPath() + " "+ tableName);
3132 }
3133 }
3134
3135 public Multimap<byte[], HbckInfo> getOverlapGroups(
3136 TableName table) {
3137 TableInfo ti = tablesInfo.get(table);
3138 return ti.overlapGroups;
3139 }
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150 HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3151 List<TableName> tableNames = new ArrayList<TableName>();
3152 long now = EnvironmentEdgeManager.currentTime();
3153
3154 for (HbckInfo hbi : regionInfoMap.values()) {
3155 MetaEntry info = hbi.metaEntry;
3156
3157
3158
3159 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3160 if (info.modTime + timelag < now) {
3161 tableNames.add(info.getTable());
3162 } else {
3163 numSkipped.incrementAndGet();
3164 }
3165 }
3166 }
3167 return getHTableDescriptors(tableNames);
3168 }
3169
3170 HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3171 HTableDescriptor[] htd = new HTableDescriptor[0];
3172 Admin admin = null;
3173 try {
3174 LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3175 admin = new HBaseAdmin(getConf());
3176 htd = admin.getTableDescriptorsByTableName(tableNames);
3177 } catch (IOException e) {
3178 LOG.debug("Exception getting table descriptors", e);
3179 } finally {
3180 if (admin != null) {
3181 try {
3182 admin.close();
3183 } catch (IOException e) {
3184 LOG.debug("Exception closing HBaseAdmin", e);
3185 }
3186 }
3187 }
3188 return htd;
3189 }
3190
3191
3192
3193
3194
3195
3196 private synchronized HbckInfo getOrCreateInfo(String name) {
3197 HbckInfo hbi = regionInfoMap.get(name);
3198 if (hbi == null) {
3199 hbi = new HbckInfo(null);
3200 regionInfoMap.put(name, hbi);
3201 }
3202 return hbi;
3203 }
3204
3205 private void checkAndFixTableLocks() throws IOException {
3206 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3207
3208 try {
3209 TableLockChecker checker = new TableLockChecker(zkw, errors);
3210 checker.checkTableLocks();
3211
3212 if (this.fixTableLocks) {
3213 checker.fixExpiredTableLocks();
3214 }
3215 } finally {
3216 zkw.close();
3217 }
3218 }
3219
3220
3221
3222
3223
3224
3225
3226 private void checkAndFixOrphanedTableZNodes()
3227 throws IOException, KeeperException, InterruptedException {
3228 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3229
3230 try {
3231 Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3232 String msg;
3233 TableInfo tableInfo;
3234
3235 for (TableName tableName : enablingTables) {
3236
3237 tableInfo = tablesInfo.get(tableName);
3238 if (tableInfo != null) {
3239
3240 continue;
3241 }
3242
3243 msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3244 LOG.warn(msg);
3245 orphanedTableZNodes.add(tableName);
3246 errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3247 }
3248
3249 if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3250 ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3251
3252 for (TableName tableName : orphanedTableZNodes) {
3253 try {
3254
3255
3256
3257
3258 zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3259 } catch (CoordinatedStateException e) {
3260
3261 LOG.error(
3262 "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3263 e);
3264 }
3265 }
3266 }
3267 } finally {
3268 zkw.close();
3269 }
3270 }
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3282 Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3283 for (HbckInfo value : regionInfoMap.values()) {
3284 if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3285 metaRegions.put(value.getReplicaId(), value);
3286 }
3287 }
3288 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3289 .getRegionReplication();
3290 boolean noProblem = true;
3291
3292
3293 for (int i = 0; i < metaReplication; i++) {
3294 HbckInfo metaHbckInfo = metaRegions.remove(i);
3295 List<ServerName> servers = new ArrayList<ServerName>();
3296 if (metaHbckInfo != null) {
3297 servers = metaHbckInfo.deployedOn;
3298 }
3299 if (servers.size() != 1) {
3300 noProblem = false;
3301 if (servers.size() == 0) {
3302 assignMetaReplica(i);
3303 } else if (servers.size() > 1) {
3304 errors
3305 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3306 metaHbckInfo.getReplicaId() + " is found on more than one region.");
3307 if (shouldFixAssignments()) {
3308 errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3309 metaHbckInfo.getReplicaId() +"..");
3310 setShouldRerun();
3311
3312 HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3313 }
3314 }
3315 }
3316 }
3317
3318 for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3319 noProblem = false;
3320 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3321 "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3322 ", deployed " + metaRegions.size());
3323 if (shouldFixAssignments()) {
3324 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3325 " of hbase:meta..");
3326 setShouldRerun();
3327 unassignMetaReplica(entry.getValue());
3328 }
3329 }
3330
3331
3332 return noProblem;
3333 }
3334
3335 private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3336 KeeperException {
3337 undeployRegions(hi);
3338 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3339 try {
3340 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3341 } finally {
3342 zkw.close();
3343 }
3344 }
3345
3346 private void assignMetaReplica(int replicaId)
3347 throws IOException, KeeperException, InterruptedException {
3348 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3349 replicaId +" is not found on any region.");
3350 if (shouldFixAssignments()) {
3351 errors.print("Trying to fix a problem with hbase:meta..");
3352 setShouldRerun();
3353
3354 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3355 HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3356 HBaseFsckRepair.fixUnassigned(admin, h);
3357 HBaseFsckRepair.waitUntilAssigned(admin, h);
3358 }
3359 }
3360
3361
3362
3363
3364
3365 boolean loadMetaEntries() throws IOException {
3366 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3367 int countRecord = 1;
3368
3369
3370 final Comparator<Cell> comp = new Comparator<Cell>() {
3371 @Override
3372 public int compare(Cell k1, Cell k2) {
3373 return (int)(k1.getTimestamp() - k2.getTimestamp());
3374 }
3375 };
3376
3377 @Override
3378 public boolean processRow(Result result) throws IOException {
3379 try {
3380
3381
3382 long ts = Collections.max(result.listCells(), comp).getTimestamp();
3383 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3384 if (rl == null) {
3385 emptyRegionInfoQualifiers.add(result);
3386 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3387 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3388 return true;
3389 }
3390 ServerName sn = null;
3391 if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3392 rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3393 emptyRegionInfoQualifiers.add(result);
3394 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3395 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3396 return true;
3397 }
3398 HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3399 if (!(isTableIncluded(hri.getTable())
3400 || hri.isMetaRegion())) {
3401 return true;
3402 }
3403 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3404 for (HRegionLocation h : rl.getRegionLocations()) {
3405 if (h == null || h.getRegionInfo() == null) {
3406 continue;
3407 }
3408 sn = h.getServerName();
3409 hri = h.getRegionInfo();
3410
3411 MetaEntry m = null;
3412 if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3413 m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3414 } else {
3415 m = new MetaEntry(hri, sn, ts, null, null);
3416 }
3417 HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3418 if (previous == null) {
3419 regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3420 } else if (previous.metaEntry == null) {
3421 previous.metaEntry = m;
3422 } else {
3423 throw new IOException("Two entries in hbase:meta are same " + previous);
3424 }
3425 }
3426 PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3427 for (HRegionInfo mergeRegion : new HRegionInfo[] {
3428 mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3429 if (mergeRegion != null) {
3430
3431 HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3432 hbInfo.setMerged(true);
3433 }
3434 }
3435
3436
3437 if (countRecord % 100 == 0) {
3438 errors.progress();
3439 }
3440 countRecord++;
3441 return true;
3442 } catch (RuntimeException e) {
3443 LOG.error("Result=" + result);
3444 throw e;
3445 }
3446 }
3447 };
3448 if (!checkMetaOnly) {
3449
3450 MetaScanner.metaScan(connection, visitor);
3451 }
3452
3453 errors.print("");
3454 return true;
3455 }
3456
3457
3458
3459
3460 static class MetaEntry extends HRegionInfo {
3461 ServerName regionServer;
3462 long modTime;
3463 HRegionInfo splitA, splitB;
3464
3465 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3466 this(rinfo, regionServer, modTime, null, null);
3467 }
3468
3469 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3470 HRegionInfo splitA, HRegionInfo splitB) {
3471 super(rinfo);
3472 this.regionServer = regionServer;
3473 this.modTime = modTime;
3474 this.splitA = splitA;
3475 this.splitB = splitB;
3476 }
3477
3478 @Override
3479 public boolean equals(Object o) {
3480 boolean superEq = super.equals(o);
3481 if (!superEq) {
3482 return superEq;
3483 }
3484
3485 MetaEntry me = (MetaEntry) o;
3486 if (!regionServer.equals(me.regionServer)) {
3487 return false;
3488 }
3489 return (modTime == me.modTime);
3490 }
3491
3492 @Override
3493 public int hashCode() {
3494 int hash = Arrays.hashCode(getRegionName());
3495 hash ^= getRegionId();
3496 hash ^= Arrays.hashCode(getStartKey());
3497 hash ^= Arrays.hashCode(getEndKey());
3498 hash ^= Boolean.valueOf(isOffline()).hashCode();
3499 hash ^= getTable().hashCode();
3500 if (regionServer != null) {
3501 hash ^= regionServer.hashCode();
3502 }
3503 hash ^= modTime;
3504 return hash;
3505 }
3506 }
3507
3508
3509
3510
3511 static class HdfsEntry {
3512 HRegionInfo hri;
3513 Path hdfsRegionDir = null;
3514 long hdfsRegionDirModTime = 0;
3515 boolean hdfsRegioninfoFilePresent = false;
3516 boolean hdfsOnlyEdits = false;
3517 }
3518
3519
3520
3521
3522 static class OnlineEntry {
3523 HRegionInfo hri;
3524 ServerName hsa;
3525
3526 @Override
3527 public String toString() {
3528 return hsa.toString() + ";" + hri.getRegionNameAsString();
3529 }
3530 }
3531
3532
3533
3534
3535
3536 public static class HbckInfo implements KeyRange {
3537 private MetaEntry metaEntry = null;
3538 private HdfsEntry hdfsEntry = null;
3539 private List<OnlineEntry> deployedEntries = Lists.newArrayList();
3540 private List<ServerName> deployedOn = Lists.newArrayList();
3541 private boolean skipChecks = false;
3542 private boolean isMerged = false;
3543 private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3544 private HRegionInfo primaryHRIForDeployedReplica = null;
3545
3546 HbckInfo(MetaEntry metaEntry) {
3547 this.metaEntry = metaEntry;
3548 }
3549
3550 public int getReplicaId() {
3551 if (metaEntry != null) return metaEntry.getReplicaId();
3552 return deployedReplicaId;
3553 }
3554
3555 public synchronized void addServer(HRegionInfo hri, ServerName server) {
3556 OnlineEntry rse = new OnlineEntry() ;
3557 rse.hri = hri;
3558 rse.hsa = server;
3559 this.deployedEntries.add(rse);
3560 this.deployedOn.add(server);
3561
3562 this.deployedReplicaId = hri.getReplicaId();
3563 this.primaryHRIForDeployedReplica =
3564 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3565 }
3566
3567 @Override
3568 public synchronized String toString() {
3569 StringBuilder sb = new StringBuilder();
3570 sb.append("{ meta => ");
3571 sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3572 sb.append( ", hdfs => " + getHdfsRegionDir());
3573 sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3574 sb.append( ", replicaId => " + getReplicaId());
3575 sb.append(" }");
3576 return sb.toString();
3577 }
3578
3579 @Override
3580 public byte[] getStartKey() {
3581 if (this.metaEntry != null) {
3582 return this.metaEntry.getStartKey();
3583 } else if (this.hdfsEntry != null) {
3584 return this.hdfsEntry.hri.getStartKey();
3585 } else {
3586 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3587 return null;
3588 }
3589 }
3590
3591 @Override
3592 public byte[] getEndKey() {
3593 if (this.metaEntry != null) {
3594 return this.metaEntry.getEndKey();
3595 } else if (this.hdfsEntry != null) {
3596 return this.hdfsEntry.hri.getEndKey();
3597 } else {
3598 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3599 return null;
3600 }
3601 }
3602
3603 public TableName getTableName() {
3604 if (this.metaEntry != null) {
3605 return this.metaEntry.getTable();
3606 } else if (this.hdfsEntry != null) {
3607
3608
3609 Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3610 return FSUtils.getTableName(tableDir);
3611 } else {
3612
3613 for (OnlineEntry e : deployedEntries) {
3614 return e.hri.getTable();
3615 }
3616 return null;
3617 }
3618 }
3619
3620 public String getRegionNameAsString() {
3621 if (metaEntry != null) {
3622 return metaEntry.getRegionNameAsString();
3623 } else if (hdfsEntry != null) {
3624 if (hdfsEntry.hri != null) {
3625 return hdfsEntry.hri.getRegionNameAsString();
3626 }
3627 } else {
3628
3629 for (OnlineEntry e : deployedEntries) {
3630 return e.hri.getRegionNameAsString();
3631 }
3632 }
3633 return null;
3634 }
3635
3636 public byte[] getRegionName() {
3637 if (metaEntry != null) {
3638 return metaEntry.getRegionName();
3639 } else if (hdfsEntry != null) {
3640 return hdfsEntry.hri.getRegionName();
3641 } else {
3642
3643 for (OnlineEntry e : deployedEntries) {
3644 return e.hri.getRegionName();
3645 }
3646 return null;
3647 }
3648 }
3649
3650 public HRegionInfo getPrimaryHRIForDeployedReplica() {
3651 return primaryHRIForDeployedReplica;
3652 }
3653
3654 Path getHdfsRegionDir() {
3655 if (hdfsEntry == null) {
3656 return null;
3657 }
3658 return hdfsEntry.hdfsRegionDir;
3659 }
3660
3661 boolean containsOnlyHdfsEdits() {
3662 if (hdfsEntry == null) {
3663 return false;
3664 }
3665 return hdfsEntry.hdfsOnlyEdits;
3666 }
3667
3668 boolean isHdfsRegioninfoPresent() {
3669 if (hdfsEntry == null) {
3670 return false;
3671 }
3672 return hdfsEntry.hdfsRegioninfoFilePresent;
3673 }
3674
3675 long getModTime() {
3676 if (hdfsEntry == null) {
3677 return 0;
3678 }
3679 return hdfsEntry.hdfsRegionDirModTime;
3680 }
3681
3682 HRegionInfo getHdfsHRI() {
3683 if (hdfsEntry == null) {
3684 return null;
3685 }
3686 return hdfsEntry.hri;
3687 }
3688
3689 public void setSkipChecks(boolean skipChecks) {
3690 this.skipChecks = skipChecks;
3691 }
3692
3693 public boolean isSkipChecks() {
3694 return skipChecks;
3695 }
3696
3697 public void setMerged(boolean isMerged) {
3698 this.isMerged = isMerged;
3699 }
3700
3701 public boolean isMerged() {
3702 return this.isMerged;
3703 }
3704 }
3705
3706 final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3707 @Override
3708 public int compare(HbckInfo l, HbckInfo r) {
3709 if (l == r) {
3710
3711 return 0;
3712 }
3713
3714 int tableCompare = l.getTableName().compareTo(r.getTableName());
3715 if (tableCompare != 0) {
3716 return tableCompare;
3717 }
3718
3719 int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3720 l.getStartKey(), r.getStartKey());
3721 if (startComparison != 0) {
3722 return startComparison;
3723 }
3724
3725
3726 byte[] endKey = r.getEndKey();
3727 endKey = (endKey.length == 0) ? null : endKey;
3728 byte[] endKey2 = l.getEndKey();
3729 endKey2 = (endKey2.length == 0) ? null : endKey2;
3730 int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3731 endKey2, endKey);
3732
3733 if (endComparison != 0) {
3734 return endComparison;
3735 }
3736
3737
3738
3739 if (l.hdfsEntry == null && r.hdfsEntry == null) {
3740 return 0;
3741 }
3742 if (l.hdfsEntry == null && r.hdfsEntry != null) {
3743 return 1;
3744 }
3745
3746 if (r.hdfsEntry == null) {
3747 return -1;
3748 }
3749
3750 return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3751 }
3752 };
3753
3754
3755
3756
3757 private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3758 StringBuilder sb = new StringBuilder();
3759 int numOfSkippedRegions;
3760 errors.print("Summary:");
3761 for (TableInfo tInfo : tablesInfo.values()) {
3762 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3763 skippedRegions.get(tInfo.getName()).size() : 0;
3764
3765 if (errors.tableHasErrors(tInfo)) {
3766 errors.print("Table " + tInfo.getName() + " is inconsistent.");
3767 } else if (numOfSkippedRegions > 0){
3768 errors.print("Table " + tInfo.getName() + " is okay (with "
3769 + numOfSkippedRegions + " skipped regions).");
3770 }
3771 else {
3772 errors.print("Table " + tInfo.getName() + " is okay.");
3773 }
3774 errors.print(" Number of regions: " + tInfo.getNumRegions());
3775 if (numOfSkippedRegions > 0) {
3776 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3777 System.out.println(" Number of skipped regions: " + numOfSkippedRegions);
3778 System.out.println(" List of skipped regions:");
3779 for(String sr : skippedRegionStrings) {
3780 System.out.println(" " + sr);
3781 }
3782 }
3783 sb.setLength(0);
3784 sb.append(" Deployed on: ");
3785 for (ServerName server : tInfo.deployedOn) {
3786 sb.append(" " + server.toString());
3787 }
3788 errors.print(sb.toString());
3789 }
3790 }
3791
3792 static ErrorReporter getErrorReporter(
3793 final Configuration conf) throws ClassNotFoundException {
3794 Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3795 return ReflectionUtils.newInstance(reporter, conf);
3796 }
3797
3798 public interface ErrorReporter {
3799 enum ERROR_CODE {
3800 UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3801 NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3802 MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3803 FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3804 HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3805 ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3806 WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR
3807 }
3808 void clear();
3809 void report(String message);
3810 void reportError(String message);
3811 void reportError(ERROR_CODE errorCode, String message);
3812 void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3813 void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3814 void reportError(
3815 ERROR_CODE errorCode,
3816 String message,
3817 TableInfo table,
3818 HbckInfo info1,
3819 HbckInfo info2
3820 );
3821 int summarize();
3822 void detail(String details);
3823 ArrayList<ERROR_CODE> getErrorList();
3824 void progress();
3825 void print(String message);
3826 void resetErrors();
3827 boolean tableHasErrors(TableInfo table);
3828 }
3829
3830 static class PrintingErrorReporter implements ErrorReporter {
3831 public int errorCount = 0;
3832 private int showProgress;
3833
3834 private static final int progressThreshold = 100;
3835
3836 Set<TableInfo> errorTables = new HashSet<TableInfo>();
3837
3838
3839 private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3840
3841 @Override
3842 public void clear() {
3843 errorTables.clear();
3844 errorList.clear();
3845 errorCount = 0;
3846 }
3847
3848 @Override
3849 public synchronized void reportError(ERROR_CODE errorCode, String message) {
3850 if (errorCode == ERROR_CODE.WRONG_USAGE) {
3851 System.err.println(message);
3852 return;
3853 }
3854
3855 errorList.add(errorCode);
3856 if (!summary) {
3857 System.out.println("ERROR: " + message);
3858 }
3859 errorCount++;
3860 showProgress = 0;
3861 }
3862
3863 @Override
3864 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3865 errorTables.add(table);
3866 reportError(errorCode, message);
3867 }
3868
3869 @Override
3870 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3871 HbckInfo info) {
3872 errorTables.add(table);
3873 String reference = "(region " + info.getRegionNameAsString() + ")";
3874 reportError(errorCode, reference + " " + message);
3875 }
3876
3877 @Override
3878 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3879 HbckInfo info1, HbckInfo info2) {
3880 errorTables.add(table);
3881 String reference = "(regions " + info1.getRegionNameAsString()
3882 + " and " + info2.getRegionNameAsString() + ")";
3883 reportError(errorCode, reference + " " + message);
3884 }
3885
3886 @Override
3887 public synchronized void reportError(String message) {
3888 reportError(ERROR_CODE.UNKNOWN, message);
3889 }
3890
3891
3892
3893
3894
3895
3896 @Override
3897 public synchronized void report(String message) {
3898 if (! summary) {
3899 System.out.println("ERROR: " + message);
3900 }
3901 showProgress = 0;
3902 }
3903
3904 @Override
3905 public synchronized int summarize() {
3906 System.out.println(Integer.toString(errorCount) +
3907 " inconsistencies detected.");
3908 if (errorCount == 0) {
3909 System.out.println("Status: OK");
3910 return 0;
3911 } else {
3912 System.out.println("Status: INCONSISTENT");
3913 return -1;
3914 }
3915 }
3916
3917 @Override
3918 public ArrayList<ERROR_CODE> getErrorList() {
3919 return errorList;
3920 }
3921
3922 @Override
3923 public synchronized void print(String message) {
3924 if (!summary) {
3925 System.out.println(message);
3926 }
3927 }
3928
3929 @Override
3930 public boolean tableHasErrors(TableInfo table) {
3931 return errorTables.contains(table);
3932 }
3933
3934 @Override
3935 public void resetErrors() {
3936 errorCount = 0;
3937 }
3938
3939 @Override
3940 public synchronized void detail(String message) {
3941 if (details) {
3942 System.out.println(message);
3943 }
3944 showProgress = 0;
3945 }
3946
3947 @Override
3948 public synchronized void progress() {
3949 if (showProgress++ == progressThreshold) {
3950 if (!summary) {
3951 System.out.print(".");
3952 }
3953 showProgress = 0;
3954 }
3955 }
3956 }
3957
3958
3959
3960
3961 static class WorkItemRegion implements Callable<Void> {
3962 private HBaseFsck hbck;
3963 private ServerName rsinfo;
3964 private ErrorReporter errors;
3965 private HConnection connection;
3966
3967 WorkItemRegion(HBaseFsck hbck, ServerName info,
3968 ErrorReporter errors, HConnection connection) {
3969 this.hbck = hbck;
3970 this.rsinfo = info;
3971 this.errors = errors;
3972 this.connection = connection;
3973 }
3974
3975 @Override
3976 public synchronized Void call() throws IOException {
3977 errors.progress();
3978 try {
3979 BlockingInterface server = connection.getAdmin(rsinfo);
3980
3981
3982 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3983 regions = filterRegions(regions);
3984
3985 if (details) {
3986 errors.detail("RegionServer: " + rsinfo.getServerName() +
3987 " number of regions: " + regions.size());
3988 for (HRegionInfo rinfo: regions) {
3989 errors.detail(" " + rinfo.getRegionNameAsString() +
3990 " id: " + rinfo.getRegionId() +
3991 " encoded_name: " + rinfo.getEncodedName() +
3992 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3993 " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3994 }
3995 }
3996
3997
3998 for (HRegionInfo r:regions) {
3999 HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4000 hbi.addServer(r, rsinfo);
4001 }
4002 } catch (IOException e) {
4003 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4004 " Unable to fetch region information. " + e);
4005 throw e;
4006 }
4007 return null;
4008 }
4009
4010 private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4011 List<HRegionInfo> ret = Lists.newArrayList();
4012 for (HRegionInfo hri : regions) {
4013 if (hri.isMetaTable() || (!hbck.checkMetaOnly
4014 && hbck.isTableIncluded(hri.getTable()))) {
4015 ret.add(hri);
4016 }
4017 }
4018 return ret;
4019 }
4020 }
4021
4022
4023
4024
4025
4026 static class WorkItemHdfsDir implements Callable<Void> {
4027 private HBaseFsck hbck;
4028 private FileStatus tableDir;
4029 private ErrorReporter errors;
4030 private FileSystem fs;
4031
4032 WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4033 FileStatus status) {
4034 this.hbck = hbck;
4035 this.fs = fs;
4036 this.tableDir = status;
4037 this.errors = errors;
4038 }
4039
4040 @Override
4041 public synchronized Void call() throws IOException {
4042 try {
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099 static class WorkItemHdfsRegionInfo implements Callable<Void> {
4100 private HbckInfo hbi;
4101 private HBaseFsck hbck;
4102 private ErrorReporter errors;
4103
4104 WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4105 this.hbi = hbi;
4106 this.hbck = hbck;
4107 this.errors = errors;
4108 }
4109
4110 @Override
4111 public synchronized Void call() throws IOException {
4112
4113 if (hbi.getHdfsHRI() == null) {
4114 try {
4115 errors.progress();
4116 hbck.loadHdfsRegioninfo(hbi);
4117 } catch (IOException ioe) {
4118 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4119 + hbi.getTableName() + " in hdfs dir "
4120 + hbi.getHdfsRegionDir()
4121 + "! It may be an invalid format or version file. Treating as "
4122 + "an orphaned regiondir.";
4123 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4124 try {
4125 hbck.debugLsr(hbi.getHdfsRegionDir());
4126 } catch (IOException ioe2) {
4127 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4128 throw ioe2;
4129 }
4130 hbck.orphanHdfsDirs.add(hbi);
4131 throw ioe;
4132 }
4133 }
4134 return null;
4135 }
4136 };
4137
4138
4139
4140
4141
4142 public static void setDisplayFullReport() {
4143 details = true;
4144 }
4145
4146
4147
4148
4149
4150 void setSummary() {
4151 summary = true;
4152 }
4153
4154
4155
4156
4157
4158 void setCheckMetaOnly() {
4159 checkMetaOnly = true;
4160 }
4161
4162
4163
4164
4165 void setRegionBoundariesCheck() {
4166 checkRegionBoundaries = true;
4167 }
4168
4169
4170
4171
4172
4173 public void setFixTableLocks(boolean shouldFix) {
4174 fixTableLocks = shouldFix;
4175 fixAny |= shouldFix;
4176 }
4177
4178
4179
4180
4181
4182 public void setFixTableZNodes(boolean shouldFix) {
4183 fixTableZNodes = shouldFix;
4184 fixAny |= shouldFix;
4185 }
4186
4187
4188
4189
4190
4191
4192
4193 void setShouldRerun() {
4194 rerun = true;
4195 }
4196
4197 boolean shouldRerun() {
4198 return rerun;
4199 }
4200
4201
4202
4203
4204
4205 public void setFixAssignments(boolean shouldFix) {
4206 fixAssignments = shouldFix;
4207 fixAny |= shouldFix;
4208 }
4209
4210 boolean shouldFixAssignments() {
4211 return fixAssignments;
4212 }
4213
4214 public void setFixMeta(boolean shouldFix) {
4215 fixMeta = shouldFix;
4216 fixAny |= shouldFix;
4217 }
4218
4219 boolean shouldFixMeta() {
4220 return fixMeta;
4221 }
4222
4223 public void setFixEmptyMetaCells(boolean shouldFix) {
4224 fixEmptyMetaCells = shouldFix;
4225 fixAny |= shouldFix;
4226 }
4227
4228 boolean shouldFixEmptyMetaCells() {
4229 return fixEmptyMetaCells;
4230 }
4231
4232 public void setCheckHdfs(boolean checking) {
4233 checkHdfs = checking;
4234 }
4235
4236 boolean shouldCheckHdfs() {
4237 return checkHdfs;
4238 }
4239
4240 public void setFixHdfsHoles(boolean shouldFix) {
4241 fixHdfsHoles = shouldFix;
4242 fixAny |= shouldFix;
4243 }
4244
4245 boolean shouldFixHdfsHoles() {
4246 return fixHdfsHoles;
4247 }
4248
4249 public void setFixTableOrphans(boolean shouldFix) {
4250 fixTableOrphans = shouldFix;
4251 fixAny |= shouldFix;
4252 }
4253
4254 boolean shouldFixTableOrphans() {
4255 return fixTableOrphans;
4256 }
4257
4258 public void setFixHdfsOverlaps(boolean shouldFix) {
4259 fixHdfsOverlaps = shouldFix;
4260 fixAny |= shouldFix;
4261 }
4262
4263 boolean shouldFixHdfsOverlaps() {
4264 return fixHdfsOverlaps;
4265 }
4266
4267 public void setFixHdfsOrphans(boolean shouldFix) {
4268 fixHdfsOrphans = shouldFix;
4269 fixAny |= shouldFix;
4270 }
4271
4272 boolean shouldFixHdfsOrphans() {
4273 return fixHdfsOrphans;
4274 }
4275
4276 public void setFixVersionFile(boolean shouldFix) {
4277 fixVersionFile = shouldFix;
4278 fixAny |= shouldFix;
4279 }
4280
4281 public boolean shouldFixVersionFile() {
4282 return fixVersionFile;
4283 }
4284
4285 public void setSidelineBigOverlaps(boolean sbo) {
4286 this.sidelineBigOverlaps = sbo;
4287 }
4288
4289 public boolean shouldSidelineBigOverlaps() {
4290 return sidelineBigOverlaps;
4291 }
4292
4293 public void setFixSplitParents(boolean shouldFix) {
4294 fixSplitParents = shouldFix;
4295 fixAny |= shouldFix;
4296 }
4297
4298 boolean shouldFixSplitParents() {
4299 return fixSplitParents;
4300 }
4301
4302 public void setFixReferenceFiles(boolean shouldFix) {
4303 fixReferenceFiles = shouldFix;
4304 fixAny |= shouldFix;
4305 }
4306
4307 boolean shouldFixReferenceFiles() {
4308 return fixReferenceFiles;
4309 }
4310
4311 public boolean shouldIgnorePreCheckPermission() {
4312 return !fixAny || ignorePreCheckPermission;
4313 }
4314
4315 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4316 this.ignorePreCheckPermission = ignorePreCheckPermission;
4317 }
4318
4319
4320
4321
4322 public void setMaxMerge(int mm) {
4323 this.maxMerge = mm;
4324 }
4325
4326 public int getMaxMerge() {
4327 return maxMerge;
4328 }
4329
4330 public void setMaxOverlapsToSideline(int mo) {
4331 this.maxOverlapsToSideline = mo;
4332 }
4333
4334 public int getMaxOverlapsToSideline() {
4335 return maxOverlapsToSideline;
4336 }
4337
4338
4339
4340
4341
4342 boolean isTableIncluded(TableName table) {
4343 return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4344 }
4345
4346 public void includeTable(TableName table) {
4347 tablesIncluded.add(table);
4348 }
4349
4350 Set<TableName> getIncludedTables() {
4351 return new HashSet<TableName>(tablesIncluded);
4352 }
4353
4354
4355
4356
4357
4358
4359 public void setTimeLag(long seconds) {
4360 timelag = seconds * 1000;
4361 }
4362
4363
4364
4365
4366
4367 public void setSidelineDir(String sidelineDir) {
4368 this.sidelineDir = new Path(sidelineDir);
4369 }
4370
4371 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4372 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4373 }
4374
4375 public HFileCorruptionChecker getHFilecorruptionChecker() {
4376 return hfcc;
4377 }
4378
4379 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4380 this.hfcc = hfcc;
4381 }
4382
4383 public void setRetCode(int code) {
4384 this.retcode = code;
4385 }
4386
4387 public int getRetCode() {
4388 return retcode;
4389 }
4390
4391 protected HBaseFsck printUsageAndExit() {
4392 StringWriter sw = new StringWriter(2048);
4393 PrintWriter out = new PrintWriter(sw);
4394 out.println("Usage: fsck [opts] {only tables}");
4395 out.println(" where [opts] are:");
4396 out.println(" -help Display help options (this)");
4397 out.println(" -details Display full report of all regions.");
4398 out.println(" -timelag <timeInSeconds> Process only regions that " +
4399 " have not experienced any metadata updates in the last " +
4400 " <timeInSeconds> seconds.");
4401 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4402 " before checking if the fix worked if run with -fix");
4403 out.println(" -summary Print only summary of the tables and status.");
4404 out.println(" -metaonly Only check the state of the hbase:meta table.");
4405 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4406 out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
4407
4408 out.println("");
4409 out.println(" Metadata Repair options: (expert features, use with caution!)");
4410 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity");
4411 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");
4412 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");
4413 out.println(" -noHdfsChecking Don't load/check region info from HDFS."
4414 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4415 out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
4416 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");
4417 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4418 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
4419 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
4420 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4421 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
4422 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4423 out.println(" -fixSplitParents Try to force offline split parents to be online.");
4424 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
4425 out.println(" -fixReferenceFiles Try to offline lingering reference store files");
4426 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"
4427 + " (empty REGIONINFO_QUALIFIER rows)");
4428
4429 out.println("");
4430 out.println(" Datafile Repair options: (expert features, use with caution!)");
4431 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");
4432 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");
4433
4434 out.println("");
4435 out.println(" Metadata Repair shortcuts");
4436 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4437 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4438 "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4439 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4440
4441 out.println("");
4442 out.println(" Table lock options");
4443 out.println(" -fixTableLocks Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4444
4445 out.println("");
4446 out.println(" Table Znode options");
4447 out.println(" -fixOrphanedTableZnodes Set table state in ZNode to disabled if table does not exists");
4448
4449 out.flush();
4450 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4451
4452 setRetCode(-2);
4453 return this;
4454 }
4455
4456
4457
4458
4459
4460
4461
4462 public static void main(String[] args) throws Exception {
4463
4464 Configuration conf = HBaseConfiguration.create();
4465 Path hbasedir = FSUtils.getRootDir(conf);
4466 URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4467 FSUtils.setFsDefault(conf, new Path(defaultFs));
4468 int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4469 System.exit(ret);
4470 }
4471
4472
4473
4474
4475 static class HBaseFsckTool extends Configured implements Tool {
4476 HBaseFsckTool(Configuration conf) { super(conf); }
4477 @Override
4478 public int run(String[] args) throws Exception {
4479 HBaseFsck hbck = new HBaseFsck(getConf());
4480 hbck.exec(hbck.executor, args);
4481 hbck.close();
4482 return hbck.getRetCode();
4483 }
4484 };
4485
4486
4487 public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4488 ServiceException, InterruptedException {
4489 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4490
4491 boolean checkCorruptHFiles = false;
4492 boolean sidelineCorruptHFiles = false;
4493
4494
4495 for (int i = 0; i < args.length; i++) {
4496 String cmd = args[i];
4497 if (cmd.equals("-help") || cmd.equals("-h")) {
4498 return printUsageAndExit();
4499 } else if (cmd.equals("-details")) {
4500 setDisplayFullReport();
4501 } else if (cmd.equals("-timelag")) {
4502 if (i == args.length - 1) {
4503 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4504 return printUsageAndExit();
4505 }
4506 try {
4507 long timelag = Long.parseLong(args[i+1]);
4508 setTimeLag(timelag);
4509 } catch (NumberFormatException e) {
4510 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4511 return printUsageAndExit();
4512 }
4513 i++;
4514 } else if (cmd.equals("-sleepBeforeRerun")) {
4515 if (i == args.length - 1) {
4516 errors.reportError(ERROR_CODE.WRONG_USAGE,
4517 "HBaseFsck: -sleepBeforeRerun needs a value.");
4518 return printUsageAndExit();
4519 }
4520 try {
4521 sleepBeforeRerun = Long.parseLong(args[i+1]);
4522 } catch (NumberFormatException e) {
4523 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4524 return printUsageAndExit();
4525 }
4526 i++;
4527 } else if (cmd.equals("-sidelineDir")) {
4528 if (i == args.length - 1) {
4529 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4530 return printUsageAndExit();
4531 }
4532 i++;
4533 setSidelineDir(args[i]);
4534 } else if (cmd.equals("-fix")) {
4535 errors.reportError(ERROR_CODE.WRONG_USAGE,
4536 "This option is deprecated, please use -fixAssignments instead.");
4537 setFixAssignments(true);
4538 } else if (cmd.equals("-fixAssignments")) {
4539 setFixAssignments(true);
4540 } else if (cmd.equals("-fixMeta")) {
4541 setFixMeta(true);
4542 } else if (cmd.equals("-noHdfsChecking")) {
4543 setCheckHdfs(false);
4544 } else if (cmd.equals("-fixHdfsHoles")) {
4545 setFixHdfsHoles(true);
4546 } else if (cmd.equals("-fixHdfsOrphans")) {
4547 setFixHdfsOrphans(true);
4548 } else if (cmd.equals("-fixTableOrphans")) {
4549 setFixTableOrphans(true);
4550 } else if (cmd.equals("-fixHdfsOverlaps")) {
4551 setFixHdfsOverlaps(true);
4552 } else if (cmd.equals("-fixVersionFile")) {
4553 setFixVersionFile(true);
4554 } else if (cmd.equals("-sidelineBigOverlaps")) {
4555 setSidelineBigOverlaps(true);
4556 } else if (cmd.equals("-fixSplitParents")) {
4557 setFixSplitParents(true);
4558 } else if (cmd.equals("-ignorePreCheckPermission")) {
4559 setIgnorePreCheckPermission(true);
4560 } else if (cmd.equals("-checkCorruptHFiles")) {
4561 checkCorruptHFiles = true;
4562 } else if (cmd.equals("-sidelineCorruptHFiles")) {
4563 sidelineCorruptHFiles = true;
4564 } else if (cmd.equals("-fixReferenceFiles")) {
4565 setFixReferenceFiles(true);
4566 } else if (cmd.equals("-fixEmptyMetaCells")) {
4567 setFixEmptyMetaCells(true);
4568 } else if (cmd.equals("-repair")) {
4569
4570
4571 setFixHdfsHoles(true);
4572 setFixHdfsOrphans(true);
4573 setFixMeta(true);
4574 setFixAssignments(true);
4575 setFixHdfsOverlaps(true);
4576 setFixVersionFile(true);
4577 setSidelineBigOverlaps(true);
4578 setFixSplitParents(false);
4579 setCheckHdfs(true);
4580 setFixReferenceFiles(true);
4581 setFixTableLocks(true);
4582 setFixTableZNodes(true);
4583 } else if (cmd.equals("-repairHoles")) {
4584
4585 setFixHdfsHoles(true);
4586 setFixHdfsOrphans(false);
4587 setFixMeta(true);
4588 setFixAssignments(true);
4589 setFixHdfsOverlaps(false);
4590 setSidelineBigOverlaps(false);
4591 setFixSplitParents(false);
4592 setCheckHdfs(true);
4593 } else if (cmd.equals("-maxOverlapsToSideline")) {
4594 if (i == args.length - 1) {
4595 errors.reportError(ERROR_CODE.WRONG_USAGE,
4596 "-maxOverlapsToSideline needs a numeric value argument.");
4597 return printUsageAndExit();
4598 }
4599 try {
4600 int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4601 setMaxOverlapsToSideline(maxOverlapsToSideline);
4602 } catch (NumberFormatException e) {
4603 errors.reportError(ERROR_CODE.WRONG_USAGE,
4604 "-maxOverlapsToSideline needs a numeric value argument.");
4605 return printUsageAndExit();
4606 }
4607 i++;
4608 } else if (cmd.equals("-maxMerge")) {
4609 if (i == args.length - 1) {
4610 errors.reportError(ERROR_CODE.WRONG_USAGE,
4611 "-maxMerge needs a numeric value argument.");
4612 return printUsageAndExit();
4613 }
4614 try {
4615 int maxMerge = Integer.parseInt(args[i+1]);
4616 setMaxMerge(maxMerge);
4617 } catch (NumberFormatException e) {
4618 errors.reportError(ERROR_CODE.WRONG_USAGE,
4619 "-maxMerge needs a numeric value argument.");
4620 return printUsageAndExit();
4621 }
4622 i++;
4623 } else if (cmd.equals("-summary")) {
4624 setSummary();
4625 } else if (cmd.equals("-metaonly")) {
4626 setCheckMetaOnly();
4627 } else if (cmd.equals("-boundaries")) {
4628 setRegionBoundariesCheck();
4629 } else if (cmd.equals("-fixTableLocks")) {
4630 setFixTableLocks(true);
4631 } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4632 setFixTableZNodes(true);
4633 } else if (cmd.startsWith("-")) {
4634 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4635 return printUsageAndExit();
4636 } else {
4637 includeTable(TableName.valueOf(cmd));
4638 errors.print("Allow checking/fixes for table: " + cmd);
4639 }
4640 }
4641
4642 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4643
4644
4645 try {
4646 preCheckPermission();
4647 } catch (AccessDeniedException ace) {
4648 Runtime.getRuntime().exit(-1);
4649 } catch (IOException ioe) {
4650 Runtime.getRuntime().exit(-1);
4651 }
4652
4653
4654 connect();
4655
4656 try {
4657
4658 if (checkCorruptHFiles || sidelineCorruptHFiles) {
4659 LOG.info("Checking all hfiles for corruption");
4660 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4661 setHFileCorruptionChecker(hfcc);
4662 Collection<TableName> tables = getIncludedTables();
4663 Collection<Path> tableDirs = new ArrayList<Path>();
4664 Path rootdir = FSUtils.getRootDir(getConf());
4665 if (tables.size() > 0) {
4666 for (TableName t : tables) {
4667 tableDirs.add(FSUtils.getTableDir(rootdir, t));
4668 }
4669 } else {
4670 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4671 }
4672 hfcc.checkTables(tableDirs);
4673 hfcc.report(errors);
4674 }
4675
4676
4677 int code = onlineHbck();
4678 setRetCode(code);
4679
4680
4681
4682
4683 if (shouldRerun()) {
4684 try {
4685 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4686 Thread.sleep(sleepBeforeRerun);
4687 } catch (InterruptedException ie) {
4688 LOG.warn("Interrupted while sleeping");
4689 return this;
4690 }
4691
4692 setFixAssignments(false);
4693 setFixMeta(false);
4694 setFixHdfsHoles(false);
4695 setFixHdfsOverlaps(false);
4696 setFixVersionFile(false);
4697 setFixTableOrphans(false);
4698 errors.resetErrors();
4699 code = onlineHbck();
4700 setRetCode(code);
4701 }
4702 } finally {
4703 IOUtils.cleanup(null, this);
4704 }
4705 return this;
4706 }
4707
4708
4709
4710
4711 void debugLsr(Path p) throws IOException {
4712 debugLsr(getConf(), p, errors);
4713 }
4714
4715
4716
4717
4718 public static void debugLsr(Configuration conf,
4719 Path p) throws IOException {
4720 debugLsr(conf, p, new PrintingErrorReporter());
4721 }
4722
4723
4724
4725
4726 public static void debugLsr(Configuration conf,
4727 Path p, ErrorReporter errors) throws IOException {
4728 if (!LOG.isDebugEnabled() || p == null) {
4729 return;
4730 }
4731 FileSystem fs = p.getFileSystem(conf);
4732
4733 if (!fs.exists(p)) {
4734
4735 return;
4736 }
4737 errors.print(p.toString());
4738
4739 if (fs.isFile(p)) {
4740 return;
4741 }
4742
4743 if (fs.getFileStatus(p).isDirectory()) {
4744 FileStatus[] fss= fs.listStatus(p);
4745 for (FileStatus status : fss) {
4746 debugLsr(conf, status.getPath(), errors);
4747 }
4748 }
4749 }
4750 }