1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.io.PrintWriter;
25 import java.io.StringWriter;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Locale;
38 import java.util.Map;
39 import java.util.Map.Entry;
40 import java.util.Set;
41 import java.util.SortedMap;
42 import java.util.SortedSet;
43 import java.util.TreeMap;
44 import java.util.TreeSet;
45 import java.util.Vector;
46 import java.util.concurrent.Callable;
47 import java.util.concurrent.ConcurrentSkipListMap;
48 import java.util.concurrent.ExecutionException;
49 import java.util.concurrent.ExecutorService;
50 import java.util.concurrent.Executors;
51 import java.util.concurrent.Future;
52 import java.util.concurrent.FutureTask;
53 import java.util.concurrent.ScheduledThreadPoolExecutor;
54 import java.util.concurrent.TimeUnit;
55 import java.util.concurrent.TimeoutException;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58
59 import org.apache.commons.io.IOUtils;
60 import org.apache.commons.lang.StringUtils;
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.conf.Configured;
65 import org.apache.hadoop.fs.FSDataOutputStream;
66 import org.apache.hadoop.fs.FileStatus;
67 import org.apache.hadoop.fs.FileSystem;
68 import org.apache.hadoop.fs.Path;
69 import org.apache.hadoop.fs.permission.FsAction;
70 import org.apache.hadoop.fs.permission.FsPermission;
71 import org.apache.hadoop.hbase.Abortable;
72 import org.apache.hadoop.hbase.Cell;
73 import org.apache.hadoop.hbase.ClusterStatus;
74 import org.apache.hadoop.hbase.CoordinatedStateException;
75 import org.apache.hadoop.hbase.HBaseConfiguration;
76 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
77 import org.apache.hadoop.hbase.HColumnDescriptor;
78 import org.apache.hadoop.hbase.HConstants;
79 import org.apache.hadoop.hbase.HRegionInfo;
80 import org.apache.hadoop.hbase.HRegionLocation;
81 import org.apache.hadoop.hbase.HTableDescriptor;
82 import org.apache.hadoop.hbase.KeyValue;
83 import org.apache.hadoop.hbase.MasterNotRunningException;
84 import org.apache.hadoop.hbase.MetaTableAccessor;
85 import org.apache.hadoop.hbase.RegionLocations;
86 import org.apache.hadoop.hbase.ServerName;
87 import org.apache.hadoop.hbase.TableName;
88 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
89 import org.apache.hadoop.hbase.classification.InterfaceAudience;
90 import org.apache.hadoop.hbase.classification.InterfaceStability;
91 import org.apache.hadoop.hbase.client.Admin;
92 import org.apache.hadoop.hbase.client.ClusterConnection;
93 import org.apache.hadoop.hbase.client.ConnectionFactory;
94 import org.apache.hadoop.hbase.client.Delete;
95 import org.apache.hadoop.hbase.client.Get;
96 import org.apache.hadoop.hbase.client.HBaseAdmin;
97 import org.apache.hadoop.hbase.client.HConnectable;
98 import org.apache.hadoop.hbase.client.HConnection;
99 import org.apache.hadoop.hbase.client.HConnectionManager;
100 import org.apache.hadoop.hbase.client.MetaScanner;
101 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
102 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
103 import org.apache.hadoop.hbase.client.Put;
104 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
105 import org.apache.hadoop.hbase.client.Result;
106 import org.apache.hadoop.hbase.client.RowMutations;
107 import org.apache.hadoop.hbase.client.Table;
108 import org.apache.hadoop.hbase.io.FileLink;
109 import org.apache.hadoop.hbase.io.HFileLink;
110 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
111 import org.apache.hadoop.hbase.io.hfile.HFile;
112 import org.apache.hadoop.hbase.master.MasterFileSystem;
113 import org.apache.hadoop.hbase.master.RegionState;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
116 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
117 import org.apache.hadoop.hbase.regionserver.HRegion;
118 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
119 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
120 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
121 import org.apache.hadoop.hbase.security.AccessDeniedException;
122 import org.apache.hadoop.hbase.security.UserProvider;
123 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
124 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
125 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
126 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
127 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
128 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
129 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
130 import org.apache.hadoop.hbase.wal.WALSplitter;
131 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
132 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
133 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
134 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
135 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
136 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
137 import org.apache.hadoop.ipc.RemoteException;
138 import org.apache.hadoop.security.UserGroupInformation;
139 import org.apache.hadoop.util.ReflectionUtils;
140 import org.apache.hadoop.util.Tool;
141 import org.apache.hadoop.util.ToolRunner;
142 import org.apache.zookeeper.KeeperException;
143
144 import com.google.common.base.Joiner;
145 import com.google.common.base.Preconditions;
146 import com.google.common.collect.ImmutableList;
147 import com.google.common.collect.Lists;
148 import com.google.common.collect.Multimap;
149 import com.google.common.collect.Ordering;
150 import com.google.common.collect.TreeMultimap;
151 import com.google.protobuf.ServiceException;
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
199 @InterfaceStability.Evolving
200 public class HBaseFsck extends Configured implements Closeable {
201 public static final long DEFAULT_TIME_LAG = 60000;
202 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
203 private static final int MAX_NUM_THREADS = 50;
204 private static boolean rsSupportsOffline = true;
205 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
206 private static final int DEFAULT_MAX_MERGE = 5;
207 private static final String TO_BE_LOADED = "to_be_loaded";
208 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
209 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
210 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200;
211 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000;
212
213
214
215
216 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80;
217 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;
218 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200;
219 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000;
220
221
222
223
224 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
225 private ClusterStatus status;
226 private ClusterConnection connection;
227 private Admin admin;
228 private Table meta;
229
230 protected ExecutorService executor;
231 private long startMillis = EnvironmentEdgeManager.currentTime();
232 private HFileCorruptionChecker hfcc;
233 private int retcode = 0;
234 private Path HBCK_LOCK_PATH;
235 private FSDataOutputStream hbckOutFd;
236
237
238
239 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
240
241
242
243
244 private static boolean details = false;
245 private long timelag = DEFAULT_TIME_LAG;
246 private static boolean forceExclusive = false;
247 private boolean fixAssignments = false;
248 private boolean fixMeta = false;
249 private boolean checkHdfs = true;
250 private boolean fixHdfsHoles = false;
251 private boolean fixHdfsOverlaps = false;
252 private boolean fixHdfsOrphans = false;
253 private boolean fixTableOrphans = false;
254 private boolean fixVersionFile = false;
255 private boolean fixSplitParents = false;
256 private boolean removeParents = false;
257 private boolean fixReferenceFiles = false;
258 private boolean fixHFileLinks = false;
259 private boolean fixEmptyMetaCells = false;
260 private boolean fixTableLocks = false;
261 private boolean fixTableZNodes = false;
262 private boolean fixReplication = false;
263 private boolean fixAny = false;
264
265
266
267 private Set<TableName> tablesIncluded = new HashSet<TableName>();
268 private int maxMerge = DEFAULT_MAX_MERGE;
269 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
270 private boolean sidelineBigOverlaps = false;
271 private Path sidelineDir = null;
272
273 private boolean rerun = false;
274 private static boolean SUMMARY = false;
275 private boolean checkMetaOnly = false;
276 private boolean checkRegionBoundaries = false;
277 private boolean ignorePreCheckPermission = false;
278
279
280
281
282 final private ErrorReporter errors;
283 int fixes = 0;
284
285
286
287
288
289
290 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
291 private TreeSet<TableName> disabledTables =
292 new TreeSet<TableName>();
293
294 private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
295
296
297
298
299
300
301
302
303
304
305
306 private SortedMap<TableName, TableInfo> tablesInfo =
307 new ConcurrentSkipListMap<TableName, TableInfo>();
308
309
310
311
312 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
313
314 private Map<TableName, Set<String>> orphanTableDirs =
315 new HashMap<TableName, Set<String>>();
316
317 private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
318
319
320
321
322 private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
323 private final RetryCounterFactory lockFileRetryCounterFactory;
324 private final RetryCounterFactory createZNodeRetryCounterFactory;
325
326 private ZooKeeperWatcher zkw = null;
327 private String hbckEphemeralNodePath = null;
328 private boolean hbckZodeCreated = false;
329
330
331
332
333
334
335
336
337 public HBaseFsck(Configuration conf) throws MasterNotRunningException,
338 ZooKeeperConnectionException, IOException, ClassNotFoundException {
339 this(conf, createThreadPool(conf));
340 }
341
342 private static ExecutorService createThreadPool(Configuration conf) {
343 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
344 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
345 }
346
347
348
349
350
351
352
353
354
355
356
357 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
358 ZooKeeperConnectionException, IOException, ClassNotFoundException {
359 super(conf);
360 errors = getErrorReporter(getConf());
361 this.executor = exec;
362 lockFileRetryCounterFactory = new RetryCounterFactory(
363 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
364 getConf().getInt(
365 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
366 getConf().getInt(
367 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
368 createZNodeRetryCounterFactory = new RetryCounterFactory(
369 getConf().getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),
370 getConf().getInt(
371 "hbase.hbck.createznode.attempt.sleep.interval",
372 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),
373 getConf().getInt(
374 "hbase.hbck.createznode.attempt.maxsleeptime",
375 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));
376 zkw = createZooKeeperWatcher();
377 }
378
379 private class FileLockCallable implements Callable<FSDataOutputStream> {
380 RetryCounter retryCounter;
381
382 public FileLockCallable(RetryCounter retryCounter) {
383 this.retryCounter = retryCounter;
384 }
385 @Override
386 public FSDataOutputStream call() throws IOException {
387 try {
388 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
389 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
390 HConstants.DATA_FILE_UMASK_KEY);
391 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
392 fs.mkdirs(tmpDir);
393 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
394 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
395 out.writeBytes(InetAddress.getLocalHost().toString());
396 out.flush();
397 return out;
398 } catch(RemoteException e) {
399 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
400 return null;
401 } else {
402 throw e;
403 }
404 }
405 }
406
407 private FSDataOutputStream createFileWithRetries(final FileSystem fs,
408 final Path hbckLockFilePath, final FsPermission defaultPerms)
409 throws IOException {
410
411 IOException exception = null;
412 do {
413 try {
414 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
415 } catch (IOException ioe) {
416 LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
417 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
418 + retryCounter.getMaxAttempts());
419 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
420 ioe);
421 try {
422 exception = ioe;
423 retryCounter.sleepUntilNextRetry();
424 } catch (InterruptedException ie) {
425 throw (InterruptedIOException) new InterruptedIOException(
426 "Can't create lock file " + hbckLockFilePath.getName())
427 .initCause(ie);
428 }
429 }
430 } while (retryCounter.shouldRetry());
431
432 throw exception;
433 }
434 }
435
436
437
438
439
440
441
442 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
443 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
444 FileLockCallable callable = new FileLockCallable(retryCounter);
445 ExecutorService executor = Executors.newFixedThreadPool(1);
446 FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
447 executor.execute(futureTask);
448 final int timeoutInSeconds = getConf().getInt(
449 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
450 FSDataOutputStream stream = null;
451 try {
452 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
453 } catch (ExecutionException ee) {
454 LOG.warn("Encountered exception when opening lock file", ee);
455 } catch (InterruptedException ie) {
456 LOG.warn("Interrupted when opening lock file", ie);
457 Thread.currentThread().interrupt();
458 } catch (TimeoutException exception) {
459
460 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
461 futureTask.cancel(true);
462 } finally {
463 executor.shutdownNow();
464 }
465 return stream;
466 }
467
468 private void unlockHbck() {
469 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
470 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
471 do {
472 try {
473 IOUtils.closeQuietly(hbckOutFd);
474 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
475 HBCK_LOCK_PATH, true);
476 LOG.info("Finishing hbck");
477 return;
478 } catch (IOException ioe) {
479 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
480 + (retryCounter.getAttemptTimes() + 1) + " of "
481 + retryCounter.getMaxAttempts());
482 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
483 try {
484 retryCounter.sleepUntilNextRetry();
485 } catch (InterruptedException ie) {
486 Thread.currentThread().interrupt();
487 LOG.warn("Interrupted while deleting lock file" +
488 HBCK_LOCK_PATH);
489 return;
490 }
491 }
492 } while (retryCounter.shouldRetry());
493 }
494 }
495
496
497
498
499
500 public void connect() throws IOException {
501
502 if (isExclusive()) {
503
504 hbckOutFd = checkAndMarkRunningHbck();
505 if (hbckOutFd == null) {
506 setRetCode(-1);
507 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
508 "[If you are sure no other instance is running, delete the lock file " +
509 HBCK_LOCK_PATH + " and rerun the tool]");
510 throw new IOException("Duplicate hbck - Abort");
511 }
512
513
514 hbckLockCleanup.set(true);
515 }
516
517
518
519
520
521 Runtime.getRuntime().addShutdownHook(new Thread() {
522 @Override
523 public void run() {
524 IOUtils.closeQuietly(HBaseFsck.this);
525 cleanupHbckZnode();
526 unlockHbck();
527 }
528 });
529
530 LOG.info("Launching hbck");
531
532 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
533 admin = connection.getAdmin();
534 meta = connection.getTable(TableName.META_TABLE_NAME);
535 status = admin.getClusterStatus();
536 }
537
538
539
540
541 private void loadDeployedRegions() throws IOException, InterruptedException {
542
543 Collection<ServerName> regionServers = status.getServers();
544 errors.print("Number of live region servers: " + regionServers.size());
545 if (details) {
546 for (ServerName rsinfo: regionServers) {
547 errors.print(" " + rsinfo.getServerName());
548 }
549 }
550
551
552 Collection<ServerName> deadRegionServers = status.getDeadServerNames();
553 errors.print("Number of dead region servers: " + deadRegionServers.size());
554 if (details) {
555 for (ServerName name: deadRegionServers) {
556 errors.print(" " + name);
557 }
558 }
559
560
561 errors.print("Master: " + status.getMaster());
562
563
564 Collection<ServerName> backupMasters = status.getBackupMasters();
565 errors.print("Number of backup masters: " + backupMasters.size());
566 if (details) {
567 for (ServerName name: backupMasters) {
568 errors.print(" " + name);
569 }
570 }
571
572 errors.print("Average load: " + status.getAverageLoad());
573 errors.print("Number of requests: " + status.getRequestsCount());
574 errors.print("Number of regions: " + status.getRegionsCount());
575
576 Set<RegionState> rits = status.getRegionsInTransition();
577 if (rits != null) {
578 errors.print("Number of regions in transition: " + rits.size());
579 if (details) {
580 for (RegionState state: rits) {
581 errors.print(" " + state.toDescriptiveString());
582 }
583 }
584 }
585
586
587 processRegionServers(regionServers);
588 }
589
590
591
592
593 private void clearState() {
594
595 fixes = 0;
596 regionInfoMap.clear();
597 emptyRegionInfoQualifiers.clear();
598 disabledTables.clear();
599 errors.clear();
600 tablesInfo.clear();
601 orphanHdfsDirs.clear();
602 skippedRegions.clear();
603 }
604
605
606
607
608
609
610 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
611
612 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
613 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
614 LOG.info("Loading regioninfos HDFS");
615
616 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
617 int curIter = 0;
618 do {
619 clearState();
620
621 restoreHdfsIntegrity();
622 curIter++;
623 } while (fixes > 0 && curIter <= maxIterations);
624
625
626
627 if (curIter > 2) {
628 if (curIter == maxIterations) {
629 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
630 + "Tables integrity may not be fully repaired!");
631 } else {
632 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
633 }
634 }
635 }
636 }
637
638
639
640
641
642
643
644
645
646 public int onlineConsistencyRepair() throws IOException, KeeperException,
647 InterruptedException {
648
649
650 loadDeployedRegions();
651
652 recordMetaRegion();
653
654 if (!checkMetaRegion()) {
655 String errorMsg = "hbase:meta table is not consistent. ";
656 if (shouldFixAssignments()) {
657 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
658 } else {
659 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
660 }
661 errors.reportError(errorMsg + " Exiting...");
662 return -2;
663 }
664
665 LOG.info("Loading regionsinfo from the hbase:meta table");
666 boolean success = loadMetaEntries();
667 if (!success) return -1;
668
669
670 reportEmptyMetaCells();
671
672
673 if (shouldFixEmptyMetaCells()) {
674 fixEmptyMetaCells();
675 }
676
677
678 if (!checkMetaOnly) {
679 reportTablesInFlux();
680 }
681
682
683 if (shouldCheckHdfs()) {
684 LOG.info("Loading region directories from HDFS");
685 loadHdfsRegionDirs();
686 LOG.info("Loading region information from HDFS");
687 loadHdfsRegionInfos();
688 }
689
690
691 loadDisabledTables();
692
693
694 fixOrphanTables();
695
696 LOG.info("Checking and fixing region consistency");
697
698 checkAndFixConsistency();
699
700
701 checkIntegrity();
702 return errors.getErrorList().size();
703 }
704
705
706
707
708
709
710
711
712 private boolean setMasterInMaintenanceMode() throws IOException {
713 RetryCounter retryCounter = createZNodeRetryCounterFactory.create();
714 hbckEphemeralNodePath = ZKUtil.joinZNode(
715 ZooKeeperWatcher.masterMaintZNode,
716 "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));
717 do {
718 try {
719 hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);
720 if (hbckZodeCreated) {
721 break;
722 }
723 } catch (KeeperException e) {
724 if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) {
725 throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);
726 }
727
728 }
729
730 LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" +
731 (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());
732
733 try {
734 retryCounter.sleepUntilNextRetry();
735 } catch (InterruptedException ie) {
736 throw (InterruptedIOException) new InterruptedIOException(
737 "Can't create znode " + hbckEphemeralNodePath).initCause(ie);
738 }
739 } while (retryCounter.shouldRetry());
740 return hbckZodeCreated;
741 }
742
743 private void cleanupHbckZnode() {
744 try {
745 if (zkw != null && hbckZodeCreated) {
746 ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);
747 hbckZodeCreated = false;
748 }
749 } catch (KeeperException e) {
750
751 if (!e.code().equals(KeeperException.Code.NONODE)) {
752 LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);
753 }
754 }
755 }
756
757
758
759
760
761 public int onlineHbck()
762 throws IOException, KeeperException, InterruptedException, ServiceException {
763
764 errors.print("Version: " + status.getHBaseVersion());
765
766
767 clearState();
768
769 offlineHdfsIntegrityRepair();
770 offlineReferenceFileRepair();
771 offlineHLinkFileRepair();
772
773
774
775
776 if (!setMasterInMaintenanceMode()) {
777 LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "
778 + "error. Please run HBCK multiple times to reduce the chance of transient error.");
779 }
780
781 onlineConsistencyRepair();
782
783 if (checkRegionBoundaries) {
784 checkRegionBoundaries();
785 }
786
787 checkAndFixTableLocks();
788
789
790 checkAndFixOrphanedTableZNodes();
791
792 checkAndFixReplication();
793
794
795 cleanupHbckZnode();
796
797
798 unlockHbck();
799
800
801 printTableSummary(tablesInfo);
802 return errors.summarize();
803 }
804
805 public static byte[] keyOnly (byte[] b) {
806 if (b == null)
807 return b;
808 int rowlength = Bytes.toShort(b, 0);
809 byte[] result = new byte[rowlength];
810 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
811 return result;
812 }
813
814 @Override
815 public void close() throws IOException {
816 try {
817 cleanupHbckZnode();
818 unlockHbck();
819 } catch (Exception io) {
820 LOG.warn(io);
821 } finally {
822 if (zkw != null) {
823 zkw.close();
824 zkw = null;
825 }
826 IOUtils.closeQuietly(admin);
827 IOUtils.closeQuietly(meta);
828 IOUtils.closeQuietly(connection);
829 }
830 }
831
832 private static class RegionBoundariesInformation {
833 public byte [] regionName;
834 public byte [] metaFirstKey;
835 public byte [] metaLastKey;
836 public byte [] storesFirstKey;
837 public byte [] storesLastKey;
838 @Override
839 public String toString () {
840 return "regionName=" + Bytes.toStringBinary(regionName) +
841 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
842 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
843 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
844 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
845 }
846 }
847
848 public void checkRegionBoundaries() {
849 try {
850 ByteArrayComparator comparator = new ByteArrayComparator();
851 List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
852 final RegionBoundariesInformation currentRegionBoundariesInformation =
853 new RegionBoundariesInformation();
854 Path hbaseRoot = FSUtils.getRootDir(getConf());
855 for (HRegionInfo regionInfo : regions) {
856 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
857 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
858
859
860 Path path = new Path(tableDir, regionInfo.getEncodedName());
861 FileSystem fs = path.getFileSystem(getConf());
862 FileStatus[] files = fs.listStatus(path);
863
864 byte[] storeFirstKey = null;
865 byte[] storeLastKey = null;
866 for (FileStatus file : files) {
867 String fileName = file.getPath().toString();
868 fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
869 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
870 FileStatus[] storeFiles = fs.listStatus(file.getPath());
871
872 for (FileStatus storeFile : storeFiles) {
873 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
874 getConf()), getConf());
875 if (reader.getFirstKey() != null) {
876 byte[] firstKey = keyOnly(reader.getFirstKey());
877 if (storeFirstKey == null || comparator.compare(storeFirstKey, firstKey) > 0) {
878 storeFirstKey = firstKey;
879 }
880 }
881 if (reader.getLastKey() != null) {
882 byte[] lastKey = keyOnly(reader.getLastKey());
883 if (storeLastKey == null || comparator.compare(storeLastKey, lastKey) < 0) {
884 storeLastKey = lastKey;
885 }
886 }
887 reader.close();
888 }
889 }
890 }
891 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
892 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
893 currentRegionBoundariesInformation.storesFirstKey = storeFirstKey;
894 currentRegionBoundariesInformation.storesLastKey = storeLastKey;
895 if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
896 currentRegionBoundariesInformation.metaFirstKey = null;
897 if (currentRegionBoundariesInformation.metaLastKey.length == 0)
898 currentRegionBoundariesInformation.metaLastKey = null;
899
900
901
902
903
904
905 boolean valid = true;
906
907 if ((currentRegionBoundariesInformation.storesFirstKey != null)
908 && (currentRegionBoundariesInformation.metaFirstKey != null)) {
909 valid = valid
910 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
911 currentRegionBoundariesInformation.metaFirstKey) >= 0;
912 }
913
914 if ((currentRegionBoundariesInformation.storesLastKey != null)
915 && (currentRegionBoundariesInformation.metaLastKey != null)) {
916 valid = valid
917 && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
918 currentRegionBoundariesInformation.metaLastKey) < 0;
919 }
920 if (!valid) {
921 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
922 tablesInfo.get(regionInfo.getTable()));
923 LOG.warn("Region's boundaries not aligned between stores and META for:");
924 LOG.warn(currentRegionBoundariesInformation);
925 }
926 }
927 } catch (IOException e) {
928 LOG.error(e);
929 }
930 }
931
932
933
934
935 private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
936 for (HbckInfo hi : orphanHdfsDirs) {
937 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
938 adoptHdfsOrphan(hi);
939 }
940 }
941
942
943
944
945
946
947
948
949
950
951 @SuppressWarnings("deprecation")
952 private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
953 Path p = hi.getHdfsRegionDir();
954 FileSystem fs = p.getFileSystem(getConf());
955 FileStatus[] dirs = fs.listStatus(p);
956 if (dirs == null) {
957 LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
958 p + ". This dir could probably be deleted.");
959 return ;
960 }
961
962 TableName tableName = hi.getTableName();
963 TableInfo tableInfo = tablesInfo.get(tableName);
964 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
965 HTableDescriptor template = tableInfo.getHTD();
966
967
968 Pair<byte[],byte[]> orphanRegionRange = null;
969 for (FileStatus cf : dirs) {
970 String cfName= cf.getPath().getName();
971
972 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
973
974 FileStatus[] hfiles = fs.listStatus(cf.getPath());
975 for (FileStatus hfile : hfiles) {
976 byte[] start, end;
977 HFile.Reader hf = null;
978 try {
979 CacheConfig cacheConf = new CacheConfig(getConf());
980 hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
981 hf.loadFileInfo();
982 KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
983 start = startKv.getRow();
984 KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
985 end = endKv.getRow();
986 } catch (IOException ioe) {
987 LOG.warn("Problem reading orphan file " + hfile + ", skipping");
988 continue;
989 } catch (NullPointerException ioe) {
990 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
991 continue;
992 } finally {
993 if (hf != null) {
994 hf.close();
995 }
996 }
997
998
999 if (orphanRegionRange == null) {
1000
1001 orphanRegionRange = new Pair<byte[], byte[]>(start, end);
1002 } else {
1003
1004
1005
1006 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
1007 orphanRegionRange.setFirst(start);
1008 }
1009 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
1010 orphanRegionRange.setSecond(end);
1011 }
1012 }
1013 }
1014 }
1015 if (orphanRegionRange == null) {
1016 LOG.warn("No data in dir " + p + ", sidelining data");
1017 fixes++;
1018 sidelineRegionDir(fs, hi);
1019 return;
1020 }
1021 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
1022 Bytes.toString(orphanRegionRange.getSecond()) + ")");
1023
1024
1025 HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(),
1026 Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
1027 LOG.info("Creating new region : " + hri);
1028 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
1029 Path target = region.getRegionFileSystem().getRegionDir();
1030
1031
1032 mergeRegionDirs(target, hi);
1033 fixes++;
1034 }
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044 private int restoreHdfsIntegrity() throws IOException, InterruptedException {
1045
1046 LOG.info("Loading HBase regioninfo from HDFS...");
1047 loadHdfsRegionDirs();
1048
1049 int errs = errors.getErrorList().size();
1050
1051 tablesInfo = loadHdfsRegionInfos();
1052 checkHdfsIntegrity(false, false);
1053
1054 if (errors.getErrorList().size() == errs) {
1055 LOG.info("No integrity errors. We are done with this phase. Glorious.");
1056 return 0;
1057 }
1058
1059 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
1060 adoptHdfsOrphans(orphanHdfsDirs);
1061
1062 }
1063
1064
1065 if (shouldFixHdfsHoles()) {
1066 clearState();
1067 loadHdfsRegionDirs();
1068 tablesInfo = loadHdfsRegionInfos();
1069 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1070 }
1071
1072
1073 if (shouldFixHdfsOverlaps()) {
1074
1075 clearState();
1076 loadHdfsRegionDirs();
1077 tablesInfo = loadHdfsRegionInfos();
1078 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1079 }
1080
1081 return errors.getErrorList().size();
1082 }
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092 private void offlineReferenceFileRepair() throws IOException, InterruptedException {
1093 clearState();
1094 Configuration conf = getConf();
1095 Path hbaseRoot = FSUtils.getRootDir(conf);
1096 FileSystem fs = hbaseRoot.getFileSystem(conf);
1097 LOG.info("Computing mapping of all store files");
1098 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,
1099 new FSUtils.ReferenceFileFilter(fs), executor, errors);
1100 errors.print("");
1101 LOG.info("Validating mapping using HDFS state");
1102 for (Path path: allFiles.values()) {
1103 Path referredToFile = StoreFileInfo.getReferredToFile(path);
1104 if (fs.exists(referredToFile)) continue;
1105
1106
1107 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1108 "Found lingering reference file " + path);
1109 if (!shouldFixReferenceFiles()) continue;
1110
1111
1112 boolean success = false;
1113 String pathStr = path.toString();
1114
1115
1116
1117
1118
1119 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1120 for (int i = 0; index > 0 && i < 5; i++) {
1121 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1122 }
1123 if (index > 0) {
1124 Path rootDir = getSidelineDir();
1125 Path dst = new Path(rootDir, pathStr.substring(index + 1));
1126 fs.mkdirs(dst.getParent());
1127 LOG.info("Trying to sideline reference file "
1128 + path + " to " + dst);
1129 setShouldRerun();
1130
1131 success = fs.rename(path, dst);
1132 debugLsr(dst);
1133
1134 }
1135 if (!success) {
1136 LOG.error("Failed to sideline reference file " + path);
1137 }
1138 }
1139 }
1140
1141
1142
1143
1144
1145
1146 private void offlineHLinkFileRepair() throws IOException, InterruptedException {
1147 Configuration conf = getConf();
1148 Path hbaseRoot = FSUtils.getRootDir(conf);
1149 FileSystem fs = hbaseRoot.getFileSystem(conf);
1150 LOG.info("Computing mapping of all link files");
1151 Map<String, Path> allFiles = FSUtils
1152 .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors);
1153 errors.print("");
1154
1155 LOG.info("Validating mapping using HDFS state");
1156 for (Path path : allFiles.values()) {
1157
1158 HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path);
1159 if (actualLink.exists(fs)) continue;
1160
1161
1162 errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path);
1163 if (!shouldFixHFileLinks()) continue;
1164
1165
1166 setShouldRerun();
1167
1168
1169
1170
1171 boolean success = sidelineFile(fs, hbaseRoot, path);
1172
1173 if (!success) {
1174 LOG.error("Failed to sideline HFileLink file " + path);
1175 }
1176
1177
1178
1179
1180 Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil
1181 .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()),
1182 HFileLink.getReferencedRegionName(path.getName().toString()),
1183 path.getParent().getName()),
1184 HFileLink.getReferencedHFileName(path.getName().toString()));
1185 success = sidelineFile(fs, hbaseRoot, backRefPath);
1186
1187 if (!success) {
1188 LOG.error("Failed to sideline HFileLink backreference file " + path);
1189 }
1190 }
1191 }
1192
1193 private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException {
1194 URI uri = hbaseRoot.toUri().relativize(path.toUri());
1195 if (uri.isAbsolute()) return false;
1196 String relativePath = uri.getPath();
1197 Path rootDir = getSidelineDir();
1198 Path dst = new Path(rootDir, relativePath);
1199 boolean pathCreated = fs.mkdirs(dst.getParent());
1200 if (!pathCreated) {
1201 LOG.error("Failed to create path: " + dst.getParent());
1202 return false;
1203 }
1204 LOG.info("Trying to sideline file " + path + " to " + dst);
1205 return fs.rename(path, dst);
1206 }
1207
1208
1209
1210
1211 private void reportEmptyMetaCells() {
1212 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1213 emptyRegionInfoQualifiers.size());
1214 if (details) {
1215 for (Result r: emptyRegionInfoQualifiers) {
1216 errors.print(" " + r);
1217 }
1218 }
1219 }
1220
1221
1222
1223
1224 private void reportTablesInFlux() {
1225 AtomicInteger numSkipped = new AtomicInteger(0);
1226 HTableDescriptor[] allTables = getTables(numSkipped);
1227 errors.print("Number of Tables: " + allTables.length);
1228 if (details) {
1229 if (numSkipped.get() > 0) {
1230 errors.detail("Number of Tables in flux: " + numSkipped.get());
1231 }
1232 for (HTableDescriptor td : allTables) {
1233 errors.detail(" Table: " + td.getTableName() + "\t" +
1234 (td.isReadOnly() ? "ro" : "rw") + "\t" +
1235 (td.isMetaRegion() ? "META" : " ") + "\t" +
1236 " families: " + td.getFamilies().size());
1237 }
1238 }
1239 }
1240
1241 public ErrorReporter getErrors() {
1242 return errors;
1243 }
1244
1245
1246
1247
1248
1249 private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1250 Path regionDir = hbi.getHdfsRegionDir();
1251 if (regionDir == null) {
1252 if (hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1253
1254 LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1255 }
1256 return;
1257 }
1258
1259 if (hbi.hdfsEntry.hri != null) {
1260
1261 return;
1262 }
1263
1264 FileSystem fs = FileSystem.get(getConf());
1265 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1266 LOG.debug("HRegionInfo read: " + hri.toString());
1267 hbi.hdfsEntry.hri = hri;
1268 }
1269
1270
1271
1272
1273
1274 public static class RegionRepairException extends IOException {
1275 private static final long serialVersionUID = 1L;
1276 final IOException ioe;
1277 public RegionRepairException(String s, IOException ioe) {
1278 super(s);
1279 this.ioe = ioe;
1280 }
1281 }
1282
1283
1284
1285
1286 private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1287 throws IOException, InterruptedException {
1288 tablesInfo.clear();
1289
1290 Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1291
1292
1293 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1294 List<Future<Void>> hbiFutures;
1295
1296 for (HbckInfo hbi : hbckInfos) {
1297 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1298 hbis.add(work);
1299 }
1300
1301
1302 hbiFutures = executor.invokeAll(hbis);
1303
1304 for(int i=0; i<hbiFutures.size(); i++) {
1305 WorkItemHdfsRegionInfo work = hbis.get(i);
1306 Future<Void> f = hbiFutures.get(i);
1307 try {
1308 f.get();
1309 } catch(ExecutionException e) {
1310 LOG.warn("Failed to read .regioninfo file for region " +
1311 work.hbi.getRegionNameAsString(), e.getCause());
1312 }
1313 }
1314
1315 Path hbaseRoot = FSUtils.getRootDir(getConf());
1316 FileSystem fs = hbaseRoot.getFileSystem(getConf());
1317
1318 for (HbckInfo hbi: hbckInfos) {
1319
1320 if (hbi.getHdfsHRI() == null) {
1321
1322 continue;
1323 }
1324
1325
1326
1327 TableName tableName = hbi.getTableName();
1328 if (tableName == null) {
1329
1330 LOG.warn("tableName was null for: " + hbi);
1331 continue;
1332 }
1333
1334 TableInfo modTInfo = tablesInfo.get(tableName);
1335 if (modTInfo == null) {
1336
1337 modTInfo = new TableInfo(tableName);
1338 tablesInfo.put(tableName, modTInfo);
1339 try {
1340 HTableDescriptor htd =
1341 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1342 modTInfo.htds.add(htd);
1343 } catch (IOException ioe) {
1344 if (!orphanTableDirs.containsKey(tableName)) {
1345 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1346
1347 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1348 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1349 Set<String> columns = new HashSet<String>();
1350 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1351 }
1352 }
1353 }
1354 if (!hbi.isSkipChecks()) {
1355 modTInfo.addRegionInfo(hbi);
1356 }
1357 }
1358
1359 loadTableInfosForTablesWithNoRegion();
1360 errors.print("");
1361
1362 return tablesInfo;
1363 }
1364
1365
1366
1367
1368
1369
1370
1371
1372 private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1373 Path regionDir = hbi.getHdfsRegionDir();
1374 FileSystem fs = regionDir.getFileSystem(getConf());
1375 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1376 for (FileStatus subdir : subDirs) {
1377 String columnfamily = subdir.getPath().getName();
1378 columns.add(columnfamily);
1379 }
1380 return columns;
1381 }
1382
1383
1384
1385
1386
1387
1388
1389
1390 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1391 Set<String> columns) throws IOException {
1392 if (columns ==null || columns.isEmpty()) return false;
1393 HTableDescriptor htd = new HTableDescriptor(tableName);
1394 for (String columnfamimly : columns) {
1395 htd.addFamily(new HColumnDescriptor(columnfamimly));
1396 }
1397 fstd.createTableDescriptor(htd, true);
1398 return true;
1399 }
1400
1401
1402
1403
1404
1405 public void fixEmptyMetaCells() throws IOException {
1406 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1407 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1408 for (Result region : emptyRegionInfoQualifiers) {
1409 deleteMetaRegion(region.getRow());
1410 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1411 }
1412 emptyRegionInfoQualifiers.clear();
1413 }
1414 }
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425 public void fixOrphanTables() throws IOException {
1426 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1427
1428 List<TableName> tmpList = new ArrayList<TableName>();
1429 tmpList.addAll(orphanTableDirs.keySet());
1430 HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1431 Iterator<Entry<TableName, Set<String>>> iter =
1432 orphanTableDirs.entrySet().iterator();
1433 int j = 0;
1434 int numFailedCase = 0;
1435 FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1436 while (iter.hasNext()) {
1437 Entry<TableName, Set<String>> entry =
1438 iter.next();
1439 TableName tableName = entry.getKey();
1440 LOG.info("Trying to fix orphan table error: " + tableName);
1441 if (j < htds.length) {
1442 if (tableName.equals(htds[j].getTableName())) {
1443 HTableDescriptor htd = htds[j];
1444 LOG.info("fixing orphan table: " + tableName + " from cache");
1445 fstd.createTableDescriptor(htd, true);
1446 j++;
1447 iter.remove();
1448 }
1449 } else {
1450 if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1451 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1452 LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1453 iter.remove();
1454 } else {
1455 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1456 numFailedCase++;
1457 }
1458 }
1459 fixes++;
1460 }
1461
1462 if (orphanTableDirs.isEmpty()) {
1463
1464
1465 setShouldRerun();
1466 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1467 } else if (numFailedCase > 0) {
1468 LOG.error("Failed to fix " + numFailedCase
1469 + " OrphanTables with default .tableinfo files");
1470 }
1471
1472 }
1473
1474 orphanTableDirs.clear();
1475
1476 }
1477
1478
1479
1480
1481
1482
1483 private HRegion createNewMeta() throws IOException {
1484 Path rootdir = FSUtils.getRootDir(getConf());
1485 Configuration c = getConf();
1486 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1487 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1488 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1489 HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1490 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1491 return meta;
1492 }
1493
1494
1495
1496
1497
1498
1499
1500 private ArrayList<Put> generatePuts(
1501 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1502 ArrayList<Put> puts = new ArrayList<Put>();
1503 boolean hasProblems = false;
1504 for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1505 TableName name = e.getKey();
1506
1507
1508 if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1509 continue;
1510 }
1511
1512 TableInfo ti = e.getValue();
1513 for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1514 .entrySet()) {
1515 Collection<HbckInfo> his = spl.getValue();
1516 int sz = his.size();
1517 if (sz != 1) {
1518
1519 LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1520 + " had " + sz + " regions instead of exactly 1." );
1521 hasProblems = true;
1522 continue;
1523 }
1524
1525
1526 HbckInfo hi = his.iterator().next();
1527 HRegionInfo hri = hi.getHdfsHRI();
1528 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1529 puts.add(p);
1530 }
1531 }
1532 return hasProblems ? null : puts;
1533 }
1534
1535
1536
1537
1538 private void suggestFixes(
1539 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1540 logParallelMerge();
1541 for (TableInfo tInfo : tablesInfo.values()) {
1542 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1543 tInfo.checkRegionChain(handler);
1544 }
1545 }
1546
1547
1548
1549
1550
1551
1552
1553
1554 public boolean rebuildMeta(boolean fix) throws IOException,
1555 InterruptedException {
1556
1557
1558
1559
1560
1561 LOG.info("Loading HBase regioninfo from HDFS...");
1562 loadHdfsRegionDirs();
1563
1564 int errs = errors.getErrorList().size();
1565 tablesInfo = loadHdfsRegionInfos();
1566 checkHdfsIntegrity(false, false);
1567
1568
1569 if (errors.getErrorList().size() != errs) {
1570
1571 while(true) {
1572 fixes = 0;
1573 suggestFixes(tablesInfo);
1574 errors.clear();
1575 loadHdfsRegionInfos();
1576 checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1577
1578 int errCount = errors.getErrorList().size();
1579
1580 if (fixes == 0) {
1581 if (errCount > 0) {
1582 return false;
1583 } else {
1584 break;
1585 }
1586 }
1587 }
1588 }
1589
1590
1591 LOG.info("HDFS regioninfo's seems good. Sidelining old hbase:meta");
1592 Path backupDir = sidelineOldMeta();
1593
1594 LOG.info("Creating new hbase:meta");
1595 HRegion meta = createNewMeta();
1596
1597
1598 List<Put> puts = generatePuts(tablesInfo);
1599 if (puts == null) {
1600 LOG.fatal("Problem encountered when creating new hbase:meta entries. " +
1601 "You may need to restore the previously sidelined hbase:meta");
1602 return false;
1603 }
1604 meta.batchMutate(puts.toArray(new Put[puts.size()]));
1605 HRegion.closeHRegion(meta);
1606
1607 removeHBCKMetaRecoveryWALDir(meta);
1608 LOG.info("Success! hbase:meta table rebuilt.");
1609 LOG.info("Old hbase:meta is moved into " + backupDir);
1610 return true;
1611 }
1612
1613
1614
1615
1616
1617 private void removeHBCKMetaRecoveryWALDir(HRegion meta) throws IOException {
1618
1619
1620 Path walLogDir = ((FSHLog) meta.getWAL()).getCurrentFileName().getParent();
1621 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
1622 FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null);
1623 if (walFiles == null || walFiles.length == 0) {
1624 LOG.info("HBCK meta recovery WAL directory is empty, removing it now.");
1625 if (!FSUtils.deleteDirectory(fs, walLogDir)) {
1626 LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir);
1627 }
1628 }
1629 }
1630
1631
1632
1633
1634 private void logParallelMerge() {
1635 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1636 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1637 " false to run serially.");
1638 } else {
1639 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
1640 " true to run in parallel.");
1641 }
1642 }
1643
1644 private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1645 boolean fixOverlaps) throws IOException {
1646 LOG.info("Checking HBase region split map from HDFS data...");
1647 logParallelMerge();
1648 for (TableInfo tInfo : tablesInfo.values()) {
1649 TableIntegrityErrorHandler handler;
1650 if (fixHoles || fixOverlaps) {
1651 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1652 fixHoles, fixOverlaps);
1653 } else {
1654 handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1655 }
1656 if (!tInfo.checkRegionChain(handler)) {
1657
1658 errors.report("Found inconsistency in table " + tInfo.getName());
1659 }
1660 }
1661 return tablesInfo;
1662 }
1663
1664 private Path getSidelineDir() throws IOException {
1665 if (sidelineDir == null) {
1666 Path hbaseDir = FSUtils.getRootDir(getConf());
1667 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1668 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1669 + startMillis);
1670 }
1671 return sidelineDir;
1672 }
1673
1674
1675
1676
1677 Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1678 return sidelineRegionDir(fs, null, hi);
1679 }
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689 Path sidelineRegionDir(FileSystem fs,
1690 String parentDir, HbckInfo hi) throws IOException {
1691 TableName tableName = hi.getTableName();
1692 Path regionDir = hi.getHdfsRegionDir();
1693
1694 if (!fs.exists(regionDir)) {
1695 LOG.warn("No previous " + regionDir + " exists. Continuing.");
1696 return null;
1697 }
1698
1699 Path rootDir = getSidelineDir();
1700 if (parentDir != null) {
1701 rootDir = new Path(rootDir, parentDir);
1702 }
1703 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1704 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1705 fs.mkdirs(sidelineRegionDir);
1706 boolean success = false;
1707 FileStatus[] cfs = fs.listStatus(regionDir);
1708 if (cfs == null) {
1709 LOG.info("Region dir is empty: " + regionDir);
1710 } else {
1711 for (FileStatus cf : cfs) {
1712 Path src = cf.getPath();
1713 Path dst = new Path(sidelineRegionDir, src.getName());
1714 if (fs.isFile(src)) {
1715
1716 success = fs.rename(src, dst);
1717 if (!success) {
1718 String msg = "Unable to rename file " + src + " to " + dst;
1719 LOG.error(msg);
1720 throw new IOException(msg);
1721 }
1722 continue;
1723 }
1724
1725
1726 fs.mkdirs(dst);
1727
1728 LOG.info("Sidelining files from " + src + " into containing region " + dst);
1729
1730
1731
1732
1733 FileStatus[] hfiles = fs.listStatus(src);
1734 if (hfiles != null && hfiles.length > 0) {
1735 for (FileStatus hfile : hfiles) {
1736 success = fs.rename(hfile.getPath(), dst);
1737 if (!success) {
1738 String msg = "Unable to rename file " + src + " to " + dst;
1739 LOG.error(msg);
1740 throw new IOException(msg);
1741 }
1742 }
1743 }
1744 LOG.debug("Sideline directory contents:");
1745 debugLsr(sidelineRegionDir);
1746 }
1747 }
1748
1749 LOG.info("Removing old region dir: " + regionDir);
1750 success = fs.delete(regionDir, true);
1751 if (!success) {
1752 String msg = "Unable to delete dir " + regionDir;
1753 LOG.error(msg);
1754 throw new IOException(msg);
1755 }
1756 return sidelineRegionDir;
1757 }
1758
1759
1760
1761
1762 void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1763 Path backupHbaseDir) throws IOException {
1764 Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1765 if (fs.exists(tableDir)) {
1766 Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1767 fs.mkdirs(backupTableDir.getParent());
1768 boolean success = fs.rename(tableDir, backupTableDir);
1769 if (!success) {
1770 throw new IOException("Failed to move " + tableName + " from "
1771 + tableDir + " to " + backupTableDir);
1772 }
1773 } else {
1774 LOG.info("No previous " + tableName + " exists. Continuing.");
1775 }
1776 }
1777
1778
1779
1780
1781 Path sidelineOldMeta() throws IOException {
1782
1783 Path hbaseDir = FSUtils.getRootDir(getConf());
1784 FileSystem fs = hbaseDir.getFileSystem(getConf());
1785 Path backupDir = getSidelineDir();
1786 fs.mkdirs(backupDir);
1787
1788 try {
1789 sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1790 } catch (IOException e) {
1791 LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore "
1792 + "try to rename hbase:meta in " + backupDir.getName() + " to "
1793 + hbaseDir.getName() + ".", e);
1794 throw e;
1795 }
1796 return backupDir;
1797 }
1798
1799
1800
1801
1802
1803
1804 private void loadDisabledTables()
1805 throws ZooKeeperConnectionException, IOException {
1806 HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1807 @Override
1808 public Void connect(HConnection connection) throws IOException {
1809 try {
1810 for (TableName tableName :
1811 ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1812 disabledTables.add(tableName);
1813 }
1814 } catch (KeeperException ke) {
1815 throw new IOException(ke);
1816 } catch (InterruptedException e) {
1817 throw new InterruptedIOException();
1818 }
1819 return null;
1820 }
1821 });
1822 }
1823
1824
1825
1826
1827 private boolean isTableDisabled(HRegionInfo regionInfo) {
1828 return disabledTables.contains(regionInfo.getTable());
1829 }
1830
1831
1832
1833
1834
1835 public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1836 Path rootDir = FSUtils.getRootDir(getConf());
1837 FileSystem fs = rootDir.getFileSystem(getConf());
1838
1839
1840 List<FileStatus> tableDirs = Lists.newArrayList();
1841
1842 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1843
1844 List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1845 for (Path path : paths) {
1846 TableName tableName = FSUtils.getTableName(path);
1847 if ((!checkMetaOnly &&
1848 isTableIncluded(tableName)) ||
1849 tableName.equals(TableName.META_TABLE_NAME)) {
1850 tableDirs.add(fs.getFileStatus(path));
1851 }
1852 }
1853
1854
1855 if (!foundVersionFile) {
1856 errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1857 "Version file does not exist in root dir " + rootDir);
1858 if (shouldFixVersionFile()) {
1859 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1860 + " file.");
1861 setShouldRerun();
1862 FSUtils.setVersion(fs, rootDir, getConf().getInt(
1863 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1864 HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1865 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1866 }
1867 }
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877 for (FileStatus tableDir : tableDirs) {
1878 LOG.debug("Loading region dirs from " +tableDir.getPath());
1879 WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir);
1880 try {
1881 item.call();
1882 } catch (ExecutionException e) {
1883 LOG.warn("Could not completely load table dir " +
1884 tableDir.getPath(), e.getCause());
1885 }
1886 }
1887 errors.print("");
1888 }
1889
1890
1891
1892
1893 private boolean recordMetaRegion() throws IOException {
1894 RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1895 HConstants.EMPTY_START_ROW, false, false);
1896 if (rl == null) {
1897 errors.reportError(ERROR_CODE.NULL_META_REGION,
1898 "META region or some of its attributes are null.");
1899 return false;
1900 }
1901 for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1902
1903 if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1904 metaLocation.getHostname() == null) {
1905 errors.reportError(ERROR_CODE.NULL_META_REGION,
1906 "META region or some of its attributes are null.");
1907 return false;
1908 }
1909 ServerName sn = metaLocation.getServerName();
1910 MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1911 HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1912 if (hbckInfo == null) {
1913 regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1914 } else {
1915 hbckInfo.metaEntry = m;
1916 }
1917 }
1918 return true;
1919 }
1920
1921 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1922 return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1923 @Override
1924 public void abort(String why, Throwable e) {
1925 LOG.error(why, e);
1926 System.exit(1);
1927 }
1928
1929 @Override
1930 public boolean isAborted() {
1931 return false;
1932 }
1933
1934 });
1935 }
1936
1937
1938
1939
1940
1941
1942
1943 void processRegionServers(Collection<ServerName> regionServerList)
1944 throws IOException, InterruptedException {
1945
1946 List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1947 List<Future<Void>> workFutures;
1948
1949
1950 for (ServerName rsinfo: regionServerList) {
1951 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1952 }
1953
1954 workFutures = executor.invokeAll(workItems);
1955
1956 for(int i=0; i<workFutures.size(); i++) {
1957 WorkItemRegion item = workItems.get(i);
1958 Future<Void> f = workFutures.get(i);
1959 try {
1960 f.get();
1961 } catch(ExecutionException e) {
1962 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1963 e.getCause());
1964 }
1965 }
1966 }
1967
1968
1969
1970
1971 private void checkAndFixConsistency()
1972 throws IOException, KeeperException, InterruptedException {
1973
1974
1975 List<CheckRegionConsistencyWorkItem> workItems =
1976 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1977 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1978 if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1979 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1980 }
1981 }
1982 checkRegionConsistencyConcurrently(workItems);
1983
1984 boolean prevHdfsCheck = shouldCheckHdfs();
1985 setCheckHdfs(false);
1986
1987
1988 List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1989 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1990 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1991 if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1992 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1993 }
1994 }
1995 checkRegionConsistencyConcurrently(replicaWorkItems);
1996 setCheckHdfs(prevHdfsCheck);
1997
1998
1999
2000
2001
2002 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
2003 int numOfSkippedRegions = skippedRegions.size();
2004 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
2005 throw new IOException(numOfSkippedRegions
2006 + " region(s) could not be checked or repaired. See logs for detail.");
2007 }
2008 }
2009
2010
2011
2012
2013 private void checkRegionConsistencyConcurrently(
2014 final List<CheckRegionConsistencyWorkItem> workItems)
2015 throws IOException, KeeperException, InterruptedException {
2016 if (workItems.isEmpty()) {
2017 return;
2018 }
2019
2020 List<Future<Void>> workFutures = executor.invokeAll(workItems);
2021 for(Future<Void> f: workFutures) {
2022 try {
2023 f.get();
2024 } catch(ExecutionException e1) {
2025 LOG.warn("Could not check region consistency " , e1.getCause());
2026 if (e1.getCause() instanceof IOException) {
2027 throw (IOException)e1.getCause();
2028 } else if (e1.getCause() instanceof KeeperException) {
2029 throw (KeeperException)e1.getCause();
2030 } else if (e1.getCause() instanceof InterruptedException) {
2031 throw (InterruptedException)e1.getCause();
2032 } else {
2033 throw new IOException(e1.getCause());
2034 }
2035 }
2036 }
2037 }
2038
2039 class CheckRegionConsistencyWorkItem implements Callable<Void> {
2040 private final String key;
2041 private final HbckInfo hbi;
2042
2043 CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
2044 this.key = key;
2045 this.hbi = hbi;
2046 }
2047
2048 @Override
2049 public synchronized Void call() throws Exception {
2050 try {
2051 checkRegionConsistency(key, hbi);
2052 } catch (Exception e) {
2053
2054
2055 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
2056 + "'.", e);
2057 if (hbi.getHdfsHRI().isMetaRegion()) {
2058 throw e;
2059 }
2060 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
2061 addSkippedRegion(hbi);
2062 }
2063 return null;
2064 }
2065 }
2066
2067 private void addSkippedRegion(final HbckInfo hbi) {
2068 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
2069 if (skippedRegionNames == null) {
2070 skippedRegionNames = new HashSet<String>();
2071 }
2072 skippedRegionNames.add(hbi.getRegionNameAsString());
2073 skippedRegions.put(hbi.getTableName(), skippedRegionNames);
2074 }
2075
2076 private void preCheckPermission() throws IOException, AccessDeniedException {
2077 if (shouldIgnorePreCheckPermission()) {
2078 return;
2079 }
2080
2081 Path hbaseDir = FSUtils.getRootDir(getConf());
2082 FileSystem fs = hbaseDir.getFileSystem(getConf());
2083 UserProvider userProvider = UserProvider.instantiate(getConf());
2084 UserGroupInformation ugi = userProvider.getCurrent().getUGI();
2085 FileStatus[] files = fs.listStatus(hbaseDir);
2086 for (FileStatus file : files) {
2087 try {
2088 FSUtils.checkAccess(ugi, file, FsAction.WRITE);
2089 } catch (AccessDeniedException ace) {
2090 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
2091 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
2092 + " does not have write perms to " + file.getPath()
2093 + ". Please rerun hbck as hdfs user " + file.getOwner());
2094 throw ace;
2095 }
2096 }
2097 }
2098
2099
2100
2101
2102 private void deleteMetaRegion(HbckInfo hi) throws IOException {
2103 deleteMetaRegion(hi.metaEntry.getRegionName());
2104 }
2105
2106
2107
2108
2109 private void deleteMetaRegion(byte[] metaKey) throws IOException {
2110 Delete d = new Delete(metaKey);
2111 meta.delete(d);
2112 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
2113 }
2114
2115
2116
2117
2118 private void resetSplitParent(HbckInfo hi) throws IOException {
2119 RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
2120 Delete d = new Delete(hi.metaEntry.getRegionName());
2121 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
2122 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
2123 mutations.add(d);
2124
2125 HRegionInfo hri = new HRegionInfo(hi.metaEntry);
2126 hri.setOffline(false);
2127 hri.setSplit(false);
2128 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
2129 mutations.add(p);
2130
2131 meta.mutateRow(mutations);
2132 LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2133 }
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143 private void offline(byte[] regionName) throws IOException {
2144 String regionString = Bytes.toStringBinary(regionName);
2145 if (!rsSupportsOffline) {
2146 LOG.warn("Using unassign region " + regionString
2147 + " instead of using offline method, you should"
2148 + " restart HMaster after these repairs");
2149 admin.unassign(regionName, true);
2150 return;
2151 }
2152
2153
2154 try {
2155 LOG.info("Offlining region " + regionString);
2156 admin.offline(regionName);
2157 } catch (IOException ioe) {
2158 String notFoundMsg = "java.lang.NoSuchMethodException: " +
2159 "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2160 if (ioe.getMessage().contains(notFoundMsg)) {
2161 LOG.warn("Using unassign region " + regionString
2162 + " instead of using offline method, you should"
2163 + " restart HMaster after these repairs");
2164 rsSupportsOffline = false;
2165 admin.unassign(regionName, true);
2166 return;
2167 }
2168 throw ioe;
2169 }
2170 }
2171
2172 private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2173 undeployRegionsForHbi(hi);
2174
2175 if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2176 return;
2177 }
2178 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2179 for (int i = 1; i < numReplicas; i++) {
2180 if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2181 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2182 hi.getPrimaryHRIForDeployedReplica(), i);
2183 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2184 if (h != null) {
2185 undeployRegionsForHbi(h);
2186
2187
2188 h.setSkipChecks(true);
2189 }
2190 }
2191 }
2192
2193 private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2194 for (OnlineEntry rse : hi.deployedEntries) {
2195 LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa);
2196 try {
2197 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2198 offline(rse.hri.getRegionName());
2199 } catch (IOException ioe) {
2200 LOG.warn("Got exception when attempting to offline region "
2201 + Bytes.toString(rse.hri.getRegionName()), ioe);
2202 }
2203 }
2204 }
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218 private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2219 if (hi.metaEntry == null && hi.hdfsEntry == null) {
2220 undeployRegions(hi);
2221 return;
2222 }
2223
2224
2225 Get get = new Get(hi.getRegionName());
2226 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2227 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2228 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2229
2230 if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2231 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2232 for (int i = 0; i < numReplicas; i++) {
2233 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2234 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2235 }
2236 }
2237 Result r = meta.get(get);
2238 RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2239 if (rl == null) {
2240 LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2241 " since meta does not have handle to reach it");
2242 return;
2243 }
2244 for (HRegionLocation h : rl.getRegionLocations()) {
2245 ServerName serverName = h.getServerName();
2246 if (serverName == null) {
2247 errors.reportError("Unable to close region "
2248 + hi.getRegionNameAsString() + " because meta does not "
2249 + "have handle to reach it.");
2250 continue;
2251 }
2252 HRegionInfo hri = h.getRegionInfo();
2253 if (hri == null) {
2254 LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2255 + " because hbase:meta had invalid or missing "
2256 + HConstants.CATALOG_FAMILY_STR + ":"
2257 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2258 + " qualifier value.");
2259 continue;
2260 }
2261
2262 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2263 }
2264 }
2265
2266 private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2267 KeeperException, InterruptedException {
2268
2269 if (shouldFixAssignments()) {
2270 errors.print(msg);
2271 undeployRegions(hbi);
2272 setShouldRerun();
2273 HRegionInfo hri = hbi.getHdfsHRI();
2274 if (hri == null) {
2275 hri = hbi.metaEntry;
2276 }
2277 HBaseFsckRepair.fixUnassigned(admin, hri);
2278 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2279
2280
2281 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2282 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2283 for (int i = 1; i < replicationCount; i++) {
2284 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2285 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2286 if (h != null) {
2287 undeployRegions(h);
2288
2289
2290 h.setSkipChecks(true);
2291 }
2292 HBaseFsckRepair.fixUnassigned(admin, hri);
2293 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2294 }
2295
2296 }
2297 }
2298
2299
2300
2301
2302 private void checkRegionConsistency(final String key, final HbckInfo hbi)
2303 throws IOException, KeeperException, InterruptedException {
2304
2305 if (hbi.isSkipChecks()) return;
2306 String descriptiveName = hbi.toString();
2307 boolean inMeta = hbi.metaEntry != null;
2308
2309 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2310 boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2311 boolean isDeployed = !hbi.deployedOn.isEmpty();
2312 boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2313 boolean deploymentMatchesMeta =
2314 hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2315 hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2316 boolean splitParent =
2317 (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2318 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2319 boolean recentlyModified = inHdfs &&
2320 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2321
2322
2323 if (hbi.containsOnlyHdfsEdits()) {
2324 return;
2325 }
2326 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2327 return;
2328 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2329 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2330 "tabled that is not deployed");
2331 return;
2332 } else if (recentlyModified) {
2333 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2334 return;
2335 }
2336
2337 else if (!inMeta && !inHdfs && !isDeployed) {
2338
2339 assert false : "Entry for region with no data";
2340 } else if (!inMeta && !inHdfs && isDeployed) {
2341 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2342 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2343 "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2344 if (shouldFixAssignments()) {
2345 undeployRegions(hbi);
2346 }
2347
2348 } else if (!inMeta && inHdfs && !isDeployed) {
2349 if (hbi.isMerged()) {
2350
2351
2352 hbi.setSkipChecks(true);
2353 LOG.info("Region " + descriptiveName
2354 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2355 return;
2356 }
2357 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2358 + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2359 "or deployed on any region server");
2360
2361 if (shouldFixMeta()) {
2362 if (!hbi.isHdfsRegioninfoPresent()) {
2363 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2364 + " in table integrity repair phase if -fixHdfsOrphans was" +
2365 " used.");
2366 return;
2367 }
2368
2369 HRegionInfo hri = hbi.getHdfsHRI();
2370 TableInfo tableInfo = tablesInfo.get(hri.getTable());
2371
2372 for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2373 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2374 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2375 hri.getEndKey()) >= 0)
2376 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2377 if(region.isSplit() || region.isOffline()) continue;
2378 Path regionDir = hbi.getHdfsRegionDir();
2379 FileSystem fs = regionDir.getFileSystem(getConf());
2380 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2381 for (Path familyDir : familyDirs) {
2382 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2383 for (Path referenceFilePath : referenceFilePaths) {
2384 Path parentRegionDir =
2385 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2386 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2387 LOG.warn(hri + " start and stop keys are in the range of " + region
2388 + ". The region might not be cleaned up from hdfs when region " + region
2389 + " split failed. Hence deleting from hdfs.");
2390 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2391 regionDir.getParent(), hri);
2392 return;
2393 }
2394 }
2395 }
2396 }
2397 }
2398
2399 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2400 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2401 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2402 admin.getClusterStatus().getServers(), numReplicas);
2403
2404 tryAssignmentRepair(hbi, "Trying to reassign region...");
2405 }
2406
2407 } else if (!inMeta && inHdfs && isDeployed) {
2408 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2409 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2410 debugLsr(hbi.getHdfsRegionDir());
2411 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2412
2413
2414
2415
2416 if (shouldFixAssignments()) {
2417 undeployRegionsForHbi(hbi);
2418 }
2419 }
2420 if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2421 if (!hbi.isHdfsRegioninfoPresent()) {
2422 LOG.error("This should have been repaired in table integrity repair phase");
2423 return;
2424 }
2425
2426 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2427 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2428 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2429 admin.getClusterStatus().getServers(), numReplicas);
2430 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2431 }
2432
2433
2434 } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2435
2436
2437 if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2438
2439 HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2440 HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2441 if (infoA != null && infoB != null) {
2442
2443 hbi.setSkipChecks(true);
2444 return;
2445 }
2446 }
2447
2448
2449
2450 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2451 LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "
2452 + "and not deployed on any region server. This may be transient.");
2453 hbi.setSkipChecks(true);
2454 return;
2455 }
2456
2457 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2458 + descriptiveName + " is a split parent in META, in HDFS, "
2459 + "and not deployed on any region server. This could be transient, "
2460 + "consider to run the catalog janitor first!");
2461 if (shouldFixSplitParents()) {
2462 setShouldRerun();
2463 resetSplitParent(hbi);
2464 }
2465 } else if (inMeta && !inHdfs && !isDeployed) {
2466 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2467 + descriptiveName + " found in META, but not in HDFS "
2468 + "or deployed on any region server.");
2469 if (shouldFixMeta()) {
2470 deleteMetaRegion(hbi);
2471 }
2472 } else if (inMeta && !inHdfs && isDeployed) {
2473 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2474 + " found in META, but not in HDFS, " +
2475 "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2476
2477
2478
2479 if (shouldFixAssignments()) {
2480 errors.print("Trying to fix unassigned region...");
2481 undeployRegions(hbi);
2482 }
2483 if (shouldFixMeta()) {
2484
2485 deleteMetaRegion(hbi);
2486 }
2487 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2488 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2489 + " not deployed on any region server.");
2490 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2491 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2492 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2493 "Region " + descriptiveName + " should not be deployed according " +
2494 "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2495 if (shouldFixAssignments()) {
2496 errors.print("Trying to close the region " + descriptiveName);
2497 setShouldRerun();
2498 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2499 }
2500 } else if (inMeta && inHdfs && isMultiplyDeployed) {
2501 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2502 + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2503 + " but is multiply assigned to region servers " +
2504 Joiner.on(", ").join(hbi.deployedOn));
2505
2506 if (shouldFixAssignments()) {
2507 errors.print("Trying to fix assignment error...");
2508 setShouldRerun();
2509 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2510 }
2511 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2512 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2513 + descriptiveName + " listed in hbase:meta on region server " +
2514 hbi.metaEntry.regionServer + " but found on region server " +
2515 hbi.deployedOn.get(0));
2516
2517 if (shouldFixAssignments()) {
2518 errors.print("Trying to fix assignment error...");
2519 setShouldRerun();
2520 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2521 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2522 }
2523 } else {
2524 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2525 " is in an unforeseen state:" +
2526 " inMeta=" + inMeta +
2527 " inHdfs=" + inHdfs +
2528 " isDeployed=" + isDeployed +
2529 " isMultiplyDeployed=" + isMultiplyDeployed +
2530 " deploymentMatchesMeta=" + deploymentMatchesMeta +
2531 " shouldBeDeployed=" + shouldBeDeployed);
2532 }
2533 }
2534
2535
2536
2537
2538
2539
2540
2541 SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2542 tablesInfo = new TreeMap<TableName,TableInfo> ();
2543 LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2544 for (HbckInfo hbi : regionInfoMap.values()) {
2545
2546 if (hbi.metaEntry == null) {
2547
2548 Path p = hbi.getHdfsRegionDir();
2549 if (p == null) {
2550 errors.report("No regioninfo in Meta or HDFS. " + hbi);
2551 }
2552
2553
2554 continue;
2555 }
2556 if (hbi.metaEntry.regionServer == null) {
2557 errors.detail("Skipping region because no region server: " + hbi);
2558 continue;
2559 }
2560 if (hbi.metaEntry.isOffline()) {
2561 errors.detail("Skipping region because it is offline: " + hbi);
2562 continue;
2563 }
2564 if (hbi.containsOnlyHdfsEdits()) {
2565 errors.detail("Skipping region because it only contains edits" + hbi);
2566 continue;
2567 }
2568
2569
2570
2571
2572
2573
2574 if (hbi.deployedOn.size() == 0) continue;
2575
2576
2577 TableName tableName = hbi.metaEntry.getTable();
2578 TableInfo modTInfo = tablesInfo.get(tableName);
2579 if (modTInfo == null) {
2580 modTInfo = new TableInfo(tableName);
2581 }
2582 for (ServerName server : hbi.deployedOn) {
2583 modTInfo.addServer(server);
2584 }
2585
2586 if (!hbi.isSkipChecks()) {
2587 modTInfo.addRegionInfo(hbi);
2588 }
2589
2590 tablesInfo.put(tableName, modTInfo);
2591 }
2592
2593 loadTableInfosForTablesWithNoRegion();
2594
2595 logParallelMerge();
2596 for (TableInfo tInfo : tablesInfo.values()) {
2597 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2598 if (!tInfo.checkRegionChain(handler)) {
2599 errors.report("Found inconsistency in table " + tInfo.getName());
2600 }
2601 }
2602 return tablesInfo;
2603 }
2604
2605
2606
2607
2608 private void loadTableInfosForTablesWithNoRegion() throws IOException {
2609 Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2610 for (HTableDescriptor htd : allTables.values()) {
2611 if (checkMetaOnly && !htd.isMetaTable()) {
2612 continue;
2613 }
2614
2615 TableName tableName = htd.getTableName();
2616 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2617 TableInfo tableInfo = new TableInfo(tableName);
2618 tableInfo.htds.add(htd);
2619 tablesInfo.put(htd.getTableName(), tableInfo);
2620 }
2621 }
2622 }
2623
2624
2625
2626
2627
2628 public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2629 int fileMoves = 0;
2630 String thread = Thread.currentThread().getName();
2631 LOG.debug("[" + thread + "] Contained region dir after close and pause");
2632 debugLsr(contained.getHdfsRegionDir());
2633
2634
2635 FileSystem fs = targetRegionDir.getFileSystem(getConf());
2636 FileStatus[] dirs = null;
2637 try {
2638 dirs = fs.listStatus(contained.getHdfsRegionDir());
2639 } catch (FileNotFoundException fnfe) {
2640
2641
2642 if (!fs.exists(contained.getHdfsRegionDir())) {
2643 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2644 + " is missing. Assuming already sidelined or moved.");
2645 } else {
2646 sidelineRegionDir(fs, contained);
2647 }
2648 return fileMoves;
2649 }
2650
2651 if (dirs == null) {
2652 if (!fs.exists(contained.getHdfsRegionDir())) {
2653 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2654 + " already sidelined.");
2655 } else {
2656 sidelineRegionDir(fs, contained);
2657 }
2658 return fileMoves;
2659 }
2660
2661 for (FileStatus cf : dirs) {
2662 Path src = cf.getPath();
2663 Path dst = new Path(targetRegionDir, src.getName());
2664
2665 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2666
2667 continue;
2668 }
2669
2670 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2671
2672 continue;
2673 }
2674
2675 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2676
2677
2678
2679
2680 for (FileStatus hfile : fs.listStatus(src)) {
2681 boolean success = fs.rename(hfile.getPath(), dst);
2682 if (success) {
2683 fileMoves++;
2684 }
2685 }
2686 LOG.debug("[" + thread + "] Sideline directory contents:");
2687 debugLsr(targetRegionDir);
2688 }
2689
2690
2691 sidelineRegionDir(fs, contained);
2692 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2693 getSidelineDir());
2694 debugLsr(contained.getHdfsRegionDir());
2695
2696 return fileMoves;
2697 }
2698
2699
2700 static class WorkItemOverlapMerge implements Callable<Void> {
2701 private TableIntegrityErrorHandler handler;
2702 Collection<HbckInfo> overlapgroup;
2703
2704 WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2705 this.handler = handler;
2706 this.overlapgroup = overlapgroup;
2707 }
2708
2709 @Override
2710 public Void call() throws Exception {
2711 handler.handleOverlapGroup(overlapgroup);
2712 return null;
2713 }
2714 };
2715
2716
2717
2718
2719
2720 public class TableInfo {
2721 TableName tableName;
2722 TreeSet <ServerName> deployedOn;
2723
2724
2725 final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2726
2727
2728 final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2729
2730
2731 final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2732
2733
2734 final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2735
2736
2737 final Multimap<byte[], HbckInfo> overlapGroups =
2738 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2739
2740
2741 private ImmutableList<HRegionInfo> regionsFromMeta = null;
2742
2743 TableInfo(TableName name) {
2744 this.tableName = name;
2745 deployedOn = new TreeSet <ServerName>();
2746 }
2747
2748
2749
2750
2751 private HTableDescriptor getHTD() {
2752 if (htds.size() == 1) {
2753 return (HTableDescriptor)htds.toArray()[0];
2754 } else {
2755 LOG.error("None/Multiple table descriptors found for table '"
2756 + tableName + "' regions: " + htds);
2757 }
2758 return null;
2759 }
2760
2761 public void addRegionInfo(HbckInfo hir) {
2762 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2763
2764
2765 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2766 return;
2767 }
2768
2769
2770 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2771 errors.reportError(
2772 ERROR_CODE.REGION_CYCLE,
2773 String.format("The endkey for this region comes before the "
2774 + "startkey, startkey=%s, endkey=%s",
2775 Bytes.toStringBinary(hir.getStartKey()),
2776 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2777 backwards.add(hir);
2778 return;
2779 }
2780
2781
2782
2783 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2784 }
2785
2786 public void addServer(ServerName server) {
2787 this.deployedOn.add(server);
2788 }
2789
2790 public TableName getName() {
2791 return tableName;
2792 }
2793
2794 public int getNumRegions() {
2795 return sc.getStarts().size() + backwards.size();
2796 }
2797
2798 public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2799
2800 if (regionsFromMeta == null) {
2801 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2802 for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2803 if (tableName.equals(h.getTableName())) {
2804 if (h.metaEntry != null) {
2805 regions.add((HRegionInfo) h.metaEntry);
2806 }
2807 }
2808 }
2809 regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2810 }
2811
2812 return regionsFromMeta;
2813 }
2814
2815
2816 private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2817 ErrorReporter errors;
2818
2819 IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2820 this.errors = errors;
2821 setTableInfo(ti);
2822 }
2823
2824 @Override
2825 public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2826 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2827 "First region should start with an empty key. You need to "
2828 + " create a new region and regioninfo in HDFS to plug the hole.",
2829 getTableInfo(), hi);
2830 }
2831
2832 @Override
2833 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2834 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2835 "Last region should end with an empty key. You need to "
2836 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2837 }
2838
2839 @Override
2840 public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2841 errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2842 "Region has the same start and end key.", getTableInfo(), hi);
2843 }
2844
2845 @Override
2846 public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2847 byte[] key = r1.getStartKey();
2848
2849 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2850 "Multiple regions have the same startkey: "
2851 + Bytes.toStringBinary(key), getTableInfo(), r1);
2852 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2853 "Multiple regions have the same startkey: "
2854 + Bytes.toStringBinary(key), getTableInfo(), r2);
2855 }
2856
2857 @Override
2858 public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
2859 byte[] key = r1.getStartKey();
2860
2861 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2862 "Multiple regions have the same regionID: "
2863 + Bytes.toStringBinary(key), getTableInfo(), r1);
2864 errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2865 "Multiple regions have the same regionID: "
2866 + Bytes.toStringBinary(key), getTableInfo(), r2);
2867 }
2868
2869 @Override
2870 public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2871 errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2872 "There is an overlap in the region chain.",
2873 getTableInfo(), hi1, hi2);
2874 }
2875
2876 @Override
2877 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2878 errors.reportError(
2879 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2880 "There is a hole in the region chain between "
2881 + Bytes.toStringBinary(holeStart) + " and "
2882 + Bytes.toStringBinary(holeStop)
2883 + ". You need to create a new .regioninfo and region "
2884 + "dir in hdfs to plug the hole.");
2885 }
2886 };
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900 private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2901 Configuration conf;
2902
2903 boolean fixOverlaps = true;
2904
2905 HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2906 boolean fixHoles, boolean fixOverlaps) {
2907 super(ti, errors);
2908 this.conf = conf;
2909 this.fixOverlaps = fixOverlaps;
2910
2911 }
2912
2913
2914
2915
2916
2917
2918 @Override
2919 public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2920 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2921 "First region should start with an empty key. Creating a new " +
2922 "region and regioninfo in HDFS to plug the hole.",
2923 getTableInfo(), next);
2924 HTableDescriptor htd = getTableInfo().getHTD();
2925
2926 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2927 HConstants.EMPTY_START_ROW, next.getStartKey());
2928
2929
2930 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2931 LOG.info("Table region start key was not empty. Created new empty region: "
2932 + newRegion + " " +region);
2933 fixes++;
2934 }
2935
2936 @Override
2937 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2938 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2939 "Last region should end with an empty key. Creating a new "
2940 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2941 HTableDescriptor htd = getTableInfo().getHTD();
2942
2943 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2944 HConstants.EMPTY_START_ROW);
2945
2946 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2947 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
2948 + " " + region);
2949 fixes++;
2950 }
2951
2952
2953
2954
2955
2956 @Override
2957 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2958 errors.reportError(
2959 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2960 "There is a hole in the region chain between "
2961 + Bytes.toStringBinary(holeStartKey) + " and "
2962 + Bytes.toStringBinary(holeStopKey)
2963 + ". Creating a new regioninfo and region "
2964 + "dir in hdfs to plug the hole.");
2965 HTableDescriptor htd = getTableInfo().getHTD();
2966 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2967 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2968 LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2969 fixes++;
2970 }
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983 @Override
2984 public void handleOverlapGroup(Collection<HbckInfo> overlap)
2985 throws IOException {
2986 Preconditions.checkNotNull(overlap);
2987 Preconditions.checkArgument(overlap.size() >0);
2988
2989 if (!this.fixOverlaps) {
2990 LOG.warn("Not attempting to repair overlaps.");
2991 return;
2992 }
2993
2994 if (overlap.size() > maxMerge) {
2995 LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2996 "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2997 if (sidelineBigOverlaps) {
2998
2999 sidelineBigOverlaps(overlap);
3000 }
3001 return;
3002 }
3003 if (shouldRemoveParents()) {
3004 removeParentsAndFixSplits(overlap);
3005 }
3006 mergeOverlaps(overlap);
3007 }
3008
3009 void removeParentsAndFixSplits(Collection<HbckInfo> overlap) throws IOException {
3010 Pair<byte[], byte[]> range = null;
3011 HbckInfo parent = null;
3012 HbckInfo daughterA = null;
3013 HbckInfo daughterB = null;
3014 Collection<HbckInfo> daughters = new ArrayList<HbckInfo>(overlap);
3015
3016 String thread = Thread.currentThread().getName();
3017 LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
3018
3019
3020 if (overlap.size() > 3) {
3021 LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
3022 return;
3023 }
3024
3025 for (HbckInfo hi : overlap) {
3026 if (range == null) {
3027 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
3028 } else {
3029 if (RegionSplitCalculator.BYTES_COMPARATOR
3030 .compare(hi.getStartKey(), range.getFirst()) < 0) {
3031 range.setFirst(hi.getStartKey());
3032 }
3033 if (RegionSplitCalculator.BYTES_COMPARATOR
3034 .compare(hi.getEndKey(), range.getSecond()) > 0) {
3035 range.setSecond(hi.getEndKey());
3036 }
3037 }
3038 }
3039
3040 LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
3041 + Bytes.toStringBinary(range.getSecond()) + "]");
3042
3043
3044 for (HbckInfo hi : overlap) {
3045 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
3046 && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
3047 LOG.info("This is a parent for this group: " + hi.toString());
3048 parent = hi;
3049 }
3050 }
3051
3052
3053 if (parent != null) {
3054 daughters.remove(parent);
3055 }
3056
3057
3058
3059 for (HbckInfo hi : daughters) {
3060 if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
3061 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
3062 daughterA = hi;
3063 }
3064 }
3065 if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
3066 if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
3067 daughterB = hi;
3068 }
3069 }
3070 }
3071
3072
3073 if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() || parent == null)
3074 return;
3075
3076 FileSystem fs = FileSystem.get(conf);
3077 LOG.info("Found parent: " + parent.getRegionNameAsString());
3078 LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
3079 LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
3080 LOG.info("Trying to fix parent in overlap by removing the parent.");
3081 try {
3082 closeRegion(parent);
3083 } catch (IOException ioe) {
3084 LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
3085 return;
3086 } catch (InterruptedException ie) {
3087 LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
3088 return;
3089 }
3090
3091 try {
3092 offline(parent.getRegionName());
3093 } catch (IOException ioe) {
3094 LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
3095 + ". Just continuing with regular merge... ", ioe);
3096 return;
3097 }
3098
3099 try {
3100 HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
3101 } catch (IOException ioe) {
3102 LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
3103 + ". Just continuing with regular merge... ", ioe);
3104 return;
3105 }
3106
3107 sidelineRegionDir(fs, parent);
3108 LOG.info("[" + thread + "] Sidelined parent region dir "+ parent.getHdfsRegionDir() + " into " +
3109 getSidelineDir());
3110 debugLsr(parent.getHdfsRegionDir());
3111
3112
3113 overlap.remove(parent);
3114 overlap.remove(daughterA);
3115 overlap.remove(daughterB);
3116
3117 LOG.info("Done fixing split.");
3118
3119 }
3120
3121 void mergeOverlaps(Collection<HbckInfo> overlap)
3122 throws IOException {
3123 String thread = Thread.currentThread().getName();
3124 LOG.info("== [" + thread + "] Merging regions into one region: "
3125 + Joiner.on(",").join(overlap));
3126
3127 Pair<byte[], byte[]> range = null;
3128 for (HbckInfo hi : overlap) {
3129 if (range == null) {
3130 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
3131 } else {
3132 if (RegionSplitCalculator.BYTES_COMPARATOR
3133 .compare(hi.getStartKey(), range.getFirst()) < 0) {
3134 range.setFirst(hi.getStartKey());
3135 }
3136 if (RegionSplitCalculator.BYTES_COMPARATOR
3137 .compare(hi.getEndKey(), range.getSecond()) > 0) {
3138 range.setSecond(hi.getEndKey());
3139 }
3140 }
3141
3142 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
3143 LOG.debug("[" + thread + "] Contained region dir before close");
3144 debugLsr(hi.getHdfsRegionDir());
3145 try {
3146 LOG.info("[" + thread + "] Closing region: " + hi);
3147 closeRegion(hi);
3148 } catch (IOException ioe) {
3149 LOG.warn("[" + thread + "] Was unable to close region " + hi
3150 + ". Just continuing... ", ioe);
3151 } catch (InterruptedException e) {
3152 LOG.warn("[" + thread + "] Was unable to close region " + hi
3153 + ". Just continuing... ", e);
3154 }
3155
3156 try {
3157 LOG.info("[" + thread + "] Offlining region: " + hi);
3158 offline(hi.getRegionName());
3159 } catch (IOException ioe) {
3160 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
3161 + ". Just continuing... ", ioe);
3162 }
3163 }
3164
3165
3166 HTableDescriptor htd = getTableInfo().getHTD();
3167
3168 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
3169 range.getSecond());
3170 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
3171 LOG.info("[" + thread + "] Created new empty container region: " +
3172 newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
3173 debugLsr(region.getRegionFileSystem().getRegionDir());
3174
3175
3176 boolean didFix= false;
3177 Path target = region.getRegionFileSystem().getRegionDir();
3178 for (HbckInfo contained : overlap) {
3179 LOG.info("[" + thread + "] Merging " + contained + " into " + target );
3180 int merges = mergeRegionDirs(target, contained);
3181 if (merges > 0) {
3182 didFix = true;
3183 }
3184 }
3185 if (didFix) {
3186 fixes++;
3187 }
3188 }
3189
3190
3191
3192
3193
3194
3195
3196
3197 void sidelineBigOverlaps(
3198 Collection<HbckInfo> bigOverlap) throws IOException {
3199 int overlapsToSideline = bigOverlap.size() - maxMerge;
3200 if (overlapsToSideline > maxOverlapsToSideline) {
3201 overlapsToSideline = maxOverlapsToSideline;
3202 }
3203 List<HbckInfo> regionsToSideline =
3204 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
3205 FileSystem fs = FileSystem.get(conf);
3206 for (HbckInfo regionToSideline: regionsToSideline) {
3207 try {
3208 LOG.info("Closing region: " + regionToSideline);
3209 closeRegion(regionToSideline);
3210 } catch (IOException ioe) {
3211 LOG.warn("Was unable to close region " + regionToSideline
3212 + ". Just continuing... ", ioe);
3213 } catch (InterruptedException e) {
3214 LOG.warn("Was unable to close region " + regionToSideline
3215 + ". Just continuing... ", e);
3216 }
3217
3218 try {
3219 LOG.info("Offlining region: " + regionToSideline);
3220 offline(regionToSideline.getRegionName());
3221 } catch (IOException ioe) {
3222 LOG.warn("Unable to offline region from master: " + regionToSideline
3223 + ". Just continuing... ", ioe);
3224 }
3225
3226 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
3227 Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
3228 if (sidelineRegionDir != null) {
3229 sidelinedRegions.put(sidelineRegionDir, regionToSideline);
3230 LOG.info("After sidelined big overlapped region: "
3231 + regionToSideline.getRegionNameAsString()
3232 + " to " + sidelineRegionDir.toString());
3233 fixes++;
3234 }
3235 }
3236 }
3237 }
3238
3239
3240
3241
3242
3243
3244
3245 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3246
3247
3248
3249 if (disabledTables.contains(this.tableName)) {
3250 return true;
3251 }
3252 int originalErrorsCount = errors.getErrorList().size();
3253 Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3254 SortedSet<byte[]> splits = sc.getSplits();
3255
3256 byte[] prevKey = null;
3257 byte[] problemKey = null;
3258
3259 if (splits.size() == 0) {
3260
3261 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3262 }
3263
3264 for (byte[] key : splits) {
3265 Collection<HbckInfo> ranges = regions.get(key);
3266 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3267 for (HbckInfo rng : ranges) {
3268 handler.handleRegionStartKeyNotEmpty(rng);
3269 }
3270 }
3271
3272
3273 for (HbckInfo rng : ranges) {
3274
3275 byte[] endKey = rng.getEndKey();
3276 endKey = (endKey.length == 0) ? null : endKey;
3277 if (Bytes.equals(rng.getStartKey(),endKey)) {
3278 handler.handleDegenerateRegion(rng);
3279 }
3280 }
3281
3282 if (ranges.size() == 1) {
3283
3284 if (problemKey != null) {
3285 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3286 }
3287 problemKey = null;
3288 } else if (ranges.size() > 1) {
3289
3290
3291 if (problemKey == null) {
3292
3293 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3294 problemKey = key;
3295 }
3296 overlapGroups.putAll(problemKey, ranges);
3297
3298
3299 ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3300
3301 for (HbckInfo r1 : ranges) {
3302 if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3303 subRange.remove(r1);
3304 for (HbckInfo r2 : subRange) {
3305 if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3306
3307 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3308 handler.handleDuplicateStartKeys(r1,r2);
3309 } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 &&
3310 r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) {
3311 LOG.info("this is a split, log to splits");
3312 handler.handleSplit(r1, r2);
3313 } else {
3314
3315 handler.handleOverlapInRegionChain(r1, r2);
3316 }
3317 }
3318 }
3319
3320 } else if (ranges.size() == 0) {
3321 if (problemKey != null) {
3322 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3323 }
3324 problemKey = null;
3325
3326 byte[] holeStopKey = sc.getSplits().higher(key);
3327
3328 if (holeStopKey != null) {
3329
3330 handler.handleHoleInRegionChain(key, holeStopKey);
3331 }
3332 }
3333 prevKey = key;
3334 }
3335
3336
3337
3338 if (prevKey != null) {
3339 handler.handleRegionEndKeyNotEmpty(prevKey);
3340 }
3341
3342
3343 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3344 boolean ok = handleOverlapsParallel(handler, prevKey);
3345 if (!ok) {
3346 return false;
3347 }
3348 } else {
3349 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3350 handler.handleOverlapGroup(overlap);
3351 }
3352 }
3353
3354 if (details) {
3355
3356 errors.print("---- Table '" + this.tableName
3357 + "': region split map");
3358 dump(splits, regions);
3359 errors.print("---- Table '" + this.tableName
3360 + "': overlap groups");
3361 dumpOverlapProblems(overlapGroups);
3362 errors.print("There are " + overlapGroups.keySet().size()
3363 + " overlap groups with " + overlapGroups.size()
3364 + " overlapping regions");
3365 }
3366 if (!sidelinedRegions.isEmpty()) {
3367 LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3368 errors.print("---- Table '" + this.tableName
3369 + "': sidelined big overlapped regions");
3370 dumpSidelinedRegions(sidelinedRegions);
3371 }
3372 return errors.getErrorList().size() == originalErrorsCount;
3373 }
3374
3375 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3376 throws IOException {
3377
3378
3379 List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3380 List<Future<Void>> rets;
3381 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3382
3383 merges.add(new WorkItemOverlapMerge(overlap, handler));
3384 }
3385 try {
3386 rets = executor.invokeAll(merges);
3387 } catch (InterruptedException e) {
3388 LOG.error("Overlap merges were interrupted", e);
3389 return false;
3390 }
3391 for(int i=0; i<merges.size(); i++) {
3392 WorkItemOverlapMerge work = merges.get(i);
3393 Future<Void> f = rets.get(i);
3394 try {
3395 f.get();
3396 } catch(ExecutionException e) {
3397 LOG.warn("Failed to merge overlap group" + work, e.getCause());
3398 } catch (InterruptedException e) {
3399 LOG.error("Waiting for overlap merges was interrupted", e);
3400 return false;
3401 }
3402 }
3403 return true;
3404 }
3405
3406
3407
3408
3409
3410
3411
3412 void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3413
3414 StringBuilder sb = new StringBuilder();
3415 for (byte[] k : splits) {
3416 sb.setLength(0);
3417 sb.append(Bytes.toStringBinary(k) + ":\t");
3418 for (HbckInfo r : regions.get(k)) {
3419 sb.append("[ "+ r.toString() + ", "
3420 + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3421 }
3422 errors.print(sb.toString());
3423 }
3424 }
3425 }
3426
3427 public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3428
3429
3430 for (byte[] k : regions.keySet()) {
3431 errors.print(Bytes.toStringBinary(k) + ":");
3432 for (HbckInfo r : regions.get(k)) {
3433 errors.print("[ " + r.toString() + ", "
3434 + Bytes.toStringBinary(r.getEndKey()) + "]");
3435 }
3436 errors.print("----");
3437 }
3438 }
3439
3440 public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3441 for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3442 TableName tableName = entry.getValue().getTableName();
3443 Path path = entry.getKey();
3444 errors.print("This sidelined region dir should be bulk loaded: "
3445 + path.toString());
3446 errors.print("Bulk load command looks like: "
3447 + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3448 + path.toUri().getPath() + " "+ tableName);
3449 }
3450 }
3451
3452 public Multimap<byte[], HbckInfo> getOverlapGroups(
3453 TableName table) {
3454 TableInfo ti = tablesInfo.get(table);
3455 return ti.overlapGroups;
3456 }
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467 HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3468 List<TableName> tableNames = new ArrayList<TableName>();
3469 long now = EnvironmentEdgeManager.currentTime();
3470
3471 for (HbckInfo hbi : regionInfoMap.values()) {
3472 MetaEntry info = hbi.metaEntry;
3473
3474
3475
3476 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3477 if (info.modTime + timelag < now) {
3478 tableNames.add(info.getTable());
3479 } else {
3480 numSkipped.incrementAndGet();
3481 }
3482 }
3483 }
3484 return getHTableDescriptors(tableNames);
3485 }
3486
3487 HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3488 HTableDescriptor[] htd = new HTableDescriptor[0];
3489 Admin admin = null;
3490 try {
3491 LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3492 admin = new HBaseAdmin(getConf());
3493 htd = admin.getTableDescriptorsByTableName(tableNames);
3494 } catch (IOException e) {
3495 LOG.debug("Exception getting table descriptors", e);
3496 } finally {
3497 if (admin != null) {
3498 try {
3499 admin.close();
3500 } catch (IOException e) {
3501 LOG.debug("Exception closing HBaseAdmin", e);
3502 }
3503 }
3504 }
3505 return htd;
3506 }
3507
3508
3509
3510
3511
3512
3513 private synchronized HbckInfo getOrCreateInfo(String name) {
3514 HbckInfo hbi = regionInfoMap.get(name);
3515 if (hbi == null) {
3516 hbi = new HbckInfo(null);
3517 regionInfoMap.put(name, hbi);
3518 }
3519 return hbi;
3520 }
3521
3522 private void checkAndFixTableLocks() throws IOException {
3523 TableLockChecker checker = new TableLockChecker(zkw, errors);
3524 checker.checkTableLocks();
3525
3526 if (this.fixTableLocks) {
3527 checker.fixExpiredTableLocks();
3528 }
3529 }
3530
3531 private void checkAndFixReplication() throws IOException {
3532 ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3533 checker.checkUnDeletedQueues();
3534
3535 if (checker.hasUnDeletedQueues() && this.fixReplication) {
3536 checker.fixUnDeletedQueues();
3537 setShouldRerun();
3538 }
3539 }
3540
3541
3542
3543
3544
3545
3546
3547 private void checkAndFixOrphanedTableZNodes()
3548 throws IOException, KeeperException, InterruptedException {
3549 Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3550 String msg;
3551 TableInfo tableInfo;
3552
3553 for (TableName tableName : enablingTables) {
3554
3555 tableInfo = tablesInfo.get(tableName);
3556 if (tableInfo != null) {
3557
3558 continue;
3559 }
3560
3561 msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3562 LOG.warn(msg);
3563 orphanedTableZNodes.add(tableName);
3564 errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3565 }
3566
3567 if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3568 ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3569
3570 for (TableName tableName : orphanedTableZNodes) {
3571 try {
3572
3573
3574
3575
3576 zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3577 } catch (CoordinatedStateException e) {
3578
3579 LOG.error(
3580 "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3581 e);
3582 }
3583 }
3584 }
3585 }
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3597 Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3598 for (HbckInfo value : regionInfoMap.values()) {
3599 if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3600 metaRegions.put(value.getReplicaId(), value);
3601 }
3602 }
3603 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3604 .getRegionReplication();
3605 boolean noProblem = true;
3606
3607
3608 for (int i = 0; i < metaReplication; i++) {
3609 HbckInfo metaHbckInfo = metaRegions.remove(i);
3610 List<ServerName> servers = new ArrayList<ServerName>();
3611 if (metaHbckInfo != null) {
3612 servers = metaHbckInfo.deployedOn;
3613 }
3614 if (servers.size() != 1) {
3615 noProblem = false;
3616 if (servers.size() == 0) {
3617 assignMetaReplica(i);
3618 } else if (servers.size() > 1) {
3619 errors
3620 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3621 metaHbckInfo.getReplicaId() + " is found on more than one region.");
3622 if (shouldFixAssignments()) {
3623 errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3624 metaHbckInfo.getReplicaId() +"..");
3625 setShouldRerun();
3626
3627 HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3628 }
3629 }
3630 }
3631 }
3632
3633 for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3634 noProblem = false;
3635 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3636 "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3637 ", deployed " + metaRegions.size());
3638 if (shouldFixAssignments()) {
3639 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3640 " of hbase:meta..");
3641 setShouldRerun();
3642 unassignMetaReplica(entry.getValue());
3643 }
3644 }
3645
3646
3647 return noProblem;
3648 }
3649
3650 private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3651 KeeperException {
3652 undeployRegions(hi);
3653 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3654 }
3655
3656 private void assignMetaReplica(int replicaId)
3657 throws IOException, KeeperException, InterruptedException {
3658 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3659 replicaId +" is not found on any region.");
3660 if (shouldFixAssignments()) {
3661 errors.print("Trying to fix a problem with hbase:meta..");
3662 setShouldRerun();
3663
3664 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3665 HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3666 HBaseFsckRepair.fixUnassigned(admin, h);
3667 HBaseFsckRepair.waitUntilAssigned(admin, h);
3668 }
3669 }
3670
3671
3672
3673
3674
3675 boolean loadMetaEntries() throws IOException {
3676 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3677 int countRecord = 1;
3678
3679
3680 final Comparator<Cell> comp = new Comparator<Cell>() {
3681 @Override
3682 public int compare(Cell k1, Cell k2) {
3683 return Long.compare(k1.getTimestamp(), k2.getTimestamp());
3684 }
3685 };
3686
3687 @Override
3688 public boolean processRow(Result result) throws IOException {
3689 try {
3690
3691
3692 long ts = Collections.max(result.listCells(), comp).getTimestamp();
3693 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3694 if (rl == null) {
3695 emptyRegionInfoQualifiers.add(result);
3696 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3697 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3698 return true;
3699 }
3700 ServerName sn = null;
3701 if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3702 rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3703 emptyRegionInfoQualifiers.add(result);
3704 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3705 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3706 return true;
3707 }
3708 HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3709 if (!(isTableIncluded(hri.getTable())
3710 || hri.isMetaRegion())) {
3711 return true;
3712 }
3713 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3714 for (HRegionLocation h : rl.getRegionLocations()) {
3715 if (h == null || h.getRegionInfo() == null) {
3716 continue;
3717 }
3718 sn = h.getServerName();
3719 hri = h.getRegionInfo();
3720
3721 MetaEntry m = null;
3722 if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3723 m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3724 } else {
3725 m = new MetaEntry(hri, sn, ts, null, null);
3726 }
3727 HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3728 if (previous == null) {
3729 regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3730 } else if (previous.metaEntry == null) {
3731 previous.metaEntry = m;
3732 } else {
3733 throw new IOException("Two entries in hbase:meta are same " + previous);
3734 }
3735 }
3736 PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3737 for (HRegionInfo mergeRegion : new HRegionInfo[] {
3738 mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3739 if (mergeRegion != null) {
3740
3741 HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3742 hbInfo.setMerged(true);
3743 }
3744 }
3745
3746
3747 if (countRecord % 100 == 0) {
3748 errors.progress();
3749 }
3750 countRecord++;
3751 return true;
3752 } catch (RuntimeException e) {
3753 LOG.error("Result=" + result);
3754 throw e;
3755 }
3756 }
3757 };
3758 if (!checkMetaOnly) {
3759
3760 MetaScanner.metaScan(connection, visitor);
3761 }
3762
3763 errors.print("");
3764 return true;
3765 }
3766
3767
3768
3769
3770 static class MetaEntry extends HRegionInfo {
3771 ServerName regionServer;
3772 long modTime;
3773 HRegionInfo splitA, splitB;
3774
3775 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3776 this(rinfo, regionServer, modTime, null, null);
3777 }
3778
3779 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3780 HRegionInfo splitA, HRegionInfo splitB) {
3781 super(rinfo);
3782 this.regionServer = regionServer;
3783 this.modTime = modTime;
3784 this.splitA = splitA;
3785 this.splitB = splitB;
3786 }
3787
3788 @Override
3789 public boolean equals(Object o) {
3790 boolean superEq = super.equals(o);
3791 if (!superEq) {
3792 return superEq;
3793 }
3794
3795 MetaEntry me = (MetaEntry) o;
3796 if (!regionServer.equals(me.regionServer)) {
3797 return false;
3798 }
3799 return (modTime == me.modTime);
3800 }
3801
3802 @Override
3803 public int hashCode() {
3804 int hash = Arrays.hashCode(getRegionName());
3805 hash = (int) (hash ^ getRegionId());
3806 hash ^= Arrays.hashCode(getStartKey());
3807 hash ^= Arrays.hashCode(getEndKey());
3808 hash ^= Boolean.valueOf(isOffline()).hashCode();
3809 hash ^= getTable().hashCode();
3810 if (regionServer != null) {
3811 hash ^= regionServer.hashCode();
3812 }
3813 hash = (int) (hash ^ modTime);
3814 return hash;
3815 }
3816 }
3817
3818
3819
3820
3821 static class HdfsEntry {
3822 HRegionInfo hri;
3823 Path hdfsRegionDir = null;
3824 long hdfsRegionDirModTime = 0;
3825 boolean hdfsRegioninfoFilePresent = false;
3826 boolean hdfsOnlyEdits = false;
3827 }
3828
3829
3830
3831
3832 static class OnlineEntry {
3833 HRegionInfo hri;
3834 ServerName hsa;
3835
3836 @Override
3837 public String toString() {
3838 return hsa.toString() + ";" + hri.getRegionNameAsString();
3839 }
3840 }
3841
3842
3843
3844
3845
3846 public static class HbckInfo implements KeyRange {
3847 private MetaEntry metaEntry = null;
3848 private HdfsEntry hdfsEntry = null;
3849 private List<OnlineEntry> deployedEntries = Lists.newArrayList();
3850 private List<ServerName> deployedOn = Lists.newArrayList();
3851 private boolean skipChecks = false;
3852 private boolean isMerged = false;
3853 private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3854 private HRegionInfo primaryHRIForDeployedReplica = null;
3855
3856 HbckInfo(MetaEntry metaEntry) {
3857 this.metaEntry = metaEntry;
3858 }
3859
3860 public synchronized int getReplicaId() {
3861 return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3862 }
3863
3864 public synchronized void addServer(HRegionInfo hri, ServerName server) {
3865 OnlineEntry rse = new OnlineEntry() ;
3866 rse.hri = hri;
3867 rse.hsa = server;
3868 this.deployedEntries.add(rse);
3869 this.deployedOn.add(server);
3870
3871 this.deployedReplicaId = hri.getReplicaId();
3872 this.primaryHRIForDeployedReplica =
3873 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3874 }
3875
3876 @Override
3877 public synchronized String toString() {
3878 StringBuilder sb = new StringBuilder();
3879 sb.append("{ meta => ");
3880 sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3881 sb.append( ", hdfs => " + getHdfsRegionDir());
3882 sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3883 sb.append( ", replicaId => " + getReplicaId());
3884 sb.append(" }");
3885 return sb.toString();
3886 }
3887
3888 @Override
3889 public byte[] getStartKey() {
3890 if (this.metaEntry != null) {
3891 return this.metaEntry.getStartKey();
3892 } else if (this.hdfsEntry != null) {
3893 return this.hdfsEntry.hri.getStartKey();
3894 } else {
3895 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3896 return null;
3897 }
3898 }
3899
3900 @Override
3901 public byte[] getEndKey() {
3902 if (this.metaEntry != null) {
3903 return this.metaEntry.getEndKey();
3904 } else if (this.hdfsEntry != null) {
3905 return this.hdfsEntry.hri.getEndKey();
3906 } else {
3907 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3908 return null;
3909 }
3910 }
3911
3912 public TableName getTableName() {
3913 if (this.metaEntry != null) {
3914 return this.metaEntry.getTable();
3915 } else if (this.hdfsEntry != null) {
3916
3917
3918 Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3919 return FSUtils.getTableName(tableDir);
3920 } else {
3921
3922 for (OnlineEntry e : deployedEntries) {
3923 return e.hri.getTable();
3924 }
3925 return null;
3926 }
3927 }
3928
3929 public String getRegionNameAsString() {
3930 if (metaEntry != null) {
3931 return metaEntry.getRegionNameAsString();
3932 } else if (hdfsEntry != null) {
3933 if (hdfsEntry.hri != null) {
3934 return hdfsEntry.hri.getRegionNameAsString();
3935 }
3936 } else {
3937
3938 for (OnlineEntry e : deployedEntries) {
3939 return e.hri.getRegionNameAsString();
3940 }
3941 }
3942 return null;
3943 }
3944
3945 public byte[] getRegionName() {
3946 if (metaEntry != null) {
3947 return metaEntry.getRegionName();
3948 } else if (hdfsEntry != null) {
3949 return hdfsEntry.hri.getRegionName();
3950 } else {
3951
3952 for (OnlineEntry e : deployedEntries) {
3953 return e.hri.getRegionName();
3954 }
3955 return null;
3956 }
3957 }
3958
3959 public HRegionInfo getPrimaryHRIForDeployedReplica() {
3960 return primaryHRIForDeployedReplica;
3961 }
3962
3963 Path getHdfsRegionDir() {
3964 if (hdfsEntry == null) {
3965 return null;
3966 }
3967 return hdfsEntry.hdfsRegionDir;
3968 }
3969
3970 boolean containsOnlyHdfsEdits() {
3971 if (hdfsEntry == null) {
3972 return false;
3973 }
3974 return hdfsEntry.hdfsOnlyEdits;
3975 }
3976
3977 boolean isHdfsRegioninfoPresent() {
3978 if (hdfsEntry == null) {
3979 return false;
3980 }
3981 return hdfsEntry.hdfsRegioninfoFilePresent;
3982 }
3983
3984 long getModTime() {
3985 if (hdfsEntry == null) {
3986 return 0;
3987 }
3988 return hdfsEntry.hdfsRegionDirModTime;
3989 }
3990
3991 HRegionInfo getHdfsHRI() {
3992 if (hdfsEntry == null) {
3993 return null;
3994 }
3995 return hdfsEntry.hri;
3996 }
3997
3998 public void setSkipChecks(boolean skipChecks) {
3999 this.skipChecks = skipChecks;
4000 }
4001
4002 public boolean isSkipChecks() {
4003 return skipChecks;
4004 }
4005
4006 public void setMerged(boolean isMerged) {
4007 this.isMerged = isMerged;
4008 }
4009
4010 public boolean isMerged() {
4011 return this.isMerged;
4012 }
4013 }
4014
4015 final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
4016 @Override
4017 public int compare(HbckInfo l, HbckInfo r) {
4018 if (l == r) {
4019
4020 return 0;
4021 }
4022
4023 int tableCompare = l.getTableName().compareTo(r.getTableName());
4024 if (tableCompare != 0) {
4025 return tableCompare;
4026 }
4027
4028 int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
4029 l.getStartKey(), r.getStartKey());
4030 if (startComparison != 0) {
4031 return startComparison;
4032 }
4033
4034
4035 byte[] endKey = r.getEndKey();
4036 endKey = (endKey.length == 0) ? null : endKey;
4037 byte[] endKey2 = l.getEndKey();
4038 endKey2 = (endKey2.length == 0) ? null : endKey2;
4039 int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
4040 endKey2, endKey);
4041
4042 if (endComparison != 0) {
4043 return endComparison;
4044 }
4045
4046
4047
4048 if (l.hdfsEntry == null && r.hdfsEntry == null) {
4049 return 0;
4050 }
4051 if (l.hdfsEntry == null && r.hdfsEntry != null) {
4052 return 1;
4053 }
4054
4055 if (r.hdfsEntry == null) {
4056 return -1;
4057 }
4058
4059 return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
4060 }
4061 };
4062
4063
4064
4065
4066 private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
4067 StringBuilder sb = new StringBuilder();
4068 int numOfSkippedRegions;
4069 errors.print("Summary:");
4070 for (TableInfo tInfo : tablesInfo.values()) {
4071 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
4072 skippedRegions.get(tInfo.getName()).size() : 0;
4073
4074 if (errors.tableHasErrors(tInfo)) {
4075 errors.print("Table " + tInfo.getName() + " is inconsistent.");
4076 } else if (numOfSkippedRegions > 0){
4077 errors.print("Table " + tInfo.getName() + " is okay (with "
4078 + numOfSkippedRegions + " skipped regions).");
4079 }
4080 else {
4081 errors.print("Table " + tInfo.getName() + " is okay.");
4082 }
4083 errors.print(" Number of regions: " + tInfo.getNumRegions());
4084 if (numOfSkippedRegions > 0) {
4085 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
4086 System.out.println(" Number of skipped regions: " + numOfSkippedRegions);
4087 System.out.println(" List of skipped regions:");
4088 for(String sr : skippedRegionStrings) {
4089 System.out.println(" " + sr);
4090 }
4091 }
4092 sb.setLength(0);
4093 sb.append(" Deployed on: ");
4094 for (ServerName server : tInfo.deployedOn) {
4095 sb.append(" " + server.toString());
4096 }
4097 errors.print(sb.toString());
4098 }
4099 }
4100
4101 static ErrorReporter getErrorReporter(
4102 final Configuration conf) throws ClassNotFoundException {
4103 Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
4104 return ReflectionUtils.newInstance(reporter, conf);
4105 }
4106
4107 public interface ErrorReporter {
4108 enum ERROR_CODE {
4109 UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
4110 NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
4111 MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
4112 FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
4113 HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
4114 ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
4115 LINGERING_HFILELINK, WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK,
4116 ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR, UNDELETED_REPLICATION_QUEUE, DUPE_ENDKEYS
4117 }
4118 void clear();
4119 void report(String message);
4120 void reportError(String message);
4121 void reportError(ERROR_CODE errorCode, String message);
4122 void reportError(ERROR_CODE errorCode, String message, TableInfo table);
4123 void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
4124 void reportError(
4125 ERROR_CODE errorCode,
4126 String message,
4127 TableInfo table,
4128 HbckInfo info1,
4129 HbckInfo info2
4130 );
4131 int summarize();
4132 void detail(String details);
4133 ArrayList<ERROR_CODE> getErrorList();
4134 void progress();
4135 void print(String message);
4136 void resetErrors();
4137 boolean tableHasErrors(TableInfo table);
4138 }
4139
4140 static class PrintingErrorReporter implements ErrorReporter {
4141 public int errorCount = 0;
4142 private int showProgress;
4143
4144 private static final int progressThreshold = 100;
4145
4146 Set<TableInfo> errorTables = new HashSet<TableInfo>();
4147
4148
4149 private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
4150
4151 @Override
4152 public void clear() {
4153 errorTables.clear();
4154 errorList.clear();
4155 errorCount = 0;
4156 }
4157
4158 @Override
4159 public synchronized void reportError(ERROR_CODE errorCode, String message) {
4160 if (errorCode == ERROR_CODE.WRONG_USAGE) {
4161 System.err.println(message);
4162 return;
4163 }
4164
4165 errorList.add(errorCode);
4166 if (!getSUMMARY()) {
4167 System.out.println("ERROR: " + message);
4168 }
4169 errorCount++;
4170 showProgress = 0;
4171 }
4172
4173 @Override
4174 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
4175 errorTables.add(table);
4176 reportError(errorCode, message);
4177 }
4178
4179 @Override
4180 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4181 HbckInfo info) {
4182 errorTables.add(table);
4183 String reference = "(region " + info.getRegionNameAsString() + ")";
4184 reportError(errorCode, reference + " " + message);
4185 }
4186
4187 @Override
4188 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4189 HbckInfo info1, HbckInfo info2) {
4190 errorTables.add(table);
4191 String reference = "(regions " + info1.getRegionNameAsString()
4192 + " and " + info2.getRegionNameAsString() + ")";
4193 reportError(errorCode, reference + " " + message);
4194 }
4195
4196 @Override
4197 public synchronized void reportError(String message) {
4198 reportError(ERROR_CODE.UNKNOWN, message);
4199 }
4200
4201
4202
4203
4204
4205
4206 @Override
4207 public synchronized void report(String message) {
4208 if (!getSUMMARY()) {
4209 System.out.println("ERROR: " + message);
4210 }
4211 showProgress = 0;
4212 }
4213
4214 @Override
4215 public synchronized int summarize() {
4216 System.out.println(Integer.toString(errorCount) +
4217 " inconsistencies detected.");
4218 if (errorCount == 0) {
4219 System.out.println("Status: OK");
4220 return 0;
4221 } else {
4222 System.out.println("Status: INCONSISTENT");
4223 return -1;
4224 }
4225 }
4226
4227 @Override
4228 public ArrayList<ERROR_CODE> getErrorList() {
4229 return errorList;
4230 }
4231
4232 @Override
4233 public synchronized void print(String message) {
4234 if (!getSUMMARY()) {
4235 System.out.println(message);
4236 }
4237 }
4238
4239 private synchronized static boolean getSUMMARY() {
4240 return SUMMARY;
4241 }
4242
4243 @Override
4244 public boolean tableHasErrors(TableInfo table) {
4245 return errorTables.contains(table);
4246 }
4247
4248 @Override
4249 public void resetErrors() {
4250 errorCount = 0;
4251 }
4252
4253 @Override
4254 public synchronized void detail(String message) {
4255 if (details) {
4256 System.out.println(message);
4257 }
4258 showProgress = 0;
4259 }
4260
4261 @Override
4262 public synchronized void progress() {
4263 if (showProgress++ == progressThreshold) {
4264 if (!getSUMMARY()) {
4265 System.out.print(".");
4266 }
4267 showProgress = 0;
4268 }
4269 }
4270 }
4271
4272
4273
4274
4275 static class WorkItemRegion implements Callable<Void> {
4276 private HBaseFsck hbck;
4277 private ServerName rsinfo;
4278 private ErrorReporter errors;
4279 private HConnection connection;
4280
4281 WorkItemRegion(HBaseFsck hbck, ServerName info,
4282 ErrorReporter errors, HConnection connection) {
4283 this.hbck = hbck;
4284 this.rsinfo = info;
4285 this.errors = errors;
4286 this.connection = connection;
4287 }
4288
4289 @Override
4290 public synchronized Void call() throws IOException {
4291 errors.progress();
4292 try {
4293 BlockingInterface server = connection.getAdmin(rsinfo);
4294
4295
4296 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4297 regions = filterRegions(regions);
4298
4299 if (details) {
4300 errors.detail("RegionServer: " + rsinfo.getServerName() +
4301 " number of regions: " + regions.size());
4302 for (HRegionInfo rinfo: regions) {
4303 errors.detail(" " + rinfo.getRegionNameAsString() +
4304 " id: " + rinfo.getRegionId() +
4305 " encoded_name: " + rinfo.getEncodedName() +
4306 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4307 " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4308 }
4309 }
4310
4311
4312 for (HRegionInfo r:regions) {
4313 HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4314 hbi.addServer(r, rsinfo);
4315 }
4316 } catch (IOException e) {
4317 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4318 " Unable to fetch region information. " + e);
4319 throw e;
4320 }
4321 return null;
4322 }
4323
4324 private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4325 List<HRegionInfo> ret = Lists.newArrayList();
4326 for (HRegionInfo hri : regions) {
4327 if (hri.isMetaTable() || (!hbck.checkMetaOnly
4328 && hbck.isTableIncluded(hri.getTable()))) {
4329 ret.add(hri);
4330 }
4331 }
4332 return ret;
4333 }
4334 }
4335
4336
4337
4338
4339
4340 class WorkItemHdfsDir implements Callable<Void> {
4341 private FileStatus tableDir;
4342 private ErrorReporter errors;
4343 private FileSystem fs;
4344
4345 WorkItemHdfsDir(FileSystem fs, ErrorReporter errors,
4346 FileStatus status) {
4347 this.fs = fs;
4348 this.tableDir = status;
4349 this.errors = errors;
4350 }
4351
4352 @Override
4353 public synchronized Void call() throws InterruptedException, ExecutionException {
4354 final Vector<Exception> exceptions = new Vector<Exception>();
4355
4356 try {
4357 final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4358 final List<Future<?>> futures = new ArrayList<Future<?>>(regionDirs.length);
4359
4360 for (final FileStatus regionDir : regionDirs) {
4361 errors.progress();
4362 final String encodedName = regionDir.getPath().getName();
4363
4364 if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
4365 continue;
4366 }
4367
4368 if (!exceptions.isEmpty()) {
4369 break;
4370 }
4371
4372 futures.add(executor.submit(new Runnable() {
4373 @Override
4374 public void run() {
4375 try {
4376 LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4377
4378 Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
4379 boolean regioninfoFileExists = fs.exists(regioninfoFile);
4380
4381 if (!regioninfoFileExists) {
4382
4383
4384 if (!fs.exists(regionDir.getPath())) {
4385 LOG.warn("By the time we tried to process this region dir it was already gone: "
4386 + regionDir.getPath());
4387 return;
4388 }
4389 }
4390
4391 HbckInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName);
4392 HdfsEntry he = new HdfsEntry();
4393 synchronized (hbi) {
4394 if (hbi.getHdfsRegionDir() != null) {
4395 errors.print("Directory " + encodedName + " duplicate??" +
4396 hbi.getHdfsRegionDir());
4397 }
4398
4399 he.hdfsRegionDir = regionDir.getPath();
4400 he.hdfsRegionDirModTime = regionDir.getModificationTime();
4401 he.hdfsRegioninfoFilePresent = regioninfoFileExists;
4402
4403
4404
4405
4406 he.hdfsOnlyEdits = true;
4407 FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4408 Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4409 for (FileStatus subDir : subDirs) {
4410 errors.progress();
4411 String sdName = subDir.getPath().getName();
4412 if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4413 he.hdfsOnlyEdits = false;
4414 break;
4415 }
4416 }
4417 hbi.hdfsEntry = he;
4418 }
4419 } catch (Exception e) {
4420 LOG.error("Could not load region dir", e);
4421 exceptions.add(e);
4422 }
4423 }
4424 }));
4425 }
4426
4427
4428 for (Future<?> f : futures) {
4429 if (!exceptions.isEmpty()) {
4430 break;
4431 }
4432 try {
4433 f.get();
4434 } catch (ExecutionException e) {
4435 LOG.error("Unexpected exec exception! Should've been caught already. (Bug?)", e);
4436
4437 };
4438 }
4439 } catch (IOException e) {
4440 LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e);
4441 exceptions.add(e);
4442 } finally {
4443 if (!exceptions.isEmpty()) {
4444 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4445 + tableDir.getPath().getName()
4446 + " Unable to fetch all HDFS region information. ");
4447
4448
4449 throw new ExecutionException("First exception in WorkItemHdfsDir", exceptions.firstElement());
4450 }
4451 }
4452 return null;
4453 }
4454 }
4455
4456
4457
4458
4459
4460 static class WorkItemHdfsRegionInfo implements Callable<Void> {
4461 private HbckInfo hbi;
4462 private HBaseFsck hbck;
4463 private ErrorReporter errors;
4464
4465 WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4466 this.hbi = hbi;
4467 this.hbck = hbck;
4468 this.errors = errors;
4469 }
4470
4471 @Override
4472 public synchronized Void call() throws IOException {
4473
4474 if (hbi.getHdfsHRI() == null) {
4475 try {
4476 errors.progress();
4477 hbck.loadHdfsRegioninfo(hbi);
4478 } catch (IOException ioe) {
4479 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4480 + hbi.getTableName() + " in hdfs dir "
4481 + hbi.getHdfsRegionDir()
4482 + "! It may be an invalid format or version file. Treating as "
4483 + "an orphaned regiondir.";
4484 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4485 try {
4486 hbck.debugLsr(hbi.getHdfsRegionDir());
4487 } catch (IOException ioe2) {
4488 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4489 throw ioe2;
4490 }
4491 hbck.orphanHdfsDirs.add(hbi);
4492 throw ioe;
4493 }
4494 }
4495 return null;
4496 }
4497 };
4498
4499
4500
4501
4502
4503 public static void setDisplayFullReport() {
4504 details = true;
4505 }
4506
4507
4508
4509
4510 public static void setForceExclusive() {
4511 forceExclusive = true;
4512 }
4513
4514
4515
4516
4517 public boolean isExclusive() {
4518 return fixAny || forceExclusive;
4519 }
4520
4521
4522
4523
4524
4525 synchronized static void setSummary() {
4526 SUMMARY = true;
4527 }
4528
4529
4530
4531
4532
4533 void setCheckMetaOnly() {
4534 checkMetaOnly = true;
4535 }
4536
4537
4538
4539
4540 void setRegionBoundariesCheck() {
4541 checkRegionBoundaries = true;
4542 }
4543
4544
4545
4546
4547
4548 public void setFixTableLocks(boolean shouldFix) {
4549 fixTableLocks = shouldFix;
4550 fixAny |= shouldFix;
4551 }
4552
4553
4554
4555
4556 public void setFixReplication(boolean shouldFix) {
4557 fixReplication = shouldFix;
4558 fixAny |= shouldFix;
4559 }
4560
4561
4562
4563
4564
4565 public void setFixTableZNodes(boolean shouldFix) {
4566 fixTableZNodes = shouldFix;
4567 fixAny |= shouldFix;
4568 }
4569
4570
4571
4572
4573
4574
4575
4576 void setShouldRerun() {
4577 rerun = true;
4578 }
4579
4580 boolean shouldRerun() {
4581 return rerun;
4582 }
4583
4584
4585
4586
4587
4588 public void setFixAssignments(boolean shouldFix) {
4589 fixAssignments = shouldFix;
4590 fixAny |= shouldFix;
4591 }
4592
4593 boolean shouldFixAssignments() {
4594 return fixAssignments;
4595 }
4596
4597 public void setFixMeta(boolean shouldFix) {
4598 fixMeta = shouldFix;
4599 fixAny |= shouldFix;
4600 }
4601
4602 boolean shouldFixMeta() {
4603 return fixMeta;
4604 }
4605
4606 public void setFixEmptyMetaCells(boolean shouldFix) {
4607 fixEmptyMetaCells = shouldFix;
4608 fixAny |= shouldFix;
4609 }
4610
4611 boolean shouldFixEmptyMetaCells() {
4612 return fixEmptyMetaCells;
4613 }
4614
4615 public void setCheckHdfs(boolean checking) {
4616 checkHdfs = checking;
4617 }
4618
4619 boolean shouldCheckHdfs() {
4620 return checkHdfs;
4621 }
4622
4623 public void setFixHdfsHoles(boolean shouldFix) {
4624 fixHdfsHoles = shouldFix;
4625 fixAny |= shouldFix;
4626 }
4627
4628 boolean shouldFixHdfsHoles() {
4629 return fixHdfsHoles;
4630 }
4631
4632 public void setFixTableOrphans(boolean shouldFix) {
4633 fixTableOrphans = shouldFix;
4634 fixAny |= shouldFix;
4635 }
4636
4637 boolean shouldFixTableOrphans() {
4638 return fixTableOrphans;
4639 }
4640
4641 public void setFixHdfsOverlaps(boolean shouldFix) {
4642 fixHdfsOverlaps = shouldFix;
4643 fixAny |= shouldFix;
4644 }
4645
4646 boolean shouldFixHdfsOverlaps() {
4647 return fixHdfsOverlaps;
4648 }
4649
4650 public void setFixHdfsOrphans(boolean shouldFix) {
4651 fixHdfsOrphans = shouldFix;
4652 fixAny |= shouldFix;
4653 }
4654
4655 boolean shouldFixHdfsOrphans() {
4656 return fixHdfsOrphans;
4657 }
4658
4659 public void setFixVersionFile(boolean shouldFix) {
4660 fixVersionFile = shouldFix;
4661 fixAny |= shouldFix;
4662 }
4663
4664 public boolean shouldFixVersionFile() {
4665 return fixVersionFile;
4666 }
4667
4668 public void setSidelineBigOverlaps(boolean sbo) {
4669 this.sidelineBigOverlaps = sbo;
4670 }
4671
4672 public boolean shouldSidelineBigOverlaps() {
4673 return sidelineBigOverlaps;
4674 }
4675
4676 public void setFixSplitParents(boolean shouldFix) {
4677 fixSplitParents = shouldFix;
4678 fixAny |= shouldFix;
4679 }
4680
4681 public void setRemoveParents(boolean shouldFix) {
4682 removeParents = shouldFix;
4683 fixAny |= shouldFix;
4684 }
4685
4686 boolean shouldFixSplitParents() {
4687 return fixSplitParents;
4688 }
4689
4690 boolean shouldRemoveParents() {
4691 return removeParents;
4692 }
4693
4694 public void setFixReferenceFiles(boolean shouldFix) {
4695 fixReferenceFiles = shouldFix;
4696 fixAny |= shouldFix;
4697 }
4698
4699 boolean shouldFixReferenceFiles() {
4700 return fixReferenceFiles;
4701 }
4702
4703 public void setFixHFileLinks(boolean shouldFix) {
4704 fixHFileLinks = shouldFix;
4705 fixAny |= shouldFix;
4706 }
4707
4708 boolean shouldFixHFileLinks() {
4709 return fixHFileLinks;
4710 }
4711
4712 public boolean shouldIgnorePreCheckPermission() {
4713 return !fixAny || ignorePreCheckPermission;
4714 }
4715
4716 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4717 this.ignorePreCheckPermission = ignorePreCheckPermission;
4718 }
4719
4720
4721
4722
4723 public void setMaxMerge(int mm) {
4724 this.maxMerge = mm;
4725 }
4726
4727 public int getMaxMerge() {
4728 return maxMerge;
4729 }
4730
4731 public void setMaxOverlapsToSideline(int mo) {
4732 this.maxOverlapsToSideline = mo;
4733 }
4734
4735 public int getMaxOverlapsToSideline() {
4736 return maxOverlapsToSideline;
4737 }
4738
4739
4740
4741
4742
4743 boolean isTableIncluded(TableName table) {
4744 return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4745 }
4746
4747 public void includeTable(TableName table) {
4748 tablesIncluded.add(table);
4749 }
4750
4751 Set<TableName> getIncludedTables() {
4752 return new HashSet<TableName>(tablesIncluded);
4753 }
4754
4755
4756
4757
4758
4759
4760 public void setTimeLag(long seconds) {
4761 timelag = seconds * 1000;
4762 }
4763
4764
4765
4766
4767
4768 public void setSidelineDir(String sidelineDir) {
4769 this.sidelineDir = new Path(sidelineDir);
4770 }
4771
4772 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4773 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4774 }
4775
4776 public HFileCorruptionChecker getHFilecorruptionChecker() {
4777 return hfcc;
4778 }
4779
4780 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4781 this.hfcc = hfcc;
4782 }
4783
4784 public void setRetCode(int code) {
4785 this.retcode = code;
4786 }
4787
4788 public int getRetCode() {
4789 return retcode;
4790 }
4791
4792 protected HBaseFsck printUsageAndExit() {
4793 StringWriter sw = new StringWriter(2048);
4794 PrintWriter out = new PrintWriter(sw);
4795 out.println("Usage: fsck [opts] {only tables}");
4796 out.println(" where [opts] are:");
4797 out.println(" -help Display help options (this)");
4798 out.println(" -details Display full report of all regions.");
4799 out.println(" -timelag <timeInSeconds> Process only regions that " +
4800 " have not experienced any metadata updates in the last " +
4801 " <timeInSeconds> seconds.");
4802 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4803 " before checking if the fix worked if run with -fix");
4804 out.println(" -summary Print only summary of the tables and status.");
4805 out.println(" -metaonly Only check the state of the hbase:meta table.");
4806 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4807 out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
4808 out.println(" -exclusive Abort if another hbck is exclusive or fixing.");
4809
4810 out.println("");
4811 out.println(" Metadata Repair options: (expert features, use with caution!)");
4812 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity");
4813 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");
4814 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");
4815 out.println(" -noHdfsChecking Don't load/check region info from HDFS."
4816 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4817 out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
4818 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");
4819 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4820 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
4821 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
4822 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4823 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
4824 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4825 out.println(" -fixSplitParents Try to force offline split parents to be online.");
4826 out.println(" -removeParents Try to offline and sideline lingering parents and keep daughter regions.");
4827 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
4828 out.println(" -fixReferenceFiles Try to offline lingering reference store files");
4829 out.println(" -fixHFileLinks Try to offline lingering HFileLinks");
4830 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"
4831 + " (empty REGIONINFO_QUALIFIER rows)");
4832
4833 out.println("");
4834 out.println(" Datafile Repair options: (expert features, use with caution!)");
4835 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");
4836 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");
4837
4838 out.println("");
4839 out.println(" Metadata Repair shortcuts");
4840 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4841 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4842 "-fixReferenceFiles -fixHFileLinks -fixTableLocks -fixOrphanedTableZnodes");
4843
4844 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4845
4846 out.println("");
4847 out.println(" Table lock options");
4848 out.println(" -fixTableLocks Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4849
4850 out.println("");
4851 out.println(" Table Znode options");
4852 out.println(" -fixOrphanedTableZnodes Set table state in ZNode to disabled if table does not exists");
4853
4854 out.println("");
4855 out.println(" Replication options");
4856 out.println(" -fixReplication Deletes replication queues for removed peers");
4857
4858 out.flush();
4859 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4860
4861 setRetCode(-2);
4862 return this;
4863 }
4864
4865
4866
4867
4868
4869
4870
4871 public static void main(String[] args) throws Exception {
4872
4873 Configuration conf = HBaseConfiguration.create();
4874 Path hbasedir = FSUtils.getRootDir(conf);
4875 URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4876 FSUtils.setFsDefault(conf, new Path(defaultFs));
4877 int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4878 System.exit(ret);
4879 }
4880
4881
4882
4883
4884 static class HBaseFsckTool extends Configured implements Tool {
4885 HBaseFsckTool(Configuration conf) { super(conf); }
4886 @Override
4887 public int run(String[] args) throws Exception {
4888 HBaseFsck hbck = new HBaseFsck(getConf());
4889 hbck.exec(hbck.executor, args);
4890 hbck.close();
4891 return hbck.getRetCode();
4892 }
4893 };
4894
4895
4896 public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4897 ServiceException, InterruptedException {
4898 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4899
4900 boolean checkCorruptHFiles = false;
4901 boolean sidelineCorruptHFiles = false;
4902
4903
4904 for (int i = 0; i < args.length; i++) {
4905 String cmd = args[i];
4906 if (cmd.equals("-help") || cmd.equals("-h")) {
4907 return printUsageAndExit();
4908 } else if (cmd.equals("-details")) {
4909 setDisplayFullReport();
4910 } else if (cmd.equals("-exclusive")) {
4911 setForceExclusive();
4912 } else if (cmd.equals("-timelag")) {
4913 if (i == args.length - 1) {
4914 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4915 return printUsageAndExit();
4916 }
4917 try {
4918 long timelag = Long.parseLong(args[i+1]);
4919 setTimeLag(timelag);
4920 } catch (NumberFormatException e) {
4921 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4922 return printUsageAndExit();
4923 }
4924 i++;
4925 } else if (cmd.equals("-sleepBeforeRerun")) {
4926 if (i == args.length - 1) {
4927 errors.reportError(ERROR_CODE.WRONG_USAGE,
4928 "HBaseFsck: -sleepBeforeRerun needs a value.");
4929 return printUsageAndExit();
4930 }
4931 try {
4932 sleepBeforeRerun = Long.parseLong(args[i+1]);
4933 } catch (NumberFormatException e) {
4934 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4935 return printUsageAndExit();
4936 }
4937 i++;
4938 } else if (cmd.equals("-sidelineDir")) {
4939 if (i == args.length - 1) {
4940 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4941 return printUsageAndExit();
4942 }
4943 i++;
4944 setSidelineDir(args[i]);
4945 } else if (cmd.equals("-fix")) {
4946 errors.reportError(ERROR_CODE.WRONG_USAGE,
4947 "This option is deprecated, please use -fixAssignments instead.");
4948 setFixAssignments(true);
4949 } else if (cmd.equals("-fixAssignments")) {
4950 setFixAssignments(true);
4951 } else if (cmd.equals("-fixMeta")) {
4952 setFixMeta(true);
4953 } else if (cmd.equals("-noHdfsChecking")) {
4954 setCheckHdfs(false);
4955 } else if (cmd.equals("-fixHdfsHoles")) {
4956 setFixHdfsHoles(true);
4957 } else if (cmd.equals("-fixHdfsOrphans")) {
4958 setFixHdfsOrphans(true);
4959 } else if (cmd.equals("-fixTableOrphans")) {
4960 setFixTableOrphans(true);
4961 } else if (cmd.equals("-fixHdfsOverlaps")) {
4962 setFixHdfsOverlaps(true);
4963 } else if (cmd.equals("-fixVersionFile")) {
4964 setFixVersionFile(true);
4965 } else if (cmd.equals("-sidelineBigOverlaps")) {
4966 setSidelineBigOverlaps(true);
4967 } else if (cmd.equals("-fixSplitParents")) {
4968 setFixSplitParents(true);
4969 } else if (cmd.equals("-removeParents")) {
4970 setRemoveParents(true);
4971 } else if (cmd.equals("-ignorePreCheckPermission")) {
4972 setIgnorePreCheckPermission(true);
4973 } else if (cmd.equals("-checkCorruptHFiles")) {
4974 checkCorruptHFiles = true;
4975 } else if (cmd.equals("-sidelineCorruptHFiles")) {
4976 sidelineCorruptHFiles = true;
4977 } else if (cmd.equals("-fixReferenceFiles")) {
4978 setFixReferenceFiles(true);
4979 } else if (cmd.equals("-fixHFileLinks")) {
4980 setFixHFileLinks(true);
4981 } else if (cmd.equals("-fixEmptyMetaCells")) {
4982 setFixEmptyMetaCells(true);
4983 } else if (cmd.equals("-repair")) {
4984
4985
4986 setFixHdfsHoles(true);
4987 setFixHdfsOrphans(true);
4988 setFixMeta(true);
4989 setFixAssignments(true);
4990 setFixHdfsOverlaps(true);
4991 setFixVersionFile(true);
4992 setSidelineBigOverlaps(true);
4993 setFixSplitParents(false);
4994 setCheckHdfs(true);
4995 setFixReferenceFiles(true);
4996 setFixHFileLinks(true);
4997 setFixTableLocks(true);
4998 setFixTableZNodes(true);
4999 } else if (cmd.equals("-repairHoles")) {
5000
5001 setFixHdfsHoles(true);
5002 setFixHdfsOrphans(false);
5003 setFixMeta(true);
5004 setFixAssignments(true);
5005 setFixHdfsOverlaps(false);
5006 setSidelineBigOverlaps(false);
5007 setFixSplitParents(false);
5008 setCheckHdfs(true);
5009 } else if (cmd.equals("-maxOverlapsToSideline")) {
5010 if (i == args.length - 1) {
5011 errors.reportError(ERROR_CODE.WRONG_USAGE,
5012 "-maxOverlapsToSideline needs a numeric value argument.");
5013 return printUsageAndExit();
5014 }
5015 try {
5016 int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
5017 setMaxOverlapsToSideline(maxOverlapsToSideline);
5018 } catch (NumberFormatException e) {
5019 errors.reportError(ERROR_CODE.WRONG_USAGE,
5020 "-maxOverlapsToSideline needs a numeric value argument.");
5021 return printUsageAndExit();
5022 }
5023 i++;
5024 } else if (cmd.equals("-maxMerge")) {
5025 if (i == args.length - 1) {
5026 errors.reportError(ERROR_CODE.WRONG_USAGE,
5027 "-maxMerge needs a numeric value argument.");
5028 return printUsageAndExit();
5029 }
5030 try {
5031 int maxMerge = Integer.parseInt(args[i+1]);
5032 setMaxMerge(maxMerge);
5033 } catch (NumberFormatException e) {
5034 errors.reportError(ERROR_CODE.WRONG_USAGE,
5035 "-maxMerge needs a numeric value argument.");
5036 return printUsageAndExit();
5037 }
5038 i++;
5039 } else if (cmd.equals("-summary")) {
5040 setSummary();
5041 } else if (cmd.equals("-metaonly")) {
5042 setCheckMetaOnly();
5043 } else if (cmd.equals("-boundaries")) {
5044 setRegionBoundariesCheck();
5045 } else if (cmd.equals("-fixTableLocks")) {
5046 setFixTableLocks(true);
5047 } else if (cmd.equals("-fixReplication")) {
5048 setFixReplication(true);
5049 } else if (cmd.equals("-fixOrphanedTableZnodes")) {
5050 setFixTableZNodes(true);
5051 } else if (cmd.startsWith("-")) {
5052 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
5053 return printUsageAndExit();
5054 } else {
5055 includeTable(TableName.valueOf(cmd));
5056 errors.print("Allow checking/fixes for table: " + cmd);
5057 }
5058 }
5059
5060 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
5061
5062
5063 try {
5064 preCheckPermission();
5065 } catch (AccessDeniedException ace) {
5066 Runtime.getRuntime().exit(-1);
5067 } catch (IOException ioe) {
5068 Runtime.getRuntime().exit(-1);
5069 }
5070
5071
5072 connect();
5073
5074 try {
5075
5076 if (checkCorruptHFiles || sidelineCorruptHFiles) {
5077 LOG.info("Checking all hfiles for corruption");
5078 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
5079 setHFileCorruptionChecker(hfcc);
5080 Collection<TableName> tables = getIncludedTables();
5081 Collection<Path> tableDirs = new ArrayList<Path>();
5082 Path rootdir = FSUtils.getRootDir(getConf());
5083 if (tables.size() > 0) {
5084 for (TableName t : tables) {
5085 tableDirs.add(FSUtils.getTableDir(rootdir, t));
5086 }
5087 } else {
5088 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
5089 }
5090 hfcc.checkTables(tableDirs);
5091 hfcc.report(errors);
5092 }
5093
5094
5095 int code = onlineHbck();
5096 setRetCode(code);
5097
5098
5099
5100
5101 if (shouldRerun()) {
5102 try {
5103 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
5104 Thread.sleep(sleepBeforeRerun);
5105 } catch (InterruptedException ie) {
5106 LOG.warn("Interrupted while sleeping");
5107 return this;
5108 }
5109
5110 setFixAssignments(false);
5111 setFixMeta(false);
5112 setFixHdfsHoles(false);
5113 setFixHdfsOverlaps(false);
5114 setFixVersionFile(false);
5115 setFixTableOrphans(false);
5116 errors.resetErrors();
5117 code = onlineHbck();
5118 setRetCode(code);
5119 }
5120 } finally {
5121 IOUtils.closeQuietly(this);
5122 }
5123 return this;
5124 }
5125
5126
5127
5128
5129 void debugLsr(Path p) throws IOException {
5130 debugLsr(getConf(), p, errors);
5131 }
5132
5133
5134
5135
5136 public static void debugLsr(Configuration conf,
5137 Path p) throws IOException {
5138 debugLsr(conf, p, new PrintingErrorReporter());
5139 }
5140
5141
5142
5143
5144 public static void debugLsr(Configuration conf,
5145 Path p, ErrorReporter errors) throws IOException {
5146 if (!LOG.isDebugEnabled() || p == null) {
5147 return;
5148 }
5149 FileSystem fs = p.getFileSystem(conf);
5150
5151 if (!fs.exists(p)) {
5152
5153 return;
5154 }
5155 errors.print(p.toString());
5156
5157 if (fs.isFile(p)) {
5158 return;
5159 }
5160
5161 if (fs.getFileStatus(p).isDirectory()) {
5162 FileStatus[] fss= fs.listStatus(p);
5163 for (FileStatus status : fss) {
5164 debugLsr(conf, status.getPath(), errors);
5165 }
5166 }
5167 }
5168 }