1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotEquals;
27 import static org.junit.Assert.assertNotNull;
28 import static org.junit.Assert.assertTrue;
29 import static org.junit.Assert.fail;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.HashMap;
36 import java.util.HashSet;
37 import java.util.LinkedList;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Random;
41 import java.util.Map.Entry;
42 import java.util.NavigableMap;
43 import java.util.Set;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.CountDownLatch;
46 import java.util.concurrent.ExecutorService;
47 import java.util.concurrent.Executors;
48 import java.util.concurrent.Future;
49 import java.util.concurrent.ScheduledThreadPoolExecutor;
50 import java.util.concurrent.SynchronousQueue;
51 import java.util.concurrent.ThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.atomic.AtomicBoolean;
54
55 import org.apache.commons.io.IOUtils;
56 import org.apache.commons.logging.Log;
57 import org.apache.commons.logging.LogFactory;
58 import org.apache.hadoop.conf.Configuration;
59 import org.apache.hadoop.fs.FileStatus;
60 import org.apache.hadoop.fs.FileSystem;
61 import org.apache.hadoop.fs.Path;
62 import org.apache.hadoop.hbase.ClusterStatus;
63 import org.apache.hadoop.hbase.HBaseTestingUtility;
64 import org.apache.hadoop.hbase.HColumnDescriptor;
65 import org.apache.hadoop.hbase.HConstants;
66 import org.apache.hadoop.hbase.HRegionInfo;
67 import org.apache.hadoop.hbase.HRegionLocation;
68 import org.apache.hadoop.hbase.HTableDescriptor;
69 import org.apache.hadoop.hbase.TableExistsException;
70 import org.apache.hadoop.hbase.testclassification.LargeTests;
71 import org.apache.hadoop.hbase.MiniHBaseCluster;
72 import org.apache.hadoop.hbase.RegionLocations;
73 import org.apache.hadoop.hbase.ServerName;
74 import org.apache.hadoop.hbase.TableName;
75 import org.apache.hadoop.hbase.MetaTableAccessor;
76 import org.apache.hadoop.hbase.client.Admin;
77 import org.apache.hadoop.hbase.client.ClusterConnection;
78 import org.apache.hadoop.hbase.client.Connection;
79 import org.apache.hadoop.hbase.client.ConnectionFactory;
80 import org.apache.hadoop.hbase.client.Delete;
81 import org.apache.hadoop.hbase.client.Durability;
82 import org.apache.hadoop.hbase.client.Get;
83 import org.apache.hadoop.hbase.client.HBaseAdmin;
84 import org.apache.hadoop.hbase.client.HConnection;
85 import org.apache.hadoop.hbase.client.HTable;
86 import org.apache.hadoop.hbase.client.MetaScanner;
87 import org.apache.hadoop.hbase.client.Mutation;
88 import org.apache.hadoop.hbase.client.Put;
89 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
90 import org.apache.hadoop.hbase.client.Result;
91 import org.apache.hadoop.hbase.client.ResultScanner;
92 import org.apache.hadoop.hbase.client.RowMutations;
93 import org.apache.hadoop.hbase.client.Scan;
94 import org.apache.hadoop.hbase.client.Table;
95 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
96 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
97 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
98 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
99 import org.apache.hadoop.hbase.io.hfile.TestHFile;
100 import org.apache.hadoop.hbase.master.AssignmentManager;
101 import org.apache.hadoop.hbase.master.HMaster;
102 import org.apache.hadoop.hbase.master.RegionState;
103 import org.apache.hadoop.hbase.master.RegionStates;
104 import org.apache.hadoop.hbase.master.TableLockManager;
105 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
106 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
107 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
108 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
109 import org.apache.hadoop.hbase.regionserver.HRegion;
110 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
111 import org.apache.hadoop.hbase.regionserver.HRegionServer;
112 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl;
113 import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
114 import org.apache.hadoop.hbase.testclassification.LargeTests;
115 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
116 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
117 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
118 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
119 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
122 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
123 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
124 import org.apache.zookeeper.KeeperException;
125 import org.junit.AfterClass;
126 import org.junit.Assert;
127 import org.junit.Before;
128 import org.junit.BeforeClass;
129 import org.junit.Ignore;
130 import org.junit.Test;
131 import org.junit.experimental.categories.Category;
132 import org.junit.rules.TestName;
133
134 import com.google.common.collect.Multimap;
135
136
137
138
139 @Category(LargeTests.class)
140 public class TestHBaseFsck {
141 static final int POOL_SIZE = 7;
142
143 final static Log LOG = LogFactory.getLog(TestHBaseFsck.class);
144 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
145 private final static Configuration conf = TEST_UTIL.getConfiguration();
146 private final static String FAM_STR = "fam";
147 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
148 private final static int REGION_ONLINE_TIMEOUT = 800;
149 private static RegionStates regionStates;
150 private static ExecutorService tableExecutorService;
151 private static ScheduledThreadPoolExecutor hbfsckExecutorService;
152 private static ClusterConnection connection;
153 private static Admin admin;
154
155
156 private HTable tbl;
157 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
158 Bytes.toBytes("B"), Bytes.toBytes("C") };
159
160 private final static byte[][] ROWKEYS= new byte[][] {
161 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
162 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
163
164 @BeforeClass
165 public static void setUpBeforeClass() throws Exception {
166 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
167 MasterSyncObserver.class.getName());
168
169 conf.setInt("hbase.regionserver.handler.count", 2);
170 conf.setInt("hbase.regionserver.metahandler.count", 30);
171
172 conf.setInt("hbase.htable.threads.max", POOL_SIZE);
173 conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
174 conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
175 conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
176 conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
177 TEST_UTIL.startMiniCluster(3);
178
179 tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
180 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
181
182 hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
183
184 AssignmentManager assignmentManager =
185 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
186 regionStates = assignmentManager.getRegionStates();
187
188 connection = (ClusterConnection) TEST_UTIL.getConnection();
189
190 admin = connection.getAdmin();
191 admin.setBalancerRunning(false, true);
192
193 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
194 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
195 }
196
197 @AfterClass
198 public static void tearDownAfterClass() throws Exception {
199 tableExecutorService.shutdown();
200 hbfsckExecutorService.shutdown();
201 admin.close();
202 TEST_UTIL.shutdownMiniCluster();
203 }
204
205 @Before
206 public void setUp() {
207 EnvironmentEdgeManager.reset();
208 }
209
210 @Test (timeout=180000)
211 public void testHBaseFsck() throws Exception {
212 assertNoErrors(doFsck(conf, false));
213 TableName table = TableName.valueOf("tableBadMetaAssign");
214 HTableDescriptor desc = new HTableDescriptor(table);
215 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
216 desc.addFamily(hcd);
217 createTable(TEST_UTIL, desc, null);
218
219
220 assertNoErrors(doFsck(conf, false));
221
222
223
224 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
225 Scan scan = new Scan();
226 scan.setStartRow(Bytes.toBytes(table+",,"));
227 ResultScanner scanner = meta.getScanner(scan);
228 HRegionInfo hri = null;
229
230 Result res = scanner.next();
231 ServerName currServer =
232 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
233 HConstants.SERVER_QUALIFIER));
234 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
235 HConstants.STARTCODE_QUALIFIER));
236
237 for (JVMClusterUtil.RegionServerThread rs :
238 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
239
240 ServerName sn = rs.getRegionServer().getServerName();
241
242
243 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
244 startCode != sn.getStartcode()) {
245 Put put = new Put(res.getRow());
246 put.setDurability(Durability.SKIP_WAL);
247 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
248 Bytes.toBytes(sn.getHostAndPort()));
249 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
250 Bytes.toBytes(sn.getStartcode()));
251 meta.put(put);
252 hri = MetaTableAccessor.getHRegionInfo(res);
253 break;
254 }
255 }
256
257
258 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
259 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
260
261 TEST_UTIL.getHBaseCluster().getMaster()
262 .getAssignmentManager().waitForAssignment(hri);
263
264
265 assertNoErrors(doFsck(conf, false));
266
267
268 Table t = connection.getTable(table, tableExecutorService);
269 ResultScanner s = t.getScanner(new Scan());
270 s.close();
271 t.close();
272
273 scanner.close();
274 meta.close();
275 }
276
277 @Test(timeout=180000)
278 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
279 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
280 admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO);
281 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
282 new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper());
283 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
284 HBaseFsck hbck = doFsck(conf, true);
285 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
286 ERROR_CODE.NULL_META_REGION });
287 assertNoErrors(doFsck(conf, false));
288 }
289
290
291
292
293 private HRegionInfo createRegion(final HTableDescriptor
294 htd, byte[] startKey, byte[] endKey)
295 throws IOException {
296 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
297 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
298 MetaTableAccessor.addRegionToMeta(meta, hri);
299 meta.close();
300 return hri;
301 }
302
303
304
305
306 private void dumpMeta(TableName tableName) throws IOException {
307 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
308 for (byte[] row : metaRows) {
309 LOG.info(Bytes.toString(row));
310 }
311 }
312
313
314
315
316
317 private void undeployRegion(Connection conn, ServerName sn,
318 HRegionInfo hri) throws IOException, InterruptedException {
319 try {
320 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri);
321 if (!hri.isMetaTable()) {
322 admin.offline(hri.getRegionName());
323 }
324 } catch (IOException ioe) {
325 LOG.warn("Got exception when attempting to offline region "
326 + Bytes.toString(hri.getRegionName()), ioe);
327 }
328 }
329
330
331
332
333
334
335 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
336 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
337 boolean hdfs) throws IOException, InterruptedException {
338 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, HRegionInfo.DEFAULT_REPLICA_ID);
339 }
340
341
342
343
344
345
346
347
348
349 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
350 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
351 boolean hdfs, boolean regionInfoOnly, int replicaId)
352 throws IOException, InterruptedException {
353 LOG.info("** Before delete:");
354 dumpMeta(htd.getTableName());
355
356 List<HRegionLocation> locations = tbl.getAllRegionLocations();
357 for (HRegionLocation location : locations) {
358 HRegionInfo hri = location.getRegionInfo();
359 ServerName hsa = location.getServerName();
360 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
361 && Bytes.compareTo(hri.getEndKey(), endKey) == 0
362 && hri.getReplicaId() == replicaId) {
363
364 LOG.info("RegionName: " +hri.getRegionNameAsString());
365 byte[] deleteRow = hri.getRegionName();
366
367 if (unassign) {
368 LOG.info("Undeploying region " + hri + " from server " + hsa);
369 undeployRegion(connection, hsa, hri);
370 }
371
372 if (regionInfoOnly) {
373 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
374 Path rootDir = FSUtils.getRootDir(conf);
375 FileSystem fs = rootDir.getFileSystem(conf);
376 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
377 hri.getEncodedName());
378 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
379 fs.delete(hriPath, true);
380 }
381
382 if (hdfs) {
383 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
384 Path rootDir = FSUtils.getRootDir(conf);
385 FileSystem fs = rootDir.getFileSystem(conf);
386 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
387 hri.getEncodedName());
388 HBaseFsck.debugLsr(conf, p);
389 boolean success = fs.delete(p, true);
390 LOG.info("Deleted " + p + " sucessfully? " + success);
391 HBaseFsck.debugLsr(conf, p);
392 }
393
394 if (metaRow) {
395 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
396 Delete delete = new Delete(deleteRow);
397 meta.delete(delete);
398 }
399 }
400 }
401 LOG.info(hri.toString() + hsa.toString());
402 }
403
404 TEST_UTIL.getMetaTableRows(htd.getTableName());
405 LOG.info("*** After delete:");
406 dumpMeta(htd.getTableName());
407 }
408
409
410
411
412
413
414
415
416
417
418 void setupTable(TableName tablename) throws Exception {
419 setupTableWithRegionReplica(tablename, 1);
420 }
421
422
423
424
425
426
427
428
429
430
431 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
432 HTableDescriptor desc = new HTableDescriptor(tablename);
433 desc.setRegionReplication(replicaCount);
434 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
435 desc.addFamily(hcd);
436 createTable(TEST_UTIL, desc, SPLITS);
437
438 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
439 List<Put> puts = new ArrayList<Put>();
440 for (byte[] row : ROWKEYS) {
441 Put p = new Put(row);
442 p.add(FAM, Bytes.toBytes("val"), row);
443 puts.add(p);
444 }
445 tbl.put(puts);
446 tbl.flushCommits();
447 }
448
449
450
451
452 int countRows() throws IOException {
453 Scan s = new Scan();
454 ResultScanner rs = tbl.getScanner(s);
455 int i = 0;
456 while(rs.next() !=null) {
457 i++;
458 }
459 return i;
460 }
461
462
463
464
465
466
467
468 void cleanupTable(TableName tablename) throws Exception {
469 if (tbl != null) {
470 tbl.close();
471 tbl = null;
472 }
473
474 ((ClusterConnection) connection).clearRegionCache();
475 deleteTable(TEST_UTIL, tablename);
476 }
477
478
479
480
481 @Test (timeout=180000)
482 public void testHBaseFsckClean() throws Exception {
483 assertNoErrors(doFsck(conf, false));
484 TableName table = TableName.valueOf("tableClean");
485 try {
486 HBaseFsck hbck = doFsck(conf, false);
487 assertNoErrors(hbck);
488
489 setupTable(table);
490 assertEquals(ROWKEYS.length, countRows());
491
492
493 hbck = doFsck(conf, false);
494 assertNoErrors(hbck);
495 assertEquals(0, hbck.getOverlapGroups(table).size());
496 assertEquals(ROWKEYS.length, countRows());
497 } finally {
498 cleanupTable(table);
499 }
500 }
501
502
503
504
505 @Test (timeout=180000)
506 public void testHbckThreadpooling() throws Exception {
507 TableName table =
508 TableName.valueOf("tableDupeStartKey");
509 try {
510
511 setupTable(table);
512
513
514 Configuration newconf = new Configuration(conf);
515 newconf.setInt("hbasefsck.numthreads", 1);
516 assertNoErrors(doFsck(newconf, false));
517
518
519 } finally {
520 cleanupTable(table);
521 }
522 }
523
524 @Test (timeout=180000)
525 public void testHbckFixOrphanTable() throws Exception {
526 TableName table = TableName.valueOf("tableInfo");
527 FileSystem fs = null;
528 Path tableinfo = null;
529 try {
530 setupTable(table);
531
532 Path hbaseTableDir = FSUtils.getTableDir(
533 FSUtils.getRootDir(conf), table);
534 fs = hbaseTableDir.getFileSystem(conf);
535 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
536 tableinfo = status.getPath();
537 fs.rename(tableinfo, new Path("/.tableinfo"));
538
539
540 HBaseFsck hbck = doFsck(conf, false);
541 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
542
543
544 hbck = doFsck(conf, true);
545 assertNoErrors(hbck);
546 status = null;
547 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
548 assertNotNull(status);
549
550 HTableDescriptor htd = admin.getTableDescriptor(table);
551 htd.setValue("NOT_DEFAULT", "true");
552 admin.disableTable(table);
553 admin.modifyTable(table, htd);
554 admin.enableTable(table);
555 fs.delete(status.getPath(), true);
556
557
558 htd = admin.getTableDescriptor(table);
559 hbck = doFsck(conf, true);
560 assertNoErrors(hbck);
561 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
562 assertNotNull(status);
563 htd = admin.getTableDescriptor(table);
564 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
565 } finally {
566 fs.rename(new Path("/.tableinfo"), tableinfo);
567 cleanupTable(table);
568 }
569 }
570
571
572
573
574
575
576 @Test (timeout=180000)
577 public void testParallelHbck() throws Exception {
578 final ExecutorService service;
579 final Future<HBaseFsck> hbck1,hbck2;
580
581 class RunHbck implements Callable<HBaseFsck>{
582 boolean fail = true;
583 @Override
584 public HBaseFsck call(){
585 Configuration c = new Configuration(conf);
586 c.setInt("hbase.hbck.lockfile.attempts", 1);
587
588
589 c.setInt("hbase.hbck.lockfile.maxwaittime", 3);
590 try{
591 return doFsck(c, false);
592 } catch(Exception e){
593 if (e.getMessage().contains("Duplicate hbck")) {
594 fail = false;
595 }
596 }
597
598 if (fail) fail();
599 return null;
600 }
601 }
602 service = Executors.newFixedThreadPool(2);
603 hbck1 = service.submit(new RunHbck());
604 hbck2 = service.submit(new RunHbck());
605 service.shutdown();
606
607 service.awaitTermination(15, TimeUnit.SECONDS);
608 HBaseFsck h1 = hbck1.get();
609 HBaseFsck h2 = hbck2.get();
610
611 assert(h1 == null || h2 == null);
612 if (h1 != null) {
613 assert(h1.getRetCode() >= 0);
614 }
615 if (h2 != null) {
616 assert(h2.getRetCode() >= 0);
617 }
618 }
619
620
621
622
623
624
625
626 @Test (timeout=180000)
627 public void testParallelWithRetriesHbck() throws Exception {
628 final ExecutorService service;
629 final Future<HBaseFsck> hbck1,hbck2;
630
631
632
633
634
635
636
637 final int timeoutInSeconds = 80;
638 final int sleepIntervalInMilliseconds = 200;
639 final int maxSleepTimeInMilliseconds = 6000;
640 final int maxRetryAttempts = 15;
641
642 class RunHbck implements Callable<HBaseFsck>{
643
644 @Override
645 public HBaseFsck call() throws Exception {
646
647 Configuration c = new Configuration(conf);
648 c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds);
649 c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds);
650 c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds);
651 c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts);
652 return doFsck(c, false);
653 }
654 }
655
656 service = Executors.newFixedThreadPool(2);
657 hbck1 = service.submit(new RunHbck());
658 hbck2 = service.submit(new RunHbck());
659 service.shutdown();
660
661 service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS);
662 HBaseFsck h1 = hbck1.get();
663 HBaseFsck h2 = hbck2.get();
664
665 assertNotNull(h1);
666 assertNotNull(h2);
667 assert(h1.getRetCode() >= 0);
668 assert(h2.getRetCode() >= 0);
669
670 }
671
672
673
674
675
676 @Test (timeout=180000)
677 public void testDupeStartKey() throws Exception {
678 TableName table =
679 TableName.valueOf("tableDupeStartKey");
680 try {
681 setupTable(table);
682 assertNoErrors(doFsck(conf, false));
683 assertEquals(ROWKEYS.length, countRows());
684
685
686 HRegionInfo hriDupe =
687 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("A2"));
688 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
689 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
690 .waitForAssignment(hriDupe);
691 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
692 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
693
694 HBaseFsck hbck = doFsck(conf, false);
695 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
696 ERROR_CODE.DUPE_STARTKEYS});
697 assertEquals(2, hbck.getOverlapGroups(table).size());
698 assertEquals(ROWKEYS.length, countRows());
699
700
701 doFsck(conf,true);
702
703
704 HBaseFsck hbck2 = doFsck(conf,false);
705 assertNoErrors(hbck2);
706 assertEquals(0, hbck2.getOverlapGroups(table).size());
707 assertEquals(ROWKEYS.length, countRows());
708 } finally {
709 cleanupTable(table);
710 }
711 }
712
713
714
715
716
717 @Test (timeout=180000)
718 public void testHbckWithRegionReplica() throws Exception {
719 TableName table =
720 TableName.valueOf("testHbckWithRegionReplica");
721 try {
722 setupTableWithRegionReplica(table, 2);
723 TEST_UTIL.getHBaseAdmin().flush(table.getName());
724 assertNoErrors(doFsck(conf, false));
725 } finally {
726 cleanupTable(table);
727 }
728 }
729
730 @Test
731 public void testHbckWithFewerReplica() throws Exception {
732 TableName table =
733 TableName.valueOf("testHbckWithFewerReplica");
734 try {
735 setupTableWithRegionReplica(table, 2);
736 TEST_UTIL.getHBaseAdmin().flush(table.getName());
737 assertNoErrors(doFsck(conf, false));
738 assertEquals(ROWKEYS.length, countRows());
739 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
740 Bytes.toBytes("C"), true, false, false, false, 1);
741
742 HBaseFsck hbck = doFsck(conf, false);
743 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_DEPLOYED});
744
745 hbck = doFsck(conf, true);
746
747 hbck = doFsck(conf, false);
748 assertErrors(hbck, new ERROR_CODE[]{});
749 } finally {
750 cleanupTable(table);
751 }
752 }
753
754 @Test
755 public void testHbckWithExcessReplica() throws Exception {
756 TableName table =
757 TableName.valueOf("testHbckWithExcessReplica");
758 try {
759 setupTableWithRegionReplica(table, 2);
760 TEST_UTIL.getHBaseAdmin().flush(table.getName());
761 assertNoErrors(doFsck(conf, false));
762 assertEquals(ROWKEYS.length, countRows());
763
764
765
766
767 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
768 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(table);
769 byte[] startKey = Bytes.toBytes("B");
770 byte[] endKey = Bytes.toBytes("C");
771 byte[] metaKey = null;
772 HRegionInfo newHri = null;
773 for (HRegionInfo h : regions) {
774 if (Bytes.compareTo(h.getStartKey(), startKey) == 0 &&
775 Bytes.compareTo(h.getEndKey(), endKey) == 0 &&
776 h.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
777 metaKey = h.getRegionName();
778
779 newHri = RegionReplicaUtil.getRegionInfoForReplica(h, 2);
780 break;
781 }
782 }
783 Put put = new Put(metaKey);
784 ServerName sn = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers()
785 .toArray(new ServerName[0])[0];
786
787 MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, 2);
788 meta.put(put);
789 meta.flushCommits();
790
791 HBaseFsckRepair.fixUnassigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
792 HBaseFsckRepair.waitUntilAssigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
793
794 Delete delete = new Delete(metaKey);
795 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(2));
796 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(2));
797 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(2));
798 meta.delete(delete);
799 meta.flushCommits();
800 meta.close();
801
802 HBaseFsck hbck = doFsck(conf, false);
803 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_IN_META});
804
805 hbck = doFsck(conf, true);
806
807 hbck = doFsck(conf, false);
808 assertErrors(hbck, new ERROR_CODE[]{});
809 } finally {
810 cleanupTable(table);
811 }
812 }
813
814
815
816 Map<ServerName, List<String>> getDeployedHRIs(final HBaseAdmin admin) throws IOException {
817 ClusterStatus status = admin.getClusterStatus();
818 Collection<ServerName> regionServers = status.getServers();
819 Map<ServerName, List<String>> mm =
820 new HashMap<ServerName, List<String>>();
821 for (ServerName hsi : regionServers) {
822 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi);
823
824
825 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
826 List<String> regionNames = new ArrayList<String>();
827 for (HRegionInfo hri : regions) {
828 regionNames.add(hri.getRegionNameAsString());
829 }
830 mm.put(hsi, regionNames);
831 }
832 return mm;
833 }
834
835
836
837
838 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
839 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
840 if (e.getValue().contains(hri.getRegionNameAsString())) {
841 return e.getKey();
842 }
843 }
844 return null;
845 }
846
847
848
849
850
851 @Test (timeout=180000)
852 public void testDupeRegion() throws Exception {
853 TableName table =
854 TableName.valueOf("tableDupeRegion");
855 try {
856 setupTable(table);
857 assertNoErrors(doFsck(conf, false));
858 assertEquals(ROWKEYS.length, countRows());
859
860
861 HRegionInfo hriDupe =
862 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"));
863
864 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
865 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
866 .waitForAssignment(hriDupe);
867 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
868 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
869
870
871
872
873
874 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriDupe) == null) {
875 Thread.sleep(250);
876 }
877
878 LOG.debug("Finished assignment of dupe region");
879
880
881 HBaseFsck hbck = doFsck(conf, false);
882 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
883 ERROR_CODE.DUPE_STARTKEYS});
884 assertEquals(2, hbck.getOverlapGroups(table).size());
885 assertEquals(ROWKEYS.length, countRows());
886
887
888 doFsck(conf,true);
889
890
891 HBaseFsck hbck2 = doFsck(conf,false);
892 assertNoErrors(hbck2);
893 assertEquals(0, hbck2.getOverlapGroups(table).size());
894 assertEquals(ROWKEYS.length, countRows());
895 } finally {
896 cleanupTable(table);
897 }
898 }
899
900
901
902
903 @Test (timeout=180000)
904 public void testDegenerateRegions() throws Exception {
905 TableName table = TableName.valueOf("tableDegenerateRegions");
906 try {
907 setupTable(table);
908 assertNoErrors(doFsck(conf,false));
909 assertEquals(ROWKEYS.length, countRows());
910
911
912 HRegionInfo hriDupe =
913 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B"));
914 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
915 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
916 .waitForAssignment(hriDupe);
917 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
918 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
919
920 HBaseFsck hbck = doFsck(conf,false);
921 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, ERROR_CODE.DUPE_STARTKEYS,
922 ERROR_CODE.DUPE_STARTKEYS });
923 assertEquals(2, hbck.getOverlapGroups(table).size());
924 assertEquals(ROWKEYS.length, countRows());
925
926
927 doFsck(conf,true);
928
929
930 HBaseFsck hbck2 = doFsck(conf,false);
931 assertNoErrors(hbck2);
932 assertEquals(0, hbck2.getOverlapGroups(table).size());
933 assertEquals(ROWKEYS.length, countRows());
934 } finally {
935 cleanupTable(table);
936 }
937 }
938
939
940
941
942
943 @Test (timeout=180000)
944 public void testContainedRegionOverlap() throws Exception {
945 TableName table =
946 TableName.valueOf("tableContainedRegionOverlap");
947 try {
948 setupTable(table);
949 assertEquals(ROWKEYS.length, countRows());
950
951
952 HRegionInfo hriOverlap =
953 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
954 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
955 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
956 .waitForAssignment(hriOverlap);
957 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
958 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
959
960 HBaseFsck hbck = doFsck(conf, false);
961 assertErrors(hbck, new ERROR_CODE[] {
962 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
963 assertEquals(2, hbck.getOverlapGroups(table).size());
964 assertEquals(ROWKEYS.length, countRows());
965
966
967 doFsck(conf, true);
968
969
970 HBaseFsck hbck2 = doFsck(conf,false);
971 assertNoErrors(hbck2);
972 assertEquals(0, hbck2.getOverlapGroups(table).size());
973 assertEquals(ROWKEYS.length, countRows());
974 } finally {
975 cleanupTable(table);
976 }
977 }
978
979
980
981
982
983
984
985 @Test (timeout=180000)
986 public void testSidelineOverlapRegion() throws Exception {
987 TableName table =
988 TableName.valueOf("testSidelineOverlapRegion");
989 try {
990 setupTable(table);
991 assertEquals(ROWKEYS.length, countRows());
992
993
994 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
995 HMaster master = cluster.getMaster();
996 HRegionInfo hriOverlap1 =
997 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB"));
998 master.assignRegion(hriOverlap1);
999 master.getAssignmentManager().waitForAssignment(hriOverlap1);
1000 HRegionInfo hriOverlap2 =
1001 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("AB"), Bytes.toBytes("B"));
1002 master.assignRegion(hriOverlap2);
1003 master.getAssignmentManager().waitForAssignment(hriOverlap2);
1004
1005 HBaseFsck hbck = doFsck(conf, false);
1006 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
1007 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
1008 assertEquals(3, hbck.getOverlapGroups(table).size());
1009 assertEquals(ROWKEYS.length, countRows());
1010
1011
1012 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
1013 ServerName serverName = null;
1014 byte[] regionName = null;
1015 for (HbckInfo hbi: overlapGroups.values()) {
1016 if ("A".equals(Bytes.toString(hbi.getStartKey()))
1017 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
1018 regionName = hbi.getRegionName();
1019
1020
1021 int k = cluster.getServerWith(regionName);
1022 for (int i = 0; i < 3; i++) {
1023 if (i != k) {
1024 HRegionServer rs = cluster.getRegionServer(i);
1025 serverName = rs.getServerName();
1026 break;
1027 }
1028 }
1029
1030 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) connection,
1031 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
1032 admin.offline(regionName);
1033 break;
1034 }
1035 }
1036
1037 assertNotNull(regionName);
1038 assertNotNull(serverName);
1039 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
1040 Put put = new Put(regionName);
1041 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
1042 Bytes.toBytes(serverName.getHostAndPort()));
1043 meta.put(put);
1044 }
1045
1046
1047 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1048 fsck.connect();
1049 fsck.setDisplayFullReport();
1050 fsck.setTimeLag(0);
1051 fsck.setFixAssignments(true);
1052 fsck.setFixMeta(true);
1053 fsck.setFixHdfsHoles(true);
1054 fsck.setFixHdfsOverlaps(true);
1055 fsck.setFixHdfsOrphans(true);
1056 fsck.setFixVersionFile(true);
1057 fsck.setSidelineBigOverlaps(true);
1058 fsck.setMaxMerge(2);
1059 fsck.onlineHbck();
1060 fsck.close();
1061
1062
1063
1064 HBaseFsck hbck2 = doFsck(conf,false);
1065 assertNoErrors(hbck2);
1066 assertEquals(0, hbck2.getOverlapGroups(table).size());
1067 assertTrue(ROWKEYS.length > countRows());
1068 } finally {
1069 cleanupTable(table);
1070 }
1071 }
1072
1073
1074
1075
1076
1077 @Test (timeout=180000)
1078 public void testOverlapAndOrphan() throws Exception {
1079 TableName table =
1080 TableName.valueOf("tableOverlapAndOrphan");
1081 try {
1082 setupTable(table);
1083 assertEquals(ROWKEYS.length, countRows());
1084
1085
1086 admin.disableTable(table);
1087 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1088 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1089 TEST_UTIL.getHBaseAdmin().enableTable(table);
1090
1091 HRegionInfo hriOverlap =
1092 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1093 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1094 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1095 .waitForAssignment(hriOverlap);
1096 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1097 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1098
1099 HBaseFsck hbck = doFsck(conf, false);
1100 assertErrors(hbck, new ERROR_CODE[] {
1101 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1102 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1103
1104
1105 doFsck(conf, true);
1106
1107
1108 HBaseFsck hbck2 = doFsck(conf,false);
1109 assertNoErrors(hbck2);
1110 assertEquals(0, hbck2.getOverlapGroups(table).size());
1111 assertEquals(ROWKEYS.length, countRows());
1112 } finally {
1113 cleanupTable(table);
1114 }
1115 }
1116
1117
1118
1119
1120
1121
1122 @Test (timeout=180000)
1123 public void testCoveredStartKey() throws Exception {
1124 TableName table =
1125 TableName.valueOf("tableCoveredStartKey");
1126 try {
1127 setupTable(table);
1128 assertEquals(ROWKEYS.length, countRows());
1129
1130
1131 HRegionInfo hriOverlap =
1132 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2"));
1133 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1134 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1135 .waitForAssignment(hriOverlap);
1136 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1137 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1138
1139 HBaseFsck hbck = doFsck(conf, false);
1140 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
1141 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1142 assertEquals(3, hbck.getOverlapGroups(table).size());
1143 assertEquals(ROWKEYS.length, countRows());
1144
1145
1146 doFsck(conf, true);
1147
1148
1149 HBaseFsck hbck2 = doFsck(conf, false);
1150 assertErrors(hbck2, new ERROR_CODE[0]);
1151 assertEquals(0, hbck2.getOverlapGroups(table).size());
1152 assertEquals(ROWKEYS.length, countRows());
1153 } finally {
1154 cleanupTable(table);
1155 }
1156 }
1157
1158
1159
1160
1161
1162 @Test (timeout=180000)
1163 public void testRegionHole() throws Exception {
1164 TableName table =
1165 TableName.valueOf("tableRegionHole");
1166 try {
1167 setupTable(table);
1168 assertEquals(ROWKEYS.length, countRows());
1169
1170
1171 admin.disableTable(table);
1172 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1173 Bytes.toBytes("C"), true, true, true);
1174 admin.enableTable(table);
1175
1176 HBaseFsck hbck = doFsck(conf, false);
1177 assertErrors(hbck, new ERROR_CODE[] {
1178 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1179
1180 assertEquals(0, hbck.getOverlapGroups(table).size());
1181
1182
1183 doFsck(conf, true);
1184
1185
1186 assertNoErrors(doFsck(conf,false));
1187 assertEquals(ROWKEYS.length - 2 , countRows());
1188 } finally {
1189 cleanupTable(table);
1190 }
1191 }
1192
1193
1194
1195
1196
1197 @Test (timeout=180000)
1198 public void testHDFSRegioninfoMissing() throws Exception {
1199 TableName table = TableName.valueOf("tableHDFSRegioninfoMissing");
1200 try {
1201 setupTable(table);
1202 assertEquals(ROWKEYS.length, countRows());
1203
1204
1205 admin.disableTable(table);
1206 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1207 Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1208 TEST_UTIL.getHBaseAdmin().enableTable(table);
1209
1210 HBaseFsck hbck = doFsck(conf, false);
1211 assertErrors(hbck, new ERROR_CODE[] {
1212 ERROR_CODE.ORPHAN_HDFS_REGION,
1213 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1214 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1215
1216 assertEquals(0, hbck.getOverlapGroups(table).size());
1217
1218
1219 doFsck(conf, true);
1220
1221
1222 assertNoErrors(doFsck(conf, false));
1223 assertEquals(ROWKEYS.length, countRows());
1224 } finally {
1225 cleanupTable(table);
1226 }
1227 }
1228
1229
1230
1231
1232
1233 @Test (timeout=180000)
1234 public void testNotInMetaOrDeployedHole() throws Exception {
1235 TableName table =
1236 TableName.valueOf("tableNotInMetaOrDeployedHole");
1237 try {
1238 setupTable(table);
1239 assertEquals(ROWKEYS.length, countRows());
1240
1241
1242 admin.disableTable(table);
1243 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1244 Bytes.toBytes("C"), true, true, false);
1245 admin.enableTable(table);
1246
1247 HBaseFsck hbck = doFsck(conf, false);
1248 assertErrors(hbck, new ERROR_CODE[] {
1249 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1250
1251 assertEquals(0, hbck.getOverlapGroups(table).size());
1252
1253
1254 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1255 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1256
1257
1258 assertNoErrors(doFsck(conf,false));
1259 assertEquals(ROWKEYS.length, countRows());
1260 } finally {
1261 cleanupTable(table);
1262 }
1263 }
1264
1265
1266
1267
1268 @Test (timeout=180000)
1269 public void testNotInMetaHole() throws Exception {
1270 TableName table =
1271 TableName.valueOf("tableNotInMetaHole");
1272 try {
1273 setupTable(table);
1274 assertEquals(ROWKEYS.length, countRows());
1275
1276
1277 admin.disableTable(table);
1278 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1279 Bytes.toBytes("C"), false, true, false);
1280 admin.enableTable(table);
1281
1282 HBaseFsck hbck = doFsck(conf, false);
1283 assertErrors(hbck, new ERROR_CODE[] {
1284 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1285
1286 assertEquals(0, hbck.getOverlapGroups(table).size());
1287
1288
1289 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1290 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1291
1292
1293 assertNoErrors(doFsck(conf,false));
1294 assertEquals(ROWKEYS.length, countRows());
1295 } finally {
1296 cleanupTable(table);
1297 }
1298 }
1299
1300
1301
1302
1303
1304 @Test (timeout=180000)
1305 public void testNotInHdfs() throws Exception {
1306 TableName table =
1307 TableName.valueOf("tableNotInHdfs");
1308 try {
1309 setupTable(table);
1310 assertEquals(ROWKEYS.length, countRows());
1311
1312
1313 admin.flush(table);
1314
1315
1316 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1317 Bytes.toBytes("C"), false, false, true);
1318
1319 HBaseFsck hbck = doFsck(conf, false);
1320 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1321
1322 assertEquals(0, hbck.getOverlapGroups(table).size());
1323
1324
1325 doFsck(conf, true);
1326
1327
1328 assertNoErrors(doFsck(conf,false));
1329 assertEquals(ROWKEYS.length - 2, countRows());
1330 } finally {
1331 cleanupTable(table);
1332 }
1333 }
1334
1335
1336
1337
1338
1339 @Test (timeout=180000)
1340 public void testNotInHdfsWithReplicas() throws Exception {
1341 TableName table =
1342 TableName.valueOf("tableNotInHdfs");
1343 HBaseAdmin admin = new HBaseAdmin(conf);
1344 try {
1345 HRegionInfo[] oldHris = new HRegionInfo[2];
1346 setupTableWithRegionReplica(table, 2);
1347 assertEquals(ROWKEYS.length, countRows());
1348 NavigableMap<HRegionInfo, ServerName> map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(),
1349 tbl.getName());
1350 int i = 0;
1351
1352 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1353 if (m.getKey().getStartKey().length > 0 &&
1354 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1355 LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
1356 oldHris[i++] = m.getKey();
1357 }
1358 }
1359
1360 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1361
1362
1363 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1364 Bytes.toBytes("C"), false, false, true);
1365
1366 HBaseFsck hbck = doFsck(conf, false);
1367 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1368
1369
1370 doFsck(conf, true);
1371
1372
1373 assertNoErrors(doFsck(conf,false));
1374 assertEquals(ROWKEYS.length - 2, countRows());
1375
1376
1377
1378 i = 0;
1379 HRegionInfo[] newHris = new HRegionInfo[2];
1380
1381 map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
1382
1383 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1384 if (m.getKey().getStartKey().length > 0 &&
1385 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1386 newHris[i++] = m.getKey();
1387 }
1388 }
1389
1390 Collection<ServerName> servers = admin.getClusterStatus().getServers();
1391 Set<HRegionInfo> onlineRegions = new HashSet<HRegionInfo>();
1392 for (ServerName s : servers) {
1393 List<HRegionInfo> list = admin.getOnlineRegions(s);
1394 onlineRegions.addAll(list);
1395 }
1396
1397 assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
1398
1399
1400 assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
1401 } finally {
1402 cleanupTable(table);
1403 admin.close();
1404 }
1405 }
1406
1407
1408
1409
1410
1411
1412 @Test (timeout=180000)
1413 public void testNoHdfsTable() throws Exception {
1414 TableName table = TableName.valueOf("NoHdfsTable");
1415 setupTable(table);
1416 assertEquals(ROWKEYS.length, countRows());
1417
1418
1419 admin.flush(table);
1420
1421
1422 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1423 Bytes.toBytes("A"), false, false, true);
1424 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1425 Bytes.toBytes("B"), false, false, true);
1426 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1427 Bytes.toBytes("C"), false, false, true);
1428 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1429 Bytes.toBytes(""), false, false, true);
1430
1431
1432 deleteTableDir(table);
1433
1434 HBaseFsck hbck = doFsck(conf, false);
1435 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1436 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1437 ERROR_CODE.NOT_IN_HDFS,});
1438
1439 assertEquals(0, hbck.getOverlapGroups(table).size());
1440
1441
1442 doFsck(conf, true);
1443
1444
1445 assertNoErrors(doFsck(conf,false));
1446 assertFalse("Table " + table + " should have been deleted", admin.tableExists(table));
1447 }
1448
1449 public void deleteTableDir(TableName table) throws IOException {
1450 Path rootDir = FSUtils.getRootDir(conf);
1451 FileSystem fs = rootDir.getFileSystem(conf);
1452 Path p = FSUtils.getTableDir(rootDir, table);
1453 HBaseFsck.debugLsr(conf, p);
1454 boolean success = fs.delete(p, true);
1455 LOG.info("Deleted " + p + " sucessfully? " + success);
1456 }
1457
1458
1459
1460
1461 @Test (timeout=180000)
1462 public void testNoVersionFile() throws Exception {
1463
1464 Path rootDir = FSUtils.getRootDir(conf);
1465 FileSystem fs = rootDir.getFileSystem(conf);
1466 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1467 fs.delete(versionFile, true);
1468
1469
1470 HBaseFsck hbck = doFsck(conf, false);
1471 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1472
1473 doFsck(conf, true);
1474
1475
1476 assertNoErrors(doFsck(conf, false));
1477 }
1478
1479
1480
1481
1482 @Test (timeout=180000)
1483 public void testRegionShouldNotBeDeployed() throws Exception {
1484 TableName table =
1485 TableName.valueOf("tableRegionShouldNotBeDeployed");
1486 try {
1487 LOG.info("Starting testRegionShouldNotBeDeployed.");
1488 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1489 assertTrue(cluster.waitForActiveAndReadyMaster());
1490
1491
1492 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1493 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1494 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1495 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1496
1497
1498 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1499 fstd.createTableDescriptor(htdDisabled);
1500 List<HRegionInfo> disabledRegions =
1501 TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
1502
1503
1504 HRegionServer hrs = cluster.getRegionServer(0);
1505
1506
1507 admin.disableTable(table);
1508 admin.enableTable(table);
1509
1510
1511 admin.disableTable(table);
1512 HRegionInfo region = disabledRegions.remove(0);
1513 byte[] regionName = region.getRegionName();
1514
1515
1516 assertTrue(cluster.getServerWith(regionName) == -1);
1517
1518
1519
1520
1521
1522 HRegion r = HRegion.openHRegion(
1523 region, htdDisabled, hrs.getWAL(region), conf);
1524 hrs.addToOnlineRegions(r);
1525
1526 HBaseFsck hbck = doFsck(conf, false);
1527 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1528
1529
1530 doFsck(conf, true);
1531
1532
1533 assertNoErrors(doFsck(conf, false));
1534 } finally {
1535 admin.enableTable(table);
1536 cleanupTable(table);
1537 }
1538 }
1539
1540
1541
1542
1543 @Test (timeout=180000)
1544 public void testFixByTable() throws Exception {
1545 TableName table1 =
1546 TableName.valueOf("testFixByTable1");
1547 TableName table2 =
1548 TableName.valueOf("testFixByTable2");
1549 try {
1550 setupTable(table1);
1551
1552 admin.flush(table1);
1553
1554 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1555 Bytes.toBytes("C"), false, false, true);
1556
1557 setupTable(table2);
1558
1559 admin.flush(table2);
1560
1561 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1562 Bytes.toBytes("C"), false, false, true);
1563
1564 HBaseFsck hbck = doFsck(conf, false);
1565 assertErrors(hbck, new ERROR_CODE[] {
1566 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1567
1568
1569 doFsck(conf, true, table1);
1570
1571 assertNoErrors(doFsck(conf, false, table1));
1572
1573 assertErrors(doFsck(conf, false, table2),
1574 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1575
1576
1577 doFsck(conf, true, table2);
1578
1579 assertNoErrors(doFsck(conf, false));
1580 assertEquals(ROWKEYS.length - 2, countRows());
1581 } finally {
1582 cleanupTable(table1);
1583 cleanupTable(table2);
1584 }
1585 }
1586
1587
1588
1589 @Test (timeout=180000)
1590 public void testLingeringSplitParent() throws Exception {
1591 TableName table =
1592 TableName.valueOf("testLingeringSplitParent");
1593 Table meta = null;
1594 try {
1595 setupTable(table);
1596 assertEquals(ROWKEYS.length, countRows());
1597
1598
1599 admin.flush(table);
1600 HRegionLocation location = tbl.getRegionLocation("B");
1601
1602
1603 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1604 Bytes.toBytes("C"), true, true, false);
1605
1606
1607 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1608 HRegionInfo hri = location.getRegionInfo();
1609
1610 HRegionInfo a = new HRegionInfo(tbl.getName(),
1611 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1612 HRegionInfo b = new HRegionInfo(tbl.getName(),
1613 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1614
1615 hri.setOffline(true);
1616 hri.setSplit(true);
1617
1618 MetaTableAccessor.addRegionToMeta(meta, hri, a, b);
1619 meta.close();
1620 admin.flush(TableName.META_TABLE_NAME);
1621
1622 HBaseFsck hbck = doFsck(conf, false);
1623 assertErrors(hbck, new ERROR_CODE[] {
1624 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1625
1626
1627 hbck = doFsck(conf, true);
1628 assertErrors(hbck, new ERROR_CODE[] {
1629 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1630 assertFalse(hbck.shouldRerun());
1631 hbck = doFsck(conf, false);
1632 assertErrors(hbck, new ERROR_CODE[] {
1633 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1634
1635
1636 hbck = new HBaseFsck(conf, hbfsckExecutorService);
1637 hbck.connect();
1638 hbck.setDisplayFullReport();
1639 hbck.setTimeLag(0);
1640 hbck.setFixSplitParents(true);
1641 hbck.onlineHbck();
1642 assertTrue(hbck.shouldRerun());
1643 hbck.close();
1644
1645 Get get = new Get(hri.getRegionName());
1646 Result result = meta.get(get);
1647 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1648 HConstants.SPLITA_QUALIFIER).isEmpty());
1649 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1650 HConstants.SPLITB_QUALIFIER).isEmpty());
1651 admin.flush(TableName.META_TABLE_NAME);
1652
1653
1654 doFsck(conf, true);
1655
1656
1657 assertNoErrors(doFsck(conf, false));
1658 assertEquals(ROWKEYS.length, countRows());
1659 } finally {
1660 cleanupTable(table);
1661 IOUtils.closeQuietly(meta);
1662 }
1663 }
1664
1665
1666
1667
1668
1669 @Test (timeout=180000)
1670 public void testValidLingeringSplitParent() throws Exception {
1671 TableName table =
1672 TableName.valueOf("testLingeringSplitParent");
1673 Table meta = null;
1674 try {
1675 setupTable(table);
1676 assertEquals(ROWKEYS.length, countRows());
1677
1678
1679 admin.flush(table);
1680 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1681
1682 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1683 HRegionInfo hri = location.getRegionInfo();
1684
1685
1686 byte[] regionName = location.getRegionInfo().getRegionName();
1687 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1688 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1689
1690
1691
1692
1693 HBaseFsck hbck = doFsck(
1694 conf, true, true, false, false, false, true, true, true, false, false, false, null);
1695 assertErrors(hbck, new ERROR_CODE[] {});
1696
1697
1698 Get get = new Get(hri.getRegionName());
1699 Result result = meta.get(get);
1700 assertNotNull(result);
1701 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1702
1703 assertEquals(ROWKEYS.length, countRows());
1704
1705
1706 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1707 assertNoErrors(doFsck(conf, false));
1708 } finally {
1709 cleanupTable(table);
1710 IOUtils.closeQuietly(meta);
1711 }
1712 }
1713
1714
1715
1716
1717
1718 @Test(timeout=75000)
1719 public void testSplitDaughtersNotInMeta() throws Exception {
1720 TableName table = TableName.valueOf("testSplitdaughtersNotInMeta");
1721 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1722 try {
1723 setupTable(table);
1724 assertEquals(ROWKEYS.length, countRows());
1725
1726
1727 admin.flush(table);
1728 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1729
1730 HRegionInfo hri = location.getRegionInfo();
1731
1732
1733 byte[] regionName = location.getRegionInfo().getRegionName();
1734 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1735 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1736
1737 PairOfSameType<HRegionInfo> daughters =
1738 MetaTableAccessor.getDaughterRegions(meta.get(new Get(regionName)));
1739
1740
1741 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
1742 undeployRegion(connection, hris.get(daughters.getFirst()), daughters.getFirst());
1743 undeployRegion(connection, hris.get(daughters.getSecond()), daughters.getSecond());
1744
1745 List<Delete> deletes = new ArrayList<>();
1746 deletes.add(new Delete(daughters.getFirst().getRegionName()));
1747 deletes.add(new Delete(daughters.getSecond().getRegionName()));
1748 meta.delete(deletes);
1749
1750
1751 RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().
1752 getAssignmentManager().getRegionStates();
1753 regionStates.deleteRegion(daughters.getFirst());
1754 regionStates.deleteRegion(daughters.getSecond());
1755
1756 HBaseFsck hbck = doFsck(conf, false);
1757 assertErrors(hbck,
1758 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1759 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1760
1761
1762 hbck = doFsck(
1763 conf, true, true, false, false, false, false, false, false, false, false, false, null);
1764 assertErrors(hbck,
1765 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1766 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1767
1768
1769 Get get = new Get(hri.getRegionName());
1770 Result result = meta.get(get);
1771 assertNotNull(result);
1772 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1773
1774 assertEquals(ROWKEYS.length, countRows());
1775
1776
1777 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1778 assertNoErrors(doFsck(conf, false));
1779 } finally {
1780 meta.close();
1781 cleanupTable(table);
1782 }
1783 }
1784
1785
1786
1787
1788
1789 @Test(timeout=120000)
1790 public void testMissingFirstRegion() throws Exception {
1791 TableName table = TableName.valueOf("testMissingFirstRegion");
1792 try {
1793 setupTable(table);
1794 assertEquals(ROWKEYS.length, countRows());
1795
1796
1797 admin.disableTable(table);
1798 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
1799 true, true);
1800 admin.enableTable(table);
1801
1802 HBaseFsck hbck = doFsck(conf, false);
1803 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
1804
1805 doFsck(conf, true);
1806
1807 assertNoErrors(doFsck(conf, false));
1808 } finally {
1809 cleanupTable(table);
1810 }
1811 }
1812
1813
1814
1815
1816
1817 @Test(timeout=120000)
1818 public void testRegionDeployedNotInHdfs() throws Exception {
1819 TableName table =
1820 TableName.valueOf("testSingleRegionDeployedNotInHdfs");
1821 try {
1822 setupTable(table);
1823 admin.flush(table);
1824
1825
1826 deleteRegion(conf, tbl.getTableDescriptor(),
1827 HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false,
1828 false, true);
1829
1830 HBaseFsck hbck = doFsck(conf, false);
1831 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
1832
1833 doFsck(conf, true);
1834
1835 assertNoErrors(doFsck(conf, false));
1836 } finally {
1837 cleanupTable(table);
1838 }
1839 }
1840
1841
1842
1843
1844
1845 @Test(timeout=120000)
1846 public void testMissingLastRegion() throws Exception {
1847 TableName table =
1848 TableName.valueOf("testMissingLastRegion");
1849 try {
1850 setupTable(table);
1851 assertEquals(ROWKEYS.length, countRows());
1852
1853
1854 admin.disableTable(table);
1855 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
1856 true, true);
1857 admin.enableTable(table);
1858
1859 HBaseFsck hbck = doFsck(conf, false);
1860 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
1861
1862 doFsck(conf, true);
1863
1864 assertNoErrors(doFsck(conf, false));
1865 } finally {
1866 cleanupTable(table);
1867 }
1868 }
1869
1870
1871
1872
1873 @Test (timeout=180000)
1874 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
1875 TableName table =
1876 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
1877 try {
1878 setupTable(table);
1879 assertEquals(ROWKEYS.length, countRows());
1880
1881
1882 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1883 Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
1884
1885
1886 HBaseFsck hbck = doFsck(conf, false);
1887 assertErrors(hbck, new ERROR_CODE[] {
1888 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1889
1890
1891 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1892 fsck.connect();
1893 fsck.setDisplayFullReport();
1894 fsck.setTimeLag(0);
1895 fsck.setCheckHdfs(false);
1896 fsck.onlineHbck();
1897 assertErrors(fsck, new ERROR_CODE[] {
1898 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1899 fsck.close();
1900
1901
1902 fsck = new HBaseFsck(conf, hbfsckExecutorService);
1903 fsck.connect();
1904 fsck.setDisplayFullReport();
1905 fsck.setTimeLag(0);
1906 fsck.setCheckHdfs(false);
1907 fsck.setFixAssignments(true);
1908 fsck.onlineHbck();
1909 assertTrue(fsck.shouldRerun());
1910 fsck.onlineHbck();
1911 assertNoErrors(fsck);
1912
1913 assertEquals(ROWKEYS.length, countRows());
1914
1915 fsck.close();
1916 } finally {
1917 cleanupTable(table);
1918 }
1919 }
1920
1921
1922
1923
1924
1925
1926 @Test (timeout=180000)
1927 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
1928 TableName table =
1929 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
1930 try {
1931 setupTable(table);
1932 assertEquals(ROWKEYS.length, countRows());
1933
1934
1935 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1936 Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
1937
1938
1939 HBaseFsck hbck = doFsck(conf, false);
1940 assertErrors(hbck,
1941 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1942
1943
1944 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1945 fsck.connect();
1946 fsck.setDisplayFullReport();
1947 fsck.setTimeLag(0);
1948 fsck.setCheckHdfs(false);
1949 fsck.onlineHbck();
1950 assertErrors(fsck,
1951 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1952 fsck.close();
1953
1954
1955 fsck = new HBaseFsck(conf, hbfsckExecutorService);
1956 fsck.connect();
1957 fsck.setDisplayFullReport();
1958 fsck.setTimeLag(0);
1959 fsck.setCheckHdfs(false);
1960 fsck.setFixAssignments(true);
1961 fsck.setFixMeta(true);
1962 fsck.onlineHbck();
1963 assertFalse(fsck.shouldRerun());
1964 assertErrors(fsck,
1965 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1966 fsck.close();
1967
1968
1969 fsck = doFsck(conf, true);
1970 assertTrue(fsck.shouldRerun());
1971 fsck = doFsck(conf, true);
1972 assertNoErrors(fsck);
1973 } finally {
1974 cleanupTable(table);
1975 }
1976 }
1977
1978
1979
1980
1981
1982 @Test (timeout=180000)
1983 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
1984 TableName table =
1985 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
1986 try {
1987 setupTable(table);
1988 assertEquals(ROWKEYS.length, countRows());
1989
1990
1991 admin.disableTable(table);
1992 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1993 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1994 TEST_UTIL.getHBaseAdmin().enableTable(table);
1995
1996 HRegionInfo hriOverlap =
1997 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1998 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1999 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
2000 .waitForAssignment(hriOverlap);
2001 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
2002 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
2003
2004 HBaseFsck hbck = doFsck(conf, false);
2005 assertErrors(hbck, new ERROR_CODE[] {
2006 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2007 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2008
2009
2010 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2011 fsck.connect();
2012 fsck.setDisplayFullReport();
2013 fsck.setTimeLag(0);
2014 fsck.setCheckHdfs(false);
2015 fsck.onlineHbck();
2016 assertErrors(fsck, new ERROR_CODE[] {
2017 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2018 fsck.close();
2019
2020
2021 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2022 fsck.connect();
2023 fsck.setDisplayFullReport();
2024 fsck.setTimeLag(0);
2025 fsck.setCheckHdfs(false);
2026 fsck.setFixHdfsHoles(true);
2027 fsck.setFixHdfsOverlaps(true);
2028 fsck.setFixHdfsOrphans(true);
2029 fsck.onlineHbck();
2030 assertFalse(fsck.shouldRerun());
2031 assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN});
2032 fsck.close();
2033 } finally {
2034 if (admin.isTableDisabled(table)) {
2035 admin.enableTable(table);
2036 }
2037 cleanupTable(table);
2038 }
2039 }
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
2050 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2051 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2052 Path famDir = new Path(regionDir, FAM_STR);
2053
2054
2055 while (true) {
2056 FileStatus[] hfFss = fs.listStatus(famDir);
2057 if (hfFss.length == 0) {
2058 continue;
2059 }
2060 for (FileStatus hfs : hfFss) {
2061 if (!hfs.isDirectory()) {
2062 return hfs.getPath();
2063 }
2064 }
2065 }
2066 }
2067
2068
2069
2070
2071 @Test(timeout=180000)
2072 public void testQuarantineCorruptHFile() throws Exception {
2073 TableName table = TableName.valueOf(name.getMethodName());
2074 try {
2075 setupTable(table);
2076 assertEquals(ROWKEYS.length, countRows());
2077 admin.flush(table);
2078
2079 FileSystem fs = FileSystem.get(conf);
2080 Path hfile = getFlushedHFile(fs, table);
2081
2082
2083 admin.disableTable(table);
2084
2085
2086 Path corrupt = new Path(hfile.getParent(), "deadbeef");
2087 TestHFile.truncateFile(fs, hfile, corrupt);
2088 LOG.info("Created corrupted file " + corrupt);
2089 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2090
2091
2092 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2093 assertEquals(res.getRetCode(), 0);
2094 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2095 assertEquals(hfcc.getHFilesChecked(), 5);
2096 assertEquals(hfcc.getCorrupted().size(), 1);
2097 assertEquals(hfcc.getFailures().size(), 0);
2098 assertEquals(hfcc.getQuarantined().size(), 1);
2099 assertEquals(hfcc.getMissing().size(), 0);
2100
2101
2102 admin.enableTable(table);
2103 } finally {
2104 cleanupTable(table);
2105 }
2106 }
2107
2108
2109
2110
2111 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
2112 int corrupt, int fail, int quar, int missing) throws Exception {
2113 try {
2114 setupTable(table);
2115 assertEquals(ROWKEYS.length, countRows());
2116 admin.flush(table);
2117
2118
2119 admin.disableTable(table);
2120
2121 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
2122 table.getNameAsString()};
2123 HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
2124
2125 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2126 assertEquals(hfcc.getHFilesChecked(), check);
2127 assertEquals(hfcc.getCorrupted().size(), corrupt);
2128 assertEquals(hfcc.getFailures().size(), fail);
2129 assertEquals(hfcc.getQuarantined().size(), quar);
2130 assertEquals(hfcc.getMissing().size(), missing);
2131
2132
2133 admin.enableTableAsync(table);
2134 while (!admin.isTableEnabled(table)) {
2135 try {
2136 Thread.sleep(250);
2137 } catch (InterruptedException e) {
2138 e.printStackTrace();
2139 fail("Interrupted when trying to enable table " + table);
2140 }
2141 }
2142 } finally {
2143 cleanupTable(table);
2144 }
2145 }
2146
2147
2148
2149
2150
2151 @Test(timeout=180000)
2152 public void testQuarantineMissingHFile() throws Exception {
2153 TableName table = TableName.valueOf(name.getMethodName());
2154
2155
2156 final FileSystem fs = FileSystem.get(conf);
2157 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2158 @Override
2159 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2160 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2161 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2162 @Override
2163 protected void checkHFile(Path p) throws IOException {
2164 if (attemptedFirstHFile.compareAndSet(false, true)) {
2165 assertTrue(fs.delete(p, true));
2166 }
2167 super.checkHFile(p);
2168 }
2169 };
2170 }
2171 };
2172 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
2173 hbck.close();
2174 }
2175
2176
2177
2178
2179
2180
2181
2182 @Ignore @Test(timeout=180000)
2183 public void testQuarantineMissingFamdir() throws Exception {
2184 TableName table = TableName.valueOf(name.getMethodName());
2185
2186 final FileSystem fs = FileSystem.get(conf);
2187 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2188 @Override
2189 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2190 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2191 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2192 @Override
2193 protected void checkColFamDir(Path p) throws IOException {
2194 if (attemptedFirstHFile.compareAndSet(false, true)) {
2195 assertTrue(fs.delete(p, true));
2196 }
2197 super.checkColFamDir(p);
2198 }
2199 };
2200 }
2201 };
2202 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2203 hbck.close();
2204 }
2205
2206
2207
2208
2209
2210 @Test(timeout=180000)
2211 public void testQuarantineMissingRegionDir() throws Exception {
2212 TableName table = TableName.valueOf(name.getMethodName());
2213
2214 final FileSystem fs = FileSystem.get(conf);
2215 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2216 @Override
2217 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
2218 throws IOException {
2219 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2220 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2221 @Override
2222 protected void checkRegionDir(Path p) throws IOException {
2223 if (attemptedFirstHFile.compareAndSet(false, true)) {
2224 assertTrue(fs.delete(p, true));
2225 }
2226 super.checkRegionDir(p);
2227 }
2228 };
2229 }
2230 };
2231 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2232 hbck.close();
2233 }
2234
2235
2236
2237
2238 @Test (timeout=180000)
2239 public void testLingeringReferenceFile() throws Exception {
2240 TableName table =
2241 TableName.valueOf("testLingeringReferenceFile");
2242 try {
2243 setupTable(table);
2244 assertEquals(ROWKEYS.length, countRows());
2245
2246
2247 FileSystem fs = FileSystem.get(conf);
2248 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2249 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2250 Path famDir = new Path(regionDir, FAM_STR);
2251 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
2252 fs.create(fakeReferenceFile);
2253
2254 HBaseFsck hbck = doFsck(conf, false);
2255 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
2256
2257 doFsck(conf, true);
2258
2259 assertNoErrors(doFsck(conf, false));
2260 } finally {
2261 cleanupTable(table);
2262 }
2263 }
2264
2265
2266
2267
2268 @Test (timeout=180000)
2269 public void testMissingRegionInfoQualifier() throws Exception {
2270 Connection connection = ConnectionFactory.createConnection(conf);
2271 TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
2272 try {
2273 setupTable(table);
2274
2275
2276 final List<Delete> deletes = new LinkedList<Delete>();
2277 Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService);
2278 MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
2279
2280 @Override
2281 public boolean processRow(Result rowResult) throws IOException {
2282 HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult);
2283 if (hri != null && !hri.getTable().isSystemTable()) {
2284 Delete delete = new Delete(rowResult.getRow());
2285 delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2286 deletes.add(delete);
2287 }
2288 return true;
2289 }
2290
2291 @Override
2292 public void close() throws IOException {
2293 }
2294 });
2295 meta.delete(deletes);
2296
2297
2298 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2299 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
2300 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2301 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
2302 meta.close();
2303
2304 HBaseFsck hbck = doFsck(conf, false);
2305 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2306
2307
2308 hbck = doFsck(conf, true);
2309
2310
2311 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2312 } finally {
2313 cleanupTable(table);
2314 }
2315 connection.close();
2316 }
2317
2318
2319
2320
2321
2322 @Test (timeout=180000)
2323 public void testErrorReporter() throws Exception {
2324 try {
2325 MockErrorReporter.calledCount = 0;
2326 doFsck(conf, false);
2327 assertEquals(MockErrorReporter.calledCount, 0);
2328
2329 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
2330 doFsck(conf, false);
2331 assertTrue(MockErrorReporter.calledCount > 20);
2332 } finally {
2333 conf.set("hbasefsck.errorreporter",
2334 PrintingErrorReporter.class.getName());
2335 MockErrorReporter.calledCount = 0;
2336 }
2337 }
2338
2339 static class MockErrorReporter implements ErrorReporter {
2340 static int calledCount = 0;
2341
2342 @Override
2343 public void clear() {
2344 calledCount++;
2345 }
2346
2347 @Override
2348 public void report(String message) {
2349 calledCount++;
2350 }
2351
2352 @Override
2353 public void reportError(String message) {
2354 calledCount++;
2355 }
2356
2357 @Override
2358 public void reportError(ERROR_CODE errorCode, String message) {
2359 calledCount++;
2360 }
2361
2362 @Override
2363 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2364 calledCount++;
2365 }
2366
2367 @Override
2368 public void reportError(ERROR_CODE errorCode,
2369 String message, TableInfo table, HbckInfo info) {
2370 calledCount++;
2371 }
2372
2373 @Override
2374 public void reportError(ERROR_CODE errorCode, String message,
2375 TableInfo table, HbckInfo info1, HbckInfo info2) {
2376 calledCount++;
2377 }
2378
2379 @Override
2380 public int summarize() {
2381 return ++calledCount;
2382 }
2383
2384 @Override
2385 public void detail(String details) {
2386 calledCount++;
2387 }
2388
2389 @Override
2390 public ArrayList<ERROR_CODE> getErrorList() {
2391 calledCount++;
2392 return new ArrayList<ERROR_CODE>();
2393 }
2394
2395 @Override
2396 public void progress() {
2397 calledCount++;
2398 }
2399
2400 @Override
2401 public void print(String message) {
2402 calledCount++;
2403 }
2404
2405 @Override
2406 public void resetErrors() {
2407 calledCount++;
2408 }
2409
2410 @Override
2411 public boolean tableHasErrors(TableInfo table) {
2412 calledCount++;
2413 return false;
2414 }
2415 }
2416
2417 @Test(timeout=180000)
2418 public void testCheckTableLocks() throws Exception {
2419 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2420 EnvironmentEdgeManager.injectEdge(edge);
2421
2422 HBaseFsck hbck = doFsck(conf, false);
2423 assertNoErrors(hbck);
2424
2425 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2426 final TableName tableName = TableName.valueOf("foo");
2427
2428
2429 final TableLockManager tableLockManager =
2430 TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2431 TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2432 writeLock.acquire();
2433 hbck = doFsck(conf, false);
2434 assertNoErrors(hbck);
2435
2436 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2437 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2438
2439 hbck = doFsck(conf, false);
2440 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2441
2442 final CountDownLatch latch = new CountDownLatch(1);
2443 new Thread() {
2444 @Override
2445 public void run() {
2446 TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2447 try {
2448 latch.countDown();
2449 readLock.acquire();
2450 } catch (IOException ex) {
2451 fail();
2452 } catch (IllegalStateException ex) {
2453 return;
2454 }
2455 fail("should not have come here");
2456 };
2457 }.start();
2458
2459 latch.await();
2460 Threads.sleep(300);
2461
2462 hbck = doFsck(conf, false);
2463 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2464
2465 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2466 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2467
2468 hbck = doFsck(conf, false);
2469 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2470
2471 conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2472
2473 Threads.sleep(10);
2474 hbck = doFsck(conf, true);
2475
2476 hbck = doFsck(conf, false);
2477 assertNoErrors(hbck);
2478
2479
2480 writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
2481 writeLock.acquire();
2482 writeLock.release();
2483 tableLockManager.tableDeleted(tableName);
2484 }
2485
2486
2487
2488
2489 @Test
2490 public void testOrphanedTableZNode() throws Exception {
2491 TableName table = TableName.valueOf("testOrphanedZKTableEntry");
2492
2493 try {
2494 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
2495 .setTableState(table, ZooKeeperProtos.Table.State.ENABLING);
2496
2497 try {
2498 setupTable(table);
2499 Assert.fail(
2500 "Create table should fail when its ZNode has already existed with ENABLING state.");
2501 } catch(TableExistsException t) {
2502
2503 }
2504
2505 try {
2506 cleanupTable(table);
2507 } catch (IOException e) {
2508
2509
2510 }
2511
2512 HBaseFsck hbck = doFsck(conf, false);
2513 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2514
2515
2516 hbck = doFsck(conf, true);
2517
2518
2519 hbck = doFsck(conf, false);
2520 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2521
2522 setupTable(table);
2523 } finally {
2524
2525
2526 try {
2527 cleanupTable(table);
2528 } catch (IOException e) {
2529
2530
2531 }
2532 }
2533 }
2534
2535 @Test (timeout=180000)
2536 public void testMetaOffline() throws Exception {
2537
2538 HBaseFsck hbck = doFsck(conf, false);
2539 assertNoErrors(hbck);
2540 deleteMetaRegion(conf, true, false, false);
2541 hbck = doFsck(conf, false);
2542
2543
2544 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2545 hbck = doFsck(conf, true);
2546 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2547 hbck = doFsck(conf, false);
2548 assertNoErrors(hbck);
2549 }
2550
2551 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2552 boolean regionInfoOnly) throws IOException, InterruptedException {
2553 HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
2554 .getRegionLocation(HConstants.EMPTY_START_ROW);
2555 ServerName hsa = metaLocation.getServerName();
2556 HRegionInfo hri = metaLocation.getRegionInfo();
2557 if (unassign) {
2558 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2559 try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
2560 undeployRegion(unmanagedConnection, hsa, hri);
2561 }
2562 }
2563
2564 if (regionInfoOnly) {
2565 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2566 Path rootDir = FSUtils.getRootDir(conf);
2567 FileSystem fs = rootDir.getFileSystem(conf);
2568 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2569 hri.getEncodedName());
2570 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2571 fs.delete(hriPath, true);
2572 }
2573
2574 if (hdfs) {
2575 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2576 Path rootDir = FSUtils.getRootDir(conf);
2577 FileSystem fs = rootDir.getFileSystem(conf);
2578 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2579 hri.getEncodedName());
2580 HBaseFsck.debugLsr(conf, p);
2581 boolean success = fs.delete(p, true);
2582 LOG.info("Deleted " + p + " sucessfully? " + success);
2583 HBaseFsck.debugLsr(conf, p);
2584 }
2585 }
2586
2587 @Test (timeout=180000)
2588 public void testTableWithNoRegions() throws Exception {
2589
2590
2591 TableName table =
2592 TableName.valueOf(name.getMethodName());
2593 try {
2594
2595 HTableDescriptor desc = new HTableDescriptor(table);
2596 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
2597 desc.addFamily(hcd);
2598 createTable(TEST_UTIL, desc, null);
2599 tbl = (HTable) connection.getTable(table, tableExecutorService);
2600
2601
2602 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW,
2603 HConstants.EMPTY_END_ROW, false, false, true);
2604
2605 HBaseFsck hbck = doFsck(conf, false);
2606 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2607
2608 doFsck(conf, true);
2609
2610
2611 doFsck(conf, true);
2612
2613
2614 assertNoErrors(doFsck(conf, false));
2615 } finally {
2616 cleanupTable(table);
2617 }
2618
2619 }
2620
2621 @Test (timeout=180000)
2622 public void testHbckAfterRegionMerge() throws Exception {
2623 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
2624 Table meta = null;
2625 try {
2626
2627 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
2628 setupTable(table);
2629 assertEquals(ROWKEYS.length, countRows());
2630
2631
2632 admin.flush(table);
2633 HRegionInfo region1 = tbl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo();
2634 HRegionInfo region2 = tbl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo();
2635
2636 int regionCountBeforeMerge = tbl.getRegionLocations().size();
2637
2638 assertNotEquals(region1, region2);
2639
2640
2641 admin.mergeRegions(region1.getEncodedNameAsBytes(),
2642 region2.getEncodedNameAsBytes(), false);
2643
2644
2645 long timeout = System.currentTimeMillis() + 30 * 1000;
2646 while (true) {
2647 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
2648 break;
2649 } else if (System.currentTimeMillis() > timeout) {
2650 fail("Time out waiting on region " + region1.getEncodedName()
2651 + " and " + region2.getEncodedName() + " be merged");
2652 }
2653 Thread.sleep(10);
2654 }
2655
2656 assertEquals(ROWKEYS.length, countRows());
2657
2658 HBaseFsck hbck = doFsck(conf, false);
2659 assertNoErrors(hbck);
2660
2661 } finally {
2662 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
2663 cleanupTable(table);
2664 IOUtils.closeQuietly(meta);
2665 }
2666 }
2667
2668 @Test (timeout = 180000)
2669 public void testRegionBoundariesCheck() throws Exception {
2670 HBaseFsck hbck = doFsck(conf, false);
2671 assertNoErrors(hbck);
2672 try {
2673 hbck.checkRegionBoundaries();
2674 } catch (IllegalArgumentException e) {
2675 if (e.getMessage().endsWith("not a valid DFS filename.")) {
2676 fail("Table directory path is not valid." + e.getMessage());
2677 }
2678 }
2679 }
2680
2681 @org.junit.Rule
2682 public TestName name = new TestName();
2683
2684 @Test (timeout=180000)
2685 public void testReadOnlyProperty() throws Exception {
2686 HBaseFsck hbck = doFsck(conf, false);
2687 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2688 hbck.shouldIgnorePreCheckPermission());
2689
2690 hbck = doFsck(conf, true);
2691 Assert.assertEquals("shouldIgnorePreCheckPermission", false,
2692 hbck.shouldIgnorePreCheckPermission());
2693
2694 hbck = doFsck(conf, true);
2695 hbck.setIgnorePreCheckPermission(true);
2696 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2697 hbck.shouldIgnorePreCheckPermission());
2698 }
2699
2700 @Test (timeout=180000)
2701 public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
2702 TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit");
2703 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
2704 try {
2705 HTableDescriptor desc = new HTableDescriptor(table);
2706 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
2707 createTable(TEST_UTIL, desc, null);
2708 tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
2709 for (int i = 0; i < 5; i++) {
2710 Put p1 = new Put(("r" + i).getBytes());
2711 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
2712 tbl.put(p1);
2713 }
2714 admin.flush(desc.getTableName());
2715 List<HRegion> regions = cluster.getRegions(desc.getTableName());
2716 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2717 HRegionServer regionServer = cluster.getRegionServer(serverWith);
2718 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2719 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
2720 st.prepare();
2721 st.stepsBeforePONR(regionServer, regionServer, false);
2722 AssignmentManager am = cluster.getMaster().getAssignmentManager();
2723 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
2724 for (RegionState state : regionsInTransition.values()) {
2725 am.regionOffline(state.getRegion());
2726 }
2727 ZKAssign.deleteNodeFailSilent(regionServer.getZooKeeper(), regions.get(0).getRegionInfo());
2728 Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();
2729 regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
2730 am.assign(regionsMap);
2731 am.waitForAssignment(regions.get(0).getRegionInfo());
2732 HBaseFsck hbck = doFsck(conf, false);
2733 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2734 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
2735
2736 assertEquals(0, hbck.getOverlapGroups(table).size());
2737
2738
2739 assertErrors(
2740 doFsck(
2741 conf, false, true, false, false, false, false, false, false, false, false, false, null),
2742 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2743 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
2744
2745
2746 assertNoErrors(doFsck(conf, false));
2747 assertEquals(5, countRows());
2748 } finally {
2749 if (tbl != null) {
2750 tbl.close();
2751 tbl = null;
2752 }
2753 cleanupTable(table);
2754 }
2755 }
2756
2757
2758 public static class MasterSyncObserver extends BaseMasterObserver {
2759 volatile CountDownLatch tableCreationLatch = null;
2760 volatile CountDownLatch tableDeletionLatch = null;
2761
2762 @Override
2763 public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
2764 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
2765
2766 if (tableCreationLatch != null) {
2767 tableCreationLatch.countDown();
2768 }
2769 }
2770
2771 @Override
2772 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
2773 TableName tableName)
2774 throws IOException {
2775
2776 if (tableDeletionLatch != null) {
2777 tableDeletionLatch.countDown();
2778 }
2779 }
2780 }
2781
2782 public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
2783 byte [][] splitKeys) throws Exception {
2784
2785
2786 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
2787 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
2788 observer.tableCreationLatch = new CountDownLatch(1);
2789 if (splitKeys != null) {
2790 admin.createTable(htd, splitKeys);
2791 } else {
2792 admin.createTable(htd);
2793 }
2794 observer.tableCreationLatch.await();
2795 observer.tableCreationLatch = null;
2796 testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
2797 }
2798
2799 public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
2800 throws Exception {
2801
2802
2803 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
2804 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
2805 observer.tableDeletionLatch = new CountDownLatch(1);
2806 try {
2807 admin.disableTable(tableName);
2808 } catch (Exception e) {
2809 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
2810 }
2811 admin.deleteTable(tableName);
2812 observer.tableDeletionLatch.await();
2813 observer.tableDeletionLatch = null;
2814 }
2815 }