1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotEquals;
27 import static org.junit.Assert.assertNotNull;
28 import static org.junit.Assert.assertTrue;
29 import static org.junit.Assert.fail;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.HashMap;
36 import java.util.HashSet;
37 import java.util.LinkedList;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Random;
41 import java.util.Map.Entry;
42 import java.util.NavigableMap;
43 import java.util.Set;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.CountDownLatch;
46 import java.util.concurrent.ExecutorService;
47 import java.util.concurrent.Executors;
48 import java.util.concurrent.Future;
49 import java.util.concurrent.ScheduledThreadPoolExecutor;
50 import java.util.concurrent.SynchronousQueue;
51 import java.util.concurrent.ThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.atomic.AtomicBoolean;
54
55 import org.apache.commons.io.IOUtils;
56 import org.apache.commons.logging.Log;
57 import org.apache.commons.logging.LogFactory;
58 import org.apache.hadoop.conf.Configuration;
59 import org.apache.hadoop.fs.FileStatus;
60 import org.apache.hadoop.fs.FileSystem;
61 import org.apache.hadoop.fs.Path;
62 import org.apache.hadoop.hbase.ClusterStatus;
63 import org.apache.hadoop.hbase.HBaseTestingUtility;
64 import org.apache.hadoop.hbase.HColumnDescriptor;
65 import org.apache.hadoop.hbase.HConstants;
66 import org.apache.hadoop.hbase.HRegionInfo;
67 import org.apache.hadoop.hbase.HRegionLocation;
68 import org.apache.hadoop.hbase.HTableDescriptor;
69 import org.apache.hadoop.hbase.TableExistsException;
70 import org.apache.hadoop.hbase.testclassification.LargeTests;
71 import org.apache.hadoop.hbase.MiniHBaseCluster;
72 import org.apache.hadoop.hbase.RegionLocations;
73 import org.apache.hadoop.hbase.ServerName;
74 import org.apache.hadoop.hbase.TableName;
75 import org.apache.hadoop.hbase.MetaTableAccessor;
76 import org.apache.hadoop.hbase.client.Admin;
77 import org.apache.hadoop.hbase.client.ClusterConnection;
78 import org.apache.hadoop.hbase.client.Connection;
79 import org.apache.hadoop.hbase.client.ConnectionFactory;
80 import org.apache.hadoop.hbase.client.Delete;
81 import org.apache.hadoop.hbase.client.Durability;
82 import org.apache.hadoop.hbase.client.Get;
83 import org.apache.hadoop.hbase.client.HBaseAdmin;
84 import org.apache.hadoop.hbase.client.HConnection;
85 import org.apache.hadoop.hbase.client.HTable;
86 import org.apache.hadoop.hbase.client.MetaScanner;
87 import org.apache.hadoop.hbase.client.Mutation;
88 import org.apache.hadoop.hbase.client.Put;
89 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
90 import org.apache.hadoop.hbase.client.Result;
91 import org.apache.hadoop.hbase.client.ResultScanner;
92 import org.apache.hadoop.hbase.client.RowMutations;
93 import org.apache.hadoop.hbase.client.Scan;
94 import org.apache.hadoop.hbase.client.Table;
95 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
96 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
97 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
98 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
99 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
100 import org.apache.hadoop.hbase.io.hfile.TestHFile;
101 import org.apache.hadoop.hbase.master.AssignmentManager;
102 import org.apache.hadoop.hbase.master.HMaster;
103 import org.apache.hadoop.hbase.master.RegionState;
104 import org.apache.hadoop.hbase.master.RegionStates;
105 import org.apache.hadoop.hbase.master.TableLockManager;
106 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
107 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
108 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
110 import org.apache.hadoop.hbase.regionserver.HRegion;
111 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
112 import org.apache.hadoop.hbase.regionserver.HRegionServer;
113 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl;
114 import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
115 import org.apache.hadoop.hbase.replication.ReplicationFactory;
116 import org.apache.hadoop.hbase.replication.ReplicationQueues;
117 import org.apache.hadoop.hbase.testclassification.LargeTests;
118 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
121 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
122 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
123 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
124 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
125 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
126 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
127 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
128 import org.apache.zookeeper.KeeperException;
129 import org.junit.AfterClass;
130 import org.junit.Assert;
131 import org.junit.Before;
132 import org.junit.BeforeClass;
133 import org.junit.Ignore;
134 import org.junit.Test;
135 import org.junit.experimental.categories.Category;
136 import org.junit.rules.TestName;
137
138 import com.google.common.collect.Multimap;
139
140
141
142
143 @Category(LargeTests.class)
144 public class TestHBaseFsck {
145 static final int POOL_SIZE = 7;
146 private static final Log LOG = LogFactory.getLog(TestHBaseFsck.class);
147 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
148 private final static Configuration conf = TEST_UTIL.getConfiguration();
149 private final static String FAM_STR = "fam";
150 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
151 private final static int REGION_ONLINE_TIMEOUT = 800;
152 private static RegionStates regionStates;
153 private static ExecutorService tableExecutorService;
154 private static ScheduledThreadPoolExecutor hbfsckExecutorService;
155 private static ClusterConnection connection;
156 private static Admin admin;
157
158
159 private HTable tbl;
160 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
161 Bytes.toBytes("B"), Bytes.toBytes("C") };
162
163 private final static byte[][] ROWKEYS= new byte[][] {
164 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
165 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
166
167 @BeforeClass
168 public static void setUpBeforeClass() throws Exception {
169 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
170 MasterSyncObserver.class.getName());
171
172 conf.setInt("hbase.regionserver.handler.count", 2);
173 conf.setInt("hbase.regionserver.metahandler.count", 30);
174
175 conf.setInt("hbase.htable.threads.max", POOL_SIZE);
176 conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
177 conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
178 conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
179 conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
180 TEST_UTIL.startMiniCluster(3);
181
182 tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
183 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
184
185 hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
186
187 AssignmentManager assignmentManager =
188 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
189 regionStates = assignmentManager.getRegionStates();
190
191 connection = (ClusterConnection) TEST_UTIL.getConnection();
192
193 admin = connection.getAdmin();
194 admin.setBalancerRunning(false, true);
195
196 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
197 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
198 }
199
200 @AfterClass
201 public static void tearDownAfterClass() throws Exception {
202 tableExecutorService.shutdown();
203 hbfsckExecutorService.shutdown();
204 admin.close();
205 TEST_UTIL.shutdownMiniCluster();
206 }
207
208 @Before
209 public void setUp() {
210 EnvironmentEdgeManager.reset();
211 }
212
213 @Test (timeout=180000)
214 public void testHBaseFsck() throws Exception {
215 assertNoErrors(doFsck(conf, false));
216 TableName table = TableName.valueOf("tableBadMetaAssign");
217 HTableDescriptor desc = new HTableDescriptor(table);
218 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
219 desc.addFamily(hcd);
220 createTable(TEST_UTIL, desc, null);
221
222
223 assertNoErrors(doFsck(conf, false));
224
225
226
227 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
228 Scan scan = new Scan();
229 scan.setStartRow(Bytes.toBytes(table+",,"));
230 ResultScanner scanner = meta.getScanner(scan);
231 HRegionInfo hri = null;
232
233 Result res = scanner.next();
234 ServerName currServer =
235 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
236 HConstants.SERVER_QUALIFIER));
237 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
238 HConstants.STARTCODE_QUALIFIER));
239
240 for (JVMClusterUtil.RegionServerThread rs :
241 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
242
243 ServerName sn = rs.getRegionServer().getServerName();
244
245
246 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
247 startCode != sn.getStartcode()) {
248 Put put = new Put(res.getRow());
249 put.setDurability(Durability.SKIP_WAL);
250 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
251 Bytes.toBytes(sn.getHostAndPort()));
252 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
253 Bytes.toBytes(sn.getStartcode()));
254 meta.put(put);
255 hri = MetaTableAccessor.getHRegionInfo(res);
256 break;
257 }
258 }
259
260
261 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
262 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
263
264 TEST_UTIL.getHBaseCluster().getMaster()
265 .getAssignmentManager().waitForAssignment(hri);
266
267
268 assertNoErrors(doFsck(conf, false));
269
270
271 Table t = connection.getTable(table, tableExecutorService);
272 ResultScanner s = t.getScanner(new Scan());
273 s.close();
274 t.close();
275
276 scanner.close();
277 meta.close();
278 }
279
280 @Test(timeout=180000)
281 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
282 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
283 admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO);
284 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
285 new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper());
286 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
287 HBaseFsck hbck = doFsck(conf, true);
288 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
289 ERROR_CODE.NULL_META_REGION });
290 assertNoErrors(doFsck(conf, false));
291 }
292
293
294
295
296
297 @Test (timeout=180000)
298 public void testSplitAndDupeRegionWithRegionReplica() throws Exception {
299 TableName table =
300 TableName.valueOf("testSplitAndDupeRegionWithRegionReplica");
301 Table meta = null;
302
303 try {
304 setupTableWithRegionReplica(table, 2);
305
306 assertNoErrors(doFsck(conf, false));
307 assertEquals(ROWKEYS.length, countRows());
308
309
310 admin.enableCatalogJanitor(false);
311 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
312 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
313 HRegionInfo hriParent = loc.getRegionInfo();
314
315
316 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
317 Thread.sleep(1000);
318
319
320 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
321 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriParent);
322 MetaTableAccessor.addRegionToMeta(meta, hriParent);
323 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
324
325 if (server != null)
326 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
327
328 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
329 Thread.sleep(250);
330 }
331
332 LOG.debug("Finished assignment of parent region");
333
334
335 HBaseFsck hbck = doFsck(conf, false);
336 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED,
337 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
338 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
339 assertEquals(3, hbck.getOverlapGroups(table).size());
340
341
342 hbck = new HBaseFsck(conf, hbfsckExecutorService);
343 hbck.setDisplayFullReport();
344 hbck.setTimeLag(0);
345 hbck.setFixHdfsOverlaps(true);
346 hbck.setRemoveParents(true);
347 hbck.setFixReferenceFiles(true);
348 hbck.connect();
349 hbck.onlineHbck();
350 hbck.close();
351
352 hbck = doFsck(conf, false);
353
354 assertNoErrors(hbck);
355 assertEquals(0, hbck.getOverlapGroups(table).size());
356 assertEquals(ROWKEYS.length, countRows());
357 } finally {
358 cleanupTable(table);
359 }
360 }
361
362
363
364
365
366 @Test (timeout=180000)
367 public void testSplitAndDupeRegion() throws Exception {
368 TableName table =
369 TableName.valueOf("testSplitAndDupeRegion");
370 Table meta = null;
371
372 try {
373 setupTable(table);
374
375 assertNoErrors(doFsck(conf, false));
376 assertEquals(ROWKEYS.length, countRows());
377
378
379 admin.enableCatalogJanitor(false);
380 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
381 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
382 HRegionInfo hriParent = loc.getRegionInfo();
383
384
385 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
386 Thread.sleep(1000);
387
388
389 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
390 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriParent);
391
392 MetaTableAccessor.addRegionToMeta(meta, hriParent);
393 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
394
395 if (server != null)
396 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
397
398 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
399 Thread.sleep(250);
400 }
401
402 LOG.debug("Finished assignment of parent region");
403
404
405 HBaseFsck hbck = doFsck(conf, false);
406 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
407 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
408 assertEquals(3, hbck.getOverlapGroups(table).size());
409
410
411 hbck = new HBaseFsck(conf, hbfsckExecutorService);
412 hbck.setDisplayFullReport();
413 hbck.setTimeLag(0);
414 hbck.setFixHdfsOverlaps(true);
415 hbck.setRemoveParents(true);
416 hbck.setFixReferenceFiles(true);
417 hbck.connect();
418 hbck.onlineHbck();
419 hbck.close();
420
421 hbck = doFsck(conf, false);
422
423 assertNoErrors(hbck);
424 assertEquals(0, hbck.getOverlapGroups(table).size());
425 assertEquals(ROWKEYS.length, countRows());
426 } finally {
427 cleanupTable(table);
428 }
429 }
430
431
432
433
434 private HRegionInfo createRegion(final HTableDescriptor
435 htd, byte[] startKey, byte[] endKey)
436 throws IOException {
437 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
438 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
439 MetaTableAccessor.addRegionToMeta(meta, hri);
440 meta.close();
441 return hri;
442 }
443
444
445
446
447 private void dumpMeta(TableName tableName) throws IOException {
448 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
449 for (byte[] row : metaRows) {
450 LOG.info(Bytes.toString(row));
451 }
452 }
453
454
455
456
457
458 private void undeployRegion(Connection conn, ServerName sn,
459 HRegionInfo hri) throws IOException, InterruptedException {
460 try {
461 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri);
462 if (!hri.isMetaTable()) {
463 admin.offline(hri.getRegionName());
464 }
465 } catch (IOException ioe) {
466 LOG.warn("Got exception when attempting to offline region "
467 + Bytes.toString(hri.getRegionName()), ioe);
468 }
469 }
470
471
472
473
474
475
476 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
477 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
478 boolean hdfs) throws IOException, InterruptedException {
479 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, HRegionInfo.DEFAULT_REPLICA_ID);
480 }
481
482
483
484
485
486
487
488
489
490 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
491 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
492 boolean hdfs, boolean regionInfoOnly, int replicaId)
493 throws IOException, InterruptedException {
494 LOG.info("** Before delete:");
495 dumpMeta(htd.getTableName());
496
497 List<HRegionLocation> locations = tbl.getAllRegionLocations();
498 for (HRegionLocation location : locations) {
499 HRegionInfo hri = location.getRegionInfo();
500 ServerName hsa = location.getServerName();
501 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
502 && Bytes.compareTo(hri.getEndKey(), endKey) == 0
503 && hri.getReplicaId() == replicaId) {
504
505 LOG.info("RegionName: " +hri.getRegionNameAsString());
506 byte[] deleteRow = hri.getRegionName();
507
508 if (unassign) {
509 LOG.info("Undeploying region " + hri + " from server " + hsa);
510 undeployRegion(connection, hsa, hri);
511 }
512
513 if (regionInfoOnly) {
514 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
515 Path rootDir = FSUtils.getRootDir(conf);
516 FileSystem fs = rootDir.getFileSystem(conf);
517 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
518 hri.getEncodedName());
519 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
520 fs.delete(hriPath, true);
521 }
522
523 if (hdfs) {
524 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
525 Path rootDir = FSUtils.getRootDir(conf);
526 FileSystem fs = rootDir.getFileSystem(conf);
527 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
528 hri.getEncodedName());
529 HBaseFsck.debugLsr(conf, p);
530 boolean success = fs.delete(p, true);
531 LOG.info("Deleted " + p + " sucessfully? " + success);
532 HBaseFsck.debugLsr(conf, p);
533 }
534
535 if (metaRow) {
536 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
537 Delete delete = new Delete(deleteRow);
538 meta.delete(delete);
539 }
540 }
541 }
542 LOG.info(hri.toString() + hsa.toString());
543 }
544
545 TEST_UTIL.getMetaTableRows(htd.getTableName());
546 LOG.info("*** After delete:");
547 dumpMeta(htd.getTableName());
548 }
549
550
551
552
553
554
555
556
557
558
559 void setupTable(TableName tablename) throws Exception {
560 setupTableWithRegionReplica(tablename, 1);
561 }
562
563
564
565
566
567
568
569
570
571
572 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
573 HTableDescriptor desc = new HTableDescriptor(tablename);
574 desc.setRegionReplication(replicaCount);
575 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
576 desc.addFamily(hcd);
577 createTable(TEST_UTIL, desc, SPLITS);
578
579 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
580 List<Put> puts = new ArrayList<Put>();
581 for (byte[] row : ROWKEYS) {
582 Put p = new Put(row);
583 p.add(FAM, Bytes.toBytes("val"), row);
584 puts.add(p);
585 }
586 tbl.put(puts);
587 tbl.flushCommits();
588 }
589
590
591
592
593 int countRows() throws IOException {
594 Scan s = new Scan();
595 ResultScanner rs = tbl.getScanner(s);
596 int i = 0;
597 while(rs.next() !=null) {
598 i++;
599 }
600 return i;
601 }
602
603
604
605
606 int countRows(byte[] start, byte[] end) throws IOException {
607 Scan s = new Scan(start, end);
608 ResultScanner rs = tbl.getScanner(s);
609 int i = 0;
610 while (rs.next() != null) {
611 i++;
612 }
613 return i;
614 }
615
616
617
618
619
620
621 void cleanupTable(TableName tablename) throws Exception {
622 if (tbl != null) {
623 tbl.close();
624 tbl = null;
625 }
626
627 ((ClusterConnection) connection).clearRegionCache();
628 deleteTable(TEST_UTIL, tablename);
629 }
630
631
632
633
634 @Test (timeout=180000)
635 public void testHBaseFsckClean() throws Exception {
636 assertNoErrors(doFsck(conf, false));
637 TableName table = TableName.valueOf("tableClean");
638 try {
639 HBaseFsck hbck = doFsck(conf, false);
640 assertNoErrors(hbck);
641
642 setupTable(table);
643 assertEquals(ROWKEYS.length, countRows());
644
645
646 hbck = doFsck(conf, false);
647 assertNoErrors(hbck);
648 assertEquals(0, hbck.getOverlapGroups(table).size());
649 assertEquals(ROWKEYS.length, countRows());
650 } finally {
651 cleanupTable(table);
652 }
653 }
654
655
656
657
658 @Test (timeout=180000)
659 public void testHbckThreadpooling() throws Exception {
660 TableName table =
661 TableName.valueOf("tableDupeStartKey");
662 try {
663
664 setupTable(table);
665
666
667 Configuration newconf = new Configuration(conf);
668 newconf.setInt("hbasefsck.numthreads", 1);
669 assertNoErrors(doFsck(newconf, false));
670
671
672 } finally {
673 cleanupTable(table);
674 }
675 }
676
677 @Test (timeout=180000)
678 public void testHbckFixOrphanTable() throws Exception {
679 TableName table = TableName.valueOf("tableInfo");
680 FileSystem fs = null;
681 Path tableinfo = null;
682 try {
683 setupTable(table);
684
685 Path hbaseTableDir = FSUtils.getTableDir(
686 FSUtils.getRootDir(conf), table);
687 fs = hbaseTableDir.getFileSystem(conf);
688 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
689 tableinfo = status.getPath();
690 fs.rename(tableinfo, new Path("/.tableinfo"));
691
692
693 HBaseFsck hbck = doFsck(conf, false);
694 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
695
696
697 hbck = doFsck(conf, true);
698 assertNoErrors(hbck);
699 status = null;
700 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
701 assertNotNull(status);
702
703 HTableDescriptor htd = admin.getTableDescriptor(table);
704 htd.setValue("NOT_DEFAULT", "true");
705 admin.disableTable(table);
706 admin.modifyTable(table, htd);
707 admin.enableTable(table);
708 fs.delete(status.getPath(), true);
709
710
711 htd = admin.getTableDescriptor(table);
712 hbck = doFsck(conf, true);
713 assertNoErrors(hbck);
714 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
715 assertNotNull(status);
716 htd = admin.getTableDescriptor(table);
717 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
718 } finally {
719 fs.rename(new Path("/.tableinfo"), tableinfo);
720 cleanupTable(table);
721 }
722 }
723
724
725
726
727
728
729 @Test (timeout=180000)
730 public void testParallelHbck() throws Exception {
731 final ExecutorService service;
732 final Future<HBaseFsck> hbck1,hbck2;
733
734 class RunHbck implements Callable<HBaseFsck>{
735 boolean fail = true;
736 @Override
737 public HBaseFsck call(){
738 Configuration c = new Configuration(conf);
739 c.setInt("hbase.hbck.lockfile.attempts", 1);
740
741
742 c.setInt("hbase.hbck.lockfile.maxwaittime", 3);
743 try{
744 return doFsck(c, true);
745 } catch(Exception e){
746 if (e.getMessage().contains("Duplicate hbck")) {
747 fail = false;
748 }
749 }
750
751 if (fail) fail();
752 return null;
753 }
754 }
755 service = Executors.newFixedThreadPool(2);
756 hbck1 = service.submit(new RunHbck());
757 hbck2 = service.submit(new RunHbck());
758 service.shutdown();
759
760 service.awaitTermination(15, TimeUnit.SECONDS);
761 HBaseFsck h1 = hbck1.get();
762 HBaseFsck h2 = hbck2.get();
763
764 assert(h1 == null || h2 == null);
765 if (h1 != null) {
766 assert(h1.getRetCode() >= 0);
767 }
768 if (h2 != null) {
769 assert(h2.getRetCode() >= 0);
770 }
771 }
772
773
774
775
776
777
778
779 @Test (timeout=180000)
780 public void testParallelWithRetriesHbck() throws Exception {
781 final ExecutorService service;
782 final Future<HBaseFsck> hbck1,hbck2;
783
784
785
786
787
788
789
790 final int timeoutInSeconds = 80;
791 final int sleepIntervalInMilliseconds = 200;
792 final int maxSleepTimeInMilliseconds = 6000;
793 final int maxRetryAttempts = 15;
794
795 class RunHbck implements Callable<HBaseFsck>{
796
797 @Override
798 public HBaseFsck call() throws Exception {
799
800 Configuration c = new Configuration(conf);
801 c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds);
802 c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds);
803 c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds);
804 c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts);
805 return doFsck(c, false);
806 }
807 }
808
809 service = Executors.newFixedThreadPool(2);
810 hbck1 = service.submit(new RunHbck());
811 hbck2 = service.submit(new RunHbck());
812 service.shutdown();
813
814 service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS);
815 HBaseFsck h1 = hbck1.get();
816 HBaseFsck h2 = hbck2.get();
817
818 assertNotNull(h1);
819 assertNotNull(h2);
820 assert(h1.getRetCode() >= 0);
821 assert(h2.getRetCode() >= 0);
822
823 }
824
825
826
827
828
829 @Test (timeout=180000)
830 public void testDupeStartKey() throws Exception {
831 TableName table =
832 TableName.valueOf("tableDupeStartKey");
833 try {
834 setupTable(table);
835 assertNoErrors(doFsck(conf, false));
836 assertEquals(ROWKEYS.length, countRows());
837
838
839 HRegionInfo hriDupe =
840 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("A2"));
841 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
842 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
843 .waitForAssignment(hriDupe);
844 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
845 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
846
847 HBaseFsck hbck = doFsck(conf, false);
848 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
849 ERROR_CODE.DUPE_STARTKEYS});
850 assertEquals(2, hbck.getOverlapGroups(table).size());
851 assertEquals(ROWKEYS.length, countRows());
852
853
854 doFsck(conf,true);
855
856
857 HBaseFsck hbck2 = doFsck(conf,false);
858 assertNoErrors(hbck2);
859 assertEquals(0, hbck2.getOverlapGroups(table).size());
860 assertEquals(ROWKEYS.length, countRows());
861 } finally {
862 cleanupTable(table);
863 }
864 }
865
866
867
868
869
870 @Test (timeout=180000)
871 public void testHbckWithRegionReplica() throws Exception {
872 TableName table =
873 TableName.valueOf("testHbckWithRegionReplica");
874 try {
875 setupTableWithRegionReplica(table, 2);
876 TEST_UTIL.getHBaseAdmin().flush(table.getName());
877 assertNoErrors(doFsck(conf, false));
878 } finally {
879 cleanupTable(table);
880 }
881 }
882
883 @Test
884 public void testHbckWithFewerReplica() throws Exception {
885 TableName table =
886 TableName.valueOf("testHbckWithFewerReplica");
887 try {
888 setupTableWithRegionReplica(table, 2);
889 TEST_UTIL.getHBaseAdmin().flush(table.getName());
890 assertNoErrors(doFsck(conf, false));
891 assertEquals(ROWKEYS.length, countRows());
892 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
893 Bytes.toBytes("C"), true, false, false, false, 1);
894
895 HBaseFsck hbck = doFsck(conf, false);
896 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_DEPLOYED});
897
898 hbck = doFsck(conf, true);
899
900 hbck = doFsck(conf, false);
901 assertErrors(hbck, new ERROR_CODE[]{});
902 } finally {
903 cleanupTable(table);
904 }
905 }
906
907 @Test
908 public void testHbckWithExcessReplica() throws Exception {
909 TableName table =
910 TableName.valueOf("testHbckWithExcessReplica");
911 try {
912 setupTableWithRegionReplica(table, 2);
913 TEST_UTIL.getHBaseAdmin().flush(table.getName());
914 assertNoErrors(doFsck(conf, false));
915 assertEquals(ROWKEYS.length, countRows());
916
917
918
919
920 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
921 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(table);
922 byte[] startKey = Bytes.toBytes("B");
923 byte[] endKey = Bytes.toBytes("C");
924 byte[] metaKey = null;
925 HRegionInfo newHri = null;
926 for (HRegionInfo h : regions) {
927 if (Bytes.compareTo(h.getStartKey(), startKey) == 0 &&
928 Bytes.compareTo(h.getEndKey(), endKey) == 0 &&
929 h.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
930 metaKey = h.getRegionName();
931
932 newHri = RegionReplicaUtil.getRegionInfoForReplica(h, 2);
933 break;
934 }
935 }
936 Put put = new Put(metaKey);
937 ServerName sn = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers()
938 .toArray(new ServerName[0])[0];
939
940 MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, 2);
941 meta.put(put);
942 meta.flushCommits();
943
944 HBaseFsckRepair.fixUnassigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
945 HBaseFsckRepair.waitUntilAssigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
946
947 Delete delete = new Delete(metaKey);
948 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(2));
949 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(2));
950 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(2));
951 meta.delete(delete);
952 meta.flushCommits();
953 meta.close();
954
955 HBaseFsck hbck = doFsck(conf, false);
956 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_IN_META});
957
958 hbck = doFsck(conf, true);
959
960 hbck = doFsck(conf, false);
961 assertErrors(hbck, new ERROR_CODE[]{});
962 } finally {
963 cleanupTable(table);
964 }
965 }
966
967
968
969 Map<ServerName, List<String>> getDeployedHRIs(final HBaseAdmin admin) throws IOException {
970 ClusterStatus status = admin.getClusterStatus();
971 Collection<ServerName> regionServers = status.getServers();
972 Map<ServerName, List<String>> mm =
973 new HashMap<ServerName, List<String>>();
974 for (ServerName hsi : regionServers) {
975 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi);
976
977
978 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
979 List<String> regionNames = new ArrayList<String>();
980 for (HRegionInfo hri : regions) {
981 regionNames.add(hri.getRegionNameAsString());
982 }
983 mm.put(hsi, regionNames);
984 }
985 return mm;
986 }
987
988
989
990
991 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
992 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
993 if (e.getValue().contains(hri.getRegionNameAsString())) {
994 return e.getKey();
995 }
996 }
997 return null;
998 }
999
1000
1001
1002
1003
1004 @Test (timeout=180000)
1005 public void testDupeRegion() throws Exception {
1006 TableName table =
1007 TableName.valueOf("tableDupeRegion");
1008 try {
1009 setupTable(table);
1010 assertNoErrors(doFsck(conf, false));
1011 assertEquals(ROWKEYS.length, countRows());
1012
1013
1014 HRegionInfo hriDupe =
1015 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"));
1016
1017 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
1018 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1019 .waitForAssignment(hriDupe);
1020 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1021 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1022
1023
1024
1025
1026
1027 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriDupe) == null) {
1028 Thread.sleep(250);
1029 }
1030
1031 LOG.debug("Finished assignment of dupe region");
1032
1033
1034 HBaseFsck hbck = doFsck(conf, false);
1035 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
1036 ERROR_CODE.DUPE_STARTKEYS});
1037 assertEquals(2, hbck.getOverlapGroups(table).size());
1038 assertEquals(ROWKEYS.length, countRows());
1039
1040
1041 doFsck(conf,true);
1042
1043
1044 HBaseFsck hbck2 = doFsck(conf,false);
1045 assertNoErrors(hbck2);
1046 assertEquals(0, hbck2.getOverlapGroups(table).size());
1047 assertEquals(ROWKEYS.length, countRows());
1048 } finally {
1049 cleanupTable(table);
1050 }
1051 }
1052
1053
1054
1055
1056 @Test (timeout=180000)
1057 public void testDegenerateRegions() throws Exception {
1058 TableName table = TableName.valueOf("tableDegenerateRegions");
1059 try {
1060 setupTable(table);
1061 assertNoErrors(doFsck(conf,false));
1062 assertEquals(ROWKEYS.length, countRows());
1063
1064
1065 HRegionInfo hriDupe =
1066 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B"));
1067 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
1068 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1069 .waitForAssignment(hriDupe);
1070 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1071 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1072
1073 HBaseFsck hbck = doFsck(conf,false);
1074 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, ERROR_CODE.DUPE_STARTKEYS,
1075 ERROR_CODE.DUPE_STARTKEYS });
1076 assertEquals(2, hbck.getOverlapGroups(table).size());
1077 assertEquals(ROWKEYS.length, countRows());
1078
1079
1080 doFsck(conf,true);
1081
1082
1083 HBaseFsck hbck2 = doFsck(conf,false);
1084 assertNoErrors(hbck2);
1085 assertEquals(0, hbck2.getOverlapGroups(table).size());
1086 assertEquals(ROWKEYS.length, countRows());
1087 } finally {
1088 cleanupTable(table);
1089 }
1090 }
1091
1092
1093
1094
1095
1096 @Test (timeout=180000)
1097 public void testContainedRegionOverlap() throws Exception {
1098 TableName table =
1099 TableName.valueOf("tableContainedRegionOverlap");
1100 try {
1101 setupTable(table);
1102 assertEquals(ROWKEYS.length, countRows());
1103
1104
1105 HRegionInfo hriOverlap =
1106 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1107 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1108 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1109 .waitForAssignment(hriOverlap);
1110 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1111 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1112
1113 HBaseFsck hbck = doFsck(conf, false);
1114 assertErrors(hbck, new ERROR_CODE[] {
1115 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1116 assertEquals(2, hbck.getOverlapGroups(table).size());
1117 assertEquals(ROWKEYS.length, countRows());
1118
1119
1120 doFsck(conf, true);
1121
1122
1123 HBaseFsck hbck2 = doFsck(conf,false);
1124 assertNoErrors(hbck2);
1125 assertEquals(0, hbck2.getOverlapGroups(table).size());
1126 assertEquals(ROWKEYS.length, countRows());
1127 } finally {
1128 cleanupTable(table);
1129 }
1130 }
1131
1132
1133
1134
1135
1136
1137
1138 @Test (timeout=180000)
1139 public void testSidelineOverlapRegion() throws Exception {
1140 TableName table =
1141 TableName.valueOf("testSidelineOverlapRegion");
1142 try {
1143 setupTable(table);
1144 assertEquals(ROWKEYS.length, countRows());
1145
1146
1147 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1148 HMaster master = cluster.getMaster();
1149 HRegionInfo hriOverlap1 =
1150 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB"));
1151 master.assignRegion(hriOverlap1);
1152 master.getAssignmentManager().waitForAssignment(hriOverlap1);
1153 HRegionInfo hriOverlap2 =
1154 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("AB"), Bytes.toBytes("B"));
1155 master.assignRegion(hriOverlap2);
1156 master.getAssignmentManager().waitForAssignment(hriOverlap2);
1157
1158 HBaseFsck hbck = doFsck(conf, false);
1159 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
1160 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
1161 assertEquals(3, hbck.getOverlapGroups(table).size());
1162 assertEquals(ROWKEYS.length, countRows());
1163
1164
1165 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
1166 ServerName serverName = null;
1167 byte[] regionName = null;
1168 for (HbckInfo hbi: overlapGroups.values()) {
1169 if ("A".equals(Bytes.toString(hbi.getStartKey()))
1170 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
1171 regionName = hbi.getRegionName();
1172
1173
1174 int k = cluster.getServerWith(regionName);
1175 for (int i = 0; i < 3; i++) {
1176 if (i != k) {
1177 HRegionServer rs = cluster.getRegionServer(i);
1178 serverName = rs.getServerName();
1179 break;
1180 }
1181 }
1182
1183 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) connection,
1184 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
1185 admin.offline(regionName);
1186 break;
1187 }
1188 }
1189
1190 assertNotNull(regionName);
1191 assertNotNull(serverName);
1192 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
1193 Put put = new Put(regionName);
1194 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
1195 Bytes.toBytes(serverName.getHostAndPort()));
1196 meta.put(put);
1197 }
1198
1199
1200 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1201 fsck.connect();
1202 fsck.setDisplayFullReport();
1203 fsck.setTimeLag(0);
1204 fsck.setFixAssignments(true);
1205 fsck.setFixMeta(true);
1206 fsck.setFixHdfsHoles(true);
1207 fsck.setFixHdfsOverlaps(true);
1208 fsck.setFixHdfsOrphans(true);
1209 fsck.setFixVersionFile(true);
1210 fsck.setSidelineBigOverlaps(true);
1211 fsck.setMaxMerge(2);
1212 fsck.onlineHbck();
1213 fsck.close();
1214
1215
1216
1217 HBaseFsck hbck2 = doFsck(conf,false);
1218 assertNoErrors(hbck2);
1219 assertEquals(0, hbck2.getOverlapGroups(table).size());
1220 assertTrue(ROWKEYS.length > countRows());
1221 } finally {
1222 cleanupTable(table);
1223 }
1224 }
1225
1226
1227
1228
1229
1230 @Test (timeout=180000)
1231 public void testOverlapAndOrphan() throws Exception {
1232 TableName table =
1233 TableName.valueOf("tableOverlapAndOrphan");
1234 try {
1235 setupTable(table);
1236 assertEquals(ROWKEYS.length, countRows());
1237
1238
1239 admin.disableTable(table);
1240 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1241 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1242 TEST_UTIL.getHBaseAdmin().enableTable(table);
1243
1244 HRegionInfo hriOverlap =
1245 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1246 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1247 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1248 .waitForAssignment(hriOverlap);
1249 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1250 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1251
1252 HBaseFsck hbck = doFsck(conf, false);
1253 assertErrors(hbck, new ERROR_CODE[] {
1254 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1255 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1256
1257
1258 doFsck(conf, true);
1259
1260
1261 HBaseFsck hbck2 = doFsck(conf,false);
1262 assertNoErrors(hbck2);
1263 assertEquals(0, hbck2.getOverlapGroups(table).size());
1264 assertEquals(ROWKEYS.length, countRows());
1265 } finally {
1266 cleanupTable(table);
1267 }
1268 }
1269
1270
1271
1272
1273
1274
1275 @Test (timeout=180000)
1276 public void testCoveredStartKey() throws Exception {
1277 TableName table =
1278 TableName.valueOf("tableCoveredStartKey");
1279 try {
1280 setupTable(table);
1281 assertEquals(ROWKEYS.length, countRows());
1282
1283
1284 HRegionInfo hriOverlap =
1285 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2"));
1286 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1287 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1288 .waitForAssignment(hriOverlap);
1289 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1290 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1291
1292 HBaseFsck hbck = doFsck(conf, false);
1293 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
1294 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1295 assertEquals(3, hbck.getOverlapGroups(table).size());
1296 assertEquals(ROWKEYS.length, countRows());
1297
1298
1299 doFsck(conf, true);
1300
1301
1302 HBaseFsck hbck2 = doFsck(conf, false);
1303 assertErrors(hbck2, new ERROR_CODE[0]);
1304 assertEquals(0, hbck2.getOverlapGroups(table).size());
1305 assertEquals(ROWKEYS.length, countRows());
1306 } finally {
1307 cleanupTable(table);
1308 }
1309 }
1310
1311
1312
1313
1314
1315 @Test (timeout=180000)
1316 public void testRegionHole() throws Exception {
1317 TableName table =
1318 TableName.valueOf("tableRegionHole");
1319 try {
1320 setupTable(table);
1321 assertEquals(ROWKEYS.length, countRows());
1322
1323
1324 admin.disableTable(table);
1325 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1326 Bytes.toBytes("C"), true, true, true);
1327 admin.enableTable(table);
1328
1329 HBaseFsck hbck = doFsck(conf, false);
1330 assertErrors(hbck, new ERROR_CODE[] {
1331 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1332
1333 assertEquals(0, hbck.getOverlapGroups(table).size());
1334
1335
1336 doFsck(conf, true);
1337
1338
1339 assertNoErrors(doFsck(conf,false));
1340 assertEquals(ROWKEYS.length - 2 , countRows());
1341 } finally {
1342 cleanupTable(table);
1343 }
1344 }
1345
1346
1347
1348
1349
1350 @Test (timeout=180000)
1351 public void testHDFSRegioninfoMissing() throws Exception {
1352 TableName table = TableName.valueOf("tableHDFSRegioninfoMissing");
1353 try {
1354 setupTable(table);
1355 assertEquals(ROWKEYS.length, countRows());
1356
1357
1358 admin.disableTable(table);
1359 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1360 Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1361 TEST_UTIL.getHBaseAdmin().enableTable(table);
1362
1363 HBaseFsck hbck = doFsck(conf, false);
1364 assertErrors(hbck, new ERROR_CODE[] {
1365 ERROR_CODE.ORPHAN_HDFS_REGION,
1366 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1367 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1368
1369 assertEquals(0, hbck.getOverlapGroups(table).size());
1370
1371
1372 doFsck(conf, true);
1373
1374
1375 assertNoErrors(doFsck(conf, false));
1376 assertEquals(ROWKEYS.length, countRows());
1377 } finally {
1378 cleanupTable(table);
1379 }
1380 }
1381
1382
1383
1384
1385
1386
1387 @Test(timeout = 180000)
1388 public void testHDFSRegioninfoMissingAndCheckRegionBoundary() throws Exception {
1389 TableName table = TableName.valueOf("testHDFSRegioninfoMissingAndCheckRegionBoundary");
1390 try {
1391 setupTable(table);
1392 assertEquals(ROWKEYS.length, countRows());
1393
1394
1395 admin.disableTable(table);
1396 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true,
1397 true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1398 admin.enableTable(table);
1399
1400 HBaseFsck hbck = doFsck(conf, false);
1401 assertErrors(hbck,
1402 new HBaseFsck.ErrorReporter.ERROR_CODE[] {
1403 HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION,
1404 HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1405 HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN });
1406
1407 assertEquals(0, hbck.getOverlapGroups(table).size());
1408
1409
1410 doFsck(conf, true);
1411
1412
1413 assertNoErrors(doFsck(conf, false));
1414
1415
1416 for (int i = 0; i < ROWKEYS.length; i++) {
1417 if (i != ROWKEYS.length - 1) {
1418 assertEquals(1, countRows(ROWKEYS[i], ROWKEYS[i + 1]));
1419 } else {
1420 assertEquals(1, countRows(ROWKEYS[i], null));
1421 }
1422 }
1423
1424 } finally {
1425 cleanupTable(table);
1426 }
1427 }
1428
1429
1430
1431
1432
1433 @Test (timeout=180000)
1434 public void testNotInMetaOrDeployedHole() throws Exception {
1435 TableName table =
1436 TableName.valueOf("tableNotInMetaOrDeployedHole");
1437 try {
1438 setupTable(table);
1439 assertEquals(ROWKEYS.length, countRows());
1440
1441
1442 admin.disableTable(table);
1443 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1444 Bytes.toBytes("C"), true, true, false);
1445 admin.enableTable(table);
1446
1447 HBaseFsck hbck = doFsck(conf, false);
1448 assertErrors(hbck, new ERROR_CODE[] {
1449 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1450
1451 assertEquals(0, hbck.getOverlapGroups(table).size());
1452
1453
1454 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1455 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1456
1457
1458 assertNoErrors(doFsck(conf,false));
1459 assertEquals(ROWKEYS.length, countRows());
1460 } finally {
1461 cleanupTable(table);
1462 }
1463 }
1464
1465
1466
1467
1468 @Test (timeout=180000)
1469 public void testNotInMetaHole() throws Exception {
1470 TableName table =
1471 TableName.valueOf("tableNotInMetaHole");
1472 try {
1473 setupTable(table);
1474 assertEquals(ROWKEYS.length, countRows());
1475
1476
1477 admin.disableTable(table);
1478 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1479 Bytes.toBytes("C"), false, true, false);
1480 admin.enableTable(table);
1481
1482 HBaseFsck hbck = doFsck(conf, false);
1483 assertErrors(hbck, new ERROR_CODE[] {
1484 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1485
1486 assertEquals(0, hbck.getOverlapGroups(table).size());
1487
1488
1489 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1490 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1491
1492
1493 assertNoErrors(doFsck(conf,false));
1494 assertEquals(ROWKEYS.length, countRows());
1495 } finally {
1496 cleanupTable(table);
1497 }
1498 }
1499
1500
1501
1502
1503
1504 @Test (timeout=180000)
1505 public void testNotInHdfs() throws Exception {
1506 TableName table =
1507 TableName.valueOf("tableNotInHdfs");
1508 try {
1509 setupTable(table);
1510 assertEquals(ROWKEYS.length, countRows());
1511
1512
1513 admin.flush(table);
1514
1515
1516 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1517 Bytes.toBytes("C"), false, false, true);
1518
1519 HBaseFsck hbck = doFsck(conf, false);
1520 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1521
1522 assertEquals(0, hbck.getOverlapGroups(table).size());
1523
1524
1525 doFsck(conf, true);
1526
1527
1528 assertNoErrors(doFsck(conf,false));
1529 assertEquals(ROWKEYS.length - 2, countRows());
1530 } finally {
1531 cleanupTable(table);
1532 }
1533 }
1534
1535
1536
1537
1538
1539 @Test (timeout=180000)
1540 public void testNotInHdfsWithReplicas() throws Exception {
1541 TableName table =
1542 TableName.valueOf("tableNotInHdfs");
1543 HBaseAdmin admin = new HBaseAdmin(conf);
1544 try {
1545 HRegionInfo[] oldHris = new HRegionInfo[2];
1546 setupTableWithRegionReplica(table, 2);
1547 assertEquals(ROWKEYS.length, countRows());
1548 NavigableMap<HRegionInfo, ServerName> map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(),
1549 tbl.getName());
1550 int i = 0;
1551
1552 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1553 if (m.getKey().getStartKey().length > 0 &&
1554 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1555 LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
1556 oldHris[i++] = m.getKey();
1557 }
1558 }
1559
1560 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1561
1562
1563 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1564 Bytes.toBytes("C"), false, false, true);
1565
1566 HBaseFsck hbck = doFsck(conf, false);
1567 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1568
1569
1570 doFsck(conf, true);
1571
1572
1573 assertNoErrors(doFsck(conf,false));
1574 assertEquals(ROWKEYS.length - 2, countRows());
1575
1576
1577
1578 i = 0;
1579 HRegionInfo[] newHris = new HRegionInfo[2];
1580
1581 map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
1582
1583 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1584 if (m.getKey().getStartKey().length > 0 &&
1585 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1586 newHris[i++] = m.getKey();
1587 }
1588 }
1589
1590 Collection<ServerName> servers = admin.getClusterStatus().getServers();
1591 Set<HRegionInfo> onlineRegions = new HashSet<HRegionInfo>();
1592 for (ServerName s : servers) {
1593 List<HRegionInfo> list = admin.getOnlineRegions(s);
1594 onlineRegions.addAll(list);
1595 }
1596
1597 assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
1598
1599
1600 assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
1601 } finally {
1602 cleanupTable(table);
1603 admin.close();
1604 }
1605 }
1606
1607
1608
1609
1610
1611
1612 @Test (timeout=180000)
1613 public void testNoHdfsTable() throws Exception {
1614 TableName table = TableName.valueOf("NoHdfsTable");
1615 setupTable(table);
1616 assertEquals(ROWKEYS.length, countRows());
1617
1618
1619 admin.flush(table);
1620
1621
1622 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1623 Bytes.toBytes("A"), false, false, true);
1624 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1625 Bytes.toBytes("B"), false, false, true);
1626 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1627 Bytes.toBytes("C"), false, false, true);
1628 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1629 Bytes.toBytes(""), false, false, true);
1630
1631
1632 deleteTableDir(table);
1633
1634 HBaseFsck hbck = doFsck(conf, false);
1635 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1636 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1637 ERROR_CODE.NOT_IN_HDFS,});
1638
1639 assertEquals(0, hbck.getOverlapGroups(table).size());
1640
1641
1642 doFsck(conf, true);
1643
1644
1645 assertNoErrors(doFsck(conf,false));
1646 assertFalse("Table " + table + " should have been deleted", admin.tableExists(table));
1647 }
1648
1649 public void deleteTableDir(TableName table) throws IOException {
1650 Path rootDir = FSUtils.getRootDir(conf);
1651 FileSystem fs = rootDir.getFileSystem(conf);
1652 Path p = FSUtils.getTableDir(rootDir, table);
1653 HBaseFsck.debugLsr(conf, p);
1654 boolean success = fs.delete(p, true);
1655 LOG.info("Deleted " + p + " sucessfully? " + success);
1656 }
1657
1658
1659
1660
1661 @Test (timeout=180000)
1662 public void testNoVersionFile() throws Exception {
1663
1664 Path rootDir = FSUtils.getRootDir(conf);
1665 FileSystem fs = rootDir.getFileSystem(conf);
1666 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1667 fs.delete(versionFile, true);
1668
1669
1670 HBaseFsck hbck = doFsck(conf, false);
1671 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1672
1673 doFsck(conf, true);
1674
1675
1676 assertNoErrors(doFsck(conf, false));
1677 }
1678
1679
1680
1681
1682 @Test (timeout=180000)
1683 public void testRegionShouldNotBeDeployed() throws Exception {
1684 TableName table =
1685 TableName.valueOf("tableRegionShouldNotBeDeployed");
1686 try {
1687 LOG.info("Starting testRegionShouldNotBeDeployed.");
1688 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1689 assertTrue(cluster.waitForActiveAndReadyMaster());
1690
1691
1692 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1693 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1694 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1695 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1696
1697
1698 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1699 fstd.createTableDescriptor(htdDisabled);
1700 List<HRegionInfo> disabledRegions =
1701 TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
1702
1703
1704 HRegionServer hrs = cluster.getRegionServer(0);
1705
1706
1707 admin.disableTable(table);
1708 admin.enableTable(table);
1709
1710
1711 admin.disableTable(table);
1712 HRegionInfo region = disabledRegions.remove(0);
1713 byte[] regionName = region.getRegionName();
1714
1715
1716 assertTrue(cluster.getServerWith(regionName) == -1);
1717
1718
1719
1720
1721
1722 HRegion r = HRegion.openHRegion(
1723 region, htdDisabled, hrs.getWAL(region), conf);
1724 hrs.addToOnlineRegions(r);
1725
1726 HBaseFsck hbck = doFsck(conf, false);
1727 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1728
1729
1730 doFsck(conf, true);
1731
1732
1733 assertNoErrors(doFsck(conf, false));
1734 } finally {
1735 admin.enableTable(table);
1736 cleanupTable(table);
1737 }
1738 }
1739
1740
1741
1742
1743 @Test (timeout=180000)
1744 public void testFixByTable() throws Exception {
1745 TableName table1 =
1746 TableName.valueOf("testFixByTable1");
1747 TableName table2 =
1748 TableName.valueOf("testFixByTable2");
1749 try {
1750 setupTable(table1);
1751
1752 admin.flush(table1);
1753
1754 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1755 Bytes.toBytes("C"), false, false, true);
1756
1757 setupTable(table2);
1758
1759 admin.flush(table2);
1760
1761 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1762 Bytes.toBytes("C"), false, false, true);
1763
1764 HBaseFsck hbck = doFsck(conf, false);
1765 assertErrors(hbck, new ERROR_CODE[] {
1766 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1767
1768
1769 doFsck(conf, true, table1);
1770
1771 assertNoErrors(doFsck(conf, false, table1));
1772
1773 assertErrors(doFsck(conf, false, table2),
1774 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1775
1776
1777 doFsck(conf, true, table2);
1778
1779 assertNoErrors(doFsck(conf, false));
1780 assertEquals(ROWKEYS.length - 2, countRows());
1781 } finally {
1782 cleanupTable(table1);
1783 cleanupTable(table2);
1784 }
1785 }
1786
1787
1788
1789 @Test (timeout=180000)
1790 public void testLingeringSplitParent() throws Exception {
1791 TableName table =
1792 TableName.valueOf("testLingeringSplitParent");
1793 Table meta = null;
1794 try {
1795 setupTable(table);
1796 assertEquals(ROWKEYS.length, countRows());
1797
1798
1799 admin.flush(table);
1800 HRegionLocation location = tbl.getRegionLocation("B");
1801
1802
1803 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1804 Bytes.toBytes("C"), true, true, false);
1805
1806
1807 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1808 HRegionInfo hri = location.getRegionInfo();
1809
1810 HRegionInfo a = new HRegionInfo(tbl.getName(),
1811 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1812 HRegionInfo b = new HRegionInfo(tbl.getName(),
1813 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1814
1815 hri.setOffline(true);
1816 hri.setSplit(true);
1817
1818 MetaTableAccessor.addRegionToMeta(meta, hri, a, b);
1819 meta.close();
1820 admin.flush(TableName.META_TABLE_NAME);
1821
1822 HBaseFsck hbck = doFsck(conf, false);
1823 assertErrors(hbck, new ERROR_CODE[] {
1824 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1825
1826
1827 hbck = doFsck(conf, true);
1828 assertErrors(hbck, new ERROR_CODE[] {
1829 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1830 assertFalse(hbck.shouldRerun());
1831 hbck = doFsck(conf, false);
1832 assertErrors(hbck, new ERROR_CODE[] {
1833 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1834
1835
1836 hbck = new HBaseFsck(conf, hbfsckExecutorService);
1837 hbck.connect();
1838 hbck.setDisplayFullReport();
1839 hbck.setTimeLag(0);
1840 hbck.setFixSplitParents(true);
1841 hbck.onlineHbck();
1842 assertTrue(hbck.shouldRerun());
1843 hbck.close();
1844
1845 Get get = new Get(hri.getRegionName());
1846 Result result = meta.get(get);
1847 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1848 HConstants.SPLITA_QUALIFIER).isEmpty());
1849 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1850 HConstants.SPLITB_QUALIFIER).isEmpty());
1851 admin.flush(TableName.META_TABLE_NAME);
1852
1853
1854 doFsck(conf, true);
1855
1856
1857 assertNoErrors(doFsck(conf, false));
1858 assertEquals(ROWKEYS.length, countRows());
1859 } finally {
1860 cleanupTable(table);
1861 IOUtils.closeQuietly(meta);
1862 }
1863 }
1864
1865
1866
1867
1868
1869 @Test (timeout=180000)
1870 public void testValidLingeringSplitParent() throws Exception {
1871 TableName table =
1872 TableName.valueOf("testLingeringSplitParent");
1873 Table meta = null;
1874 try {
1875 setupTable(table);
1876 assertEquals(ROWKEYS.length, countRows());
1877
1878
1879 admin.flush(table);
1880 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1881
1882 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1883 HRegionInfo hri = location.getRegionInfo();
1884
1885
1886 byte[] regionName = location.getRegionInfo().getRegionName();
1887 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1888 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1889
1890
1891
1892
1893 HBaseFsck hbck = doFsck(
1894 conf, true, true, false, false, false, true, true, true, false, false, false, false, null);
1895 assertErrors(hbck, new ERROR_CODE[] {});
1896
1897
1898 Get get = new Get(hri.getRegionName());
1899 Result result = meta.get(get);
1900 assertNotNull(result);
1901 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1902
1903 assertEquals(ROWKEYS.length, countRows());
1904
1905
1906 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1907 assertNoErrors(doFsck(conf, false));
1908 } finally {
1909 cleanupTable(table);
1910 IOUtils.closeQuietly(meta);
1911 }
1912 }
1913
1914
1915
1916
1917
1918 @Test(timeout=75000)
1919 public void testSplitDaughtersNotInMeta() throws Exception {
1920 TableName table = TableName.valueOf("testSplitdaughtersNotInMeta");
1921 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1922 try {
1923 setupTable(table);
1924 assertEquals(ROWKEYS.length, countRows());
1925
1926
1927 admin.flush(table);
1928 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1929
1930 HRegionInfo hri = location.getRegionInfo();
1931
1932
1933
1934 admin.enableCatalogJanitor(false);
1935
1936
1937 byte[] regionName = location.getRegionInfo().getRegionName();
1938 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1939 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1940
1941 PairOfSameType<HRegionInfo> daughters =
1942 MetaTableAccessor.getDaughterRegions(meta.get(new Get(regionName)));
1943
1944
1945 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
1946 undeployRegion(connection, hris.get(daughters.getFirst()), daughters.getFirst());
1947 undeployRegion(connection, hris.get(daughters.getSecond()), daughters.getSecond());
1948
1949 List<Delete> deletes = new ArrayList<>();
1950 deletes.add(new Delete(daughters.getFirst().getRegionName()));
1951 deletes.add(new Delete(daughters.getSecond().getRegionName()));
1952 meta.delete(deletes);
1953
1954
1955 RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().
1956 getAssignmentManager().getRegionStates();
1957 regionStates.deleteRegion(daughters.getFirst());
1958 regionStates.deleteRegion(daughters.getSecond());
1959
1960 HBaseFsck hbck = doFsck(conf, false);
1961 assertErrors(hbck,
1962 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1963 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1964
1965
1966 hbck = doFsck(
1967 conf, true, true, false, false, false, false, false, false, false, false, false,false,null);
1968 assertErrors(hbck,
1969 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1970 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1971
1972
1973 Get get = new Get(hri.getRegionName());
1974 Result result = meta.get(get);
1975 assertNotNull(result);
1976 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1977
1978 assertEquals(ROWKEYS.length, countRows());
1979
1980
1981 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1982 assertNoErrors(doFsck(conf, false));
1983 } finally {
1984 admin.enableCatalogJanitor(true);
1985 meta.close();
1986 cleanupTable(table);
1987 }
1988 }
1989
1990
1991
1992
1993
1994 @Test(timeout=120000)
1995 public void testMissingFirstRegion() throws Exception {
1996 TableName table = TableName.valueOf("testMissingFirstRegion");
1997 try {
1998 setupTable(table);
1999 assertEquals(ROWKEYS.length, countRows());
2000
2001
2002 admin.disableTable(table);
2003 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
2004 true, true);
2005 admin.enableTable(table);
2006
2007 HBaseFsck hbck = doFsck(conf, false);
2008 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
2009
2010 doFsck(conf, true);
2011
2012 assertNoErrors(doFsck(conf, false));
2013 } finally {
2014 cleanupTable(table);
2015 }
2016 }
2017
2018
2019
2020
2021
2022 @Test(timeout=120000)
2023 public void testRegionDeployedNotInHdfs() throws Exception {
2024 TableName table =
2025 TableName.valueOf("testSingleRegionDeployedNotInHdfs");
2026 try {
2027 setupTable(table);
2028 admin.flush(table);
2029
2030
2031 deleteRegion(conf, tbl.getTableDescriptor(),
2032 HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false,
2033 false, true);
2034
2035 HBaseFsck hbck = doFsck(conf, false);
2036 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2037
2038 doFsck(conf, true);
2039
2040 assertNoErrors(doFsck(conf, false));
2041 } finally {
2042 cleanupTable(table);
2043 }
2044 }
2045
2046
2047
2048
2049
2050 @Test(timeout=120000)
2051 public void testMissingLastRegion() throws Exception {
2052 TableName table =
2053 TableName.valueOf("testMissingLastRegion");
2054 try {
2055 setupTable(table);
2056 assertEquals(ROWKEYS.length, countRows());
2057
2058
2059 admin.disableTable(table);
2060 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
2061 true, true);
2062 admin.enableTable(table);
2063
2064 HBaseFsck hbck = doFsck(conf, false);
2065 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
2066
2067 doFsck(conf, true);
2068
2069 assertNoErrors(doFsck(conf, false));
2070 } finally {
2071 cleanupTable(table);
2072 }
2073 }
2074
2075
2076
2077
2078 @Test (timeout=180000)
2079 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
2080 TableName table =
2081 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
2082 try {
2083 setupTable(table);
2084 assertEquals(ROWKEYS.length, countRows());
2085
2086
2087 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2088 Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2089
2090
2091 HBaseFsck hbck = doFsck(conf, false);
2092 assertErrors(hbck, new ERROR_CODE[] {
2093 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2094
2095
2096 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2097 fsck.connect();
2098 fsck.setDisplayFullReport();
2099 fsck.setTimeLag(0);
2100 fsck.setCheckHdfs(false);
2101 fsck.onlineHbck();
2102 assertErrors(fsck, new ERROR_CODE[] {
2103 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2104 fsck.close();
2105
2106
2107 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2108 fsck.connect();
2109 fsck.setDisplayFullReport();
2110 fsck.setTimeLag(0);
2111 fsck.setCheckHdfs(false);
2112 fsck.setFixAssignments(true);
2113 fsck.onlineHbck();
2114 assertTrue(fsck.shouldRerun());
2115 fsck.onlineHbck();
2116 assertNoErrors(fsck);
2117
2118 assertEquals(ROWKEYS.length, countRows());
2119
2120 fsck.close();
2121 } finally {
2122 cleanupTable(table);
2123 }
2124 }
2125
2126
2127
2128
2129
2130
2131 @Test (timeout=180000)
2132 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
2133 TableName table =
2134 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
2135 try {
2136 setupTable(table);
2137 assertEquals(ROWKEYS.length, countRows());
2138
2139
2140 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2141 Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2142
2143
2144 HBaseFsck hbck = doFsck(conf, false);
2145 assertErrors(hbck,
2146 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2147
2148
2149 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2150 fsck.connect();
2151 fsck.setDisplayFullReport();
2152 fsck.setTimeLag(0);
2153 fsck.setCheckHdfs(false);
2154 fsck.onlineHbck();
2155 assertErrors(fsck,
2156 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2157 fsck.close();
2158
2159
2160 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2161 fsck.connect();
2162 fsck.setDisplayFullReport();
2163 fsck.setTimeLag(0);
2164 fsck.setCheckHdfs(false);
2165 fsck.setFixAssignments(true);
2166 fsck.setFixMeta(true);
2167 fsck.onlineHbck();
2168 assertFalse(fsck.shouldRerun());
2169 assertErrors(fsck,
2170 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2171 fsck.close();
2172
2173
2174 fsck = doFsck(conf, true);
2175 assertTrue(fsck.shouldRerun());
2176 fsck = doFsck(conf, true);
2177 assertNoErrors(fsck);
2178 } finally {
2179 cleanupTable(table);
2180 }
2181 }
2182
2183
2184
2185
2186
2187 @Test (timeout=180000)
2188 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
2189 TableName table =
2190 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
2191 try {
2192 setupTable(table);
2193 assertEquals(ROWKEYS.length, countRows());
2194
2195
2196 admin.disableTable(table);
2197 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2198 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
2199 TEST_UTIL.getHBaseAdmin().enableTable(table);
2200
2201 HRegionInfo hriOverlap =
2202 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
2203 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
2204 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
2205 .waitForAssignment(hriOverlap);
2206 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
2207 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
2208
2209 HBaseFsck hbck = doFsck(conf, false);
2210 assertErrors(hbck, new ERROR_CODE[] {
2211 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2212 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2213
2214
2215 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2216 fsck.connect();
2217 fsck.setDisplayFullReport();
2218 fsck.setTimeLag(0);
2219 fsck.setCheckHdfs(false);
2220 fsck.onlineHbck();
2221 assertErrors(fsck, new ERROR_CODE[] {
2222 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2223 fsck.close();
2224
2225
2226 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2227 fsck.connect();
2228 fsck.setDisplayFullReport();
2229 fsck.setTimeLag(0);
2230 fsck.setCheckHdfs(false);
2231 fsck.setFixHdfsHoles(true);
2232 fsck.setFixHdfsOverlaps(true);
2233 fsck.setFixHdfsOrphans(true);
2234 fsck.onlineHbck();
2235 assertFalse(fsck.shouldRerun());
2236 assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN});
2237 fsck.close();
2238 } finally {
2239 if (admin.isTableDisabled(table)) {
2240 admin.enableTable(table);
2241 }
2242 cleanupTable(table);
2243 }
2244 }
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
2255 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2256 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2257 Path famDir = new Path(regionDir, FAM_STR);
2258
2259
2260 while (true) {
2261 FileStatus[] hfFss = fs.listStatus(famDir);
2262 if (hfFss.length == 0) {
2263 continue;
2264 }
2265 for (FileStatus hfs : hfFss) {
2266 if (!hfs.isDirectory()) {
2267 return hfs.getPath();
2268 }
2269 }
2270 }
2271 }
2272
2273
2274
2275
2276 @Test(timeout=180000)
2277 public void testQuarantineCorruptHFile() throws Exception {
2278 TableName table = TableName.valueOf(name.getMethodName());
2279 try {
2280 setupTable(table);
2281 assertEquals(ROWKEYS.length, countRows());
2282 admin.flush(table);
2283
2284 FileSystem fs = FileSystem.get(conf);
2285 Path hfile = getFlushedHFile(fs, table);
2286
2287
2288 admin.disableTable(table);
2289
2290
2291 Path corrupt = new Path(hfile.getParent(), "deadbeef");
2292 TestHFile.truncateFile(fs, hfile, corrupt);
2293 LOG.info("Created corrupted file " + corrupt);
2294 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2295
2296
2297 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2298 assertEquals(res.getRetCode(), 0);
2299 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2300 assertEquals(hfcc.getHFilesChecked(), 5);
2301 assertEquals(hfcc.getCorrupted().size(), 1);
2302 assertEquals(hfcc.getFailures().size(), 0);
2303 assertEquals(hfcc.getQuarantined().size(), 1);
2304 assertEquals(hfcc.getMissing().size(), 0);
2305
2306
2307 admin.enableTable(table);
2308 } finally {
2309 cleanupTable(table);
2310 }
2311 }
2312
2313
2314
2315
2316 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
2317 int corrupt, int fail, int quar, int missing) throws Exception {
2318 try {
2319 setupTable(table);
2320 assertEquals(ROWKEYS.length, countRows());
2321 admin.flush(table);
2322
2323
2324 admin.disableTable(table);
2325
2326 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
2327 table.getNameAsString()};
2328 HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
2329
2330 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2331 assertEquals(hfcc.getHFilesChecked(), check);
2332 assertEquals(hfcc.getCorrupted().size(), corrupt);
2333 assertEquals(hfcc.getFailures().size(), fail);
2334 assertEquals(hfcc.getQuarantined().size(), quar);
2335 assertEquals(hfcc.getMissing().size(), missing);
2336
2337
2338 admin.enableTableAsync(table);
2339 while (!admin.isTableEnabled(table)) {
2340 try {
2341 Thread.sleep(250);
2342 } catch (InterruptedException e) {
2343 e.printStackTrace();
2344 fail("Interrupted when trying to enable table " + table);
2345 }
2346 }
2347 } finally {
2348 cleanupTable(table);
2349 }
2350 }
2351
2352
2353
2354
2355
2356 @Test(timeout=180000)
2357 public void testQuarantineMissingHFile() throws Exception {
2358 TableName table = TableName.valueOf(name.getMethodName());
2359
2360
2361 final FileSystem fs = FileSystem.get(conf);
2362 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2363 @Override
2364 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2365 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2366 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2367 @Override
2368 protected void checkHFile(Path p) throws IOException {
2369 if (attemptedFirstHFile.compareAndSet(false, true)) {
2370 assertTrue(fs.delete(p, true));
2371 }
2372 super.checkHFile(p);
2373 }
2374 };
2375 }
2376 };
2377 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
2378 hbck.close();
2379 }
2380
2381
2382
2383
2384
2385
2386
2387 @Ignore @Test(timeout=180000)
2388 public void testQuarantineMissingFamdir() throws Exception {
2389 TableName table = TableName.valueOf(name.getMethodName());
2390
2391 final FileSystem fs = FileSystem.get(conf);
2392 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2393 @Override
2394 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2395 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2396 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2397 @Override
2398 protected void checkColFamDir(Path p) throws IOException {
2399 if (attemptedFirstHFile.compareAndSet(false, true)) {
2400 assertTrue(fs.delete(p, true));
2401 }
2402 super.checkColFamDir(p);
2403 }
2404 };
2405 }
2406 };
2407 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2408 hbck.close();
2409 }
2410
2411 @Test(timeout=60000)
2412 public void testCheckReplication() throws Exception {
2413
2414 HBaseFsck hbck = doFsck(conf, false);
2415 assertNoErrors(hbck);
2416
2417
2418 ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf);
2419 Assert.assertEquals(0, replicationAdmin.getPeersCount());
2420 String zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
2421 replicationAdmin.addPeer("1", "127.0.0.1:2181" + zkPort + ":/hbase");
2422 replicationAdmin.getPeersCount();
2423 Assert.assertEquals(1, replicationAdmin.getPeersCount());
2424
2425
2426 ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection);
2427 ReplicationQueues repQueues =
2428 ReplicationFactory.getReplicationQueues(zkw, conf, connection);
2429 repQueues.init("server1");
2430
2431 repQueues.addLog("1", "file1");
2432 repQueues.addLog("1-server2", "file1");
2433 Assert.assertEquals(2, repQueues.getAllQueues().size());
2434 hbck = doFsck(conf, false);
2435 assertNoErrors(hbck);
2436
2437
2438 repQueues.addLog("2", "file1");
2439 repQueues.addLog("2-server2", "file1");
2440 Assert.assertEquals(4, repQueues.getAllQueues().size());
2441 hbck = doFsck(conf, false);
2442 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
2443 ERROR_CODE.UNDELETED_REPLICATION_QUEUE });
2444
2445
2446 hbck = doFsck(conf, true);
2447 hbck = doFsck(conf, false);
2448 assertNoErrors(hbck);
2449
2450 Assert.assertEquals(2, repQueues.getAllQueues().size());
2451 Assert.assertNull(repQueues.getLogsInQueue("2"));
2452 Assert.assertNull(repQueues.getLogsInQueue("2-sever2"));
2453
2454 replicationAdmin.removePeer("1");
2455 repQueues.removeAllQueues();
2456 zkw.close();
2457 replicationAdmin.close();
2458 }
2459
2460
2461
2462
2463
2464 @Test(timeout=180000)
2465 public void testQuarantineMissingRegionDir() throws Exception {
2466 TableName table = TableName.valueOf(name.getMethodName());
2467
2468 final FileSystem fs = FileSystem.get(conf);
2469 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2470 @Override
2471 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
2472 throws IOException {
2473 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2474 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2475 @Override
2476 protected void checkRegionDir(Path p) throws IOException {
2477 if (attemptedFirstHFile.compareAndSet(false, true)) {
2478 assertTrue(fs.delete(p, true));
2479 }
2480 super.checkRegionDir(p);
2481 }
2482 };
2483 }
2484 };
2485 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2486 hbck.close();
2487 }
2488
2489
2490
2491
2492 @Test (timeout=180000)
2493 public void testLingeringReferenceFile() throws Exception {
2494 TableName table =
2495 TableName.valueOf("testLingeringReferenceFile");
2496 try {
2497 setupTable(table);
2498 assertEquals(ROWKEYS.length, countRows());
2499
2500
2501 FileSystem fs = FileSystem.get(conf);
2502 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2503 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2504 Path famDir = new Path(regionDir, FAM_STR);
2505 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
2506 fs.create(fakeReferenceFile);
2507
2508 HBaseFsck hbck = doFsck(conf, false);
2509 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
2510
2511 doFsck(conf, true);
2512
2513 assertNoErrors(doFsck(conf, false));
2514 } finally {
2515 cleanupTable(table);
2516 }
2517 }
2518
2519
2520
2521
2522 @Test (timeout=180000)
2523 public void testMissingRegionInfoQualifier() throws Exception {
2524 Connection connection = ConnectionFactory.createConnection(conf);
2525 TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
2526 try {
2527 setupTable(table);
2528
2529
2530 final List<Delete> deletes = new LinkedList<Delete>();
2531 Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService);
2532 MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
2533
2534 @Override
2535 public boolean processRow(Result rowResult) throws IOException {
2536 HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult);
2537 if (hri != null && !hri.getTable().isSystemTable()) {
2538 Delete delete = new Delete(rowResult.getRow());
2539 delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2540 deletes.add(delete);
2541 }
2542 return true;
2543 }
2544
2545 @Override
2546 public void close() throws IOException {
2547 }
2548 });
2549 meta.delete(deletes);
2550
2551
2552 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2553 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
2554 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2555 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
2556 meta.close();
2557
2558 HBaseFsck hbck = doFsck(conf, false);
2559 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2560
2561
2562 hbck = doFsck(conf, true);
2563
2564
2565 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2566 } finally {
2567 cleanupTable(table);
2568 }
2569 connection.close();
2570 }
2571
2572
2573
2574
2575
2576 @Test (timeout=180000)
2577 public void testErrorReporter() throws Exception {
2578 try {
2579 MockErrorReporter.calledCount = 0;
2580 doFsck(conf, false);
2581 assertEquals(MockErrorReporter.calledCount, 0);
2582
2583 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
2584 doFsck(conf, false);
2585 assertTrue(MockErrorReporter.calledCount > 20);
2586 } finally {
2587 conf.set("hbasefsck.errorreporter",
2588 PrintingErrorReporter.class.getName());
2589 MockErrorReporter.calledCount = 0;
2590 }
2591 }
2592
2593 static class MockErrorReporter implements ErrorReporter {
2594 static int calledCount = 0;
2595
2596 @Override
2597 public void clear() {
2598 calledCount++;
2599 }
2600
2601 @Override
2602 public void report(String message) {
2603 calledCount++;
2604 }
2605
2606 @Override
2607 public void reportError(String message) {
2608 calledCount++;
2609 }
2610
2611 @Override
2612 public void reportError(ERROR_CODE errorCode, String message) {
2613 calledCount++;
2614 }
2615
2616 @Override
2617 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2618 calledCount++;
2619 }
2620
2621 @Override
2622 public void reportError(ERROR_CODE errorCode,
2623 String message, TableInfo table, HbckInfo info) {
2624 calledCount++;
2625 }
2626
2627 @Override
2628 public void reportError(ERROR_CODE errorCode, String message,
2629 TableInfo table, HbckInfo info1, HbckInfo info2) {
2630 calledCount++;
2631 }
2632
2633 @Override
2634 public int summarize() {
2635 return ++calledCount;
2636 }
2637
2638 @Override
2639 public void detail(String details) {
2640 calledCount++;
2641 }
2642
2643 @Override
2644 public ArrayList<ERROR_CODE> getErrorList() {
2645 calledCount++;
2646 return new ArrayList<ERROR_CODE>();
2647 }
2648
2649 @Override
2650 public void progress() {
2651 calledCount++;
2652 }
2653
2654 @Override
2655 public void print(String message) {
2656 calledCount++;
2657 }
2658
2659 @Override
2660 public void resetErrors() {
2661 calledCount++;
2662 }
2663
2664 @Override
2665 public boolean tableHasErrors(TableInfo table) {
2666 calledCount++;
2667 return false;
2668 }
2669 }
2670
2671 @Test(timeout=180000)
2672 public void testCheckTableLocks() throws Exception {
2673 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2674 EnvironmentEdgeManager.injectEdge(edge);
2675
2676 HBaseFsck hbck = doFsck(conf, false);
2677 assertNoErrors(hbck);
2678
2679 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2680 final TableName tableName = TableName.valueOf("foo");
2681
2682
2683 final TableLockManager tableLockManager =
2684 TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2685 TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2686 writeLock.acquire();
2687 hbck = doFsck(conf, false);
2688 assertNoErrors(hbck);
2689
2690 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2691 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2692
2693 hbck = doFsck(conf, false);
2694 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2695
2696 final CountDownLatch latch = new CountDownLatch(1);
2697 new Thread() {
2698 @Override
2699 public void run() {
2700 TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2701 try {
2702 latch.countDown();
2703 readLock.acquire();
2704 } catch (IOException ex) {
2705 fail();
2706 } catch (IllegalStateException ex) {
2707 return;
2708 }
2709 fail("should not have come here");
2710 };
2711 }.start();
2712
2713 latch.await();
2714 Threads.sleep(300);
2715
2716 hbck = doFsck(conf, false);
2717
2718 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2719
2720 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2721 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2722
2723 hbck = doFsck(conf, false);
2724
2725 assertErrors(
2726 hbck,
2727 new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2728
2729 Configuration localConf = new Configuration(conf);
2730
2731
2732 localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2733 Threads.sleep(10);
2734 hbck = doFsck(localConf, true);
2735
2736 hbck = doFsck(localConf, false);
2737 assertNoErrors(hbck);
2738
2739
2740 writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
2741 writeLock.acquire();
2742 writeLock.release();
2743 tableLockManager.tableDeleted(tableName);
2744 }
2745
2746
2747
2748
2749 @Test
2750 public void testOrphanedTableZNode() throws Exception {
2751 TableName table = TableName.valueOf("testOrphanedZKTableEntry");
2752
2753 try {
2754 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
2755 .setTableState(table, ZooKeeperProtos.Table.State.ENABLING);
2756
2757 try {
2758 setupTable(table);
2759 Assert.fail(
2760 "Create table should fail when its ZNode has already existed with ENABLING state.");
2761 } catch(TableExistsException t) {
2762
2763 }
2764
2765 try {
2766 cleanupTable(table);
2767 } catch (IOException e) {
2768
2769
2770 }
2771
2772 HBaseFsck hbck = doFsck(conf, false);
2773 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2774
2775
2776 hbck = doFsck(conf, true);
2777
2778
2779 hbck = doFsck(conf, false);
2780 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2781
2782 setupTable(table);
2783 } finally {
2784
2785
2786 try {
2787 cleanupTable(table);
2788 } catch (IOException e) {
2789
2790
2791 }
2792 }
2793 }
2794
2795 @Test (timeout=180000)
2796 public void testMetaOffline() throws Exception {
2797
2798 HBaseFsck hbck = doFsck(conf, false);
2799 assertNoErrors(hbck);
2800 deleteMetaRegion(conf, true, false, false);
2801 hbck = doFsck(conf, false);
2802
2803
2804 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2805 hbck = doFsck(conf, true);
2806 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2807 hbck = doFsck(conf, false);
2808 assertNoErrors(hbck);
2809 }
2810
2811 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2812 boolean regionInfoOnly) throws IOException, InterruptedException {
2813 HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
2814 .getRegionLocation(HConstants.EMPTY_START_ROW);
2815 ServerName hsa = metaLocation.getServerName();
2816 HRegionInfo hri = metaLocation.getRegionInfo();
2817 if (unassign) {
2818 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2819 try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
2820 undeployRegion(unmanagedConnection, hsa, hri);
2821 }
2822 }
2823
2824 if (regionInfoOnly) {
2825 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2826 Path rootDir = FSUtils.getRootDir(conf);
2827 FileSystem fs = rootDir.getFileSystem(conf);
2828 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2829 hri.getEncodedName());
2830 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2831 fs.delete(hriPath, true);
2832 }
2833
2834 if (hdfs) {
2835 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2836 Path rootDir = FSUtils.getRootDir(conf);
2837 FileSystem fs = rootDir.getFileSystem(conf);
2838 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2839 hri.getEncodedName());
2840 HBaseFsck.debugLsr(conf, p);
2841 boolean success = fs.delete(p, true);
2842 LOG.info("Deleted " + p + " sucessfully? " + success);
2843 HBaseFsck.debugLsr(conf, p);
2844 }
2845 }
2846
2847 @Test (timeout=180000)
2848 public void testTableWithNoRegions() throws Exception {
2849
2850
2851 TableName table =
2852 TableName.valueOf(name.getMethodName());
2853 try {
2854
2855 HTableDescriptor desc = new HTableDescriptor(table);
2856 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
2857 desc.addFamily(hcd);
2858 createTable(TEST_UTIL, desc, null);
2859 tbl = (HTable) connection.getTable(table, tableExecutorService);
2860
2861
2862 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW,
2863 HConstants.EMPTY_END_ROW, false, false, true);
2864
2865 HBaseFsck hbck = doFsck(conf, false);
2866 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2867
2868 doFsck(conf, true);
2869
2870
2871 doFsck(conf, true);
2872
2873
2874 assertNoErrors(doFsck(conf, false));
2875 } finally {
2876 cleanupTable(table);
2877 }
2878
2879 }
2880
2881 @Test (timeout=180000)
2882 public void testHbckAfterRegionMerge() throws Exception {
2883 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
2884 Table meta = null;
2885 try {
2886
2887 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
2888 setupTable(table);
2889 assertEquals(ROWKEYS.length, countRows());
2890
2891
2892 admin.flush(table);
2893 HRegionInfo region1 = tbl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo();
2894 HRegionInfo region2 = tbl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo();
2895
2896 int regionCountBeforeMerge = tbl.getRegionLocations().size();
2897
2898 assertNotEquals(region1, region2);
2899
2900
2901 admin.mergeRegions(region1.getEncodedNameAsBytes(),
2902 region2.getEncodedNameAsBytes(), false);
2903
2904
2905 long timeout = System.currentTimeMillis() + 30 * 1000;
2906 while (true) {
2907 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
2908 break;
2909 } else if (System.currentTimeMillis() > timeout) {
2910 fail("Time out waiting on region " + region1.getEncodedName()
2911 + " and " + region2.getEncodedName() + " be merged");
2912 }
2913 Thread.sleep(10);
2914 }
2915
2916 assertEquals(ROWKEYS.length, countRows());
2917
2918 HBaseFsck hbck = doFsck(conf, false);
2919 assertNoErrors(hbck);
2920
2921 } finally {
2922 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
2923 cleanupTable(table);
2924 IOUtils.closeQuietly(meta);
2925 }
2926 }
2927
2928 @Test (timeout = 180000)
2929 public void testRegionBoundariesCheck() throws Exception {
2930 HBaseFsck hbck = doFsck(conf, false);
2931 assertNoErrors(hbck);
2932 try {
2933 hbck.checkRegionBoundaries();
2934 } catch (IllegalArgumentException e) {
2935 if (e.getMessage().endsWith("not a valid DFS filename.")) {
2936 fail("Table directory path is not valid." + e.getMessage());
2937 }
2938 }
2939 }
2940
2941 @org.junit.Rule
2942 public TestName name = new TestName();
2943
2944 @Test (timeout=180000)
2945 public void testReadOnlyProperty() throws Exception {
2946 HBaseFsck hbck = doFsck(conf, false);
2947 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2948 hbck.shouldIgnorePreCheckPermission());
2949
2950 hbck = doFsck(conf, true);
2951 Assert.assertEquals("shouldIgnorePreCheckPermission", false,
2952 hbck.shouldIgnorePreCheckPermission());
2953
2954 hbck = doFsck(conf, true);
2955 hbck.setIgnorePreCheckPermission(true);
2956 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2957 hbck.shouldIgnorePreCheckPermission());
2958 }
2959
2960 @Test (timeout=180000)
2961 public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
2962 TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit");
2963 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
2964 try {
2965 HTableDescriptor desc = new HTableDescriptor(table);
2966 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
2967 createTable(TEST_UTIL, desc, null);
2968 tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
2969 for (int i = 0; i < 5; i++) {
2970 Put p1 = new Put(("r" + i).getBytes());
2971 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
2972 tbl.put(p1);
2973 }
2974 admin.flush(desc.getTableName());
2975 List<HRegion> regions = cluster.getRegions(desc.getTableName());
2976 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2977 HRegionServer regionServer = cluster.getRegionServer(serverWith);
2978 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2979 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
2980 st.prepare();
2981 st.stepsBeforePONR(regionServer, regionServer, false);
2982 AssignmentManager am = cluster.getMaster().getAssignmentManager();
2983 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
2984 for (RegionState state : regionsInTransition.values()) {
2985 am.regionOffline(state.getRegion());
2986 }
2987 ZKAssign.deleteNodeFailSilent(regionServer.getZooKeeper(), regions.get(0).getRegionInfo());
2988 Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();
2989 regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
2990 am.assign(regionsMap);
2991 am.waitForAssignment(regions.get(0).getRegionInfo());
2992 HBaseFsck hbck = doFsck(conf, false);
2993 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2994 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
2995
2996 assertEquals(0, hbck.getOverlapGroups(table).size());
2997
2998
2999 assertErrors(
3000 doFsck(
3001 conf, false, true, false, false, false, false, false, false, false, false, false,
3002 false, null),
3003 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
3004 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
3005
3006
3007 assertNoErrors(doFsck(conf, false));
3008 assertEquals(5, countRows());
3009 } finally {
3010 if (tbl != null) {
3011 tbl.close();
3012 tbl = null;
3013 }
3014 cleanupTable(table);
3015 }
3016 }
3017
3018
3019 public static class MasterSyncObserver extends BaseMasterObserver {
3020 volatile CountDownLatch tableCreationLatch = null;
3021 volatile CountDownLatch tableDeletionLatch = null;
3022
3023 @Override
3024 public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3025 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
3026
3027 if (tableCreationLatch != null) {
3028 tableCreationLatch.countDown();
3029 }
3030 }
3031
3032 @Override
3033 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3034 TableName tableName)
3035 throws IOException {
3036
3037 if (tableDeletionLatch != null) {
3038 tableDeletionLatch.countDown();
3039 }
3040 }
3041 }
3042
3043 public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
3044 byte [][] splitKeys) throws Exception {
3045
3046
3047 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3048 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3049 observer.tableCreationLatch = new CountDownLatch(1);
3050 if (splitKeys != null) {
3051 admin.createTable(htd, splitKeys);
3052 } else {
3053 admin.createTable(htd);
3054 }
3055 observer.tableCreationLatch.await();
3056 observer.tableCreationLatch = null;
3057 testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
3058 }
3059
3060 public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
3061 throws Exception {
3062
3063
3064 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3065 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3066 observer.tableDeletionLatch = new CountDownLatch(1);
3067 try {
3068 admin.disableTable(tableName);
3069 } catch (Exception e) {
3070 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
3071 }
3072 admin.deleteTable(tableName);
3073 observer.tableDeletionLatch.await();
3074 observer.tableDeletionLatch = null;
3075 }
3076 }