1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.checkRegionBoundaries;
24 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
25 import static org.junit.Assert.assertEquals;
26 import static org.junit.Assert.assertFalse;
27 import static org.junit.Assert.assertNotEquals;
28 import static org.junit.Assert.assertNotNull;
29 import static org.junit.Assert.assertTrue;
30 import static org.junit.Assert.fail;
31
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.HashMap;
37 import java.util.HashSet;
38 import java.util.LinkedList;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.NavigableMap;
42 import java.util.Set;
43 import java.util.concurrent.Callable;
44 import java.util.concurrent.CountDownLatch;
45 import java.util.concurrent.ExecutorService;
46 import java.util.concurrent.Executors;
47 import java.util.concurrent.Future;
48 import java.util.concurrent.ScheduledThreadPoolExecutor;
49 import java.util.concurrent.SynchronousQueue;
50 import java.util.concurrent.ThreadPoolExecutor;
51 import java.util.concurrent.TimeUnit;
52 import java.util.concurrent.atomic.AtomicBoolean;
53
54 import org.apache.commons.io.IOUtils;
55 import org.apache.commons.logging.Log;
56 import org.apache.commons.logging.LogFactory;
57 import org.apache.hadoop.conf.Configuration;
58 import org.apache.hadoop.fs.FileStatus;
59 import org.apache.hadoop.fs.FileSystem;
60 import org.apache.hadoop.fs.Path;
61 import org.apache.hadoop.hbase.ClusterStatus;
62 import org.apache.hadoop.hbase.HBaseTestingUtility;
63 import org.apache.hadoop.hbase.HColumnDescriptor;
64 import org.apache.hadoop.hbase.HConstants;
65 import org.apache.hadoop.hbase.HRegionInfo;
66 import org.apache.hadoop.hbase.HRegionLocation;
67 import org.apache.hadoop.hbase.HTableDescriptor;
68 import org.apache.hadoop.hbase.MetaTableAccessor;
69 import org.apache.hadoop.hbase.MiniHBaseCluster;
70 import org.apache.hadoop.hbase.ServerName;
71 import org.apache.hadoop.hbase.TableExistsException;
72 import org.apache.hadoop.hbase.TableName;
73 import org.apache.hadoop.hbase.Waiter.Predicate;
74 import org.apache.hadoop.hbase.client.Admin;
75 import org.apache.hadoop.hbase.client.ClusterConnection;
76 import org.apache.hadoop.hbase.client.Connection;
77 import org.apache.hadoop.hbase.client.ConnectionFactory;
78 import org.apache.hadoop.hbase.client.Delete;
79 import org.apache.hadoop.hbase.client.Durability;
80 import org.apache.hadoop.hbase.client.Get;
81 import org.apache.hadoop.hbase.client.HBaseAdmin;
82 import org.apache.hadoop.hbase.client.HConnection;
83 import org.apache.hadoop.hbase.client.HTable;
84 import org.apache.hadoop.hbase.client.MetaScanner;
85 import org.apache.hadoop.hbase.client.Put;
86 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
87 import org.apache.hadoop.hbase.client.Result;
88 import org.apache.hadoop.hbase.client.ResultScanner;
89 import org.apache.hadoop.hbase.client.Scan;
90 import org.apache.hadoop.hbase.client.Table;
91 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
92 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
93 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
94 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
95 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
96 import org.apache.hadoop.hbase.io.HFileLink;
97 import org.apache.hadoop.hbase.io.hfile.HFile;
98 import org.apache.hadoop.hbase.io.hfile.HFileContext;
99 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
100 import org.apache.hadoop.hbase.io.hfile.TestHFile;
101 import org.apache.hadoop.hbase.master.AssignmentManager;
102 import org.apache.hadoop.hbase.master.HMaster;
103 import org.apache.hadoop.hbase.master.RegionState;
104 import org.apache.hadoop.hbase.master.RegionStates;
105 import org.apache.hadoop.hbase.master.TableLockManager;
106 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
107 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
108 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
110 import org.apache.hadoop.hbase.regionserver.HRegion;
111 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
112 import org.apache.hadoop.hbase.regionserver.HRegionServer;
113 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl;
114 import org.apache.hadoop.hbase.replication.ReplicationFactory;
115 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
116 import org.apache.hadoop.hbase.replication.ReplicationQueues;
117 import org.apache.hadoop.hbase.testclassification.LargeTests;
118 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
121 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
122 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
123 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
124 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
125 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
126 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
127 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
128 import org.apache.zookeeper.KeeperException;
129 import org.junit.AfterClass;
130 import org.junit.Assert;
131 import org.junit.Before;
132 import org.junit.BeforeClass;
133 import org.junit.Ignore;
134 import org.junit.Test;
135 import org.junit.experimental.categories.Category;
136 import org.junit.rules.TestName;
137
138 import com.google.common.collect.Multimap;
139
140
141
142
143 @Category(LargeTests.class)
144 public class TestHBaseFsck {
145 static final int POOL_SIZE = 7;
146 private static final Log LOG = LogFactory.getLog(TestHBaseFsck.class);
147 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
148 private final static Configuration conf = TEST_UTIL.getConfiguration();
149 private final static String FAM_STR = "fam";
150 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
151 private final static int REGION_ONLINE_TIMEOUT = 800;
152 private static RegionStates regionStates;
153 private static ExecutorService tableExecutorService;
154 private static ScheduledThreadPoolExecutor hbfsckExecutorService;
155 private static ClusterConnection connection;
156 private static Admin admin;
157
158
159 private HTable tbl;
160 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
161 Bytes.toBytes("B"), Bytes.toBytes("C") };
162
163 private final static byte[][] ROWKEYS= new byte[][] {
164 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
165 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
166
167 @BeforeClass
168 public static void setUpBeforeClass() throws Exception {
169 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
170 MasterSyncObserver.class.getName());
171
172 conf.setInt("hbase.regionserver.handler.count", 2);
173 conf.setInt("hbase.regionserver.metahandler.count", 30);
174
175 conf.setInt("hbase.htable.threads.max", POOL_SIZE);
176 conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
177 conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
178 conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
179 conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
180 TEST_UTIL.startMiniCluster(3);
181
182 tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
183 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
184
185 hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
186
187 AssignmentManager assignmentManager =
188 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
189 regionStates = assignmentManager.getRegionStates();
190
191 connection = (ClusterConnection) TEST_UTIL.getConnection();
192
193 admin = connection.getAdmin();
194 admin.setBalancerRunning(false, true);
195
196 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
197 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
198 }
199
200 @AfterClass
201 public static void tearDownAfterClass() throws Exception {
202 tableExecutorService.shutdown();
203 hbfsckExecutorService.shutdown();
204 admin.close();
205 TEST_UTIL.shutdownMiniCluster();
206 }
207
208 @Before
209 public void setUp() {
210 EnvironmentEdgeManager.reset();
211 }
212
213
214
215
216
217 @Test public void testHbckReportReplicaLingeringSplitParent() throws Exception {
218 TableName table = TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
219
220 try {
221 setupTableWithRegionReplica(table, 2);
222 TEST_UTIL.getHBaseAdmin().flush(table.getName());
223
224
225 TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
226 admin.split(table, Bytes.toBytes("A1"));
227
228 Thread.sleep(1000);
229
230 assertNoErrors(doFsck(conf, false));
231 } finally {
232 cleanupTable(table);
233
234 TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
235 }
236 }
237
238 @Test (timeout=180000)
239 public void testHBaseFsck() throws Exception {
240 assertNoErrors(doFsck(conf, false));
241 TableName table = TableName.valueOf("tableBadMetaAssign");
242 HTableDescriptor desc = new HTableDescriptor(table);
243 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
244 desc.addFamily(hcd);
245 createTable(TEST_UTIL, desc, null);
246
247
248 assertNoErrors(doFsck(conf, false));
249
250
251
252 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
253 Scan scan = new Scan();
254 scan.setStartRow(Bytes.toBytes(table+",,"));
255 ResultScanner scanner = meta.getScanner(scan);
256 HRegionInfo hri = null;
257
258 Result res = scanner.next();
259 ServerName currServer =
260 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
261 HConstants.SERVER_QUALIFIER));
262 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
263 HConstants.STARTCODE_QUALIFIER));
264
265 for (JVMClusterUtil.RegionServerThread rs :
266 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
267
268 ServerName sn = rs.getRegionServer().getServerName();
269
270
271 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
272 startCode != sn.getStartcode()) {
273 Put put = new Put(res.getRow());
274 put.setDurability(Durability.SKIP_WAL);
275 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
276 Bytes.toBytes(sn.getHostAndPort()));
277 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
278 Bytes.toBytes(sn.getStartcode()));
279 meta.put(put);
280 hri = MetaTableAccessor.getHRegionInfo(res);
281 break;
282 }
283 }
284
285
286 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
287 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
288
289 TEST_UTIL.getHBaseCluster().getMaster()
290 .getAssignmentManager().waitForAssignment(hri);
291
292
293 assertNoErrors(doFsck(conf, false));
294
295
296 Table t = connection.getTable(table, tableExecutorService);
297 ResultScanner s = t.getScanner(new Scan());
298 s.close();
299 t.close();
300
301 scanner.close();
302 meta.close();
303 }
304
305 @Test(timeout=180000)
306 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
307 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
308 admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO);
309 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
310 new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper());
311 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
312 HBaseFsck hbck = doFsck(conf, true);
313 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
314 ERROR_CODE.NULL_META_REGION });
315 assertNoErrors(doFsck(conf, false));
316 }
317
318
319
320
321
322 @Test (timeout=180000)
323 public void testSplitAndDupeRegionWithRegionReplica() throws Exception {
324 TableName table =
325 TableName.valueOf("testSplitAndDupeRegionWithRegionReplica");
326 Table meta = null;
327
328 try {
329 setupTableWithRegionReplica(table, 2);
330
331 assertNoErrors(doFsck(conf, false));
332 assertEquals(ROWKEYS.length, countRows());
333
334
335 admin.enableCatalogJanitor(false);
336 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
337 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
338 HRegionInfo hriParent = loc.getRegionInfo();
339
340
341 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
342 Thread.sleep(1000);
343
344
345 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
346 TEST_UTIL.assignRegion(hriParent);
347 MetaTableAccessor.addRegionToMeta(meta, hriParent);
348 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
349
350 if (server != null)
351 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
352
353 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
354 Thread.sleep(250);
355 }
356
357 LOG.debug("Finished assignment of parent region");
358
359
360 HBaseFsck hbck = doFsck(conf, false);
361 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED,
362 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
363 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
364 assertEquals(3, hbck.getOverlapGroups(table).size());
365
366
367 hbck = new HBaseFsck(conf, hbfsckExecutorService);
368 hbck.setDisplayFullReport();
369 hbck.setTimeLag(0);
370 hbck.setFixHdfsOverlaps(true);
371 hbck.setRemoveParents(true);
372 hbck.setFixReferenceFiles(true);
373 hbck.setFixHFileLinks(true);
374 hbck.connect();
375 hbck.onlineHbck();
376 hbck.close();
377
378 hbck = doFsck(conf, false);
379
380 assertNoErrors(hbck);
381 assertEquals(0, hbck.getOverlapGroups(table).size());
382 assertEquals(ROWKEYS.length, countRows());
383 } finally {
384 cleanupTable(table);
385 }
386 }
387
388
389
390
391
392 @Test (timeout=180000)
393 public void testSplitAndDupeRegion() throws Exception {
394 TableName table =
395 TableName.valueOf("testSplitAndDupeRegion");
396 Table meta = null;
397
398 try {
399 setupTable(table);
400
401 assertNoErrors(doFsck(conf, false));
402 assertEquals(ROWKEYS.length, countRows());
403
404
405 admin.enableCatalogJanitor(false);
406 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
407 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
408 HRegionInfo hriParent = loc.getRegionInfo();
409
410
411 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
412 Thread.sleep(1000);
413
414
415 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
416 TEST_UTIL.assignRegion(hriParent);
417 MetaTableAccessor.addRegionToMeta(meta, hriParent);
418 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
419
420 if (server != null)
421 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
422
423 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
424 Thread.sleep(250);
425 }
426
427 LOG.debug("Finished assignment of parent region");
428
429
430 HBaseFsck hbck = doFsck(conf, false);
431 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
432 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
433 assertEquals(3, hbck.getOverlapGroups(table).size());
434
435
436 hbck = new HBaseFsck(conf, hbfsckExecutorService);
437 hbck.setDisplayFullReport();
438 hbck.setTimeLag(0);
439 hbck.setFixHdfsOverlaps(true);
440 hbck.setRemoveParents(true);
441 hbck.setFixReferenceFiles(true);
442 hbck.setFixHFileLinks(true);
443 hbck.connect();
444 hbck.onlineHbck();
445 hbck.close();
446
447 hbck = doFsck(conf, false);
448
449 assertNoErrors(hbck);
450 assertEquals(0, hbck.getOverlapGroups(table).size());
451 assertEquals(ROWKEYS.length, countRows());
452 } finally {
453 cleanupTable(table);
454 }
455 }
456
457
458
459
460 private HRegionInfo createRegion(final HTableDescriptor
461 htd, byte[] startKey, byte[] endKey)
462 throws IOException {
463 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
464 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
465 MetaTableAccessor.addRegionToMeta(meta, hri);
466 meta.close();
467 return hri;
468 }
469
470
471
472
473 private void dumpMeta(TableName tableName) throws IOException {
474 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
475 for (byte[] row : metaRows) {
476 LOG.info(Bytes.toString(row));
477 }
478 }
479
480
481
482
483
484 private void undeployRegion(Connection conn, ServerName sn,
485 HRegionInfo hri) throws IOException, InterruptedException {
486 try {
487 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri);
488 if (!hri.isMetaTable()) {
489 admin.offline(hri.getRegionName());
490 }
491 } catch (IOException ioe) {
492 LOG.warn("Got exception when attempting to offline region "
493 + Bytes.toString(hri.getRegionName()), ioe);
494 }
495 }
496
497
498
499
500
501
502 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
503 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
504 boolean hdfs) throws IOException, InterruptedException {
505 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, HRegionInfo.DEFAULT_REPLICA_ID);
506 }
507
508
509
510
511
512
513
514
515
516 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
517 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
518 boolean hdfs, boolean regionInfoOnly, int replicaId)
519 throws IOException, InterruptedException {
520 LOG.info("** Before delete:");
521 dumpMeta(htd.getTableName());
522
523 List<HRegionLocation> locations = tbl.getAllRegionLocations();
524 for (HRegionLocation location : locations) {
525 HRegionInfo hri = location.getRegionInfo();
526 ServerName hsa = location.getServerName();
527 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
528 && Bytes.compareTo(hri.getEndKey(), endKey) == 0
529 && hri.getReplicaId() == replicaId) {
530
531 LOG.info("RegionName: " +hri.getRegionNameAsString());
532 byte[] deleteRow = hri.getRegionName();
533
534 if (unassign) {
535 LOG.info("Undeploying region " + hri + " from server " + hsa);
536 undeployRegion(connection, hsa, hri);
537 }
538
539 if (regionInfoOnly) {
540 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
541 Path rootDir = FSUtils.getRootDir(conf);
542 FileSystem fs = rootDir.getFileSystem(conf);
543 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
544 hri.getEncodedName());
545 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
546 fs.delete(hriPath, true);
547 }
548
549 if (hdfs) {
550 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
551 Path rootDir = FSUtils.getRootDir(conf);
552 FileSystem fs = rootDir.getFileSystem(conf);
553 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
554 hri.getEncodedName());
555 HBaseFsck.debugLsr(conf, p);
556 boolean success = fs.delete(p, true);
557 LOG.info("Deleted " + p + " sucessfully? " + success);
558 HBaseFsck.debugLsr(conf, p);
559 }
560
561 if (metaRow) {
562 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
563 Delete delete = new Delete(deleteRow);
564 meta.delete(delete);
565 }
566 }
567 }
568 LOG.info(hri.toString() + hsa.toString());
569 }
570
571 TEST_UTIL.getMetaTableRows(htd.getTableName());
572 LOG.info("*** After delete:");
573 dumpMeta(htd.getTableName());
574 }
575
576
577
578
579
580
581
582
583
584
585 void setupTable(TableName tablename) throws Exception {
586 setupTableWithRegionReplica(tablename, 1);
587 }
588
589
590
591
592
593
594
595
596
597
598 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
599 HTableDescriptor desc = new HTableDescriptor(tablename);
600 desc.setRegionReplication(replicaCount);
601 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
602 desc.addFamily(hcd);
603 createTable(TEST_UTIL, desc, SPLITS);
604
605 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
606 List<Put> puts = new ArrayList<Put>();
607 for (byte[] row : ROWKEYS) {
608 Put p = new Put(row);
609 p.add(FAM, Bytes.toBytes("val"), row);
610 puts.add(p);
611 }
612 tbl.put(puts);
613 tbl.flushCommits();
614 }
615
616
617
618
619 int countRows() throws IOException {
620 Scan s = new Scan();
621 ResultScanner rs = tbl.getScanner(s);
622 int i = 0;
623 while(rs.next() !=null) {
624 i++;
625 }
626 return i;
627 }
628
629
630
631
632 int countRows(byte[] start, byte[] end) throws IOException {
633 Scan s = new Scan(start, end);
634 ResultScanner rs = tbl.getScanner(s);
635 int i = 0;
636 while (rs.next() != null) {
637 i++;
638 }
639 return i;
640 }
641
642
643
644
645
646
647 void cleanupTable(TableName tablename) throws Exception {
648 if (tbl != null) {
649 tbl.close();
650 tbl = null;
651 }
652
653 ((ClusterConnection) connection).clearRegionCache();
654 deleteTable(TEST_UTIL, tablename);
655 }
656
657
658
659
660 @Test (timeout=180000)
661 public void testHBaseFsckClean() throws Exception {
662 assertNoErrors(doFsck(conf, false));
663 TableName table = TableName.valueOf("tableClean");
664 try {
665 HBaseFsck hbck = doFsck(conf, false);
666 assertNoErrors(hbck);
667
668 setupTable(table);
669 assertEquals(ROWKEYS.length, countRows());
670
671
672 hbck = doFsck(conf, false);
673 assertNoErrors(hbck);
674 assertEquals(0, hbck.getOverlapGroups(table).size());
675 assertEquals(ROWKEYS.length, countRows());
676 } finally {
677 cleanupTable(table);
678 }
679 }
680
681
682
683
684 @Test (timeout=180000)
685 public void testHbckThreadpooling() throws Exception {
686 TableName table =
687 TableName.valueOf("tableDupeStartKey");
688 try {
689
690 setupTable(table);
691
692
693 Configuration newconf = new Configuration(conf);
694 newconf.setInt("hbasefsck.numthreads", 1);
695 assertNoErrors(doFsck(newconf, false));
696
697
698 } finally {
699 cleanupTable(table);
700 }
701 }
702
703 @Test (timeout=180000)
704 public void testHbckFixOrphanTable() throws Exception {
705 TableName table = TableName.valueOf("tableInfo");
706 FileSystem fs = null;
707 Path tableinfo = null;
708 try {
709 setupTable(table);
710
711 Path hbaseTableDir = FSUtils.getTableDir(
712 FSUtils.getRootDir(conf), table);
713 fs = hbaseTableDir.getFileSystem(conf);
714 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
715 tableinfo = status.getPath();
716 fs.rename(tableinfo, new Path("/.tableinfo"));
717
718
719 HBaseFsck hbck = doFsck(conf, false);
720 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
721
722
723 hbck = doFsck(conf, true);
724 assertNoErrors(hbck);
725 status = null;
726 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
727 assertNotNull(status);
728
729 HTableDescriptor htd = admin.getTableDescriptor(table);
730 htd.setValue("NOT_DEFAULT", "true");
731 admin.disableTable(table);
732 admin.modifyTable(table, htd);
733 admin.enableTable(table);
734 fs.delete(status.getPath(), true);
735
736
737 htd = admin.getTableDescriptor(table);
738 hbck = doFsck(conf, true);
739 assertNoErrors(hbck);
740 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
741 assertNotNull(status);
742 htd = admin.getTableDescriptor(table);
743 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
744 } finally {
745 if (fs != null) {
746 fs.rename(new Path("/.tableinfo"), tableinfo);
747 }
748 cleanupTable(table);
749 }
750 }
751
752
753
754
755
756
757 @Test (timeout=180000)
758 public void testParallelHbck() throws Exception {
759 final ExecutorService service;
760 final Future<HBaseFsck> hbck1,hbck2;
761
762 class RunHbck implements Callable<HBaseFsck>{
763 boolean fail = true;
764 @Override
765 public HBaseFsck call(){
766 Configuration c = new Configuration(conf);
767 c.setInt("hbase.hbck.lockfile.attempts", 1);
768
769
770 c.setInt("hbase.hbck.lockfile.maxwaittime", 3);
771 try{
772 return doFsck(c, true);
773 } catch(Exception e){
774 if (e.getMessage().contains("Duplicate hbck")) {
775 fail = false;
776 }
777 }
778
779 if (fail) fail();
780 return null;
781 }
782 }
783 service = Executors.newFixedThreadPool(2);
784 hbck1 = service.submit(new RunHbck());
785 hbck2 = service.submit(new RunHbck());
786 service.shutdown();
787
788 service.awaitTermination(15, TimeUnit.SECONDS);
789 HBaseFsck h1 = hbck1.get();
790 HBaseFsck h2 = hbck2.get();
791
792 assert(h1 == null || h2 == null);
793 if (h1 != null) {
794 assert(h1.getRetCode() >= 0);
795 }
796 if (h2 != null) {
797 assert(h2.getRetCode() >= 0);
798 }
799 }
800
801
802
803
804
805
806
807 @Test (timeout=180000)
808 public void testParallelWithRetriesHbck() throws Exception {
809 final ExecutorService service;
810 final Future<HBaseFsck> hbck1,hbck2;
811
812
813
814
815
816
817
818 final int timeoutInSeconds = 80;
819 final int sleepIntervalInMilliseconds = 200;
820 final int maxSleepTimeInMilliseconds = 6000;
821 final int maxRetryAttempts = 15;
822
823 class RunHbck implements Callable<HBaseFsck>{
824
825 @Override
826 public HBaseFsck call() throws Exception {
827
828 Configuration c = new Configuration(conf);
829 c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds);
830 c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds);
831 c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds);
832 c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts);
833 return doFsck(c, false);
834 }
835 }
836
837 service = Executors.newFixedThreadPool(2);
838 hbck1 = service.submit(new RunHbck());
839 hbck2 = service.submit(new RunHbck());
840 service.shutdown();
841
842 service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS);
843 HBaseFsck h1 = hbck1.get();
844 HBaseFsck h2 = hbck2.get();
845
846 assertNotNull(h1);
847 assertNotNull(h2);
848 assert(h1.getRetCode() >= 0);
849 assert(h2.getRetCode() >= 0);
850
851 }
852
853
854
855
856
857 @Test (timeout=180000)
858 public void testDupeStartKey() throws Exception {
859 TableName table =
860 TableName.valueOf("tableDupeStartKey");
861 try {
862 setupTable(table);
863 assertNoErrors(doFsck(conf, false));
864 assertEquals(ROWKEYS.length, countRows());
865
866
867 HRegionInfo hriDupe =
868 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("A2"));
869 TEST_UTIL.assignRegion(hriDupe);
870 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
871 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
872
873 HBaseFsck hbck = doFsck(conf, false);
874 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
875 ERROR_CODE.DUPE_STARTKEYS});
876 assertEquals(2, hbck.getOverlapGroups(table).size());
877 assertEquals(ROWKEYS.length, countRows());
878
879
880 doFsck(conf,true);
881
882
883 HBaseFsck hbck2 = doFsck(conf,false);
884 assertNoErrors(hbck2);
885 assertEquals(0, hbck2.getOverlapGroups(table).size());
886 assertEquals(ROWKEYS.length, countRows());
887 } finally {
888 cleanupTable(table);
889 }
890 }
891
892
893
894
895
896 @Test (timeout=180000)
897 public void testHbckWithRegionReplica() throws Exception {
898 TableName table =
899 TableName.valueOf("testHbckWithRegionReplica");
900 try {
901 setupTableWithRegionReplica(table, 2);
902 TEST_UTIL.getHBaseAdmin().flush(table.getName());
903 assertNoErrors(doFsck(conf, false));
904 } finally {
905 cleanupTable(table);
906 }
907 }
908
909
910
911
912
913 @Test
914 public void testHbckReplicaRegionAsKeyInMeta() throws Exception {
915 TableName table = TableName.valueOf("testHbckReplicaRegionAsKeyInMeta");
916 try {
917 setupTableWithRegionReplica(table, 2);
918 TEST_UTIL.getHBaseAdmin().flush(table.getName());
919
920 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
921 HRegionInfo hri = new HRegionInfo(table, SPLITS[0], SPLITS[2], false, 1500328224175L, 1);
922 Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
923 meta.put(put);
924
925 assertErrors(doFsck(conf, false),
926 new HBaseFsck.ErrorReporter.ERROR_CODE[] {
927 HBaseFsck.ErrorReporter.ERROR_CODE.EMPTY_META_CELL });
928
929
930 doFsck(conf, true);
931
932
933 assertNoErrors(doFsck(conf, false));
934 } finally {
935 cleanupTable(table);
936 }
937 }
938
939 @Test
940 public void testHbckWithFewerReplica() throws Exception {
941 TableName table =
942 TableName.valueOf("testHbckWithFewerReplica");
943 try {
944 setupTableWithRegionReplica(table, 2);
945 TEST_UTIL.getHBaseAdmin().flush(table.getName());
946 assertNoErrors(doFsck(conf, false));
947 assertEquals(ROWKEYS.length, countRows());
948 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
949 Bytes.toBytes("C"), true, false, false, false, 1);
950
951 HBaseFsck hbck = doFsck(conf, false);
952 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_DEPLOYED});
953
954 hbck = doFsck(conf, true);
955
956 hbck = doFsck(conf, false);
957 assertErrors(hbck, new ERROR_CODE[]{});
958 } finally {
959 cleanupTable(table);
960 }
961 }
962
963 @Test
964 public void testHbckWithExcessReplica() throws Exception {
965 TableName table =
966 TableName.valueOf("testHbckWithExcessReplica");
967 try {
968 setupTableWithRegionReplica(table, 2);
969 TEST_UTIL.getHBaseAdmin().flush(table.getName());
970 assertNoErrors(doFsck(conf, false));
971 assertEquals(ROWKEYS.length, countRows());
972
973
974
975
976 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
977 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(table);
978 byte[] startKey = Bytes.toBytes("B");
979 byte[] endKey = Bytes.toBytes("C");
980 byte[] metaKey = null;
981 HRegionInfo newHri = null;
982 for (HRegionInfo h : regions) {
983 if (Bytes.compareTo(h.getStartKey(), startKey) == 0 &&
984 Bytes.compareTo(h.getEndKey(), endKey) == 0 &&
985 h.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
986 metaKey = h.getRegionName();
987
988 newHri = RegionReplicaUtil.getRegionInfoForReplica(h, 2);
989 break;
990 }
991 }
992 Put put = new Put(metaKey);
993 ServerName sn = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers()
994 .toArray(new ServerName[0])[0];
995
996 MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, 2);
997 meta.put(put);
998 meta.flushCommits();
999
1000 HBaseFsckRepair.fixUnassigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
1001 HBaseFsckRepair.waitUntilAssigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
1002
1003 Delete delete = new Delete(metaKey);
1004 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(2));
1005 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(2));
1006 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(2));
1007 meta.delete(delete);
1008 meta.flushCommits();
1009 meta.close();
1010
1011 HBaseFsck hbck = doFsck(conf, false);
1012 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_IN_META});
1013
1014 hbck = doFsck(conf, true);
1015
1016 hbck = doFsck(conf, false);
1017 assertErrors(hbck, new ERROR_CODE[]{});
1018 } finally {
1019 cleanupTable(table);
1020 }
1021 }
1022
1023
1024
1025 Map<ServerName, List<String>> getDeployedHRIs(final HBaseAdmin admin) throws IOException {
1026 ClusterStatus status = admin.getClusterStatus();
1027 Collection<ServerName> regionServers = status.getServers();
1028 Map<ServerName, List<String>> mm =
1029 new HashMap<ServerName, List<String>>();
1030 for (ServerName hsi : regionServers) {
1031 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi);
1032
1033
1034 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
1035 List<String> regionNames = new ArrayList<String>();
1036 for (HRegionInfo hri : regions) {
1037 regionNames.add(hri.getRegionNameAsString());
1038 }
1039 mm.put(hsi, regionNames);
1040 }
1041 return mm;
1042 }
1043
1044
1045
1046
1047 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
1048 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
1049 if (e.getValue().contains(hri.getRegionNameAsString())) {
1050 return e.getKey();
1051 }
1052 }
1053 return null;
1054 }
1055
1056
1057
1058
1059
1060 @Test (timeout=180000)
1061 public void testDupeRegion() throws Exception {
1062 TableName table =
1063 TableName.valueOf("tableDupeRegion");
1064 try {
1065 setupTable(table);
1066 assertNoErrors(doFsck(conf, false));
1067 assertEquals(ROWKEYS.length, countRows());
1068
1069
1070 HRegionInfo hriDupe =
1071 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"));
1072
1073 TEST_UTIL.assignRegion(hriDupe);
1074 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1075 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1076
1077
1078
1079
1080
1081 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriDupe) == null) {
1082 Thread.sleep(250);
1083 }
1084
1085 LOG.debug("Finished assignment of dupe region");
1086
1087
1088 HBaseFsck hbck = doFsck(conf, false);
1089 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
1090 ERROR_CODE.DUPE_STARTKEYS});
1091 assertEquals(2, hbck.getOverlapGroups(table).size());
1092 assertEquals(ROWKEYS.length, countRows());
1093
1094
1095 doFsck(conf,true);
1096
1097
1098 HBaseFsck hbck2 = doFsck(conf,false);
1099 assertNoErrors(hbck2);
1100 assertEquals(0, hbck2.getOverlapGroups(table).size());
1101 assertEquals(ROWKEYS.length, countRows());
1102 } finally {
1103 cleanupTable(table);
1104 }
1105 }
1106
1107
1108
1109
1110 @Test (timeout=180000)
1111 public void testDegenerateRegions() throws Exception {
1112 TableName table = TableName.valueOf("tableDegenerateRegions");
1113 try {
1114 setupTable(table);
1115 assertNoErrors(doFsck(conf,false));
1116 assertEquals(ROWKEYS.length, countRows());
1117
1118
1119 HRegionInfo hriDupe =
1120 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B"));
1121 TEST_UTIL.assignRegion(hriDupe);
1122 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1123 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1124
1125 HBaseFsck hbck = doFsck(conf,false);
1126 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, ERROR_CODE.DUPE_STARTKEYS,
1127 ERROR_CODE.DUPE_STARTKEYS });
1128 assertEquals(2, hbck.getOverlapGroups(table).size());
1129 assertEquals(ROWKEYS.length, countRows());
1130
1131
1132 doFsck(conf,true);
1133
1134
1135 HBaseFsck hbck2 = doFsck(conf,false);
1136 assertNoErrors(hbck2);
1137 assertEquals(0, hbck2.getOverlapGroups(table).size());
1138 assertEquals(ROWKEYS.length, countRows());
1139 } finally {
1140 cleanupTable(table);
1141 }
1142 }
1143
1144
1145
1146
1147
1148 @Test (timeout=180000)
1149 public void testContainedRegionOverlap() throws Exception {
1150 TableName table =
1151 TableName.valueOf("tableContainedRegionOverlap");
1152 try {
1153 setupTable(table);
1154 assertEquals(ROWKEYS.length, countRows());
1155
1156
1157 HRegionInfo hriOverlap =
1158 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1159 TEST_UTIL.assignRegion(hriOverlap);
1160
1161 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1162 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1163
1164 HBaseFsck hbck = doFsck(conf, false);
1165 assertErrors(hbck, new ERROR_CODE[] {
1166 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1167 assertEquals(2, hbck.getOverlapGroups(table).size());
1168 assertEquals(ROWKEYS.length, countRows());
1169
1170
1171 doFsck(conf, true);
1172
1173
1174 HBaseFsck hbck2 = doFsck(conf,false);
1175 assertNoErrors(hbck2);
1176 assertEquals(0, hbck2.getOverlapGroups(table).size());
1177 assertEquals(ROWKEYS.length, countRows());
1178 } finally {
1179 cleanupTable(table);
1180 }
1181 }
1182
1183
1184
1185
1186
1187
1188
1189 @Test (timeout=180000)
1190 public void testSidelineOverlapRegion() throws Exception {
1191 TableName table =
1192 TableName.valueOf("testSidelineOverlapRegion");
1193 try {
1194 setupTable(table);
1195 assertEquals(ROWKEYS.length, countRows());
1196
1197
1198 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1199 HMaster master = cluster.getMaster();
1200 HRegionInfo hriOverlap1 =
1201 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB"));
1202 TEST_UTIL.assignRegion(hriOverlap1);
1203 HRegionInfo hriOverlap2 =
1204 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("AB"), Bytes.toBytes("B"));
1205 TEST_UTIL.assignRegion(hriOverlap2);
1206
1207 HBaseFsck hbck = doFsck(conf, false);
1208 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
1209 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
1210 assertEquals(3, hbck.getOverlapGroups(table).size());
1211 assertEquals(ROWKEYS.length, countRows());
1212
1213
1214 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
1215 ServerName serverName = null;
1216 byte[] regionName = null;
1217 for (HbckInfo hbi: overlapGroups.values()) {
1218 if ("A".equals(Bytes.toString(hbi.getStartKey()))
1219 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
1220 regionName = hbi.getRegionName();
1221
1222
1223 int k = cluster.getServerWith(regionName);
1224 for (int i = 0; i < 3; i++) {
1225 if (i != k) {
1226 HRegionServer rs = cluster.getRegionServer(i);
1227 serverName = rs.getServerName();
1228 break;
1229 }
1230 }
1231
1232 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) connection,
1233 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
1234 admin.offline(regionName);
1235 break;
1236 }
1237 }
1238
1239 assertNotNull(regionName);
1240 assertNotNull(serverName);
1241 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
1242 Put put = new Put(regionName);
1243 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
1244 Bytes.toBytes(serverName.getHostAndPort()));
1245 meta.put(put);
1246 }
1247
1248
1249 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1250 fsck.connect();
1251 fsck.setDisplayFullReport();
1252 fsck.setTimeLag(0);
1253 fsck.setFixAssignments(true);
1254 fsck.setFixMeta(true);
1255 fsck.setFixHdfsHoles(true);
1256 fsck.setFixHdfsOverlaps(true);
1257 fsck.setFixHdfsOrphans(true);
1258 fsck.setFixVersionFile(true);
1259 fsck.setSidelineBigOverlaps(true);
1260 fsck.setMaxMerge(2);
1261 fsck.onlineHbck();
1262 fsck.close();
1263
1264
1265
1266 HBaseFsck hbck2 = doFsck(conf,false);
1267 assertNoErrors(hbck2);
1268 assertEquals(0, hbck2.getOverlapGroups(table).size());
1269 assertTrue(ROWKEYS.length > countRows());
1270 } finally {
1271 cleanupTable(table);
1272 }
1273 }
1274
1275
1276
1277
1278
1279 @Test (timeout=180000)
1280 public void testOverlapAndOrphan() throws Exception {
1281 TableName table =
1282 TableName.valueOf("tableOverlapAndOrphan");
1283 try {
1284 setupTable(table);
1285 assertEquals(ROWKEYS.length, countRows());
1286
1287
1288 admin.disableTable(table);
1289 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1290 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1291 TEST_UTIL.getHBaseAdmin().enableTable(table);
1292
1293 HRegionInfo hriOverlap =
1294 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1295 TEST_UTIL.assignRegion(hriOverlap);
1296 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1297 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1298
1299 HBaseFsck hbck = doFsck(conf, false);
1300 assertErrors(hbck, new ERROR_CODE[] {
1301 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1302 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1303
1304
1305 doFsck(conf, true);
1306
1307
1308 HBaseFsck hbck2 = doFsck(conf,false);
1309 assertNoErrors(hbck2);
1310 assertEquals(0, hbck2.getOverlapGroups(table).size());
1311 assertEquals(ROWKEYS.length, countRows());
1312 } finally {
1313 cleanupTable(table);
1314 }
1315 }
1316
1317
1318
1319
1320
1321
1322 @Test (timeout=180000)
1323 public void testCoveredStartKey() throws Exception {
1324 TableName table =
1325 TableName.valueOf("tableCoveredStartKey");
1326 try {
1327 setupTable(table);
1328 assertEquals(ROWKEYS.length, countRows());
1329
1330
1331 HRegionInfo hriOverlap =
1332 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2"));
1333 TEST_UTIL.assignRegion(hriOverlap);
1334 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1335 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1336
1337 HBaseFsck hbck = doFsck(conf, false);
1338 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
1339 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1340 assertEquals(3, hbck.getOverlapGroups(table).size());
1341 assertEquals(ROWKEYS.length, countRows());
1342
1343
1344 doFsck(conf, true);
1345
1346
1347 HBaseFsck hbck2 = doFsck(conf, false);
1348 assertErrors(hbck2, new ERROR_CODE[0]);
1349 assertEquals(0, hbck2.getOverlapGroups(table).size());
1350 assertEquals(ROWKEYS.length, countRows());
1351 } finally {
1352 cleanupTable(table);
1353 }
1354 }
1355
1356
1357
1358
1359
1360 @Test (timeout=180000)
1361 public void testRegionHole() throws Exception {
1362 TableName table =
1363 TableName.valueOf("tableRegionHole");
1364 try {
1365 setupTable(table);
1366 assertEquals(ROWKEYS.length, countRows());
1367
1368
1369 admin.disableTable(table);
1370 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1371 Bytes.toBytes("C"), true, true, true);
1372 admin.enableTable(table);
1373
1374 HBaseFsck hbck = doFsck(conf, false);
1375 assertErrors(hbck, new ERROR_CODE[] {
1376 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1377
1378 assertEquals(0, hbck.getOverlapGroups(table).size());
1379
1380
1381 doFsck(conf, true);
1382
1383
1384 assertNoErrors(doFsck(conf,false));
1385 assertEquals(ROWKEYS.length - 2 , countRows());
1386 } finally {
1387 cleanupTable(table);
1388 }
1389 }
1390
1391
1392
1393
1394
1395 @Test (timeout=180000)
1396 public void testHDFSRegioninfoMissing() throws Exception {
1397 TableName table = TableName.valueOf("tableHDFSRegioninfoMissing");
1398 try {
1399 setupTable(table);
1400 assertEquals(ROWKEYS.length, countRows());
1401
1402
1403 admin.disableTable(table);
1404 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1405 Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1406 TEST_UTIL.getHBaseAdmin().enableTable(table);
1407
1408 HBaseFsck hbck = doFsck(conf, false);
1409 assertErrors(hbck, new ERROR_CODE[] {
1410 ERROR_CODE.ORPHAN_HDFS_REGION,
1411 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1412 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1413
1414 assertEquals(0, hbck.getOverlapGroups(table).size());
1415
1416
1417 doFsck(conf, true);
1418
1419
1420 assertNoErrors(doFsck(conf, false));
1421 assertEquals(ROWKEYS.length, countRows());
1422 } finally {
1423 cleanupTable(table);
1424 }
1425 }
1426
1427
1428
1429
1430
1431
1432 @Test(timeout = 180000)
1433 public void testHDFSRegioninfoMissingAndCheckRegionBoundary() throws Exception {
1434 TableName table = TableName.valueOf("testHDFSRegioninfoMissingAndCheckRegionBoundary");
1435 try {
1436 setupTable(table);
1437 assertEquals(ROWKEYS.length, countRows());
1438
1439
1440 admin.disableTable(table);
1441 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true,
1442 true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1443 admin.enableTable(table);
1444
1445 HBaseFsck hbck = doFsck(conf, false);
1446 assertErrors(hbck,
1447 new HBaseFsck.ErrorReporter.ERROR_CODE[] {
1448 HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION,
1449 HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1450 HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN });
1451
1452 assertEquals(0, hbck.getOverlapGroups(table).size());
1453
1454
1455 doFsck(conf, true);
1456
1457
1458 assertNoErrors(doFsck(conf, false));
1459
1460
1461 for (int i = 0; i < ROWKEYS.length; i++) {
1462 if (i != ROWKEYS.length - 1) {
1463 assertEquals(1, countRows(ROWKEYS[i], ROWKEYS[i + 1]));
1464 } else {
1465 assertEquals(1, countRows(ROWKEYS[i], null));
1466 }
1467 }
1468
1469 } finally {
1470 cleanupTable(table);
1471 }
1472 }
1473
1474
1475
1476
1477
1478 @Test (timeout=180000)
1479 public void testNotInMetaOrDeployedHole() throws Exception {
1480 TableName table =
1481 TableName.valueOf("tableNotInMetaOrDeployedHole");
1482 try {
1483 setupTable(table);
1484 assertEquals(ROWKEYS.length, countRows());
1485
1486
1487 admin.disableTable(table);
1488 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1489 Bytes.toBytes("C"), true, true, false);
1490 admin.enableTable(table);
1491
1492 HBaseFsck hbck = doFsck(conf, false);
1493 assertErrors(hbck, new ERROR_CODE[] {
1494 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1495
1496 assertEquals(0, hbck.getOverlapGroups(table).size());
1497
1498
1499 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1500 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1501
1502
1503 assertNoErrors(doFsck(conf,false));
1504 assertEquals(ROWKEYS.length, countRows());
1505 } finally {
1506 cleanupTable(table);
1507 }
1508 }
1509
1510
1511
1512
1513 @Test (timeout=180000)
1514 public void testNotInMetaHole() throws Exception {
1515 TableName table =
1516 TableName.valueOf("tableNotInMetaHole");
1517 try {
1518 setupTable(table);
1519 assertEquals(ROWKEYS.length, countRows());
1520
1521
1522 admin.disableTable(table);
1523 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1524 Bytes.toBytes("C"), false, true, false);
1525 admin.enableTable(table);
1526
1527 HBaseFsck hbck = doFsck(conf, false);
1528 assertErrors(hbck, new ERROR_CODE[] {
1529 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1530
1531 assertEquals(0, hbck.getOverlapGroups(table).size());
1532
1533
1534 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1535 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1536
1537
1538 assertNoErrors(doFsck(conf,false));
1539 assertEquals(ROWKEYS.length, countRows());
1540 } finally {
1541 cleanupTable(table);
1542 }
1543 }
1544
1545
1546
1547
1548
1549 @Test (timeout=180000)
1550 public void testNotInHdfs() throws Exception {
1551 TableName table =
1552 TableName.valueOf("tableNotInHdfs");
1553 try {
1554 setupTable(table);
1555 assertEquals(ROWKEYS.length, countRows());
1556
1557
1558 admin.flush(table);
1559
1560
1561 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1562 Bytes.toBytes("C"), false, false, true);
1563
1564 HBaseFsck hbck = doFsck(conf, false);
1565 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1566
1567 assertEquals(0, hbck.getOverlapGroups(table).size());
1568
1569
1570 doFsck(conf, true);
1571
1572
1573 assertNoErrors(doFsck(conf,false));
1574 assertEquals(ROWKEYS.length - 2, countRows());
1575 } finally {
1576 cleanupTable(table);
1577 }
1578 }
1579
1580
1581
1582
1583
1584 @Test (timeout=180000)
1585 public void testNotInHdfsWithReplicas() throws Exception {
1586 TableName table =
1587 TableName.valueOf("tableNotInHdfs");
1588 HBaseAdmin admin = new HBaseAdmin(conf);
1589 try {
1590 HRegionInfo[] oldHris = new HRegionInfo[2];
1591 setupTableWithRegionReplica(table, 2);
1592 assertEquals(ROWKEYS.length, countRows());
1593 NavigableMap<HRegionInfo, ServerName> map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(),
1594 tbl.getName());
1595 int i = 0;
1596
1597 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1598 if (m.getKey().getStartKey().length > 0 &&
1599 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1600 LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
1601 oldHris[i++] = m.getKey();
1602 }
1603 }
1604
1605 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1606
1607
1608 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1609 Bytes.toBytes("C"), false, false, true);
1610
1611 HBaseFsck hbck = doFsck(conf, false);
1612 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1613
1614
1615 doFsck(conf, true);
1616
1617
1618 assertNoErrors(doFsck(conf,false));
1619 assertEquals(ROWKEYS.length - 2, countRows());
1620
1621
1622
1623 i = 0;
1624 HRegionInfo[] newHris = new HRegionInfo[2];
1625
1626 map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
1627
1628 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1629 if (m.getKey().getStartKey().length > 0 &&
1630 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1631 newHris[i++] = m.getKey();
1632 }
1633 }
1634
1635 Collection<ServerName> servers = admin.getClusterStatus().getServers();
1636 Set<HRegionInfo> onlineRegions = new HashSet<HRegionInfo>();
1637 for (ServerName s : servers) {
1638 List<HRegionInfo> list = admin.getOnlineRegions(s);
1639 onlineRegions.addAll(list);
1640 }
1641
1642 assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
1643
1644
1645 assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
1646 } finally {
1647 cleanupTable(table);
1648 admin.close();
1649 }
1650 }
1651
1652
1653
1654
1655
1656
1657 @Test (timeout=180000)
1658 public void testNoHdfsTable() throws Exception {
1659 TableName table = TableName.valueOf("NoHdfsTable");
1660 setupTable(table);
1661 assertEquals(ROWKEYS.length, countRows());
1662
1663
1664 admin.flush(table);
1665
1666
1667 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1668 Bytes.toBytes("A"), false, false, true);
1669 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1670 Bytes.toBytes("B"), false, false, true);
1671 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1672 Bytes.toBytes("C"), false, false, true);
1673 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1674 Bytes.toBytes(""), false, false, true);
1675
1676
1677 deleteTableDir(table);
1678
1679 HBaseFsck hbck = doFsck(conf, false);
1680 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1681 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1682 ERROR_CODE.NOT_IN_HDFS,});
1683
1684 assertEquals(0, hbck.getOverlapGroups(table).size());
1685
1686
1687 doFsck(conf, true);
1688
1689
1690 assertNoErrors(doFsck(conf,false));
1691 assertFalse("Table " + table + " should have been deleted", admin.tableExists(table));
1692 }
1693
1694 public void deleteTableDir(TableName table) throws IOException {
1695 Path rootDir = FSUtils.getRootDir(conf);
1696 FileSystem fs = rootDir.getFileSystem(conf);
1697 Path p = FSUtils.getTableDir(rootDir, table);
1698 HBaseFsck.debugLsr(conf, p);
1699 boolean success = fs.delete(p, true);
1700 LOG.info("Deleted " + p + " sucessfully? " + success);
1701 }
1702
1703
1704
1705
1706 @Test (timeout=180000)
1707 public void testNoVersionFile() throws Exception {
1708
1709 Path rootDir = FSUtils.getRootDir(conf);
1710 FileSystem fs = rootDir.getFileSystem(conf);
1711 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1712 fs.delete(versionFile, true);
1713
1714
1715 HBaseFsck hbck = doFsck(conf, false);
1716 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1717
1718 doFsck(conf, true);
1719
1720
1721 assertNoErrors(doFsck(conf, false));
1722 }
1723
1724
1725
1726
1727 @Test (timeout=180000)
1728 public void testRegionShouldNotBeDeployed() throws Exception {
1729 TableName table =
1730 TableName.valueOf("tableRegionShouldNotBeDeployed");
1731 try {
1732 LOG.info("Starting testRegionShouldNotBeDeployed.");
1733 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1734 assertTrue(cluster.waitForActiveAndReadyMaster());
1735
1736
1737 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1738 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1739 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1740 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1741
1742
1743 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1744 fstd.createTableDescriptor(htdDisabled);
1745 List<HRegionInfo> disabledRegions =
1746 TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
1747
1748
1749 HRegionServer hrs = cluster.getRegionServer(0);
1750
1751
1752 admin.disableTable(table);
1753 admin.enableTable(table);
1754
1755
1756 admin.disableTable(table);
1757 HRegionInfo region = disabledRegions.remove(0);
1758 byte[] regionName = region.getRegionName();
1759
1760
1761 assertTrue(cluster.getServerWith(regionName) == -1);
1762
1763
1764
1765
1766
1767 HRegion r = HRegion.openHRegion(
1768 region, htdDisabled, hrs.getWAL(region), conf);
1769 hrs.addToOnlineRegions(r);
1770
1771 HBaseFsck hbck = doFsck(conf, false);
1772 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1773
1774
1775 doFsck(conf, true);
1776
1777
1778 assertNoErrors(doFsck(conf, false));
1779 } finally {
1780 admin.enableTable(table);
1781 cleanupTable(table);
1782 }
1783 }
1784
1785
1786
1787
1788 @Test (timeout=180000)
1789 public void testFixByTable() throws Exception {
1790 TableName table1 =
1791 TableName.valueOf("testFixByTable1");
1792 TableName table2 =
1793 TableName.valueOf("testFixByTable2");
1794 try {
1795 setupTable(table1);
1796
1797 admin.flush(table1);
1798
1799 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1800 Bytes.toBytes("C"), false, false, true);
1801
1802 setupTable(table2);
1803
1804 admin.flush(table2);
1805
1806 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1807 Bytes.toBytes("C"), false, false, true);
1808
1809 HBaseFsck hbck = doFsck(conf, false);
1810 assertErrors(hbck, new ERROR_CODE[] {
1811 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1812
1813
1814 doFsck(conf, true, table1);
1815
1816 assertNoErrors(doFsck(conf, false, table1));
1817
1818 assertErrors(doFsck(conf, false, table2),
1819 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1820
1821
1822 doFsck(conf, true, table2);
1823
1824 assertNoErrors(doFsck(conf, false));
1825 assertEquals(ROWKEYS.length - 2, countRows());
1826 } finally {
1827 cleanupTable(table1);
1828 cleanupTable(table2);
1829 }
1830 }
1831
1832
1833
1834 @Test (timeout=180000)
1835 public void testLingeringSplitParent() throws Exception {
1836 TableName table =
1837 TableName.valueOf("testLingeringSplitParent");
1838 Table meta = null;
1839 try {
1840 setupTable(table);
1841 assertEquals(ROWKEYS.length, countRows());
1842
1843
1844 admin.flush(table);
1845 HRegionLocation location = tbl.getRegionLocation("B");
1846
1847
1848 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1849 Bytes.toBytes("C"), true, true, false);
1850
1851
1852 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1853 HRegionInfo hri = location.getRegionInfo();
1854
1855 HRegionInfo a = new HRegionInfo(tbl.getName(),
1856 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1857 HRegionInfo b = new HRegionInfo(tbl.getName(),
1858 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1859
1860 hri.setOffline(true);
1861 hri.setSplit(true);
1862
1863 MetaTableAccessor.addRegionToMeta(meta, hri, a, b);
1864 meta.close();
1865 admin.flush(TableName.META_TABLE_NAME);
1866
1867 HBaseFsck hbck = doFsck(conf, false);
1868 assertErrors(hbck, new ERROR_CODE[] {
1869 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1870
1871
1872 hbck = doFsck(conf, true);
1873 assertErrors(hbck, new ERROR_CODE[] {
1874 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1875 assertFalse(hbck.shouldRerun());
1876 hbck = doFsck(conf, false);
1877 assertErrors(hbck, new ERROR_CODE[] {
1878 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1879
1880
1881 hbck = new HBaseFsck(conf, hbfsckExecutorService);
1882 hbck.connect();
1883 hbck.setDisplayFullReport();
1884 hbck.setTimeLag(0);
1885 hbck.setFixSplitParents(true);
1886 hbck.onlineHbck();
1887 assertTrue(hbck.shouldRerun());
1888 hbck.close();
1889
1890 Get get = new Get(hri.getRegionName());
1891 Result result = meta.get(get);
1892 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1893 HConstants.SPLITA_QUALIFIER).isEmpty());
1894 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1895 HConstants.SPLITB_QUALIFIER).isEmpty());
1896 admin.flush(TableName.META_TABLE_NAME);
1897
1898
1899 doFsck(conf, true);
1900
1901
1902 assertNoErrors(doFsck(conf, false));
1903 assertEquals(ROWKEYS.length, countRows());
1904 } finally {
1905 cleanupTable(table);
1906 IOUtils.closeQuietly(meta);
1907 }
1908 }
1909
1910
1911
1912
1913
1914 @Test (timeout=180000)
1915 public void testValidLingeringSplitParent() throws Exception {
1916 final TableName table =
1917 TableName.valueOf("testLingeringSplitParent");
1918 Table meta = null;
1919 try {
1920 setupTable(table);
1921 assertEquals(ROWKEYS.length, countRows());
1922
1923
1924 admin.flush(table);
1925 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1926
1927 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1928 HRegionInfo hri = location.getRegionInfo();
1929
1930 splitAndWait(table, location);
1931
1932
1933
1934
1935 HBaseFsck hbck = doFsck(
1936 conf, true, true, false, false, false, true, true, true, false, false, false, false, false, null);
1937 assertErrors(hbck, new ERROR_CODE[] {});
1938
1939
1940 Get get = new Get(hri.getRegionName());
1941 Result result = meta.get(get);
1942 assertNotNull(result);
1943 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1944
1945 assertEquals(ROWKEYS.length, countRows());
1946
1947
1948 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1949 assertNoErrors(doFsck(conf, false));
1950 } finally {
1951 cleanupTable(table);
1952 IOUtils.closeQuietly(meta);
1953 }
1954 }
1955
1956 private byte[] splitAndWait(final TableName table, HRegionLocation location)
1957 throws IOException, Exception {
1958
1959
1960 final List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(table);
1961 byte[] regionName = location.getRegionInfo().getRegionName();
1962 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1963 TEST_UTIL.waitFor(60000, new Predicate<Exception>() {
1964 @Override
1965 public boolean evaluate() throws Exception {
1966 List<HRegion> regions1 = TEST_UTIL.getMiniHBaseCluster().getRegions(table);
1967 regions1.removeAll(regions);
1968 return regions1.size() == 2;
1969 }
1970 });
1971
1972 return regionName;
1973 }
1974
1975
1976
1977
1978
1979 @Test(timeout=75000)
1980 public void testSplitDaughtersNotInMeta() throws Exception {
1981 TableName table = TableName.valueOf("testSplitdaughtersNotInMeta");
1982 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1983 try {
1984 setupTable(table);
1985 assertEquals(ROWKEYS.length, countRows());
1986
1987
1988 admin.flush(table);
1989 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1990
1991 HRegionInfo hri = location.getRegionInfo();
1992
1993
1994
1995 admin.enableCatalogJanitor(false);
1996
1997 byte[] regionName = splitAndWait(table, location);
1998 PairOfSameType<HRegionInfo> daughters =
1999 MetaTableAccessor.getDaughterRegions(meta.get(new Get(regionName)));
2000
2001
2002 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
2003 undeployRegion(connection, hris.get(daughters.getFirst()), daughters.getFirst());
2004 undeployRegion(connection, hris.get(daughters.getSecond()), daughters.getSecond());
2005
2006 List<Delete> deletes = new ArrayList<>();
2007 deletes.add(new Delete(daughters.getFirst().getRegionName()));
2008 deletes.add(new Delete(daughters.getSecond().getRegionName()));
2009 meta.delete(deletes);
2010
2011
2012 RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().
2013 getAssignmentManager().getRegionStates();
2014 regionStates.deleteRegion(daughters.getFirst());
2015 regionStates.deleteRegion(daughters.getSecond());
2016
2017 HBaseFsck hbck = doFsck(conf, false);
2018 assertErrors(hbck,
2019 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2020 ERROR_CODE.HOLE_IN_REGION_CHAIN });
2021
2022
2023 hbck = doFsck(
2024 conf, true, true, false, false, false, false, false, false, false, false, false, false,false,null);
2025 assertErrors(hbck,
2026 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2027 ERROR_CODE.HOLE_IN_REGION_CHAIN });
2028
2029
2030 Get get = new Get(hri.getRegionName());
2031 Result result = meta.get(get);
2032 assertNotNull(result);
2033 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
2034
2035 assertEquals(ROWKEYS.length, countRows());
2036
2037
2038 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
2039 assertNoErrors(doFsck(conf, false));
2040 } finally {
2041 admin.enableCatalogJanitor(true);
2042 meta.close();
2043 cleanupTable(table);
2044 }
2045 }
2046
2047
2048
2049
2050
2051 @Test(timeout=120000)
2052 public void testMissingFirstRegion() throws Exception {
2053 TableName table = TableName.valueOf("testMissingFirstRegion");
2054 try {
2055 setupTable(table);
2056 assertEquals(ROWKEYS.length, countRows());
2057
2058
2059 admin.disableTable(table);
2060 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
2061 true, true);
2062 admin.enableTable(table);
2063
2064 HBaseFsck hbck = doFsck(conf, false);
2065 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
2066
2067 doFsck(conf, true);
2068
2069 assertNoErrors(doFsck(conf, false));
2070 } finally {
2071 cleanupTable(table);
2072 }
2073 }
2074
2075
2076
2077
2078
2079 @Test(timeout=120000)
2080 public void testRegionDeployedNotInHdfs() throws Exception {
2081 TableName table =
2082 TableName.valueOf("testSingleRegionDeployedNotInHdfs");
2083 try {
2084 setupTable(table);
2085 admin.flush(table);
2086
2087
2088 deleteRegion(conf, tbl.getTableDescriptor(),
2089 HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false,
2090 false, true);
2091
2092 HBaseFsck hbck = doFsck(conf, false);
2093 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2094
2095 doFsck(conf, true);
2096
2097 assertNoErrors(doFsck(conf, false));
2098 } finally {
2099 cleanupTable(table);
2100 }
2101 }
2102
2103
2104
2105
2106
2107 @Test(timeout=120000)
2108 public void testMissingLastRegion() throws Exception {
2109 TableName table =
2110 TableName.valueOf("testMissingLastRegion");
2111 try {
2112 setupTable(table);
2113 assertEquals(ROWKEYS.length, countRows());
2114
2115
2116 admin.disableTable(table);
2117 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
2118 true, true);
2119 admin.enableTable(table);
2120
2121 HBaseFsck hbck = doFsck(conf, false);
2122 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
2123
2124 doFsck(conf, true);
2125
2126 assertNoErrors(doFsck(conf, false));
2127 } finally {
2128 cleanupTable(table);
2129 }
2130 }
2131
2132
2133
2134
2135 @Test (timeout=180000)
2136 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
2137 TableName table =
2138 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
2139 try {
2140 setupTable(table);
2141 assertEquals(ROWKEYS.length, countRows());
2142
2143
2144 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2145 Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2146
2147
2148 HBaseFsck hbck = doFsck(conf, false);
2149 assertErrors(hbck, new ERROR_CODE[] {
2150 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2151
2152
2153 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2154 fsck.connect();
2155 fsck.setDisplayFullReport();
2156 fsck.setTimeLag(0);
2157 fsck.setCheckHdfs(false);
2158 fsck.onlineHbck();
2159 assertErrors(fsck, new ERROR_CODE[] {
2160 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2161 fsck.close();
2162
2163
2164 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2165 fsck.connect();
2166 fsck.setDisplayFullReport();
2167 fsck.setTimeLag(0);
2168 fsck.setCheckHdfs(false);
2169 fsck.setFixAssignments(true);
2170 fsck.onlineHbck();
2171 assertTrue(fsck.shouldRerun());
2172 fsck.onlineHbck();
2173 assertNoErrors(fsck);
2174
2175 assertEquals(ROWKEYS.length, countRows());
2176
2177 fsck.close();
2178 } finally {
2179 cleanupTable(table);
2180 }
2181 }
2182
2183
2184
2185
2186
2187
2188 @Test (timeout=180000)
2189 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
2190 TableName table =
2191 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
2192 try {
2193 setupTable(table);
2194 assertEquals(ROWKEYS.length, countRows());
2195
2196
2197 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2198 Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2199
2200
2201 HBaseFsck hbck = doFsck(conf, false);
2202 assertErrors(hbck,
2203 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2204
2205
2206 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2207 fsck.connect();
2208 fsck.setDisplayFullReport();
2209 fsck.setTimeLag(0);
2210 fsck.setCheckHdfs(false);
2211 fsck.onlineHbck();
2212 assertErrors(fsck,
2213 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2214 fsck.close();
2215
2216
2217 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2218 fsck.connect();
2219 fsck.setDisplayFullReport();
2220 fsck.setTimeLag(0);
2221 fsck.setCheckHdfs(false);
2222 fsck.setFixAssignments(true);
2223 fsck.setFixMeta(true);
2224 fsck.onlineHbck();
2225 assertFalse(fsck.shouldRerun());
2226 assertErrors(fsck,
2227 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2228 fsck.close();
2229
2230
2231 fsck = doFsck(conf, true);
2232 assertTrue(fsck.shouldRerun());
2233 fsck = doFsck(conf, true);
2234 assertNoErrors(fsck);
2235 } finally {
2236 cleanupTable(table);
2237 }
2238 }
2239
2240
2241
2242
2243
2244 @Test (timeout=180000)
2245 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
2246 TableName table =
2247 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
2248 try {
2249 setupTable(table);
2250 assertEquals(ROWKEYS.length, countRows());
2251
2252
2253 admin.disableTable(table);
2254 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2255 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
2256 TEST_UTIL.getHBaseAdmin().enableTable(table);
2257
2258 HRegionInfo hriOverlap =
2259 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
2260 TEST_UTIL.assignRegion(hriOverlap);
2261 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
2262 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
2263
2264 HBaseFsck hbck = doFsck(conf, false);
2265 assertErrors(hbck, new ERROR_CODE[] {
2266 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2267 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2268
2269
2270 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2271 fsck.connect();
2272 fsck.setDisplayFullReport();
2273 fsck.setTimeLag(0);
2274 fsck.setCheckHdfs(false);
2275 fsck.onlineHbck();
2276 assertErrors(fsck, new ERROR_CODE[] {
2277 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2278 fsck.close();
2279
2280
2281 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2282 fsck.connect();
2283 fsck.setDisplayFullReport();
2284 fsck.setTimeLag(0);
2285 fsck.setCheckHdfs(false);
2286 fsck.setFixHdfsHoles(true);
2287 fsck.setFixHdfsOverlaps(true);
2288 fsck.setFixHdfsOrphans(true);
2289 fsck.onlineHbck();
2290 assertFalse(fsck.shouldRerun());
2291 assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN});
2292 fsck.close();
2293 } finally {
2294 if (admin.isTableDisabled(table)) {
2295 admin.enableTable(table);
2296 }
2297 cleanupTable(table);
2298 }
2299 }
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
2310 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2311 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2312 Path famDir = new Path(regionDir, FAM_STR);
2313
2314
2315 while (true) {
2316 FileStatus[] hfFss = fs.listStatus(famDir);
2317 if (hfFss.length == 0) {
2318 continue;
2319 }
2320 for (FileStatus hfs : hfFss) {
2321 if (!hfs.isDirectory()) {
2322 return hfs.getPath();
2323 }
2324 }
2325 }
2326 }
2327
2328
2329
2330
2331 @Test(timeout=180000)
2332 public void testQuarantineCorruptHFile() throws Exception {
2333 TableName table = TableName.valueOf(name.getMethodName());
2334 try {
2335 setupTable(table);
2336 assertEquals(ROWKEYS.length, countRows());
2337 admin.flush(table);
2338
2339 FileSystem fs = FileSystem.get(conf);
2340 Path hfile = getFlushedHFile(fs, table);
2341
2342
2343 admin.disableTable(table);
2344
2345
2346 Path corrupt = new Path(hfile.getParent(), "deadbeef");
2347 TestHFile.truncateFile(fs, hfile, corrupt);
2348 LOG.info("Created corrupted file " + corrupt);
2349 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2350
2351
2352 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2353 assertEquals(res.getRetCode(), 0);
2354 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2355 assertEquals(hfcc.getHFilesChecked(), 5);
2356 assertEquals(hfcc.getCorrupted().size(), 1);
2357 assertEquals(hfcc.getFailures().size(), 0);
2358 assertEquals(hfcc.getQuarantined().size(), 1);
2359 assertEquals(hfcc.getMissing().size(), 0);
2360
2361
2362 admin.enableTable(table);
2363 } finally {
2364 cleanupTable(table);
2365 }
2366 }
2367
2368
2369
2370
2371 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
2372 int corrupt, int fail, int quar, int missing) throws Exception {
2373 try {
2374 setupTable(table);
2375 assertEquals(ROWKEYS.length, countRows());
2376 admin.flush(table);
2377
2378
2379 admin.disableTable(table);
2380
2381 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
2382 table.getNameAsString()};
2383 HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
2384
2385 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2386 assertEquals(hfcc.getHFilesChecked(), check);
2387 assertEquals(hfcc.getCorrupted().size(), corrupt);
2388 assertEquals(hfcc.getFailures().size(), fail);
2389 assertEquals(hfcc.getQuarantined().size(), quar);
2390 assertEquals(hfcc.getMissing().size(), missing);
2391
2392
2393 admin.enableTableAsync(table);
2394 while (!admin.isTableEnabled(table)) {
2395 try {
2396 Thread.sleep(250);
2397 } catch (InterruptedException e) {
2398 e.printStackTrace();
2399 fail("Interrupted when trying to enable table " + table);
2400 }
2401 }
2402 } finally {
2403 cleanupTable(table);
2404 }
2405 }
2406
2407
2408
2409
2410
2411 @Test(timeout=180000)
2412 public void testQuarantineMissingHFile() throws Exception {
2413 TableName table = TableName.valueOf(name.getMethodName());
2414
2415
2416 final FileSystem fs = FileSystem.get(conf);
2417 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2418 @Override
2419 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2420 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2421 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2422 @Override
2423 protected void checkHFile(Path p) throws IOException {
2424 if (attemptedFirstHFile.compareAndSet(false, true)) {
2425 assertTrue(fs.delete(p, true));
2426 }
2427 super.checkHFile(p);
2428 }
2429 };
2430 }
2431 };
2432 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
2433 hbck.close();
2434 }
2435
2436
2437
2438
2439
2440
2441
2442 @Ignore @Test(timeout=180000)
2443 public void testQuarantineMissingFamdir() throws Exception {
2444 TableName table = TableName.valueOf(name.getMethodName());
2445
2446 final FileSystem fs = FileSystem.get(conf);
2447 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2448 @Override
2449 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2450 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2451 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2452 @Override
2453 protected void checkColFamDir(Path p) throws IOException {
2454 if (attemptedFirstHFile.compareAndSet(false, true)) {
2455 assertTrue(fs.delete(p, true));
2456 }
2457 super.checkColFamDir(p);
2458 }
2459 };
2460 }
2461 };
2462 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2463 hbck.close();
2464 }
2465
2466 @Test(timeout=60000)
2467 public void testCheckReplication() throws Exception {
2468
2469 HBaseFsck hbck = doFsck(conf, false);
2470 assertNoErrors(hbck);
2471
2472
2473 ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf);
2474 Assert.assertEquals(0, replicationAdmin.getPeersCount());
2475 String zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
2476 ReplicationPeerConfig rpc = new ReplicationPeerConfig();
2477 rpc.setClusterKey("127.0.0.1:2181" + zkPort + ":/hbase");
2478 replicationAdmin.addPeer("1", rpc);
2479 replicationAdmin.getPeersCount();
2480 Assert.assertEquals(1, replicationAdmin.getPeersCount());
2481
2482
2483 ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection);
2484 ReplicationQueues repQueues =
2485 ReplicationFactory.getReplicationQueues(zkw, conf, connection);
2486 repQueues.init("server1");
2487
2488 repQueues.addLog("1", "file1");
2489 repQueues.addLog("1-server2", "file1");
2490 Assert.assertEquals(2, repQueues.getAllQueues().size());
2491 hbck = doFsck(conf, false);
2492 assertNoErrors(hbck);
2493
2494
2495 repQueues.addLog("2", "file1");
2496 repQueues.addLog("2-server2", "file1");
2497 Assert.assertEquals(4, repQueues.getAllQueues().size());
2498 hbck = doFsck(conf, false);
2499 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
2500 ERROR_CODE.UNDELETED_REPLICATION_QUEUE });
2501
2502
2503 hbck = doFsck(conf, true);
2504 hbck = doFsck(conf, false);
2505 assertNoErrors(hbck);
2506
2507 Assert.assertEquals(2, repQueues.getAllQueues().size());
2508 Assert.assertNull(repQueues.getLogsInQueue("2"));
2509 Assert.assertNull(repQueues.getLogsInQueue("2-sever2"));
2510
2511 replicationAdmin.removePeer("1");
2512 repQueues.removeAllQueues();
2513 zkw.close();
2514 replicationAdmin.close();
2515 }
2516
2517
2518
2519
2520
2521 @Test(timeout=180000)
2522 public void testQuarantineMissingRegionDir() throws Exception {
2523 TableName table = TableName.valueOf(name.getMethodName());
2524
2525 final FileSystem fs = FileSystem.get(conf);
2526 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2527 @Override
2528 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
2529 throws IOException {
2530 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2531 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2532 @Override
2533 protected void checkRegionDir(Path p) throws IOException {
2534 if (attemptedFirstHFile.compareAndSet(false, true)) {
2535 assertTrue(fs.delete(p, true));
2536 }
2537 super.checkRegionDir(p);
2538 }
2539 };
2540 }
2541 };
2542 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2543 hbck.close();
2544 }
2545
2546
2547
2548
2549 @Test (timeout=180000)
2550 public void testLingeringReferenceFile() throws Exception {
2551 TableName table =
2552 TableName.valueOf("testLingeringReferenceFile");
2553 try {
2554 setupTable(table);
2555 assertEquals(ROWKEYS.length, countRows());
2556
2557
2558 FileSystem fs = FileSystem.get(conf);
2559 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2560 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2561 Path famDir = new Path(regionDir, FAM_STR);
2562 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
2563 fs.create(fakeReferenceFile);
2564
2565 HBaseFsck hbck = doFsck(conf, false);
2566 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
2567
2568 doFsck(conf, true);
2569
2570 assertNoErrors(doFsck(conf, false));
2571 } finally {
2572 cleanupTable(table);
2573 }
2574 }
2575
2576
2577
2578
2579 @Test(timeout = 180000)
2580 public void testLingeringHFileLinks() throws Exception {
2581 TableName table = TableName.valueOf("testLingeringHFileLinks");
2582 try {
2583 setupTable(table);
2584
2585 FileSystem fs = FileSystem.get(conf);
2586 Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2587 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2588 String regionName = regionDir.getName();
2589 Path famDir = new Path(regionDir, FAM_STR);
2590 String HFILE_NAME = "01234567abcd";
2591 Path hFilePath = new Path(famDir, HFILE_NAME);
2592
2593
2594 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
2595 HFile.Writer w =
2596 HFile.getWriterFactoryNoCache(conf).withPath(fs, hFilePath).withFileContext(context)
2597 .create();
2598 w.close();
2599
2600 HFileLink.create(conf, fs, famDir, table, regionName, HFILE_NAME);
2601
2602
2603 HBaseFsck hbck = doFsck(conf, false);
2604 assertNoErrors(hbck);
2605
2606
2607 fs.delete(hFilePath, true);
2608
2609
2610 hbck = doFsck(conf, false);
2611 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
2612 HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_HFILELINK });
2613
2614
2615 hbck = doFsck(conf, true);
2616 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
2617 HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_HFILELINK });
2618
2619
2620 hbck = doFsck(conf, false);
2621 assertNoErrors(hbck);
2622 } finally {
2623 cleanupTable(table);
2624 }
2625 }
2626
2627 @Test(timeout = 180000)
2628 public void testCorruptLinkDirectory() throws Exception {
2629 TableName table = TableName.valueOf("testLingeringHFileLinks");
2630 try {
2631 setupTable(table);
2632 FileSystem fs = FileSystem.get(conf);
2633
2634 Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2635 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2636 Path famDir = new Path(regionDir, FAM_STR);
2637 String regionName = regionDir.getName();
2638 String HFILE_NAME = "01234567abcd";
2639 String link = HFileLink.createHFileLinkName(table, regionName, HFILE_NAME);
2640
2641
2642 HBaseFsck hbck = doFsck(conf, false);
2643 assertNoErrors(hbck);
2644
2645
2646 fs.mkdirs(new Path(famDir, link));
2647 fs.create(new Path(new Path(famDir, link), "somefile"));
2648
2649
2650 hbck = doFsck(conf, false);
2651 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
2652 HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_HFILELINK });
2653
2654
2655 hbck = doFsck(conf, true);
2656 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
2657 HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_HFILELINK });
2658
2659
2660 hbck = doFsck(conf, false);
2661 assertNoErrors(hbck);
2662 } finally {
2663 cleanupTable(table);
2664 }
2665 }
2666
2667
2668
2669
2670 @Test (timeout=180000)
2671 public void testMissingRegionInfoQualifier() throws Exception {
2672 Connection connection = ConnectionFactory.createConnection(conf);
2673 TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
2674 try {
2675 setupTable(table);
2676
2677
2678 final List<Delete> deletes = new LinkedList<Delete>();
2679 Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService);
2680 MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
2681
2682 @Override
2683 public boolean processRow(Result rowResult) throws IOException {
2684 HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult);
2685 if (hri != null && !hri.getTable().isSystemTable()) {
2686 Delete delete = new Delete(rowResult.getRow());
2687 delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2688 deletes.add(delete);
2689 }
2690 return true;
2691 }
2692
2693 @Override
2694 public void close() throws IOException {
2695 }
2696 });
2697 meta.delete(deletes);
2698
2699
2700 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2701 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
2702 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2703 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
2704 meta.close();
2705
2706 HBaseFsck hbck = doFsck(conf, false);
2707 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2708
2709
2710 hbck = doFsck(conf, true);
2711
2712
2713 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2714 } finally {
2715 cleanupTable(table);
2716 }
2717 connection.close();
2718 }
2719
2720
2721
2722
2723
2724 @Test (timeout=180000)
2725 public void testErrorReporter() throws Exception {
2726 try {
2727 MockErrorReporter.calledCount = 0;
2728 doFsck(conf, false);
2729 assertEquals(MockErrorReporter.calledCount, 0);
2730
2731 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
2732 doFsck(conf, false);
2733 assertTrue(MockErrorReporter.calledCount > 20);
2734 } finally {
2735 conf.set("hbasefsck.errorreporter",
2736 PrintingErrorReporter.class.getName());
2737 MockErrorReporter.calledCount = 0;
2738 }
2739 }
2740
2741 static class MockErrorReporter implements ErrorReporter {
2742 static int calledCount = 0;
2743
2744 @Override
2745 public void clear() {
2746 calledCount++;
2747 }
2748
2749 @Override
2750 public void report(String message) {
2751 calledCount++;
2752 }
2753
2754 @Override
2755 public void reportError(String message) {
2756 calledCount++;
2757 }
2758
2759 @Override
2760 public void reportError(ERROR_CODE errorCode, String message) {
2761 calledCount++;
2762 }
2763
2764 @Override
2765 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2766 calledCount++;
2767 }
2768
2769 @Override
2770 public void reportError(ERROR_CODE errorCode,
2771 String message, TableInfo table, HbckInfo info) {
2772 calledCount++;
2773 }
2774
2775 @Override
2776 public void reportError(ERROR_CODE errorCode, String message,
2777 TableInfo table, HbckInfo info1, HbckInfo info2) {
2778 calledCount++;
2779 }
2780
2781 @Override
2782 public int summarize() {
2783 return ++calledCount;
2784 }
2785
2786 @Override
2787 public void detail(String details) {
2788 calledCount++;
2789 }
2790
2791 @Override
2792 public ArrayList<ERROR_CODE> getErrorList() {
2793 calledCount++;
2794 return new ArrayList<ERROR_CODE>();
2795 }
2796
2797 @Override
2798 public void progress() {
2799 calledCount++;
2800 }
2801
2802 @Override
2803 public void print(String message) {
2804 calledCount++;
2805 }
2806
2807 @Override
2808 public void resetErrors() {
2809 calledCount++;
2810 }
2811
2812 @Override
2813 public boolean tableHasErrors(TableInfo table) {
2814 calledCount++;
2815 return false;
2816 }
2817 }
2818
2819 @Test(timeout=180000)
2820 public void testCheckTableLocks() throws Exception {
2821 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2822 EnvironmentEdgeManager.injectEdge(edge);
2823
2824 HBaseFsck hbck = doFsck(conf, false);
2825 assertNoErrors(hbck);
2826
2827 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2828 final TableName tableName = TableName.valueOf("foo");
2829
2830
2831 final TableLockManager tableLockManager =
2832 TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2833 TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2834 writeLock.acquire();
2835 hbck = doFsck(conf, false);
2836 assertNoErrors(hbck);
2837
2838 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2839 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2840
2841 hbck = doFsck(conf, false);
2842 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2843
2844 final CountDownLatch latch = new CountDownLatch(1);
2845 new Thread() {
2846 @Override
2847 public void run() {
2848 TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2849 try {
2850 latch.countDown();
2851 readLock.acquire();
2852 } catch (IOException ex) {
2853 fail();
2854 } catch (IllegalStateException ex) {
2855 return;
2856 }
2857 fail("should not have come here");
2858 };
2859 }.start();
2860
2861 latch.await();
2862 Threads.sleep(300);
2863
2864 hbck = doFsck(conf, false);
2865
2866 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2867
2868 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2869 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2870
2871 hbck = doFsck(conf, false);
2872
2873 assertErrors(
2874 hbck,
2875 new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2876
2877 Configuration localConf = new Configuration(conf);
2878
2879
2880 localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2881 Threads.sleep(10);
2882 hbck = doFsck(localConf, true);
2883
2884 hbck = doFsck(localConf, false);
2885 assertNoErrors(hbck);
2886
2887
2888 writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
2889 writeLock.acquire();
2890 writeLock.release();
2891 tableLockManager.tableDeleted(tableName);
2892 }
2893
2894
2895
2896
2897 @Test
2898 public void testOrphanedTableZNode() throws Exception {
2899 TableName table = TableName.valueOf("testOrphanedZKTableEntry");
2900
2901 try {
2902 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
2903 .setTableState(table, ZooKeeperProtos.Table.State.ENABLING);
2904
2905 try {
2906 setupTable(table);
2907 Assert.fail(
2908 "Create table should fail when its ZNode has already existed with ENABLING state.");
2909 } catch(TableExistsException t) {
2910
2911 }
2912
2913 try {
2914 cleanupTable(table);
2915 } catch (IOException e) {
2916
2917
2918 }
2919
2920 HBaseFsck hbck = doFsck(conf, false);
2921 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2922
2923
2924 hbck = doFsck(conf, true);
2925
2926
2927 hbck = doFsck(conf, false);
2928 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2929
2930 setupTable(table);
2931 } finally {
2932
2933
2934 try {
2935 cleanupTable(table);
2936 } catch (IOException e) {
2937
2938
2939 }
2940 }
2941 }
2942
2943 @Test (timeout=180000)
2944 public void testMetaOffline() throws Exception {
2945
2946 HBaseFsck hbck = doFsck(conf, false);
2947 assertNoErrors(hbck);
2948 deleteMetaRegion(conf, true, false, false);
2949 hbck = doFsck(conf, false);
2950
2951
2952 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2953 hbck = doFsck(conf, true);
2954 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2955 hbck = doFsck(conf, false);
2956 assertNoErrors(hbck);
2957 }
2958
2959 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2960 boolean regionInfoOnly) throws IOException, InterruptedException {
2961 HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
2962 .getRegionLocation(HConstants.EMPTY_START_ROW);
2963 ServerName hsa = metaLocation.getServerName();
2964 HRegionInfo hri = metaLocation.getRegionInfo();
2965 if (unassign) {
2966 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2967 try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
2968 undeployRegion(unmanagedConnection, hsa, hri);
2969 }
2970 }
2971
2972 if (regionInfoOnly) {
2973 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2974 Path rootDir = FSUtils.getRootDir(conf);
2975 FileSystem fs = rootDir.getFileSystem(conf);
2976 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2977 hri.getEncodedName());
2978 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2979 fs.delete(hriPath, true);
2980 }
2981
2982 if (hdfs) {
2983 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2984 Path rootDir = FSUtils.getRootDir(conf);
2985 FileSystem fs = rootDir.getFileSystem(conf);
2986 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2987 hri.getEncodedName());
2988 HBaseFsck.debugLsr(conf, p);
2989 boolean success = fs.delete(p, true);
2990 LOG.info("Deleted " + p + " sucessfully? " + success);
2991 HBaseFsck.debugLsr(conf, p);
2992 }
2993 }
2994
2995 @Test (timeout=180000)
2996 public void testTableWithNoRegions() throws Exception {
2997
2998
2999 TableName table =
3000 TableName.valueOf(name.getMethodName());
3001 try {
3002
3003 HTableDescriptor desc = new HTableDescriptor(table);
3004 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
3005 desc.addFamily(hcd);
3006 createTable(TEST_UTIL, desc, null);
3007 tbl = (HTable) connection.getTable(table, tableExecutorService);
3008
3009
3010 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW,
3011 HConstants.EMPTY_END_ROW, false, false, true);
3012
3013 HBaseFsck hbck = doFsck(conf, false);
3014 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
3015
3016 doFsck(conf, true);
3017
3018
3019 doFsck(conf, true);
3020
3021
3022 assertNoErrors(doFsck(conf, false));
3023 } finally {
3024 cleanupTable(table);
3025 }
3026
3027 }
3028
3029 @Test (timeout=180000)
3030 public void testHbckAfterRegionMerge() throws Exception {
3031 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
3032 Table meta = null;
3033 try {
3034
3035 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
3036 setupTable(table);
3037 assertEquals(ROWKEYS.length, countRows());
3038
3039
3040 admin.flush(table);
3041 HRegionInfo region1 = tbl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo();
3042 HRegionInfo region2 = tbl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo();
3043
3044 int regionCountBeforeMerge = tbl.getRegionLocations().size();
3045
3046 assertNotEquals(region1, region2);
3047
3048
3049 admin.mergeRegions(region1.getEncodedNameAsBytes(),
3050 region2.getEncodedNameAsBytes(), false);
3051
3052
3053 long timeout = System.currentTimeMillis() + 30 * 1000;
3054 while (true) {
3055 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
3056 break;
3057 } else if (System.currentTimeMillis() > timeout) {
3058 fail("Time out waiting on region " + region1.getEncodedName()
3059 + " and " + region2.getEncodedName() + " be merged");
3060 }
3061 Thread.sleep(10);
3062 }
3063
3064 assertEquals(ROWKEYS.length, countRows());
3065
3066 HBaseFsck hbck = doFsck(conf, false);
3067 assertNoErrors(hbck);
3068
3069 } finally {
3070 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
3071 cleanupTable(table);
3072 IOUtils.closeQuietly(meta);
3073 }
3074 }
3075
3076 @Test (timeout = 180000)
3077 public void testRegionBoundariesCheck() throws Exception {
3078 TableName tableName = TableName.valueOf("testRegionBoundariesCheck");
3079
3080
3081 HTableDescriptor desc = new HTableDescriptor(tableName);
3082 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
3083 desc.addFamily(hcd);
3084 createTable(TEST_UTIL, desc, SPLITS);
3085
3086 Table table = connection.getTable(tableName, tableExecutorService);
3087 List<Put> puts = new ArrayList<>();
3088
3089
3090 puts.add(new Put(Bytes.toBytes("0")).addColumn(FAM, Bytes.toBytes("col"),
3091 Bytes.toBytes("val")));
3092 puts.add(new Put(Bytes.toBytes("999")).addColumn(FAM, Bytes.toBytes("col"),
3093 Bytes.toBytes("val")));
3094
3095
3096 puts.add(new Put(Bytes.toBytes("AA")).addColumn(FAM, Bytes.toBytes("col"),
3097 Bytes.toBytes("val")));
3098 puts.add(new Put(Bytes.toBytes("AZ")).addColumn(FAM, Bytes.toBytes("col"),
3099 Bytes.toBytes("val")));
3100
3101 table.put(puts);
3102
3103
3104 admin.disableTable(tableName);
3105 admin.enableTable(tableName);
3106
3107
3108 HBaseFsck hbck = checkRegionBoundaries(conf);
3109 assertNoErrors(hbck);
3110
3111
3112 admin.disableTable(tableName);
3113
3114 List<HRegionInfo> tableRegions = admin.getTableRegions(tableName);
3115 HRegionInfo firstRegion = tableRegions.get(0);
3116 HRegionInfo secondRegion = tableRegions.get(1);
3117
3118 FileSystem fs = FileSystem.get(conf);
3119 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
3120 Path firstRegionFamDir = new Path(new Path(tableDir, firstRegion.getEncodedName()), FAM_STR);
3121 Path hfileInFirstRegion = getHFilePath(fs, firstRegionFamDir);
3122 Path secondRegionFamDir = new Path(new Path(tableDir, secondRegion.getEncodedName()), FAM_STR);
3123 Path hfileInSecondRegion = getHFilePath(fs, secondRegionFamDir);
3124
3125
3126
3127 fs.rename(hfileInFirstRegion, new Path(firstRegionFamDir, "0"));
3128 fs.rename(hfileInSecondRegion, new Path(firstRegionFamDir, "1"));
3129
3130 admin.enableTable(tableName);
3131
3132
3133 hbck = checkRegionBoundaries(conf);
3134 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.BOUNDARIES_ERROR });
3135 }
3136
3137 private static Path getHFilePath(FileSystem fs, Path famDir) throws IOException {
3138 FileStatus[] hfFss = fs.listStatus(famDir);
3139 for (FileStatus hfs : hfFss) {
3140 if (!hfs.isDirectory()) {
3141 return hfs.getPath();
3142 }
3143 }
3144 return null;
3145 }
3146
3147 @org.junit.Rule
3148 public TestName name = new TestName();
3149
3150 @Test (timeout=180000)
3151 public void testReadOnlyProperty() throws Exception {
3152 HBaseFsck hbck = doFsck(conf, false);
3153 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
3154 hbck.shouldIgnorePreCheckPermission());
3155
3156 hbck = doFsck(conf, true);
3157 Assert.assertEquals("shouldIgnorePreCheckPermission", false,
3158 hbck.shouldIgnorePreCheckPermission());
3159
3160 hbck = doFsck(conf, true);
3161 hbck.setIgnorePreCheckPermission(true);
3162 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
3163 hbck.shouldIgnorePreCheckPermission());
3164 }
3165
3166 @Test (timeout=180000)
3167 public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
3168 TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit");
3169 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
3170 try {
3171 HTableDescriptor desc = new HTableDescriptor(table);
3172 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
3173 createTable(TEST_UTIL, desc, null);
3174 tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
3175 for (int i = 0; i < 5; i++) {
3176 Put p1 = new Put(("r" + i).getBytes());
3177 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
3178 tbl.put(p1);
3179 }
3180 admin.flush(desc.getTableName());
3181 List<HRegion> regions = cluster.getRegions(desc.getTableName());
3182 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
3183 HRegionServer regionServer = cluster.getRegionServer(serverWith);
3184 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
3185 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
3186 st.prepare();
3187 st.stepsBeforePONR(regionServer, regionServer, false);
3188 AssignmentManager am = cluster.getMaster().getAssignmentManager();
3189 Set<RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
3190 for (RegionState state : regionsInTransition) {
3191 am.regionOffline(state.getRegion());
3192 }
3193 ZKAssign.deleteNodeFailSilent(regionServer.getZooKeeper(), regions.get(0).getRegionInfo());
3194 Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();
3195 regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
3196 am.assign(regionsMap);
3197 am.waitForAssignment(regions.get(0).getRegionInfo());
3198 HBaseFsck hbck = doFsck(conf, false);
3199 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
3200 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
3201
3202 assertEquals(0, hbck.getOverlapGroups(table).size());
3203
3204
3205 assertErrors(
3206 doFsck(
3207 conf, false, true, false, false, false, false, false, false, false, false, false, false,
3208 false, null),
3209 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
3210 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
3211
3212
3213 assertNoErrors(doFsck(conf, false));
3214 assertEquals(5, countRows());
3215 } finally {
3216 if (tbl != null) {
3217 tbl.close();
3218 tbl = null;
3219 }
3220 cleanupTable(table);
3221 }
3222 }
3223
3224
3225 public static class MasterSyncObserver extends BaseMasterObserver {
3226 volatile CountDownLatch tableCreationLatch = null;
3227 volatile CountDownLatch tableDeletionLatch = null;
3228
3229 @Override
3230 public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3231 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
3232
3233 if (tableCreationLatch != null) {
3234 tableCreationLatch.countDown();
3235 }
3236 }
3237
3238 @Override
3239 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3240 TableName tableName)
3241 throws IOException {
3242
3243 if (tableDeletionLatch != null) {
3244 tableDeletionLatch.countDown();
3245 }
3246 }
3247 }
3248
3249 public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
3250 byte [][] splitKeys) throws Exception {
3251
3252
3253 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3254 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3255 observer.tableCreationLatch = new CountDownLatch(1);
3256 if (splitKeys != null) {
3257 admin.createTable(htd, splitKeys);
3258 } else {
3259 admin.createTable(htd);
3260 }
3261 observer.tableCreationLatch.await();
3262 observer.tableCreationLatch = null;
3263 testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
3264 }
3265
3266 public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
3267 throws Exception {
3268
3269
3270 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3271 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3272 observer.tableDeletionLatch = new CountDownLatch(1);
3273 try {
3274 admin.disableTable(tableName);
3275 } catch (Exception e) {
3276 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
3277 }
3278 admin.deleteTable(tableName);
3279 observer.tableDeletionLatch.await();
3280 observer.tableDeletionLatch = null;
3281 }
3282 }