1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.checkRegionBoundaries;
24 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
25 import static org.junit.Assert.assertEquals;
26 import static org.junit.Assert.assertFalse;
27 import static org.junit.Assert.assertNotEquals;
28 import static org.junit.Assert.assertNotNull;
29 import static org.junit.Assert.assertTrue;
30 import static org.junit.Assert.fail;
31
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.HashMap;
37 import java.util.HashSet;
38 import java.util.LinkedList;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Random;
42 import java.util.Map.Entry;
43 import java.util.NavigableMap;
44 import java.util.Set;
45 import java.util.concurrent.Callable;
46 import java.util.concurrent.CountDownLatch;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.ScheduledThreadPoolExecutor;
51 import java.util.concurrent.SynchronousQueue;
52 import java.util.concurrent.ThreadPoolExecutor;
53 import java.util.concurrent.TimeUnit;
54 import java.util.concurrent.atomic.AtomicBoolean;
55
56 import org.apache.commons.io.IOUtils;
57 import org.apache.commons.logging.Log;
58 import org.apache.commons.logging.LogFactory;
59 import org.apache.hadoop.conf.Configuration;
60 import org.apache.hadoop.fs.FileStatus;
61 import org.apache.hadoop.fs.FileSystem;
62 import org.apache.hadoop.fs.Path;
63 import org.apache.hadoop.hbase.ClusterStatus;
64 import org.apache.hadoop.hbase.HBaseTestingUtility;
65 import org.apache.hadoop.hbase.HColumnDescriptor;
66 import org.apache.hadoop.hbase.HConstants;
67 import org.apache.hadoop.hbase.HRegionInfo;
68 import org.apache.hadoop.hbase.HRegionLocation;
69 import org.apache.hadoop.hbase.HTableDescriptor;
70 import org.apache.hadoop.hbase.MetaTableAccessor;
71 import org.apache.hadoop.hbase.MiniHBaseCluster;
72 import org.apache.hadoop.hbase.RegionLocations;
73 import org.apache.hadoop.hbase.ServerName;
74 import org.apache.hadoop.hbase.TableName;
75 import org.apache.hadoop.hbase.TableExistsException;
76 import org.apache.hadoop.hbase.testclassification.LargeTests;
77 import org.apache.hadoop.hbase.Waiter.Predicate;
78 import org.apache.hadoop.hbase.client.Admin;
79 import org.apache.hadoop.hbase.client.ClusterConnection;
80 import org.apache.hadoop.hbase.client.Connection;
81 import org.apache.hadoop.hbase.client.ConnectionFactory;
82 import org.apache.hadoop.hbase.client.Delete;
83 import org.apache.hadoop.hbase.client.Durability;
84 import org.apache.hadoop.hbase.client.Get;
85 import org.apache.hadoop.hbase.client.HBaseAdmin;
86 import org.apache.hadoop.hbase.client.HConnection;
87 import org.apache.hadoop.hbase.client.HTable;
88 import org.apache.hadoop.hbase.client.MetaScanner;
89 import org.apache.hadoop.hbase.client.Mutation;
90 import org.apache.hadoop.hbase.client.Put;
91 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
92 import org.apache.hadoop.hbase.client.Result;
93 import org.apache.hadoop.hbase.client.ResultScanner;
94 import org.apache.hadoop.hbase.client.RowMutations;
95 import org.apache.hadoop.hbase.client.Scan;
96 import org.apache.hadoop.hbase.client.Table;
97 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
98 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
99 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
100 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
101 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
102 import org.apache.hadoop.hbase.io.HFileLink;
103 import org.apache.hadoop.hbase.io.hfile.HFile;
104 import org.apache.hadoop.hbase.io.hfile.HFileContext;
105 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
106 import org.apache.hadoop.hbase.io.hfile.TestHFile;
107 import org.apache.hadoop.hbase.master.AssignmentManager;
108 import org.apache.hadoop.hbase.master.HMaster;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.master.RegionStates;
111 import org.apache.hadoop.hbase.master.TableLockManager;
112 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
113 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
114 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
116 import org.apache.hadoop.hbase.regionserver.HRegion;
117 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
118 import org.apache.hadoop.hbase.regionserver.HRegionServer;
119 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl;
120 import org.apache.hadoop.hbase.replication.ReplicationFactory;
121 import org.apache.hadoop.hbase.replication.ReplicationQueues;
122 import org.apache.hadoop.hbase.testclassification.LargeTests;
123 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
124 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
125 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
126 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
127 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
128 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
129 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
130 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
131 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
132 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
133 import org.apache.zookeeper.KeeperException;
134 import org.junit.AfterClass;
135 import org.junit.Assert;
136 import org.junit.Before;
137 import org.junit.BeforeClass;
138 import org.junit.Ignore;
139 import org.junit.Test;
140 import org.junit.experimental.categories.Category;
141 import org.junit.rules.TestName;
142
143 import com.google.common.collect.Multimap;
144
145
146
147
148 @Category(LargeTests.class)
149 public class TestHBaseFsck {
150 static final int POOL_SIZE = 7;
151 private static final Log LOG = LogFactory.getLog(TestHBaseFsck.class);
152 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
153 private final static Configuration conf = TEST_UTIL.getConfiguration();
154 private final static String FAM_STR = "fam";
155 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
156 private final static int REGION_ONLINE_TIMEOUT = 800;
157 private static RegionStates regionStates;
158 private static ExecutorService tableExecutorService;
159 private static ScheduledThreadPoolExecutor hbfsckExecutorService;
160 private static ClusterConnection connection;
161 private static Admin admin;
162
163
164 private HTable tbl;
165 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
166 Bytes.toBytes("B"), Bytes.toBytes("C") };
167
168 private final static byte[][] ROWKEYS= new byte[][] {
169 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
170 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
171
172 @BeforeClass
173 public static void setUpBeforeClass() throws Exception {
174 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
175 MasterSyncObserver.class.getName());
176
177 conf.setInt("hbase.regionserver.handler.count", 2);
178 conf.setInt("hbase.regionserver.metahandler.count", 30);
179
180 conf.setInt("hbase.htable.threads.max", POOL_SIZE);
181 conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
182 conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
183 conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
184 conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
185 TEST_UTIL.startMiniCluster(3);
186
187 tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
188 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
189
190 hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
191
192 AssignmentManager assignmentManager =
193 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
194 regionStates = assignmentManager.getRegionStates();
195
196 connection = (ClusterConnection) TEST_UTIL.getConnection();
197
198 admin = connection.getAdmin();
199 admin.setBalancerRunning(false, true);
200
201 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
202 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
203 }
204
205 @AfterClass
206 public static void tearDownAfterClass() throws Exception {
207 tableExecutorService.shutdown();
208 hbfsckExecutorService.shutdown();
209 admin.close();
210 TEST_UTIL.shutdownMiniCluster();
211 }
212
213 @Before
214 public void setUp() {
215 EnvironmentEdgeManager.reset();
216 }
217
218
219
220
221
222 @Test public void testHbckReportReplicaLingeringSplitParent() throws Exception {
223 TableName table = TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
224
225 try {
226 setupTableWithRegionReplica(table, 2);
227 TEST_UTIL.getHBaseAdmin().flush(table.getName());
228
229
230 TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
231 admin.split(table, Bytes.toBytes("A1"));
232
233 Thread.sleep(1000);
234
235 assertNoErrors(doFsck(conf, false));
236 } finally {
237 cleanupTable(table);
238
239 TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
240 }
241 }
242
243 @Test (timeout=180000)
244 public void testHBaseFsck() throws Exception {
245 assertNoErrors(doFsck(conf, false));
246 TableName table = TableName.valueOf("tableBadMetaAssign");
247 HTableDescriptor desc = new HTableDescriptor(table);
248 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
249 desc.addFamily(hcd);
250 createTable(TEST_UTIL, desc, null);
251
252
253 assertNoErrors(doFsck(conf, false));
254
255
256
257 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
258 Scan scan = new Scan();
259 scan.setStartRow(Bytes.toBytes(table+",,"));
260 ResultScanner scanner = meta.getScanner(scan);
261 HRegionInfo hri = null;
262
263 Result res = scanner.next();
264 ServerName currServer =
265 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
266 HConstants.SERVER_QUALIFIER));
267 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
268 HConstants.STARTCODE_QUALIFIER));
269
270 for (JVMClusterUtil.RegionServerThread rs :
271 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
272
273 ServerName sn = rs.getRegionServer().getServerName();
274
275
276 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
277 startCode != sn.getStartcode()) {
278 Put put = new Put(res.getRow());
279 put.setDurability(Durability.SKIP_WAL);
280 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
281 Bytes.toBytes(sn.getHostAndPort()));
282 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
283 Bytes.toBytes(sn.getStartcode()));
284 meta.put(put);
285 hri = MetaTableAccessor.getHRegionInfo(res);
286 break;
287 }
288 }
289
290
291 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
292 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
293
294 TEST_UTIL.getHBaseCluster().getMaster()
295 .getAssignmentManager().waitForAssignment(hri);
296
297
298 assertNoErrors(doFsck(conf, false));
299
300
301 Table t = connection.getTable(table, tableExecutorService);
302 ResultScanner s = t.getScanner(new Scan());
303 s.close();
304 t.close();
305
306 scanner.close();
307 meta.close();
308 }
309
310 @Test(timeout=180000)
311 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
312 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
313 admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO);
314 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
315 new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper());
316 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
317 HBaseFsck hbck = doFsck(conf, true);
318 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
319 ERROR_CODE.NULL_META_REGION });
320 assertNoErrors(doFsck(conf, false));
321 }
322
323
324
325
326
327 @Test (timeout=180000)
328 public void testSplitAndDupeRegionWithRegionReplica() throws Exception {
329 TableName table =
330 TableName.valueOf("testSplitAndDupeRegionWithRegionReplica");
331 Table meta = null;
332
333 try {
334 setupTableWithRegionReplica(table, 2);
335
336 assertNoErrors(doFsck(conf, false));
337 assertEquals(ROWKEYS.length, countRows());
338
339
340 admin.enableCatalogJanitor(false);
341 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
342 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
343 HRegionInfo hriParent = loc.getRegionInfo();
344
345
346 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
347 Thread.sleep(1000);
348
349
350 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
351 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriParent);
352 MetaTableAccessor.addRegionToMeta(meta, hriParent);
353 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
354
355 if (server != null)
356 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
357
358 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
359 Thread.sleep(250);
360 }
361
362 LOG.debug("Finished assignment of parent region");
363
364
365 HBaseFsck hbck = doFsck(conf, false);
366 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED,
367 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
368 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
369 assertEquals(3, hbck.getOverlapGroups(table).size());
370
371
372 hbck = new HBaseFsck(conf, hbfsckExecutorService);
373 hbck.setDisplayFullReport();
374 hbck.setTimeLag(0);
375 hbck.setFixHdfsOverlaps(true);
376 hbck.setRemoveParents(true);
377 hbck.setFixReferenceFiles(true);
378 hbck.connect();
379 hbck.onlineHbck();
380 hbck.close();
381
382 hbck = doFsck(conf, false);
383
384 assertNoErrors(hbck);
385 assertEquals(0, hbck.getOverlapGroups(table).size());
386 assertEquals(ROWKEYS.length, countRows());
387 } finally {
388 cleanupTable(table);
389 }
390 }
391
392
393
394
395
396 @Test (timeout=180000)
397 public void testSplitAndDupeRegion() throws Exception {
398 TableName table =
399 TableName.valueOf("testSplitAndDupeRegion");
400 Table meta = null;
401
402 try {
403 setupTable(table);
404
405 assertNoErrors(doFsck(conf, false));
406 assertEquals(ROWKEYS.length, countRows());
407
408
409 admin.enableCatalogJanitor(false);
410 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
411 HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false);
412 HRegionInfo hriParent = loc.getRegionInfo();
413
414
415 this.connection.getAdmin().split(table, Bytes.toBytes("A@"));
416 Thread.sleep(1000);
417
418
419 regionStates.updateRegionState(hriParent, RegionState.State.CLOSED);
420 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriParent);
421
422 MetaTableAccessor.addRegionToMeta(meta, hriParent);
423 ServerName server = regionStates.getRegionServerOfRegion(hriParent);
424
425 if (server != null)
426 TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT);
427
428 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) {
429 Thread.sleep(250);
430 }
431
432 LOG.debug("Finished assignment of parent region");
433
434
435 HBaseFsck hbck = doFsck(conf, false);
436 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS,
437 HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
438 assertEquals(3, hbck.getOverlapGroups(table).size());
439
440
441 hbck = new HBaseFsck(conf, hbfsckExecutorService);
442 hbck.setDisplayFullReport();
443 hbck.setTimeLag(0);
444 hbck.setFixHdfsOverlaps(true);
445 hbck.setRemoveParents(true);
446 hbck.setFixReferenceFiles(true);
447 hbck.connect();
448 hbck.onlineHbck();
449 hbck.close();
450
451 hbck = doFsck(conf, false);
452
453 assertNoErrors(hbck);
454 assertEquals(0, hbck.getOverlapGroups(table).size());
455 assertEquals(ROWKEYS.length, countRows());
456 } finally {
457 cleanupTable(table);
458 }
459 }
460
461
462
463
464 private HRegionInfo createRegion(final HTableDescriptor
465 htd, byte[] startKey, byte[] endKey)
466 throws IOException {
467 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
468 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
469 MetaTableAccessor.addRegionToMeta(meta, hri);
470 meta.close();
471 return hri;
472 }
473
474
475
476
477 private void dumpMeta(TableName tableName) throws IOException {
478 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
479 for (byte[] row : metaRows) {
480 LOG.info(Bytes.toString(row));
481 }
482 }
483
484
485
486
487
488 private void undeployRegion(Connection conn, ServerName sn,
489 HRegionInfo hri) throws IOException, InterruptedException {
490 try {
491 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri);
492 if (!hri.isMetaTable()) {
493 admin.offline(hri.getRegionName());
494 }
495 } catch (IOException ioe) {
496 LOG.warn("Got exception when attempting to offline region "
497 + Bytes.toString(hri.getRegionName()), ioe);
498 }
499 }
500
501
502
503
504
505
506 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
507 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
508 boolean hdfs) throws IOException, InterruptedException {
509 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, HRegionInfo.DEFAULT_REPLICA_ID);
510 }
511
512
513
514
515
516
517
518
519
520 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
521 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
522 boolean hdfs, boolean regionInfoOnly, int replicaId)
523 throws IOException, InterruptedException {
524 LOG.info("** Before delete:");
525 dumpMeta(htd.getTableName());
526
527 List<HRegionLocation> locations = tbl.getAllRegionLocations();
528 for (HRegionLocation location : locations) {
529 HRegionInfo hri = location.getRegionInfo();
530 ServerName hsa = location.getServerName();
531 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
532 && Bytes.compareTo(hri.getEndKey(), endKey) == 0
533 && hri.getReplicaId() == replicaId) {
534
535 LOG.info("RegionName: " +hri.getRegionNameAsString());
536 byte[] deleteRow = hri.getRegionName();
537
538 if (unassign) {
539 LOG.info("Undeploying region " + hri + " from server " + hsa);
540 undeployRegion(connection, hsa, hri);
541 }
542
543 if (regionInfoOnly) {
544 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
545 Path rootDir = FSUtils.getRootDir(conf);
546 FileSystem fs = rootDir.getFileSystem(conf);
547 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
548 hri.getEncodedName());
549 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
550 fs.delete(hriPath, true);
551 }
552
553 if (hdfs) {
554 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
555 Path rootDir = FSUtils.getRootDir(conf);
556 FileSystem fs = rootDir.getFileSystem(conf);
557 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
558 hri.getEncodedName());
559 HBaseFsck.debugLsr(conf, p);
560 boolean success = fs.delete(p, true);
561 LOG.info("Deleted " + p + " sucessfully? " + success);
562 HBaseFsck.debugLsr(conf, p);
563 }
564
565 if (metaRow) {
566 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
567 Delete delete = new Delete(deleteRow);
568 meta.delete(delete);
569 }
570 }
571 }
572 LOG.info(hri.toString() + hsa.toString());
573 }
574
575 TEST_UTIL.getMetaTableRows(htd.getTableName());
576 LOG.info("*** After delete:");
577 dumpMeta(htd.getTableName());
578 }
579
580
581
582
583
584
585
586
587
588
589 void setupTable(TableName tablename) throws Exception {
590 setupTableWithRegionReplica(tablename, 1);
591 }
592
593
594
595
596
597
598
599
600
601
602 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
603 HTableDescriptor desc = new HTableDescriptor(tablename);
604 desc.setRegionReplication(replicaCount);
605 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
606 desc.addFamily(hcd);
607 createTable(TEST_UTIL, desc, SPLITS);
608
609 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
610 List<Put> puts = new ArrayList<Put>();
611 for (byte[] row : ROWKEYS) {
612 Put p = new Put(row);
613 p.add(FAM, Bytes.toBytes("val"), row);
614 puts.add(p);
615 }
616 tbl.put(puts);
617 tbl.flushCommits();
618 }
619
620
621
622
623 int countRows() throws IOException {
624 Scan s = new Scan();
625 ResultScanner rs = tbl.getScanner(s);
626 int i = 0;
627 while(rs.next() !=null) {
628 i++;
629 }
630 return i;
631 }
632
633
634
635
636 int countRows(byte[] start, byte[] end) throws IOException {
637 Scan s = new Scan(start, end);
638 ResultScanner rs = tbl.getScanner(s);
639 int i = 0;
640 while (rs.next() != null) {
641 i++;
642 }
643 return i;
644 }
645
646
647
648
649
650
651 void cleanupTable(TableName tablename) throws Exception {
652 if (tbl != null) {
653 tbl.close();
654 tbl = null;
655 }
656
657 ((ClusterConnection) connection).clearRegionCache();
658 deleteTable(TEST_UTIL, tablename);
659 }
660
661
662
663
664 @Test (timeout=180000)
665 public void testHBaseFsckClean() throws Exception {
666 assertNoErrors(doFsck(conf, false));
667 TableName table = TableName.valueOf("tableClean");
668 try {
669 HBaseFsck hbck = doFsck(conf, false);
670 assertNoErrors(hbck);
671
672 setupTable(table);
673 assertEquals(ROWKEYS.length, countRows());
674
675
676 hbck = doFsck(conf, false);
677 assertNoErrors(hbck);
678 assertEquals(0, hbck.getOverlapGroups(table).size());
679 assertEquals(ROWKEYS.length, countRows());
680 } finally {
681 cleanupTable(table);
682 }
683 }
684
685
686
687
688 @Test (timeout=180000)
689 public void testHbckThreadpooling() throws Exception {
690 TableName table =
691 TableName.valueOf("tableDupeStartKey");
692 try {
693
694 setupTable(table);
695
696
697 Configuration newconf = new Configuration(conf);
698 newconf.setInt("hbasefsck.numthreads", 1);
699 assertNoErrors(doFsck(newconf, false));
700
701
702 } finally {
703 cleanupTable(table);
704 }
705 }
706
707 @Test (timeout=180000)
708 public void testHbckFixOrphanTable() throws Exception {
709 TableName table = TableName.valueOf("tableInfo");
710 FileSystem fs = null;
711 Path tableinfo = null;
712 try {
713 setupTable(table);
714
715 Path hbaseTableDir = FSUtils.getTableDir(
716 FSUtils.getRootDir(conf), table);
717 fs = hbaseTableDir.getFileSystem(conf);
718 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
719 tableinfo = status.getPath();
720 fs.rename(tableinfo, new Path("/.tableinfo"));
721
722
723 HBaseFsck hbck = doFsck(conf, false);
724 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
725
726
727 hbck = doFsck(conf, true);
728 assertNoErrors(hbck);
729 status = null;
730 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
731 assertNotNull(status);
732
733 HTableDescriptor htd = admin.getTableDescriptor(table);
734 htd.setValue("NOT_DEFAULT", "true");
735 admin.disableTable(table);
736 admin.modifyTable(table, htd);
737 admin.enableTable(table);
738 fs.delete(status.getPath(), true);
739
740
741 htd = admin.getTableDescriptor(table);
742 hbck = doFsck(conf, true);
743 assertNoErrors(hbck);
744 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
745 assertNotNull(status);
746 htd = admin.getTableDescriptor(table);
747 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
748 } finally {
749 fs.rename(new Path("/.tableinfo"), tableinfo);
750 cleanupTable(table);
751 }
752 }
753
754
755
756
757
758
759 @Test (timeout=180000)
760 public void testParallelHbck() throws Exception {
761 final ExecutorService service;
762 final Future<HBaseFsck> hbck1,hbck2;
763
764 class RunHbck implements Callable<HBaseFsck>{
765 boolean fail = true;
766 @Override
767 public HBaseFsck call(){
768 Configuration c = new Configuration(conf);
769 c.setInt("hbase.hbck.lockfile.attempts", 1);
770
771
772 c.setInt("hbase.hbck.lockfile.maxwaittime", 3);
773 try{
774 return doFsck(c, true);
775 } catch(Exception e){
776 if (e.getMessage().contains("Duplicate hbck")) {
777 fail = false;
778 }
779 }
780
781 if (fail) fail();
782 return null;
783 }
784 }
785 service = Executors.newFixedThreadPool(2);
786 hbck1 = service.submit(new RunHbck());
787 hbck2 = service.submit(new RunHbck());
788 service.shutdown();
789
790 service.awaitTermination(15, TimeUnit.SECONDS);
791 HBaseFsck h1 = hbck1.get();
792 HBaseFsck h2 = hbck2.get();
793
794 assert(h1 == null || h2 == null);
795 if (h1 != null) {
796 assert(h1.getRetCode() >= 0);
797 }
798 if (h2 != null) {
799 assert(h2.getRetCode() >= 0);
800 }
801 }
802
803
804
805
806
807
808
809 @Test (timeout=180000)
810 public void testParallelWithRetriesHbck() throws Exception {
811 final ExecutorService service;
812 final Future<HBaseFsck> hbck1,hbck2;
813
814
815
816
817
818
819
820 final int timeoutInSeconds = 80;
821 final int sleepIntervalInMilliseconds = 200;
822 final int maxSleepTimeInMilliseconds = 6000;
823 final int maxRetryAttempts = 15;
824
825 class RunHbck implements Callable<HBaseFsck>{
826
827 @Override
828 public HBaseFsck call() throws Exception {
829
830 Configuration c = new Configuration(conf);
831 c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds);
832 c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds);
833 c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds);
834 c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts);
835 return doFsck(c, false);
836 }
837 }
838
839 service = Executors.newFixedThreadPool(2);
840 hbck1 = service.submit(new RunHbck());
841 hbck2 = service.submit(new RunHbck());
842 service.shutdown();
843
844 service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS);
845 HBaseFsck h1 = hbck1.get();
846 HBaseFsck h2 = hbck2.get();
847
848 assertNotNull(h1);
849 assertNotNull(h2);
850 assert(h1.getRetCode() >= 0);
851 assert(h2.getRetCode() >= 0);
852
853 }
854
855
856
857
858
859 @Test (timeout=180000)
860 public void testDupeStartKey() throws Exception {
861 TableName table =
862 TableName.valueOf("tableDupeStartKey");
863 try {
864 setupTable(table);
865 assertNoErrors(doFsck(conf, false));
866 assertEquals(ROWKEYS.length, countRows());
867
868
869 HRegionInfo hriDupe =
870 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("A2"));
871 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
872 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
873 .waitForAssignment(hriDupe);
874 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
875 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
876
877 HBaseFsck hbck = doFsck(conf, false);
878 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
879 ERROR_CODE.DUPE_STARTKEYS});
880 assertEquals(2, hbck.getOverlapGroups(table).size());
881 assertEquals(ROWKEYS.length, countRows());
882
883
884 doFsck(conf,true);
885
886
887 HBaseFsck hbck2 = doFsck(conf,false);
888 assertNoErrors(hbck2);
889 assertEquals(0, hbck2.getOverlapGroups(table).size());
890 assertEquals(ROWKEYS.length, countRows());
891 } finally {
892 cleanupTable(table);
893 }
894 }
895
896
897
898
899
900 @Test (timeout=180000)
901 public void testHbckWithRegionReplica() throws Exception {
902 TableName table =
903 TableName.valueOf("testHbckWithRegionReplica");
904 try {
905 setupTableWithRegionReplica(table, 2);
906 TEST_UTIL.getHBaseAdmin().flush(table.getName());
907 assertNoErrors(doFsck(conf, false));
908 } finally {
909 cleanupTable(table);
910 }
911 }
912
913
914
915
916
917 @Test
918 public void testHbckReplicaRegionAsKeyInMeta() throws Exception {
919 TableName table = TableName.valueOf("testHbckReplicaRegionAsKeyInMeta");
920 try {
921 setupTableWithRegionReplica(table, 2);
922 TEST_UTIL.getHBaseAdmin().flush(table.getName());
923
924 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
925 HRegionInfo hri = new HRegionInfo(table, SPLITS[0], SPLITS[2], false, 1500328224175L, 1);
926 Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
927 meta.put(put);
928
929 assertErrors(doFsck(conf, false),
930 new HBaseFsck.ErrorReporter.ERROR_CODE[] {
931 HBaseFsck.ErrorReporter.ERROR_CODE.EMPTY_META_CELL });
932
933
934 doFsck(conf, true);
935
936
937 assertNoErrors(doFsck(conf, false));
938 } finally {
939 cleanupTable(table);
940 }
941 }
942
943 @Test
944 public void testHbckWithFewerReplica() throws Exception {
945 TableName table =
946 TableName.valueOf("testHbckWithFewerReplica");
947 try {
948 setupTableWithRegionReplica(table, 2);
949 TEST_UTIL.getHBaseAdmin().flush(table.getName());
950 assertNoErrors(doFsck(conf, false));
951 assertEquals(ROWKEYS.length, countRows());
952 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
953 Bytes.toBytes("C"), true, false, false, false, 1);
954
955 HBaseFsck hbck = doFsck(conf, false);
956 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_DEPLOYED});
957
958 hbck = doFsck(conf, true);
959
960 hbck = doFsck(conf, false);
961 assertErrors(hbck, new ERROR_CODE[]{});
962 } finally {
963 cleanupTable(table);
964 }
965 }
966
967 @Test
968 public void testHbckWithExcessReplica() throws Exception {
969 TableName table =
970 TableName.valueOf("testHbckWithExcessReplica");
971 try {
972 setupTableWithRegionReplica(table, 2);
973 TEST_UTIL.getHBaseAdmin().flush(table.getName());
974 assertNoErrors(doFsck(conf, false));
975 assertEquals(ROWKEYS.length, countRows());
976
977
978
979
980 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
981 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(table);
982 byte[] startKey = Bytes.toBytes("B");
983 byte[] endKey = Bytes.toBytes("C");
984 byte[] metaKey = null;
985 HRegionInfo newHri = null;
986 for (HRegionInfo h : regions) {
987 if (Bytes.compareTo(h.getStartKey(), startKey) == 0 &&
988 Bytes.compareTo(h.getEndKey(), endKey) == 0 &&
989 h.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
990 metaKey = h.getRegionName();
991
992 newHri = RegionReplicaUtil.getRegionInfoForReplica(h, 2);
993 break;
994 }
995 }
996 Put put = new Put(metaKey);
997 ServerName sn = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers()
998 .toArray(new ServerName[0])[0];
999
1000 MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, 2);
1001 meta.put(put);
1002 meta.flushCommits();
1003
1004 HBaseFsckRepair.fixUnassigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
1005 HBaseFsckRepair.waitUntilAssigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
1006
1007 Delete delete = new Delete(metaKey);
1008 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(2));
1009 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(2));
1010 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(2));
1011 meta.delete(delete);
1012 meta.flushCommits();
1013 meta.close();
1014
1015 HBaseFsck hbck = doFsck(conf, false);
1016 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_IN_META});
1017
1018 hbck = doFsck(conf, true);
1019
1020 hbck = doFsck(conf, false);
1021 assertErrors(hbck, new ERROR_CODE[]{});
1022 } finally {
1023 cleanupTable(table);
1024 }
1025 }
1026
1027
1028
1029 Map<ServerName, List<String>> getDeployedHRIs(final HBaseAdmin admin) throws IOException {
1030 ClusterStatus status = admin.getClusterStatus();
1031 Collection<ServerName> regionServers = status.getServers();
1032 Map<ServerName, List<String>> mm =
1033 new HashMap<ServerName, List<String>>();
1034 for (ServerName hsi : regionServers) {
1035 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi);
1036
1037
1038 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
1039 List<String> regionNames = new ArrayList<String>();
1040 for (HRegionInfo hri : regions) {
1041 regionNames.add(hri.getRegionNameAsString());
1042 }
1043 mm.put(hsi, regionNames);
1044 }
1045 return mm;
1046 }
1047
1048
1049
1050
1051 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
1052 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
1053 if (e.getValue().contains(hri.getRegionNameAsString())) {
1054 return e.getKey();
1055 }
1056 }
1057 return null;
1058 }
1059
1060
1061
1062
1063
1064 @Test (timeout=180000)
1065 public void testDupeRegion() throws Exception {
1066 TableName table =
1067 TableName.valueOf("tableDupeRegion");
1068 try {
1069 setupTable(table);
1070 assertNoErrors(doFsck(conf, false));
1071 assertEquals(ROWKEYS.length, countRows());
1072
1073
1074 HRegionInfo hriDupe =
1075 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"));
1076
1077 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
1078 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1079 .waitForAssignment(hriDupe);
1080 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1081 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1082
1083
1084
1085
1086
1087 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriDupe) == null) {
1088 Thread.sleep(250);
1089 }
1090
1091 LOG.debug("Finished assignment of dupe region");
1092
1093
1094 HBaseFsck hbck = doFsck(conf, false);
1095 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
1096 ERROR_CODE.DUPE_STARTKEYS});
1097 assertEquals(2, hbck.getOverlapGroups(table).size());
1098 assertEquals(ROWKEYS.length, countRows());
1099
1100
1101 doFsck(conf,true);
1102
1103
1104 HBaseFsck hbck2 = doFsck(conf,false);
1105 assertNoErrors(hbck2);
1106 assertEquals(0, hbck2.getOverlapGroups(table).size());
1107 assertEquals(ROWKEYS.length, countRows());
1108 } finally {
1109 cleanupTable(table);
1110 }
1111 }
1112
1113
1114
1115
1116 @Test (timeout=180000)
1117 public void testDegenerateRegions() throws Exception {
1118 TableName table = TableName.valueOf("tableDegenerateRegions");
1119 try {
1120 setupTable(table);
1121 assertNoErrors(doFsck(conf,false));
1122 assertEquals(ROWKEYS.length, countRows());
1123
1124
1125 HRegionInfo hriDupe =
1126 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B"));
1127 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
1128 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1129 .waitForAssignment(hriDupe);
1130 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
1131 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
1132
1133 HBaseFsck hbck = doFsck(conf,false);
1134 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, ERROR_CODE.DUPE_STARTKEYS,
1135 ERROR_CODE.DUPE_STARTKEYS });
1136 assertEquals(2, hbck.getOverlapGroups(table).size());
1137 assertEquals(ROWKEYS.length, countRows());
1138
1139
1140 doFsck(conf,true);
1141
1142
1143 HBaseFsck hbck2 = doFsck(conf,false);
1144 assertNoErrors(hbck2);
1145 assertEquals(0, hbck2.getOverlapGroups(table).size());
1146 assertEquals(ROWKEYS.length, countRows());
1147 } finally {
1148 cleanupTable(table);
1149 }
1150 }
1151
1152
1153
1154
1155
1156 @Test (timeout=180000)
1157 public void testContainedRegionOverlap() throws Exception {
1158 TableName table =
1159 TableName.valueOf("tableContainedRegionOverlap");
1160 try {
1161 setupTable(table);
1162 assertEquals(ROWKEYS.length, countRows());
1163
1164
1165 HRegionInfo hriOverlap =
1166 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1167 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1168 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1169 .waitForAssignment(hriOverlap);
1170 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1171 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1172
1173 HBaseFsck hbck = doFsck(conf, false);
1174 assertErrors(hbck, new ERROR_CODE[] {
1175 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1176 assertEquals(2, hbck.getOverlapGroups(table).size());
1177 assertEquals(ROWKEYS.length, countRows());
1178
1179
1180 doFsck(conf, true);
1181
1182
1183 HBaseFsck hbck2 = doFsck(conf,false);
1184 assertNoErrors(hbck2);
1185 assertEquals(0, hbck2.getOverlapGroups(table).size());
1186 assertEquals(ROWKEYS.length, countRows());
1187 } finally {
1188 cleanupTable(table);
1189 }
1190 }
1191
1192
1193
1194
1195
1196
1197
1198 @Test (timeout=180000)
1199 public void testSidelineOverlapRegion() throws Exception {
1200 TableName table =
1201 TableName.valueOf("testSidelineOverlapRegion");
1202 try {
1203 setupTable(table);
1204 assertEquals(ROWKEYS.length, countRows());
1205
1206
1207 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1208 HMaster master = cluster.getMaster();
1209 HRegionInfo hriOverlap1 =
1210 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB"));
1211 master.assignRegion(hriOverlap1);
1212 master.getAssignmentManager().waitForAssignment(hriOverlap1);
1213 HRegionInfo hriOverlap2 =
1214 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("AB"), Bytes.toBytes("B"));
1215 master.assignRegion(hriOverlap2);
1216 master.getAssignmentManager().waitForAssignment(hriOverlap2);
1217
1218 HBaseFsck hbck = doFsck(conf, false);
1219 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
1220 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
1221 assertEquals(3, hbck.getOverlapGroups(table).size());
1222 assertEquals(ROWKEYS.length, countRows());
1223
1224
1225 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
1226 ServerName serverName = null;
1227 byte[] regionName = null;
1228 for (HbckInfo hbi: overlapGroups.values()) {
1229 if ("A".equals(Bytes.toString(hbi.getStartKey()))
1230 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
1231 regionName = hbi.getRegionName();
1232
1233
1234 int k = cluster.getServerWith(regionName);
1235 for (int i = 0; i < 3; i++) {
1236 if (i != k) {
1237 HRegionServer rs = cluster.getRegionServer(i);
1238 serverName = rs.getServerName();
1239 break;
1240 }
1241 }
1242
1243 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) connection,
1244 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
1245 admin.offline(regionName);
1246 break;
1247 }
1248 }
1249
1250 assertNotNull(regionName);
1251 assertNotNull(serverName);
1252 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
1253 Put put = new Put(regionName);
1254 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
1255 Bytes.toBytes(serverName.getHostAndPort()));
1256 meta.put(put);
1257 }
1258
1259
1260 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1261 fsck.connect();
1262 fsck.setDisplayFullReport();
1263 fsck.setTimeLag(0);
1264 fsck.setFixAssignments(true);
1265 fsck.setFixMeta(true);
1266 fsck.setFixHdfsHoles(true);
1267 fsck.setFixHdfsOverlaps(true);
1268 fsck.setFixHdfsOrphans(true);
1269 fsck.setFixVersionFile(true);
1270 fsck.setSidelineBigOverlaps(true);
1271 fsck.setMaxMerge(2);
1272 fsck.onlineHbck();
1273 fsck.close();
1274
1275
1276
1277 HBaseFsck hbck2 = doFsck(conf,false);
1278 assertNoErrors(hbck2);
1279 assertEquals(0, hbck2.getOverlapGroups(table).size());
1280 assertTrue(ROWKEYS.length > countRows());
1281 } finally {
1282 cleanupTable(table);
1283 }
1284 }
1285
1286
1287
1288
1289
1290 @Test (timeout=180000)
1291 public void testOverlapAndOrphan() throws Exception {
1292 TableName table =
1293 TableName.valueOf("tableOverlapAndOrphan");
1294 try {
1295 setupTable(table);
1296 assertEquals(ROWKEYS.length, countRows());
1297
1298
1299 admin.disableTable(table);
1300 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1301 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1302 TEST_UTIL.getHBaseAdmin().enableTable(table);
1303
1304 HRegionInfo hriOverlap =
1305 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1306 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1307 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1308 .waitForAssignment(hriOverlap);
1309 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1310 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1311
1312 HBaseFsck hbck = doFsck(conf, false);
1313 assertErrors(hbck, new ERROR_CODE[] {
1314 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1315 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1316
1317
1318 doFsck(conf, true);
1319
1320
1321 HBaseFsck hbck2 = doFsck(conf,false);
1322 assertNoErrors(hbck2);
1323 assertEquals(0, hbck2.getOverlapGroups(table).size());
1324 assertEquals(ROWKEYS.length, countRows());
1325 } finally {
1326 cleanupTable(table);
1327 }
1328 }
1329
1330
1331
1332
1333
1334
1335 @Test (timeout=180000)
1336 public void testCoveredStartKey() throws Exception {
1337 TableName table =
1338 TableName.valueOf("tableCoveredStartKey");
1339 try {
1340 setupTable(table);
1341 assertEquals(ROWKEYS.length, countRows());
1342
1343
1344 HRegionInfo hriOverlap =
1345 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2"));
1346 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1347 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1348 .waitForAssignment(hriOverlap);
1349 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1350 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1351
1352 HBaseFsck hbck = doFsck(conf, false);
1353 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
1354 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1355 assertEquals(3, hbck.getOverlapGroups(table).size());
1356 assertEquals(ROWKEYS.length, countRows());
1357
1358
1359 doFsck(conf, true);
1360
1361
1362 HBaseFsck hbck2 = doFsck(conf, false);
1363 assertErrors(hbck2, new ERROR_CODE[0]);
1364 assertEquals(0, hbck2.getOverlapGroups(table).size());
1365 assertEquals(ROWKEYS.length, countRows());
1366 } finally {
1367 cleanupTable(table);
1368 }
1369 }
1370
1371
1372
1373
1374
1375 @Test (timeout=180000)
1376 public void testRegionHole() throws Exception {
1377 TableName table =
1378 TableName.valueOf("tableRegionHole");
1379 try {
1380 setupTable(table);
1381 assertEquals(ROWKEYS.length, countRows());
1382
1383
1384 admin.disableTable(table);
1385 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1386 Bytes.toBytes("C"), true, true, true);
1387 admin.enableTable(table);
1388
1389 HBaseFsck hbck = doFsck(conf, false);
1390 assertErrors(hbck, new ERROR_CODE[] {
1391 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1392
1393 assertEquals(0, hbck.getOverlapGroups(table).size());
1394
1395
1396 doFsck(conf, true);
1397
1398
1399 assertNoErrors(doFsck(conf,false));
1400 assertEquals(ROWKEYS.length - 2 , countRows());
1401 } finally {
1402 cleanupTable(table);
1403 }
1404 }
1405
1406
1407
1408
1409
1410 @Test (timeout=180000)
1411 public void testHDFSRegioninfoMissing() throws Exception {
1412 TableName table = TableName.valueOf("tableHDFSRegioninfoMissing");
1413 try {
1414 setupTable(table);
1415 assertEquals(ROWKEYS.length, countRows());
1416
1417
1418 admin.disableTable(table);
1419 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1420 Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1421 TEST_UTIL.getHBaseAdmin().enableTable(table);
1422
1423 HBaseFsck hbck = doFsck(conf, false);
1424 assertErrors(hbck, new ERROR_CODE[] {
1425 ERROR_CODE.ORPHAN_HDFS_REGION,
1426 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1427 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1428
1429 assertEquals(0, hbck.getOverlapGroups(table).size());
1430
1431
1432 doFsck(conf, true);
1433
1434
1435 assertNoErrors(doFsck(conf, false));
1436 assertEquals(ROWKEYS.length, countRows());
1437 } finally {
1438 cleanupTable(table);
1439 }
1440 }
1441
1442
1443
1444
1445
1446
1447 @Test(timeout = 180000)
1448 public void testHDFSRegioninfoMissingAndCheckRegionBoundary() throws Exception {
1449 TableName table = TableName.valueOf("testHDFSRegioninfoMissingAndCheckRegionBoundary");
1450 try {
1451 setupTable(table);
1452 assertEquals(ROWKEYS.length, countRows());
1453
1454
1455 admin.disableTable(table);
1456 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true,
1457 true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1458 admin.enableTable(table);
1459
1460 HBaseFsck hbck = doFsck(conf, false);
1461 assertErrors(hbck,
1462 new HBaseFsck.ErrorReporter.ERROR_CODE[] {
1463 HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION,
1464 HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1465 HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN });
1466
1467 assertEquals(0, hbck.getOverlapGroups(table).size());
1468
1469
1470 doFsck(conf, true);
1471
1472
1473 assertNoErrors(doFsck(conf, false));
1474
1475
1476 for (int i = 0; i < ROWKEYS.length; i++) {
1477 if (i != ROWKEYS.length - 1) {
1478 assertEquals(1, countRows(ROWKEYS[i], ROWKEYS[i + 1]));
1479 } else {
1480 assertEquals(1, countRows(ROWKEYS[i], null));
1481 }
1482 }
1483
1484 } finally {
1485 cleanupTable(table);
1486 }
1487 }
1488
1489
1490
1491
1492
1493 @Test (timeout=180000)
1494 public void testNotInMetaOrDeployedHole() throws Exception {
1495 TableName table =
1496 TableName.valueOf("tableNotInMetaOrDeployedHole");
1497 try {
1498 setupTable(table);
1499 assertEquals(ROWKEYS.length, countRows());
1500
1501
1502 admin.disableTable(table);
1503 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1504 Bytes.toBytes("C"), true, true, false);
1505 admin.enableTable(table);
1506
1507 HBaseFsck hbck = doFsck(conf, false);
1508 assertErrors(hbck, new ERROR_CODE[] {
1509 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1510
1511 assertEquals(0, hbck.getOverlapGroups(table).size());
1512
1513
1514 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1515 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1516
1517
1518 assertNoErrors(doFsck(conf,false));
1519 assertEquals(ROWKEYS.length, countRows());
1520 } finally {
1521 cleanupTable(table);
1522 }
1523 }
1524
1525
1526
1527
1528 @Test (timeout=180000)
1529 public void testNotInMetaHole() throws Exception {
1530 TableName table =
1531 TableName.valueOf("tableNotInMetaHole");
1532 try {
1533 setupTable(table);
1534 assertEquals(ROWKEYS.length, countRows());
1535
1536
1537 admin.disableTable(table);
1538 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1539 Bytes.toBytes("C"), false, true, false);
1540 admin.enableTable(table);
1541
1542 HBaseFsck hbck = doFsck(conf, false);
1543 assertErrors(hbck, new ERROR_CODE[] {
1544 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1545
1546 assertEquals(0, hbck.getOverlapGroups(table).size());
1547
1548
1549 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1550 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1551
1552
1553 assertNoErrors(doFsck(conf,false));
1554 assertEquals(ROWKEYS.length, countRows());
1555 } finally {
1556 cleanupTable(table);
1557 }
1558 }
1559
1560
1561
1562
1563
1564 @Test (timeout=180000)
1565 public void testNotInHdfs() throws Exception {
1566 TableName table =
1567 TableName.valueOf("tableNotInHdfs");
1568 try {
1569 setupTable(table);
1570 assertEquals(ROWKEYS.length, countRows());
1571
1572
1573 admin.flush(table);
1574
1575
1576 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1577 Bytes.toBytes("C"), false, false, true);
1578
1579 HBaseFsck hbck = doFsck(conf, false);
1580 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1581
1582 assertEquals(0, hbck.getOverlapGroups(table).size());
1583
1584
1585 doFsck(conf, true);
1586
1587
1588 assertNoErrors(doFsck(conf,false));
1589 assertEquals(ROWKEYS.length - 2, countRows());
1590 } finally {
1591 cleanupTable(table);
1592 }
1593 }
1594
1595
1596
1597
1598
1599 @Test (timeout=180000)
1600 public void testNotInHdfsWithReplicas() throws Exception {
1601 TableName table =
1602 TableName.valueOf("tableNotInHdfs");
1603 HBaseAdmin admin = new HBaseAdmin(conf);
1604 try {
1605 HRegionInfo[] oldHris = new HRegionInfo[2];
1606 setupTableWithRegionReplica(table, 2);
1607 assertEquals(ROWKEYS.length, countRows());
1608 NavigableMap<HRegionInfo, ServerName> map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(),
1609 tbl.getName());
1610 int i = 0;
1611
1612 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1613 if (m.getKey().getStartKey().length > 0 &&
1614 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1615 LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
1616 oldHris[i++] = m.getKey();
1617 }
1618 }
1619
1620 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1621
1622
1623 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1624 Bytes.toBytes("C"), false, false, true);
1625
1626 HBaseFsck hbck = doFsck(conf, false);
1627 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1628
1629
1630 doFsck(conf, true);
1631
1632
1633 assertNoErrors(doFsck(conf,false));
1634 assertEquals(ROWKEYS.length - 2, countRows());
1635
1636
1637
1638 i = 0;
1639 HRegionInfo[] newHris = new HRegionInfo[2];
1640
1641 map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
1642
1643 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1644 if (m.getKey().getStartKey().length > 0 &&
1645 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1646 newHris[i++] = m.getKey();
1647 }
1648 }
1649
1650 Collection<ServerName> servers = admin.getClusterStatus().getServers();
1651 Set<HRegionInfo> onlineRegions = new HashSet<HRegionInfo>();
1652 for (ServerName s : servers) {
1653 List<HRegionInfo> list = admin.getOnlineRegions(s);
1654 onlineRegions.addAll(list);
1655 }
1656
1657 assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
1658
1659
1660 assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
1661 } finally {
1662 cleanupTable(table);
1663 admin.close();
1664 }
1665 }
1666
1667
1668
1669
1670
1671
1672 @Test (timeout=180000)
1673 public void testNoHdfsTable() throws Exception {
1674 TableName table = TableName.valueOf("NoHdfsTable");
1675 setupTable(table);
1676 assertEquals(ROWKEYS.length, countRows());
1677
1678
1679 admin.flush(table);
1680
1681
1682 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1683 Bytes.toBytes("A"), false, false, true);
1684 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1685 Bytes.toBytes("B"), false, false, true);
1686 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1687 Bytes.toBytes("C"), false, false, true);
1688 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1689 Bytes.toBytes(""), false, false, true);
1690
1691
1692 deleteTableDir(table);
1693
1694 HBaseFsck hbck = doFsck(conf, false);
1695 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1696 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1697 ERROR_CODE.NOT_IN_HDFS,});
1698
1699 assertEquals(0, hbck.getOverlapGroups(table).size());
1700
1701
1702 doFsck(conf, true);
1703
1704
1705 assertNoErrors(doFsck(conf,false));
1706 assertFalse("Table " + table + " should have been deleted", admin.tableExists(table));
1707 }
1708
1709 public void deleteTableDir(TableName table) throws IOException {
1710 Path rootDir = FSUtils.getRootDir(conf);
1711 FileSystem fs = rootDir.getFileSystem(conf);
1712 Path p = FSUtils.getTableDir(rootDir, table);
1713 HBaseFsck.debugLsr(conf, p);
1714 boolean success = fs.delete(p, true);
1715 LOG.info("Deleted " + p + " sucessfully? " + success);
1716 }
1717
1718
1719
1720
1721 @Test (timeout=180000)
1722 public void testNoVersionFile() throws Exception {
1723
1724 Path rootDir = FSUtils.getRootDir(conf);
1725 FileSystem fs = rootDir.getFileSystem(conf);
1726 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1727 fs.delete(versionFile, true);
1728
1729
1730 HBaseFsck hbck = doFsck(conf, false);
1731 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1732
1733 doFsck(conf, true);
1734
1735
1736 assertNoErrors(doFsck(conf, false));
1737 }
1738
1739
1740
1741
1742 @Test (timeout=180000)
1743 public void testRegionShouldNotBeDeployed() throws Exception {
1744 TableName table =
1745 TableName.valueOf("tableRegionShouldNotBeDeployed");
1746 try {
1747 LOG.info("Starting testRegionShouldNotBeDeployed.");
1748 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1749 assertTrue(cluster.waitForActiveAndReadyMaster());
1750
1751
1752 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1753 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1754 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1755 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1756
1757
1758 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1759 fstd.createTableDescriptor(htdDisabled);
1760 List<HRegionInfo> disabledRegions =
1761 TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
1762
1763
1764 HRegionServer hrs = cluster.getRegionServer(0);
1765
1766
1767 admin.disableTable(table);
1768 admin.enableTable(table);
1769
1770
1771 admin.disableTable(table);
1772 HRegionInfo region = disabledRegions.remove(0);
1773 byte[] regionName = region.getRegionName();
1774
1775
1776 assertTrue(cluster.getServerWith(regionName) == -1);
1777
1778
1779
1780
1781
1782 HRegion r = HRegion.openHRegion(
1783 region, htdDisabled, hrs.getWAL(region), conf);
1784 hrs.addToOnlineRegions(r);
1785
1786 HBaseFsck hbck = doFsck(conf, false);
1787 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1788
1789
1790 doFsck(conf, true);
1791
1792
1793 assertNoErrors(doFsck(conf, false));
1794 } finally {
1795 admin.enableTable(table);
1796 cleanupTable(table);
1797 }
1798 }
1799
1800
1801
1802
1803 @Test (timeout=180000)
1804 public void testFixByTable() throws Exception {
1805 TableName table1 =
1806 TableName.valueOf("testFixByTable1");
1807 TableName table2 =
1808 TableName.valueOf("testFixByTable2");
1809 try {
1810 setupTable(table1);
1811
1812 admin.flush(table1);
1813
1814 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1815 Bytes.toBytes("C"), false, false, true);
1816
1817 setupTable(table2);
1818
1819 admin.flush(table2);
1820
1821 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1822 Bytes.toBytes("C"), false, false, true);
1823
1824 HBaseFsck hbck = doFsck(conf, false);
1825 assertErrors(hbck, new ERROR_CODE[] {
1826 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1827
1828
1829 doFsck(conf, true, table1);
1830
1831 assertNoErrors(doFsck(conf, false, table1));
1832
1833 assertErrors(doFsck(conf, false, table2),
1834 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1835
1836
1837 doFsck(conf, true, table2);
1838
1839 assertNoErrors(doFsck(conf, false));
1840 assertEquals(ROWKEYS.length - 2, countRows());
1841 } finally {
1842 cleanupTable(table1);
1843 cleanupTable(table2);
1844 }
1845 }
1846
1847
1848
1849 @Test (timeout=180000)
1850 public void testLingeringSplitParent() throws Exception {
1851 TableName table =
1852 TableName.valueOf("testLingeringSplitParent");
1853 Table meta = null;
1854 try {
1855 setupTable(table);
1856 assertEquals(ROWKEYS.length, countRows());
1857
1858
1859 admin.flush(table);
1860 HRegionLocation location = tbl.getRegionLocation("B");
1861
1862
1863 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1864 Bytes.toBytes("C"), true, true, false);
1865
1866
1867 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1868 HRegionInfo hri = location.getRegionInfo();
1869
1870 HRegionInfo a = new HRegionInfo(tbl.getName(),
1871 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1872 HRegionInfo b = new HRegionInfo(tbl.getName(),
1873 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1874
1875 hri.setOffline(true);
1876 hri.setSplit(true);
1877
1878 MetaTableAccessor.addRegionToMeta(meta, hri, a, b);
1879 meta.close();
1880 admin.flush(TableName.META_TABLE_NAME);
1881
1882 HBaseFsck hbck = doFsck(conf, false);
1883 assertErrors(hbck, new ERROR_CODE[] {
1884 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1885
1886
1887 hbck = doFsck(conf, true);
1888 assertErrors(hbck, new ERROR_CODE[] {
1889 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1890 assertFalse(hbck.shouldRerun());
1891 hbck = doFsck(conf, false);
1892 assertErrors(hbck, new ERROR_CODE[] {
1893 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1894
1895
1896 hbck = new HBaseFsck(conf, hbfsckExecutorService);
1897 hbck.connect();
1898 hbck.setDisplayFullReport();
1899 hbck.setTimeLag(0);
1900 hbck.setFixSplitParents(true);
1901 hbck.onlineHbck();
1902 assertTrue(hbck.shouldRerun());
1903 hbck.close();
1904
1905 Get get = new Get(hri.getRegionName());
1906 Result result = meta.get(get);
1907 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1908 HConstants.SPLITA_QUALIFIER).isEmpty());
1909 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1910 HConstants.SPLITB_QUALIFIER).isEmpty());
1911 admin.flush(TableName.META_TABLE_NAME);
1912
1913
1914 doFsck(conf, true);
1915
1916
1917 assertNoErrors(doFsck(conf, false));
1918 assertEquals(ROWKEYS.length, countRows());
1919 } finally {
1920 cleanupTable(table);
1921 IOUtils.closeQuietly(meta);
1922 }
1923 }
1924
1925
1926
1927
1928
1929 @Test (timeout=180000)
1930 public void testValidLingeringSplitParent() throws Exception {
1931 final TableName table =
1932 TableName.valueOf("testLingeringSplitParent");
1933 Table meta = null;
1934 try {
1935 setupTable(table);
1936 assertEquals(ROWKEYS.length, countRows());
1937
1938
1939 admin.flush(table);
1940 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1941
1942 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1943 HRegionInfo hri = location.getRegionInfo();
1944
1945 splitAndWait(table, location);
1946
1947
1948
1949
1950 HBaseFsck hbck = doFsck(
1951 conf, true, true, false, false, false, true, true, true, false, false, false, false, null);
1952 assertErrors(hbck, new ERROR_CODE[] {});
1953
1954
1955 Get get = new Get(hri.getRegionName());
1956 Result result = meta.get(get);
1957 assertNotNull(result);
1958 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1959
1960 assertEquals(ROWKEYS.length, countRows());
1961
1962
1963 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1964 assertNoErrors(doFsck(conf, false));
1965 } finally {
1966 cleanupTable(table);
1967 IOUtils.closeQuietly(meta);
1968 }
1969 }
1970
1971 private byte[] splitAndWait(final TableName table, HRegionLocation location)
1972 throws IOException, Exception {
1973
1974
1975 final List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(table);
1976 byte[] regionName = location.getRegionInfo().getRegionName();
1977 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1978 TEST_UTIL.waitFor(60000, new Predicate<Exception>() {
1979 @Override
1980 public boolean evaluate() throws Exception {
1981 List<HRegion> regions1 = TEST_UTIL.getMiniHBaseCluster().getRegions(table);
1982 regions1.removeAll(regions);
1983 return regions1.size() == 2;
1984 }
1985 });
1986
1987 return regionName;
1988 }
1989
1990
1991
1992
1993
1994 @Test(timeout=75000)
1995 public void testSplitDaughtersNotInMeta() throws Exception {
1996 TableName table = TableName.valueOf("testSplitdaughtersNotInMeta");
1997 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1998 try {
1999 setupTable(table);
2000 assertEquals(ROWKEYS.length, countRows());
2001
2002
2003 admin.flush(table);
2004 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
2005
2006 HRegionInfo hri = location.getRegionInfo();
2007
2008
2009
2010 admin.enableCatalogJanitor(false);
2011
2012 byte[] regionName = splitAndWait(table, location);
2013 PairOfSameType<HRegionInfo> daughters =
2014 MetaTableAccessor.getDaughterRegions(meta.get(new Get(regionName)));
2015
2016
2017 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
2018 undeployRegion(connection, hris.get(daughters.getFirst()), daughters.getFirst());
2019 undeployRegion(connection, hris.get(daughters.getSecond()), daughters.getSecond());
2020
2021 List<Delete> deletes = new ArrayList<>();
2022 deletes.add(new Delete(daughters.getFirst().getRegionName()));
2023 deletes.add(new Delete(daughters.getSecond().getRegionName()));
2024 meta.delete(deletes);
2025
2026
2027 RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().
2028 getAssignmentManager().getRegionStates();
2029 regionStates.deleteRegion(daughters.getFirst());
2030 regionStates.deleteRegion(daughters.getSecond());
2031
2032 HBaseFsck hbck = doFsck(conf, false);
2033 assertErrors(hbck,
2034 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2035 ERROR_CODE.HOLE_IN_REGION_CHAIN });
2036
2037
2038 hbck = doFsck(
2039 conf, true, true, false, false, false, false, false, false, false, false, false,false,null);
2040 assertErrors(hbck,
2041 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2042 ERROR_CODE.HOLE_IN_REGION_CHAIN });
2043
2044
2045 Get get = new Get(hri.getRegionName());
2046 Result result = meta.get(get);
2047 assertNotNull(result);
2048 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
2049
2050 assertEquals(ROWKEYS.length, countRows());
2051
2052
2053 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
2054 assertNoErrors(doFsck(conf, false));
2055 } finally {
2056 admin.enableCatalogJanitor(true);
2057 meta.close();
2058 cleanupTable(table);
2059 }
2060 }
2061
2062
2063
2064
2065
2066 @Test(timeout=120000)
2067 public void testMissingFirstRegion() throws Exception {
2068 TableName table = TableName.valueOf("testMissingFirstRegion");
2069 try {
2070 setupTable(table);
2071 assertEquals(ROWKEYS.length, countRows());
2072
2073
2074 admin.disableTable(table);
2075 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
2076 true, true);
2077 admin.enableTable(table);
2078
2079 HBaseFsck hbck = doFsck(conf, false);
2080 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
2081
2082 doFsck(conf, true);
2083
2084 assertNoErrors(doFsck(conf, false));
2085 } finally {
2086 cleanupTable(table);
2087 }
2088 }
2089
2090
2091
2092
2093
2094 @Test(timeout=120000)
2095 public void testRegionDeployedNotInHdfs() throws Exception {
2096 TableName table =
2097 TableName.valueOf("testSingleRegionDeployedNotInHdfs");
2098 try {
2099 setupTable(table);
2100 admin.flush(table);
2101
2102
2103 deleteRegion(conf, tbl.getTableDescriptor(),
2104 HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false,
2105 false, true);
2106
2107 HBaseFsck hbck = doFsck(conf, false);
2108 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2109
2110 doFsck(conf, true);
2111
2112 assertNoErrors(doFsck(conf, false));
2113 } finally {
2114 cleanupTable(table);
2115 }
2116 }
2117
2118
2119
2120
2121
2122 @Test(timeout=120000)
2123 public void testMissingLastRegion() throws Exception {
2124 TableName table =
2125 TableName.valueOf("testMissingLastRegion");
2126 try {
2127 setupTable(table);
2128 assertEquals(ROWKEYS.length, countRows());
2129
2130
2131 admin.disableTable(table);
2132 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
2133 true, true);
2134 admin.enableTable(table);
2135
2136 HBaseFsck hbck = doFsck(conf, false);
2137 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
2138
2139 doFsck(conf, true);
2140
2141 assertNoErrors(doFsck(conf, false));
2142 } finally {
2143 cleanupTable(table);
2144 }
2145 }
2146
2147
2148
2149
2150 @Test (timeout=180000)
2151 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
2152 TableName table =
2153 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
2154 try {
2155 setupTable(table);
2156 assertEquals(ROWKEYS.length, countRows());
2157
2158
2159 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2160 Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2161
2162
2163 HBaseFsck hbck = doFsck(conf, false);
2164 assertErrors(hbck, new ERROR_CODE[] {
2165 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2166
2167
2168 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2169 fsck.connect();
2170 fsck.setDisplayFullReport();
2171 fsck.setTimeLag(0);
2172 fsck.setCheckHdfs(false);
2173 fsck.onlineHbck();
2174 assertErrors(fsck, new ERROR_CODE[] {
2175 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
2176 fsck.close();
2177
2178
2179 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2180 fsck.connect();
2181 fsck.setDisplayFullReport();
2182 fsck.setTimeLag(0);
2183 fsck.setCheckHdfs(false);
2184 fsck.setFixAssignments(true);
2185 fsck.onlineHbck();
2186 assertTrue(fsck.shouldRerun());
2187 fsck.onlineHbck();
2188 assertNoErrors(fsck);
2189
2190 assertEquals(ROWKEYS.length, countRows());
2191
2192 fsck.close();
2193 } finally {
2194 cleanupTable(table);
2195 }
2196 }
2197
2198
2199
2200
2201
2202
2203 @Test (timeout=180000)
2204 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
2205 TableName table =
2206 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
2207 try {
2208 setupTable(table);
2209 assertEquals(ROWKEYS.length, countRows());
2210
2211
2212 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2213 Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
2214
2215
2216 HBaseFsck hbck = doFsck(conf, false);
2217 assertErrors(hbck,
2218 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2219
2220
2221 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2222 fsck.connect();
2223 fsck.setDisplayFullReport();
2224 fsck.setTimeLag(0);
2225 fsck.setCheckHdfs(false);
2226 fsck.onlineHbck();
2227 assertErrors(fsck,
2228 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2229 fsck.close();
2230
2231
2232 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2233 fsck.connect();
2234 fsck.setDisplayFullReport();
2235 fsck.setTimeLag(0);
2236 fsck.setCheckHdfs(false);
2237 fsck.setFixAssignments(true);
2238 fsck.setFixMeta(true);
2239 fsck.onlineHbck();
2240 assertFalse(fsck.shouldRerun());
2241 assertErrors(fsck,
2242 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2243 fsck.close();
2244
2245
2246 fsck = doFsck(conf, true);
2247 assertTrue(fsck.shouldRerun());
2248 fsck = doFsck(conf, true);
2249 assertNoErrors(fsck);
2250 } finally {
2251 cleanupTable(table);
2252 }
2253 }
2254
2255
2256
2257
2258
2259 @Test (timeout=180000)
2260 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
2261 TableName table =
2262 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
2263 try {
2264 setupTable(table);
2265 assertEquals(ROWKEYS.length, countRows());
2266
2267
2268 admin.disableTable(table);
2269 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2270 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
2271 TEST_UTIL.getHBaseAdmin().enableTable(table);
2272
2273 HRegionInfo hriOverlap =
2274 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
2275 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
2276 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
2277 .waitForAssignment(hriOverlap);
2278 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
2279 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
2280
2281 HBaseFsck hbck = doFsck(conf, false);
2282 assertErrors(hbck, new ERROR_CODE[] {
2283 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2284 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2285
2286
2287 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2288 fsck.connect();
2289 fsck.setDisplayFullReport();
2290 fsck.setTimeLag(0);
2291 fsck.setCheckHdfs(false);
2292 fsck.onlineHbck();
2293 assertErrors(fsck, new ERROR_CODE[] {
2294 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2295 fsck.close();
2296
2297
2298 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2299 fsck.connect();
2300 fsck.setDisplayFullReport();
2301 fsck.setTimeLag(0);
2302 fsck.setCheckHdfs(false);
2303 fsck.setFixHdfsHoles(true);
2304 fsck.setFixHdfsOverlaps(true);
2305 fsck.setFixHdfsOrphans(true);
2306 fsck.onlineHbck();
2307 assertFalse(fsck.shouldRerun());
2308 assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN});
2309 fsck.close();
2310 } finally {
2311 if (admin.isTableDisabled(table)) {
2312 admin.enableTable(table);
2313 }
2314 cleanupTable(table);
2315 }
2316 }
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
2327 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2328 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2329 Path famDir = new Path(regionDir, FAM_STR);
2330
2331
2332 while (true) {
2333 FileStatus[] hfFss = fs.listStatus(famDir);
2334 if (hfFss.length == 0) {
2335 continue;
2336 }
2337 for (FileStatus hfs : hfFss) {
2338 if (!hfs.isDirectory()) {
2339 return hfs.getPath();
2340 }
2341 }
2342 }
2343 }
2344
2345
2346
2347
2348 @Test(timeout=180000)
2349 public void testQuarantineCorruptHFile() throws Exception {
2350 TableName table = TableName.valueOf(name.getMethodName());
2351 try {
2352 setupTable(table);
2353 assertEquals(ROWKEYS.length, countRows());
2354 admin.flush(table);
2355
2356 FileSystem fs = FileSystem.get(conf);
2357 Path hfile = getFlushedHFile(fs, table);
2358
2359
2360 admin.disableTable(table);
2361
2362
2363 Path corrupt = new Path(hfile.getParent(), "deadbeef");
2364 TestHFile.truncateFile(fs, hfile, corrupt);
2365 LOG.info("Created corrupted file " + corrupt);
2366 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2367
2368
2369 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2370 assertEquals(res.getRetCode(), 0);
2371 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2372 assertEquals(hfcc.getHFilesChecked(), 5);
2373 assertEquals(hfcc.getCorrupted().size(), 1);
2374 assertEquals(hfcc.getFailures().size(), 0);
2375 assertEquals(hfcc.getQuarantined().size(), 1);
2376 assertEquals(hfcc.getMissing().size(), 0);
2377
2378
2379 admin.enableTable(table);
2380 } finally {
2381 cleanupTable(table);
2382 }
2383 }
2384
2385
2386
2387
2388 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
2389 int corrupt, int fail, int quar, int missing) throws Exception {
2390 try {
2391 setupTable(table);
2392 assertEquals(ROWKEYS.length, countRows());
2393 admin.flush(table);
2394
2395
2396 admin.disableTable(table);
2397
2398 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
2399 table.getNameAsString()};
2400 HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
2401
2402 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2403 assertEquals(hfcc.getHFilesChecked(), check);
2404 assertEquals(hfcc.getCorrupted().size(), corrupt);
2405 assertEquals(hfcc.getFailures().size(), fail);
2406 assertEquals(hfcc.getQuarantined().size(), quar);
2407 assertEquals(hfcc.getMissing().size(), missing);
2408
2409
2410 admin.enableTableAsync(table);
2411 while (!admin.isTableEnabled(table)) {
2412 try {
2413 Thread.sleep(250);
2414 } catch (InterruptedException e) {
2415 e.printStackTrace();
2416 fail("Interrupted when trying to enable table " + table);
2417 }
2418 }
2419 } finally {
2420 cleanupTable(table);
2421 }
2422 }
2423
2424
2425
2426
2427
2428 @Test(timeout=180000)
2429 public void testQuarantineMissingHFile() throws Exception {
2430 TableName table = TableName.valueOf(name.getMethodName());
2431
2432
2433 final FileSystem fs = FileSystem.get(conf);
2434 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2435 @Override
2436 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2437 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2438 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2439 @Override
2440 protected void checkHFile(Path p) throws IOException {
2441 if (attemptedFirstHFile.compareAndSet(false, true)) {
2442 assertTrue(fs.delete(p, true));
2443 }
2444 super.checkHFile(p);
2445 }
2446 };
2447 }
2448 };
2449 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
2450 hbck.close();
2451 }
2452
2453
2454
2455
2456
2457
2458
2459 @Ignore @Test(timeout=180000)
2460 public void testQuarantineMissingFamdir() throws Exception {
2461 TableName table = TableName.valueOf(name.getMethodName());
2462
2463 final FileSystem fs = FileSystem.get(conf);
2464 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2465 @Override
2466 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2467 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2468 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2469 @Override
2470 protected void checkColFamDir(Path p) throws IOException {
2471 if (attemptedFirstHFile.compareAndSet(false, true)) {
2472 assertTrue(fs.delete(p, true));
2473 }
2474 super.checkColFamDir(p);
2475 }
2476 };
2477 }
2478 };
2479 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2480 hbck.close();
2481 }
2482
2483 @Test(timeout=60000)
2484 public void testCheckReplication() throws Exception {
2485
2486 HBaseFsck hbck = doFsck(conf, false);
2487 assertNoErrors(hbck);
2488
2489
2490 ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf);
2491 Assert.assertEquals(0, replicationAdmin.getPeersCount());
2492 String zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
2493 replicationAdmin.addPeer("1", "127.0.0.1:2181" + zkPort + ":/hbase");
2494 replicationAdmin.getPeersCount();
2495 Assert.assertEquals(1, replicationAdmin.getPeersCount());
2496
2497
2498 ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection);
2499 ReplicationQueues repQueues =
2500 ReplicationFactory.getReplicationQueues(zkw, conf, connection);
2501 repQueues.init("server1");
2502
2503 repQueues.addLog("1", "file1");
2504 repQueues.addLog("1-server2", "file1");
2505 Assert.assertEquals(2, repQueues.getAllQueues().size());
2506 hbck = doFsck(conf, false);
2507 assertNoErrors(hbck);
2508
2509
2510 repQueues.addLog("2", "file1");
2511 repQueues.addLog("2-server2", "file1");
2512 Assert.assertEquals(4, repQueues.getAllQueues().size());
2513 hbck = doFsck(conf, false);
2514 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
2515 ERROR_CODE.UNDELETED_REPLICATION_QUEUE });
2516
2517
2518 hbck = doFsck(conf, true);
2519 hbck = doFsck(conf, false);
2520 assertNoErrors(hbck);
2521
2522 Assert.assertEquals(2, repQueues.getAllQueues().size());
2523 Assert.assertNull(repQueues.getLogsInQueue("2"));
2524 Assert.assertNull(repQueues.getLogsInQueue("2-sever2"));
2525
2526 replicationAdmin.removePeer("1");
2527 repQueues.removeAllQueues();
2528 zkw.close();
2529 replicationAdmin.close();
2530 }
2531
2532
2533
2534
2535
2536 @Test(timeout=180000)
2537 public void testQuarantineMissingRegionDir() throws Exception {
2538 TableName table = TableName.valueOf(name.getMethodName());
2539
2540 final FileSystem fs = FileSystem.get(conf);
2541 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2542 @Override
2543 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
2544 throws IOException {
2545 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2546 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2547 @Override
2548 protected void checkRegionDir(Path p) throws IOException {
2549 if (attemptedFirstHFile.compareAndSet(false, true)) {
2550 assertTrue(fs.delete(p, true));
2551 }
2552 super.checkRegionDir(p);
2553 }
2554 };
2555 }
2556 };
2557 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2558 hbck.close();
2559 }
2560
2561
2562
2563
2564 @Test (timeout=180000)
2565 public void testLingeringReferenceFile() throws Exception {
2566 TableName table =
2567 TableName.valueOf("testLingeringReferenceFile");
2568 try {
2569 setupTable(table);
2570 assertEquals(ROWKEYS.length, countRows());
2571
2572
2573 FileSystem fs = FileSystem.get(conf);
2574 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2575 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2576 Path famDir = new Path(regionDir, FAM_STR);
2577 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
2578 fs.create(fakeReferenceFile);
2579
2580 HBaseFsck hbck = doFsck(conf, false);
2581 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
2582
2583 doFsck(conf, true);
2584
2585 assertNoErrors(doFsck(conf, false));
2586 } finally {
2587 cleanupTable(table);
2588 }
2589 }
2590
2591
2592
2593
2594 @Test (timeout=180000)
2595 public void testMissingRegionInfoQualifier() throws Exception {
2596 Connection connection = ConnectionFactory.createConnection(conf);
2597 TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
2598 try {
2599 setupTable(table);
2600
2601
2602 final List<Delete> deletes = new LinkedList<Delete>();
2603 Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService);
2604 MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
2605
2606 @Override
2607 public boolean processRow(Result rowResult) throws IOException {
2608 HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult);
2609 if (hri != null && !hri.getTable().isSystemTable()) {
2610 Delete delete = new Delete(rowResult.getRow());
2611 delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2612 deletes.add(delete);
2613 }
2614 return true;
2615 }
2616
2617 @Override
2618 public void close() throws IOException {
2619 }
2620 });
2621 meta.delete(deletes);
2622
2623
2624 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2625 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
2626 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2627 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
2628 meta.close();
2629
2630 HBaseFsck hbck = doFsck(conf, false);
2631 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2632
2633
2634 hbck = doFsck(conf, true);
2635
2636
2637 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2638 } finally {
2639 cleanupTable(table);
2640 }
2641 connection.close();
2642 }
2643
2644
2645
2646
2647
2648 @Test (timeout=180000)
2649 public void testErrorReporter() throws Exception {
2650 try {
2651 MockErrorReporter.calledCount = 0;
2652 doFsck(conf, false);
2653 assertEquals(MockErrorReporter.calledCount, 0);
2654
2655 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
2656 doFsck(conf, false);
2657 assertTrue(MockErrorReporter.calledCount > 20);
2658 } finally {
2659 conf.set("hbasefsck.errorreporter",
2660 PrintingErrorReporter.class.getName());
2661 MockErrorReporter.calledCount = 0;
2662 }
2663 }
2664
2665 static class MockErrorReporter implements ErrorReporter {
2666 static int calledCount = 0;
2667
2668 @Override
2669 public void clear() {
2670 calledCount++;
2671 }
2672
2673 @Override
2674 public void report(String message) {
2675 calledCount++;
2676 }
2677
2678 @Override
2679 public void reportError(String message) {
2680 calledCount++;
2681 }
2682
2683 @Override
2684 public void reportError(ERROR_CODE errorCode, String message) {
2685 calledCount++;
2686 }
2687
2688 @Override
2689 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2690 calledCount++;
2691 }
2692
2693 @Override
2694 public void reportError(ERROR_CODE errorCode,
2695 String message, TableInfo table, HbckInfo info) {
2696 calledCount++;
2697 }
2698
2699 @Override
2700 public void reportError(ERROR_CODE errorCode, String message,
2701 TableInfo table, HbckInfo info1, HbckInfo info2) {
2702 calledCount++;
2703 }
2704
2705 @Override
2706 public int summarize() {
2707 return ++calledCount;
2708 }
2709
2710 @Override
2711 public void detail(String details) {
2712 calledCount++;
2713 }
2714
2715 @Override
2716 public ArrayList<ERROR_CODE> getErrorList() {
2717 calledCount++;
2718 return new ArrayList<ERROR_CODE>();
2719 }
2720
2721 @Override
2722 public void progress() {
2723 calledCount++;
2724 }
2725
2726 @Override
2727 public void print(String message) {
2728 calledCount++;
2729 }
2730
2731 @Override
2732 public void resetErrors() {
2733 calledCount++;
2734 }
2735
2736 @Override
2737 public boolean tableHasErrors(TableInfo table) {
2738 calledCount++;
2739 return false;
2740 }
2741 }
2742
2743 @Test(timeout=180000)
2744 public void testCheckTableLocks() throws Exception {
2745 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2746 EnvironmentEdgeManager.injectEdge(edge);
2747
2748 HBaseFsck hbck = doFsck(conf, false);
2749 assertNoErrors(hbck);
2750
2751 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2752 final TableName tableName = TableName.valueOf("foo");
2753
2754
2755 final TableLockManager tableLockManager =
2756 TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2757 TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2758 writeLock.acquire();
2759 hbck = doFsck(conf, false);
2760 assertNoErrors(hbck);
2761
2762 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2763 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2764
2765 hbck = doFsck(conf, false);
2766 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2767
2768 final CountDownLatch latch = new CountDownLatch(1);
2769 new Thread() {
2770 @Override
2771 public void run() {
2772 TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2773 try {
2774 latch.countDown();
2775 readLock.acquire();
2776 } catch (IOException ex) {
2777 fail();
2778 } catch (IllegalStateException ex) {
2779 return;
2780 }
2781 fail("should not have come here");
2782 };
2783 }.start();
2784
2785 latch.await();
2786 Threads.sleep(300);
2787
2788 hbck = doFsck(conf, false);
2789
2790 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2791
2792 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2793 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2794
2795 hbck = doFsck(conf, false);
2796
2797 assertErrors(
2798 hbck,
2799 new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2800
2801 Configuration localConf = new Configuration(conf);
2802
2803
2804 localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2805 Threads.sleep(10);
2806 hbck = doFsck(localConf, true);
2807
2808 hbck = doFsck(localConf, false);
2809 assertNoErrors(hbck);
2810
2811
2812 writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
2813 writeLock.acquire();
2814 writeLock.release();
2815 tableLockManager.tableDeleted(tableName);
2816 }
2817
2818
2819
2820
2821 @Test
2822 public void testOrphanedTableZNode() throws Exception {
2823 TableName table = TableName.valueOf("testOrphanedZKTableEntry");
2824
2825 try {
2826 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
2827 .setTableState(table, ZooKeeperProtos.Table.State.ENABLING);
2828
2829 try {
2830 setupTable(table);
2831 Assert.fail(
2832 "Create table should fail when its ZNode has already existed with ENABLING state.");
2833 } catch(TableExistsException t) {
2834
2835 }
2836
2837 try {
2838 cleanupTable(table);
2839 } catch (IOException e) {
2840
2841
2842 }
2843
2844 HBaseFsck hbck = doFsck(conf, false);
2845 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2846
2847
2848 hbck = doFsck(conf, true);
2849
2850
2851 hbck = doFsck(conf, false);
2852 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2853
2854 setupTable(table);
2855 } finally {
2856
2857
2858 try {
2859 cleanupTable(table);
2860 } catch (IOException e) {
2861
2862
2863 }
2864 }
2865 }
2866
2867 @Test (timeout=180000)
2868 public void testMetaOffline() throws Exception {
2869
2870 HBaseFsck hbck = doFsck(conf, false);
2871 assertNoErrors(hbck);
2872 deleteMetaRegion(conf, true, false, false);
2873 hbck = doFsck(conf, false);
2874
2875
2876 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2877 hbck = doFsck(conf, true);
2878 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2879 hbck = doFsck(conf, false);
2880 assertNoErrors(hbck);
2881 }
2882
2883 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2884 boolean regionInfoOnly) throws IOException, InterruptedException {
2885 HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
2886 .getRegionLocation(HConstants.EMPTY_START_ROW);
2887 ServerName hsa = metaLocation.getServerName();
2888 HRegionInfo hri = metaLocation.getRegionInfo();
2889 if (unassign) {
2890 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2891 try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
2892 undeployRegion(unmanagedConnection, hsa, hri);
2893 }
2894 }
2895
2896 if (regionInfoOnly) {
2897 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2898 Path rootDir = FSUtils.getRootDir(conf);
2899 FileSystem fs = rootDir.getFileSystem(conf);
2900 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2901 hri.getEncodedName());
2902 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2903 fs.delete(hriPath, true);
2904 }
2905
2906 if (hdfs) {
2907 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2908 Path rootDir = FSUtils.getRootDir(conf);
2909 FileSystem fs = rootDir.getFileSystem(conf);
2910 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2911 hri.getEncodedName());
2912 HBaseFsck.debugLsr(conf, p);
2913 boolean success = fs.delete(p, true);
2914 LOG.info("Deleted " + p + " sucessfully? " + success);
2915 HBaseFsck.debugLsr(conf, p);
2916 }
2917 }
2918
2919 @Test (timeout=180000)
2920 public void testTableWithNoRegions() throws Exception {
2921
2922
2923 TableName table =
2924 TableName.valueOf(name.getMethodName());
2925 try {
2926
2927 HTableDescriptor desc = new HTableDescriptor(table);
2928 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
2929 desc.addFamily(hcd);
2930 createTable(TEST_UTIL, desc, null);
2931 tbl = (HTable) connection.getTable(table, tableExecutorService);
2932
2933
2934 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW,
2935 HConstants.EMPTY_END_ROW, false, false, true);
2936
2937 HBaseFsck hbck = doFsck(conf, false);
2938 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2939
2940 doFsck(conf, true);
2941
2942
2943 doFsck(conf, true);
2944
2945
2946 assertNoErrors(doFsck(conf, false));
2947 } finally {
2948 cleanupTable(table);
2949 }
2950
2951 }
2952
2953 @Test (timeout=180000)
2954 public void testHbckAfterRegionMerge() throws Exception {
2955 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
2956 Table meta = null;
2957 try {
2958
2959 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
2960 setupTable(table);
2961 assertEquals(ROWKEYS.length, countRows());
2962
2963
2964 admin.flush(table);
2965 HRegionInfo region1 = tbl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo();
2966 HRegionInfo region2 = tbl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo();
2967
2968 int regionCountBeforeMerge = tbl.getRegionLocations().size();
2969
2970 assertNotEquals(region1, region2);
2971
2972
2973 admin.mergeRegions(region1.getEncodedNameAsBytes(),
2974 region2.getEncodedNameAsBytes(), false);
2975
2976
2977 long timeout = System.currentTimeMillis() + 30 * 1000;
2978 while (true) {
2979 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
2980 break;
2981 } else if (System.currentTimeMillis() > timeout) {
2982 fail("Time out waiting on region " + region1.getEncodedName()
2983 + " and " + region2.getEncodedName() + " be merged");
2984 }
2985 Thread.sleep(10);
2986 }
2987
2988 assertEquals(ROWKEYS.length, countRows());
2989
2990 HBaseFsck hbck = doFsck(conf, false);
2991 assertNoErrors(hbck);
2992
2993 } finally {
2994 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
2995 cleanupTable(table);
2996 IOUtils.closeQuietly(meta);
2997 }
2998 }
2999
3000 @Test (timeout = 180000)
3001 public void testRegionBoundariesCheck() throws Exception {
3002 TableName tableName = TableName.valueOf("testRegionBoundariesCheck");
3003
3004
3005 HTableDescriptor desc = new HTableDescriptor(tableName);
3006 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
3007 desc.addFamily(hcd);
3008 createTable(TEST_UTIL, desc, SPLITS);
3009
3010 Table table = connection.getTable(tableName, tableExecutorService);
3011 List<Put> puts = new ArrayList<>();
3012
3013
3014 puts.add(new Put(Bytes.toBytes("0")).addColumn(FAM, Bytes.toBytes("col"),
3015 Bytes.toBytes("val")));
3016 puts.add(new Put(Bytes.toBytes("999")).addColumn(FAM, Bytes.toBytes("col"),
3017 Bytes.toBytes("val")));
3018
3019
3020 puts.add(new Put(Bytes.toBytes("AA")).addColumn(FAM, Bytes.toBytes("col"),
3021 Bytes.toBytes("val")));
3022 puts.add(new Put(Bytes.toBytes("AZ")).addColumn(FAM, Bytes.toBytes("col"),
3023 Bytes.toBytes("val")));
3024
3025 table.put(puts);
3026
3027
3028 admin.disableTable(tableName);
3029 admin.enableTable(tableName);
3030
3031
3032 HBaseFsck hbck = checkRegionBoundaries(conf);
3033 assertNoErrors(hbck);
3034
3035
3036 admin.disableTable(tableName);
3037
3038 List<HRegionInfo> tableRegions = admin.getTableRegions(tableName);
3039 HRegionInfo firstRegion = tableRegions.get(0);
3040 HRegionInfo secondRegion = tableRegions.get(1);
3041
3042 FileSystem fs = FileSystem.get(conf);
3043 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
3044 Path firstRegionFamDir = new Path(new Path(tableDir, firstRegion.getEncodedName()), FAM_STR);
3045 Path hfileInFirstRegion = getHFilePath(fs, firstRegionFamDir);
3046 Path secondRegionFamDir = new Path(new Path(tableDir, secondRegion.getEncodedName()), FAM_STR);
3047 Path hfileInSecondRegion = getHFilePath(fs, secondRegionFamDir);
3048
3049
3050
3051 fs.rename(hfileInFirstRegion, new Path(firstRegionFamDir, "0"));
3052 fs.rename(hfileInSecondRegion, new Path(firstRegionFamDir, "1"));
3053
3054 admin.enableTable(tableName);
3055
3056
3057 hbck = checkRegionBoundaries(conf);
3058 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.BOUNDARIES_ERROR });
3059 }
3060
3061 private static Path getHFilePath(FileSystem fs, Path famDir) throws IOException {
3062 FileStatus[] hfFss = fs.listStatus(famDir);
3063 for (FileStatus hfs : hfFss) {
3064 if (!hfs.isDirectory()) {
3065 return hfs.getPath();
3066 }
3067 }
3068 return null;
3069 }
3070
3071 @org.junit.Rule
3072 public TestName name = new TestName();
3073
3074 @Test (timeout=180000)
3075 public void testReadOnlyProperty() throws Exception {
3076 HBaseFsck hbck = doFsck(conf, false);
3077 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
3078 hbck.shouldIgnorePreCheckPermission());
3079
3080 hbck = doFsck(conf, true);
3081 Assert.assertEquals("shouldIgnorePreCheckPermission", false,
3082 hbck.shouldIgnorePreCheckPermission());
3083
3084 hbck = doFsck(conf, true);
3085 hbck.setIgnorePreCheckPermission(true);
3086 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
3087 hbck.shouldIgnorePreCheckPermission());
3088 }
3089
3090 @Test (timeout=180000)
3091 public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
3092 TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit");
3093 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
3094 try {
3095 HTableDescriptor desc = new HTableDescriptor(table);
3096 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
3097 createTable(TEST_UTIL, desc, null);
3098 tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
3099 for (int i = 0; i < 5; i++) {
3100 Put p1 = new Put(("r" + i).getBytes());
3101 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
3102 tbl.put(p1);
3103 }
3104 admin.flush(desc.getTableName());
3105 List<HRegion> regions = cluster.getRegions(desc.getTableName());
3106 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
3107 HRegionServer regionServer = cluster.getRegionServer(serverWith);
3108 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
3109 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
3110 st.prepare();
3111 st.stepsBeforePONR(regionServer, regionServer, false);
3112 AssignmentManager am = cluster.getMaster().getAssignmentManager();
3113 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
3114 for (RegionState state : regionsInTransition.values()) {
3115 am.regionOffline(state.getRegion());
3116 }
3117 ZKAssign.deleteNodeFailSilent(regionServer.getZooKeeper(), regions.get(0).getRegionInfo());
3118 Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();
3119 regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
3120 am.assign(regionsMap);
3121 am.waitForAssignment(regions.get(0).getRegionInfo());
3122 HBaseFsck hbck = doFsck(conf, false);
3123 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
3124 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
3125
3126 assertEquals(0, hbck.getOverlapGroups(table).size());
3127
3128
3129 assertErrors(
3130 doFsck(
3131 conf, false, true, false, false, false, false, false, false, false, false, false,
3132 false, null),
3133 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
3134 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
3135
3136
3137 assertNoErrors(doFsck(conf, false));
3138 assertEquals(5, countRows());
3139 } finally {
3140 if (tbl != null) {
3141 tbl.close();
3142 tbl = null;
3143 }
3144 cleanupTable(table);
3145 }
3146 }
3147
3148
3149
3150
3151
3152
3153 @Test(timeout = 180000)
3154 public void testNoDataLossAfterRegionOverlapFix() throws Exception {
3155 int startRow = 0;
3156 int endRow = 5;
3157 TableName table = TableName.valueOf("testNoDataLossAfterRegionOverlapFix");
3158 try {
3159 TEST_UTIL.createTable(table, FAM);
3160 tbl = new HTable(TEST_UTIL.getConfiguration(), table);
3161
3162 TEST_UTIL.loadNumericRows(tbl, FAM, startRow, endRow);
3163 admin.flush(table);
3164
3165 HRegionInfo hriOverlap =
3166 createRegion(tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW, Bytes.toBytes("3"));
3167 TEST_UTIL.assignRegion(hriOverlap);
3168
3169 HBaseFsck hbck = doFsck(conf, false);
3170 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS });
3171 assertEquals(2, hbck.getOverlapGroups(table).size());
3172
3173 doFsck(conf, true);
3174
3175 HBaseFsck hbck2 = doFsck(conf, false);
3176 assertNoErrors(hbck2);
3177 assertEquals(0, hbck2.getOverlapGroups(table).size());
3178
3179 for (int i = startRow; i < endRow; i++) {
3180 assertEquals(endRow - i,
3181 countRows(Bytes.toBytes(String.valueOf(i)), HConstants.EMPTY_BYTE_ARRAY));
3182 }
3183 } finally {
3184 if (tbl != null) {
3185 tbl.close();
3186 tbl = null;
3187 }
3188 cleanupTable(table);
3189 }
3190 }
3191
3192 public static class MasterSyncObserver extends BaseMasterObserver {
3193 volatile CountDownLatch tableCreationLatch = null;
3194 volatile CountDownLatch tableDeletionLatch = null;
3195
3196 @Override
3197 public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3198 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
3199
3200 if (tableCreationLatch != null) {
3201 tableCreationLatch.countDown();
3202 }
3203 }
3204
3205 @Override
3206 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
3207 TableName tableName)
3208 throws IOException {
3209
3210 if (tableDeletionLatch != null) {
3211 tableDeletionLatch.countDown();
3212 }
3213 }
3214 }
3215
3216 public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
3217 byte [][] splitKeys) throws Exception {
3218
3219
3220 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3221 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3222 observer.tableCreationLatch = new CountDownLatch(1);
3223 if (splitKeys != null) {
3224 admin.createTable(htd, splitKeys);
3225 } else {
3226 admin.createTable(htd);
3227 }
3228 observer.tableCreationLatch.await();
3229 observer.tableCreationLatch = null;
3230 testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
3231 }
3232
3233 public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
3234 throws Exception {
3235
3236
3237 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
3238 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
3239 observer.tableDeletionLatch = new CountDownLatch(1);
3240 try {
3241 admin.disableTable(tableName);
3242 } catch (Exception e) {
3243 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
3244 }
3245 admin.deleteTable(tableName);
3246 observer.tableDeletionLatch.await();
3247 observer.tableDeletionLatch = null;
3248 }
3249 }