1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.ChoreService;
42 import org.apache.hadoop.hbase.CoordinatedStateManager;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.MetaMockingUtil;
49 import org.apache.hadoop.hbase.NamespaceDescriptor;
50 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51 import org.apache.hadoop.hbase.ProcedureInfo;
52 import org.apache.hadoop.hbase.Server;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.apache.hadoop.hbase.TableDescriptors;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.TableStateManager;
58 import org.apache.hadoop.hbase.backup.BackupType;
59 import org.apache.hadoop.hbase.client.ClusterConnection;
60 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
61 import org.apache.hadoop.hbase.client.Result;
62 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
63 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
64 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
65 import org.apache.hadoop.hbase.executor.ExecutorService;
66 import org.apache.hadoop.hbase.io.Reference;
67 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
68 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
69 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
70 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
71 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
72 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
78 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
79 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
80 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
81 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
82 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
83 import org.apache.hadoop.hbase.regionserver.HStore;
84 import org.apache.hadoop.hbase.util.Bytes;
85 import org.apache.hadoop.hbase.util.FSUtils;
86 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
87 import org.apache.hadoop.hbase.util.Pair;
88 import org.apache.hadoop.hbase.util.Triple;
89 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
90 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
91 import org.junit.Test;
92 import org.junit.experimental.categories.Category;
93 import org.mockito.Mockito;
94 import org.mockito.invocation.InvocationOnMock;
95 import org.mockito.stubbing.Answer;
96
97 import com.google.protobuf.RpcController;
98 import com.google.protobuf.Service;
99 import com.google.protobuf.ServiceException;
100
101 @Category(SmallTests.class)
102 public class TestCatalogJanitor {
103 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
104
105
106
107
108
109 class MockServer implements Server {
110 private final ClusterConnection connection;
111 private final Configuration c;
112
113 MockServer(final HBaseTestingUtility htu)
114 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
115 this.c = htu.getConfiguration();
116 ClientProtos.ClientService.BlockingInterface ri =
117 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
118 MutateResponse.Builder builder = MutateResponse.newBuilder();
119 builder.setProcessed(true);
120 try {
121 Mockito.when(ri.mutate(
122 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
123 thenReturn(builder.build());
124 } catch (ServiceException se) {
125 throw ProtobufUtil.getRemoteException(se);
126 }
127 try {
128 Mockito.when(ri.multi(
129 (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
130 thenAnswer(new Answer<MultiResponse>() {
131 @Override
132 public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
133 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
134 }
135 });
136 } catch (ServiceException se) {
137 throw ProtobufUtil.getRemoteException(se);
138 }
139
140
141
142 this.connection =
143 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
144 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
145 ServerName.valueOf("example.org,12345,6789"),
146 HRegionInfo.FIRST_META_REGIONINFO);
147
148 FileSystem fs = FileSystem.get(this.c);
149 Path rootdir = FSUtils.getRootDir(this.c);
150 FSUtils.setRootDir(this.c, rootdir);
151 AdminProtos.AdminService.BlockingInterface hri =
152 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
153 }
154
155 @Override
156 public ClusterConnection getConnection() {
157 return this.connection;
158 }
159
160 @Override
161 public MetaTableLocator getMetaTableLocator() {
162 return null;
163 }
164
165 @Override
166 public Configuration getConfiguration() {
167 return this.c;
168 }
169
170 @Override
171 public ServerName getServerName() {
172 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
173 }
174
175 @Override
176 public ZooKeeperWatcher getZooKeeper() {
177 return null;
178 }
179
180 @Override
181 public CoordinatedStateManager getCoordinatedStateManager() {
182 BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
183 SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
184 Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
185 SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
186 Mockito.when(c.getDetails()).thenReturn(d);
187 return m;
188 }
189
190 @Override
191 public void abort(String why, Throwable e) {
192
193 }
194
195 @Override
196 public boolean isAborted() {
197 return false;
198 }
199
200 @Override
201 public boolean isStopped() {
202 return false;
203 }
204
205 @Override
206 public void stop(String why) {
207 }
208
209 @Override
210 public ChoreService getChoreService() {
211 return null;
212 }
213 }
214
215
216
217
218 class MockMasterServices implements MasterServices {
219 private final MasterFileSystem mfs;
220 private final AssignmentManager asm;
221 private final ServerManager sm;
222
223 MockMasterServices(final Server server) throws IOException {
224 this.mfs = new MasterFileSystem(server, this);
225 this.asm = Mockito.mock(AssignmentManager.class);
226 this.sm = Mockito.mock(ServerManager.class);
227 }
228
229 @Override
230 public void checkTableModifiable(TableName tableName) throws IOException {
231
232 }
233
234 @Override
235 public boolean isMasterProcedureExecutorEnabled() {
236 return true;
237 }
238
239 @Override
240 public long createTable(
241 final HTableDescriptor desc,
242 final byte[][] splitKeys,
243 final long nonceGroup,
244 final long nonce) throws IOException {
245
246 return -1;
247 }
248
249 @Override
250 public SnapshotManager getSnapshotManager() {
251 return null;
252 }
253
254 @Override
255 public MasterProcedureManagerHost getMasterProcedureManagerHost() {
256 return null;
257 }
258
259 @Override
260 public AssignmentManager getAssignmentManager() {
261 return this.asm;
262 }
263
264 @Override
265 public ExecutorService getExecutorService() {
266 return null;
267 }
268
269 @Override
270 public ChoreService getChoreService() {
271 return null;
272 }
273
274 @Override
275 public MasterFileSystem getMasterFileSystem() {
276 return this.mfs;
277 }
278
279 @Override
280 public MasterCoprocessorHost getMasterCoprocessorHost() {
281 return null;
282 }
283
284 @Override
285 public MasterQuotaManager getMasterQuotaManager() {
286 return null;
287 }
288
289 @Override
290 public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
291 return null;
292 }
293
294 @Override
295 public ServerManager getServerManager() {
296 return sm;
297 }
298
299 @Override
300 public ZooKeeperWatcher getZooKeeper() {
301 return null;
302 }
303
304 @Override
305 public CoordinatedStateManager getCoordinatedStateManager() {
306 return null;
307 }
308
309 @Override
310 public MetaTableLocator getMetaTableLocator() {
311 return null;
312 }
313
314 @Override
315 public ClusterConnection getConnection() {
316 return null;
317 }
318
319 @Override
320 public Configuration getConfiguration() {
321 return mfs.conf;
322 }
323
324 @Override
325 public ServerName getServerName() {
326 return null;
327 }
328
329 @Override
330 public void abort(String why, Throwable e) {
331
332 }
333
334 @Override
335 public boolean isAborted() {
336 return false;
337 }
338
339 private boolean stopped = false;
340
341 @Override
342 public void stop(String why) {
343 stopped = true;
344 }
345
346 @Override
347 public boolean isStopped() {
348 return stopped;
349 }
350
351 @Override
352 public TableDescriptors getTableDescriptors() {
353 return new TableDescriptors() {
354 @Override
355 public HTableDescriptor remove(TableName tablename) throws IOException {
356
357 return null;
358 }
359
360 @Override
361 public Map<String, HTableDescriptor> getAll() throws IOException {
362
363 return null;
364 }
365
366 @Override
367 public HTableDescriptor get(TableName tablename)
368 throws IOException {
369 return createHTableDescriptor();
370 }
371
372 @Override
373 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
374 return null;
375 }
376
377 @Override
378 public void add(HTableDescriptor htd) throws IOException {
379
380
381 }
382 @Override
383 public void setCacheOn() throws IOException {
384 }
385
386 @Override
387 public void setCacheOff() throws IOException {
388 }
389 };
390 }
391
392 @Override
393 public boolean isServerShutdownHandlerEnabled() {
394 return true;
395 }
396
397 @Override
398 public boolean registerService(Service instance) {
399 return false;
400 }
401
402 @Override
403 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
404
405 }
406
407 @Override
408 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
409
410 }
411
412 @Override
413 public void deleteNamespace(String name) throws IOException {
414
415 }
416
417 @Override
418 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
419 return null;
420 }
421
422 @Override
423 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
424 return null;
425 }
426
427 @Override
428 public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
429 throws IOException {
430 return false;
431 }
432
433 @Override
434 public List<ProcedureInfo> listProcedures() throws IOException {
435 return null;
436 }
437
438 @Override
439 public Pair<Long, String> backupTables(
440 final BackupType type,
441 final List<TableName> tableList,
442 final String targetRootDir, final int workers,
443 final long bandwidth) throws IOException {
444 return null;
445 }
446
447
448 @Override
449 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
450 return null;
451 }
452
453 @Override
454 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
455 return null;
456 }
457
458 @Override
459 public long deleteTable(
460 final TableName tableName,
461 final long nonceGroup,
462 final long nonce) throws IOException {
463 return -1;
464 }
465 @Override
466 public LoadBalancer getLoadBalancer() {
467 return null;
468 }
469
470 public void truncateTable(
471 final TableName tableName,
472 final boolean preserveSplits,
473 final long nonceGroup,
474 final long nonce) throws IOException {
475 }
476
477 @Override
478 public void modifyTable(
479 final TableName tableName,
480 final HTableDescriptor descriptor,
481 final long nonceGroup,
482 final long nonce) throws IOException {
483 }
484
485 @Override
486 public long enableTable(
487 final TableName tableName,
488 final long nonceGroup,
489 final long nonce) throws IOException {
490 return -1;
491 }
492
493 @Override
494 public long disableTable(
495 TableName tableName,
496 final long nonceGroup,
497 final long nonce) throws IOException {
498 return -1;
499 }
500
501 @Override
502 public void addColumn(
503 final TableName tableName,
504 final HColumnDescriptor columnDescriptor,
505 final long nonceGroup,
506 final long nonce) throws IOException { }
507
508 @Override
509 public void modifyColumn(
510 final TableName tableName,
511 final HColumnDescriptor descriptor,
512 final long nonceGroup,
513 final long nonce) throws IOException { }
514
515 @Override
516 public void deleteColumn(
517 final TableName tableName,
518 final byte[] columnName,
519 final long nonceGroup,
520 final long nonce) throws IOException { }
521
522 @Override
523 public TableLockManager getTableLockManager() {
524 return null;
525 }
526
527 @Override
528 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
529 boolean forcible) throws IOException {
530 }
531
532 @Override
533 public boolean isInitialized() {
534
535 return false;
536 }
537
538 @Override
539 public boolean isNamespaceManagerInitialized() {
540 return false;
541 }
542
543 @Override
544 public boolean isInMaintenanceMode() {
545 return false;
546 }
547
548 @Override
549 public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
550
551 return 0;
552 }
553
554 @Override
555 public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
556
557 return 0;
558 }
559
560 @Override
561 public TableStateManager getTableStateManager() {
562
563 return null;
564 }
565 }
566
567 @Test
568 public void testCleanParent() throws IOException, InterruptedException {
569 HBaseTestingUtility htu = new HBaseTestingUtility();
570 setRootDirAndCleanIt(htu, "testCleanParent");
571 Server server = new MockServer(htu);
572 try {
573 MasterServices services = new MockMasterServices(server);
574 CatalogJanitor janitor = new CatalogJanitor(server, services);
575
576 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
577 htd.addFamily(new HColumnDescriptor("f"));
578 HRegionInfo parent =
579 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
580 Bytes.toBytes("eee"));
581 HRegionInfo splita =
582 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
583 Bytes.toBytes("ccc"));
584 HRegionInfo splitb =
585 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
586 Bytes.toBytes("eee"));
587
588
589 Result r = createResult(parent, splita, splitb);
590
591 Path rootdir = services.getMasterFileSystem().getRootDir();
592 Path tabledir =
593 FSUtils.getTableDir(rootdir, htd.getTableName());
594 Path storedir = HStore.getStoreHomedir(tabledir, splita,
595 htd.getColumnFamilies()[0].getName());
596 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
597 long now = System.currentTimeMillis();
598
599 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
600 FileSystem fs = services.getMasterFileSystem().getFileSystem();
601 Path path = ref.write(fs, p);
602 assertTrue(fs.exists(path));
603 assertFalse(janitor.cleanParent(parent, r));
604
605 assertTrue(fs.delete(p, true));
606 assertTrue(janitor.cleanParent(parent, r));
607 } finally {
608 server.stop("shutdown");
609 }
610 }
611
612
613
614
615
616
617 @Test
618 public void testParentCleanedEvenIfDaughterGoneFirst()
619 throws IOException, InterruptedException {
620 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
621 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
622 }
623
624
625
626
627
628
629 @Test
630 public void testLastParentCleanedEvenIfDaughterGoneFirst()
631 throws IOException, InterruptedException {
632 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
633 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
634 }
635
636
637
638
639
640
641
642
643
644 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
645 final String rootDir, final byte[] lastEndKey)
646 throws IOException, InterruptedException {
647 HBaseTestingUtility htu = new HBaseTestingUtility();
648 setRootDirAndCleanIt(htu, rootDir);
649 Server server = new MockServer(htu);
650 MasterServices services = new MockMasterServices(server);
651 CatalogJanitor janitor = new CatalogJanitor(server, services);
652 final HTableDescriptor htd = createHTableDescriptor();
653
654
655
656
657 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
658 lastEndKey);
659
660
661 Thread.sleep(1001);
662
663
664 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
665 Bytes.toBytes("ccc"));
666 Thread.sleep(1001);
667
668 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
669 Bytes.toBytes("bbb"));
670 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
671 Bytes.toBytes("ccc"));
672
673
674 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
675 lastEndKey);
676 Thread.sleep(1001);
677
678 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
679 Bytes.toBytes("ddd"));
680 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
681 lastEndKey);
682
683
684
685 SortedMap<HRegionInfo, Result> regions =
686 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
687
688 regions.put(parent, createResult(parent, splita, splitb));
689 regions.put(splitb, createResult(splitb, splitba, splitbb));
690 regions.put(splita, createResult(splita, splitaa, splitab));
691
692 int index = 0;
693 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
694 if (index == 0) {
695 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
696 } else if (index == 1) {
697 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
698 } else if (index == 2) {
699 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
700 }
701 index++;
702 }
703
704
705
706 Path splitaRef =
707 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
708
709 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
710
711
712
713 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
714
715
716
717
718
719 FileSystem fs = FileSystem.get(htu.getConfiguration());
720 assertTrue(fs.delete(splitaRef, true));
721
722 Path splitaaRef =
723 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
724 Path splitabRef =
725 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
726
727
728 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
729
730
731 assertTrue(fs.delete(splitaaRef, true));
732 assertTrue(fs.delete(splitabRef, true));
733 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
734
735
736 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
737
738 services.stop("test finished");
739 janitor.cancel(true);
740 }
741
742
743
744
745
746
747 @Test
748 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
749 HBaseTestingUtility htu = new HBaseTestingUtility();
750 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
751 Server server = new MockServer(htu);
752 MasterServices services = new MockMasterServices(server);
753
754 final HTableDescriptor htd = createHTableDescriptor();
755
756
757
758
759 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
760 new byte[0], true);
761
762
763 Thread.sleep(1001);
764
765
766 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
767 Bytes.toBytes("ccc"), true);
768 Thread.sleep(1001);
769
770 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
771 Bytes.toBytes("bbb"), false);
772 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
773 Bytes.toBytes("ccc"), false);
774
775
776 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
777 new byte[0]);
778 Thread.sleep(1001);
779
780 final Map<HRegionInfo, Result> splitParents =
781 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
782 splitParents.put(parent, createResult(parent, splita, splitb));
783 splita.setOffline(true);
784 splitParents.put(splita, createResult(splita, splitaa,splitab));
785
786 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
787 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
788 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
789 10, mergedRegions, splitParents)).when(janitor)
790 .getMergedRegionsAndSplitParents();
791
792
793 Path splitaRef =
794 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
795
796
797 assertEquals(0, janitor.scan());
798
799
800 FileSystem fs = FileSystem.get(htu.getConfiguration());
801 assertTrue(fs.delete(splitaRef, true));
802
803
804 assertEquals(2, janitor.scan());
805
806 services.stop("test finished");
807 janitor.cancel(true);
808 }
809
810
811
812
813
814 @Test
815 public void testSplitParentFirstComparator() {
816 SplitParentFirstComparator comp = new SplitParentFirstComparator();
817 final HTableDescriptor htd = createHTableDescriptor();
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
839 HConstants.EMPTY_START_ROW,
840 HConstants.EMPTY_END_ROW, true);
841 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
842 HConstants.EMPTY_START_ROW,
843 Bytes.toBytes("bbb"), true);
844 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
845 Bytes.toBytes("bbb"),
846 HConstants.EMPTY_END_ROW, true);
847
848 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
849 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
850 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
851 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
852 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
853 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
854
855
856 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
857 HConstants.EMPTY_START_ROW,
858 Bytes.toBytes("aaa"), true);
859 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
860 Bytes.toBytes("aaa"),
861 Bytes.toBytes("bbb"), true);
862
863 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
864 Bytes.toBytes("bbb"),
865 Bytes.toBytes("ddd"), true);
866 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
867 Bytes.toBytes("ddd"),
868 HConstants.EMPTY_END_ROW, true);
869
870 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
871 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
872 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
873 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
874 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
875 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
876 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
877
878 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
879 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
880 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
881 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
882 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
883 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
884 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
885
886 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
887 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
888 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
889 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
890
891 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
892 Bytes.toBytes("bbb"),
893 Bytes.toBytes("ccc"), false);
894 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
895 Bytes.toBytes("ccc"),
896 Bytes.toBytes("ddd"), false);
897
898 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
899 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
900 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
901
902 }
903
904 @Test
905 public void testArchiveOldRegion() throws Exception {
906 String table = "table";
907 HBaseTestingUtility htu = new HBaseTestingUtility();
908 setRootDirAndCleanIt(htu, "testCleanParent");
909 Server server = new MockServer(htu);
910 MasterServices services = new MockMasterServices(server);
911
912
913 CatalogJanitor janitor = new CatalogJanitor(server, services);
914
915
916 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
917 htd.addFamily(new HColumnDescriptor("f"));
918 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
919 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
920 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
921 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
922 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
923 Bytes.toBytes("ccc"),
924 Bytes.toBytes("eee"));
925
926
927
928 Result parentMetaRow = createResult(parent, splita, splitb);
929 FileSystem fs = FileSystem.get(htu.getConfiguration());
930 Path rootdir = services.getMasterFileSystem().getRootDir();
931
932
933
934 FSUtils.setRootDir(fs.getConf(), rootdir);
935 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
936 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
937 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
938 tabledir, htd.getColumnFamilies()[0].getName());
939 LOG.debug("Table dir:" + tabledir);
940 LOG.debug("Store dir:" + storedir);
941 LOG.debug("Store archive dir:" + storeArchive);
942
943
944 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
945
946 FileStatus[] storeFiles = fs.listStatus(storedir);
947 int index = 0;
948 for (FileStatus file : storeFiles) {
949 LOG.debug("Have store file:" + file.getPath());
950 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
951 storeFiles[index].getPath());
952 index++;
953 }
954
955
956 assertTrue(janitor.cleanParent(parent, parentMetaRow));
957 LOG.debug("Finished cleanup of parent region");
958
959
960 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
961 logFiles("archived files", storeFiles);
962 logFiles("archived files", archivedStoreFiles);
963
964 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
965
966
967 FSUtils.delete(fs, rootdir, true);
968 services.stop("Test finished");
969 server.stop("Test finished");
970 janitor.cancel(true);
971 }
972
973
974
975
976
977 private void logFiles(String description, FileStatus[] storeFiles) {
978 LOG.debug("Current " + description + ": ");
979 for (FileStatus file : storeFiles) {
980 LOG.debug(file.getPath());
981 }
982 }
983
984
985
986
987
988 @Test
989 public void testDuplicateHFileResolution() throws Exception {
990 String table = "table";
991 HBaseTestingUtility htu = new HBaseTestingUtility();
992 setRootDirAndCleanIt(htu, "testCleanParent");
993 Server server = new MockServer(htu);
994 MasterServices services = new MockMasterServices(server);
995
996
997
998 CatalogJanitor janitor = new CatalogJanitor(server, services);
999
1000
1001 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
1002 htd.addFamily(new HColumnDescriptor("f"));
1003 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
1004 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
1005 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
1006 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
1007 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
1008 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
1009
1010
1011 Result r = createResult(parent, splita, splitb);
1012
1013 FileSystem fs = FileSystem.get(htu.getConfiguration());
1014
1015 Path rootdir = services.getMasterFileSystem().getRootDir();
1016
1017
1018
1019 FSUtils.setRootDir(fs.getConf(), rootdir);
1020 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1021 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
1022 System.out.println("Old root:" + rootdir);
1023 System.out.println("Old table:" + tabledir);
1024 System.out.println("Old store:" + storedir);
1025
1026 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
1027 tabledir, htd.getColumnFamilies()[0].getName());
1028 System.out.println("Old archive:" + storeArchive);
1029
1030
1031 addMockStoreFiles(2, services, storedir);
1032
1033 FileStatus[] storeFiles = fs.listStatus(storedir);
1034
1035 assertTrue(janitor.cleanParent(parent, r));
1036
1037
1038 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
1039 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
1040
1041
1042
1043 addMockStoreFiles(2, services, storedir);
1044
1045
1046 assertTrue(janitor.cleanParent(parent, r));
1047
1048
1049 archivedStoreFiles = fs.listStatus(storeArchive);
1050 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1051
1052
1053 services.stop("Test finished");
1054 server.stop("shutdown");
1055 janitor.cancel(true);
1056 }
1057
1058 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1059 throws IOException {
1060
1061 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1062 fs.mkdirs(storedir);
1063
1064 for (int i = 0; i < count; i++) {
1065 Path storeFile = new Path(storedir, "_store" + i);
1066 FSDataOutputStream dos = fs.create(storeFile, true);
1067 dos.writeBytes("Some data: " + i);
1068 dos.close();
1069 }
1070 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1071
1072 FileStatus[] storeFiles = fs.listStatus(storedir);
1073 assertEquals("Didn't have expected store files", count, storeFiles.length);
1074 return storeFiles;
1075 }
1076
1077 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1078 final String subdir)
1079 throws IOException {
1080 Path testdir = htu.getDataTestDir(subdir);
1081 FileSystem fs = FileSystem.get(htu.getConfiguration());
1082 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1083 FSUtils.setRootDir(htu.getConfiguration(), testdir);
1084 return FSUtils.getRootDir(htu.getConfiguration()).toString();
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097 private Path createReferences(final MasterServices services,
1098 final HTableDescriptor htd, final HRegionInfo parent,
1099 final HRegionInfo daughter, final byte [] midkey, final boolean top)
1100 throws IOException {
1101 Path rootdir = services.getMasterFileSystem().getRootDir();
1102 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1103 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1104 htd.getColumnFamilies()[0].getName());
1105 Reference ref =
1106 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1107 long now = System.currentTimeMillis();
1108
1109 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1110 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1111 ref.write(fs, p);
1112 return p;
1113 }
1114
1115 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1116 final HRegionInfo b)
1117 throws IOException {
1118 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1119 }
1120
1121 private HTableDescriptor createHTableDescriptor() {
1122 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1123 htd.addFamily(new HColumnDescriptor("f"));
1124 return htd;
1125 }
1126
1127 private MultiResponse buildMultiResponse(MultiRequest req) {
1128 MultiResponse.Builder builder = MultiResponse.newBuilder();
1129 RegionActionResult.Builder regionActionResultBuilder =
1130 RegionActionResult.newBuilder();
1131 ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1132 for (RegionAction regionAction: req.getRegionActionList()) {
1133 regionActionResultBuilder.clear();
1134 for (ClientProtos.Action action: regionAction.getActionList()) {
1135 roeBuilder.clear();
1136 roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1137 roeBuilder.setIndex(action.getIndex());
1138 regionActionResultBuilder.addResultOrException(roeBuilder.build());
1139 }
1140 builder.addRegionActionResult(regionActionResultBuilder.build());
1141 }
1142 return builder.build();
1143 }
1144
1145 }
1146