View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertTrue;
25  import static org.mockito.Mockito.doReturn;
26  import static org.mockito.Mockito.spy;
27  
28  import java.io.IOException;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.SortedMap;
32  import java.util.TreeMap;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.ChoreService;
42  import org.apache.hadoop.hbase.CoordinatedStateManager;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.MetaMockingUtil;
49  import org.apache.hadoop.hbase.NamespaceDescriptor;
50  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51  import org.apache.hadoop.hbase.ProcedureInfo;
52  import org.apache.hadoop.hbase.Server;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.testclassification.SmallTests;
55  import org.apache.hadoop.hbase.TableDescriptors;
56  import org.apache.hadoop.hbase.TableName;
57  import org.apache.hadoop.hbase.TableStateManager;
58  import org.apache.hadoop.hbase.backup.BackupType;
59  import org.apache.hadoop.hbase.client.ClusterConnection;
60  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
61  import org.apache.hadoop.hbase.client.Result;
62  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
63  import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
64  import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
65  import org.apache.hadoop.hbase.executor.ExecutorService;
66  import org.apache.hadoop.hbase.io.Reference;
67  import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
68  import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
69  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
70  import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
71  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
74  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
80  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
81  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
82  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
83  import org.apache.hadoop.hbase.regionserver.HStore;
84  import org.apache.hadoop.hbase.util.Bytes;
85  import org.apache.hadoop.hbase.util.FSUtils;
86  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
87  import org.apache.hadoop.hbase.util.Pair;
88  import org.apache.hadoop.hbase.util.Triple;
89  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
90  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
91  import org.junit.Test;
92  import org.junit.experimental.categories.Category;
93  import org.mockito.Mockito;
94  import org.mockito.invocation.InvocationOnMock;
95  import org.mockito.stubbing.Answer;
96  
97  import com.google.protobuf.RpcController;
98  import com.google.protobuf.Service;
99  import com.google.protobuf.ServiceException;
100 
101 @Category(SmallTests.class)
102 public class TestCatalogJanitor {
103   private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
104 
105   /**
106    * Pseudo server for below tests.
107    * Be sure to call stop on the way out else could leave some mess around.
108    */
109   class MockServer implements Server {
110     private final ClusterConnection connection;
111     private final Configuration c;
112 
113     MockServer(final HBaseTestingUtility htu)
114     throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
115       this.c = htu.getConfiguration();
116       ClientProtos.ClientService.BlockingInterface ri =
117         Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
118       MutateResponse.Builder builder = MutateResponse.newBuilder();
119       builder.setProcessed(true);
120       try {
121         Mockito.when(ri.mutate(
122           (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
123             thenReturn(builder.build());
124       } catch (ServiceException se) {
125         throw ProtobufUtil.getRemoteException(se);
126       }
127       try {
128         Mockito.when(ri.multi(
129           (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
130             thenAnswer(new Answer<MultiResponse>() {
131               @Override
132               public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
133                 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
134               }
135             });
136       } catch (ServiceException se) {
137         throw ProtobufUtil.getRemoteException(se);
138       }
139       // Mock an HConnection and a AdminProtocol implementation.  Have the
140       // HConnection return the HRI.  Have the HRI return a few mocked up responses
141       // to make our test work.
142       this.connection =
143         HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
144           Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
145             ServerName.valueOf("example.org,12345,6789"),
146           HRegionInfo.FIRST_META_REGIONINFO);
147       // Set hbase.rootdir into test dir.
148       FileSystem fs = FileSystem.get(this.c);
149       Path rootdir = FSUtils.getRootDir(this.c);
150       FSUtils.setRootDir(this.c, rootdir);
151       AdminProtos.AdminService.BlockingInterface hri =
152         Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
153     }
154 
155     @Override
156     public ClusterConnection getConnection() {
157       return this.connection;
158     }
159 
160     @Override
161     public MetaTableLocator getMetaTableLocator() {
162       return null;
163     }
164 
165     @Override
166     public Configuration getConfiguration() {
167       return this.c;
168     }
169 
170     @Override
171     public ServerName getServerName() {
172       return ServerName.valueOf("mockserver.example.org", 1234, -1L);
173     }
174 
175     @Override
176     public ZooKeeperWatcher getZooKeeper() {
177       return null;
178     }
179 
180     @Override
181     public CoordinatedStateManager getCoordinatedStateManager() {
182       BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
183       SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
184       Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
185       SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
186       Mockito.when(c.getDetails()).thenReturn(d);
187       return m;
188     }
189 
190     @Override
191     public void abort(String why, Throwable e) {
192       //no-op
193     }
194 
195     @Override
196     public boolean isAborted() {
197       return false;
198     }
199 
200     @Override
201     public boolean isStopped() {
202       return false;
203     }
204 
205     @Override
206     public void stop(String why) {
207     }
208 
209     @Override
210     public ChoreService getChoreService() {
211       return null;
212     }
213   }
214 
215   /**
216    * Mock MasterServices for tests below.
217    */
218   class MockMasterServices implements MasterServices {
219     private final MasterFileSystem mfs;
220     private final AssignmentManager asm;
221     private final ServerManager sm;
222 
223     MockMasterServices(final Server server) throws IOException {
224       this.mfs = new MasterFileSystem(server, this);
225       this.asm = Mockito.mock(AssignmentManager.class);
226       this.sm = Mockito.mock(ServerManager.class);
227     }
228 
229     @Override
230     public void checkTableModifiable(TableName tableName) throws IOException {
231       //no-op
232     }
233 
234     @Override
235     public boolean isMasterProcedureExecutorEnabled() {
236       return true;
237     }
238 
239     @Override
240     public long createTable(
241         final HTableDescriptor desc,
242         final byte[][] splitKeys,
243         final long nonceGroup,
244         final long nonce) throws IOException {
245       // no-op
246       return -1;
247     }
248 
249     @Override
250     public SnapshotManager getSnapshotManager() {
251       return null;
252     }
253 
254     @Override
255     public MasterProcedureManagerHost getMasterProcedureManagerHost() {
256       return null;
257     }
258 
259     @Override
260     public AssignmentManager getAssignmentManager() {
261       return this.asm;
262     }
263 
264     @Override
265     public ExecutorService getExecutorService() {
266       return null;
267     }
268 
269     @Override
270     public ChoreService getChoreService() {
271       return null;
272     }
273 
274     @Override
275     public MasterFileSystem getMasterFileSystem() {
276       return this.mfs;
277     }
278 
279     @Override
280     public MasterCoprocessorHost getMasterCoprocessorHost() {
281       return null;
282     }
283 
284     @Override
285     public MasterQuotaManager getMasterQuotaManager() {
286       return null;
287     }
288 
289     @Override
290     public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
291       return null;
292     }
293 
294     @Override
295     public ServerManager getServerManager() {
296       return sm;
297     }
298 
299     @Override
300     public ZooKeeperWatcher getZooKeeper() {
301       return null;
302     }
303 
304     @Override
305     public CoordinatedStateManager getCoordinatedStateManager() {
306       return null;
307     }
308 
309     @Override
310     public MetaTableLocator getMetaTableLocator() {
311       return null;
312     }
313 
314     @Override
315     public ClusterConnection getConnection() {
316       return null;
317     }
318 
319     @Override
320     public Configuration getConfiguration() {
321       return mfs.conf;
322     }
323 
324     @Override
325     public ServerName getServerName() {
326       return null;
327     }
328 
329     @Override
330     public void abort(String why, Throwable e) {
331       //no-op
332     }
333 
334     @Override
335     public boolean isAborted() {
336       return false;
337     }
338 
339     private boolean stopped = false;
340 
341     @Override
342     public void stop(String why) {
343       stopped = true;
344     }
345 
346     @Override
347     public boolean isStopped() {
348       return stopped;
349     }
350 
351     @Override
352     public TableDescriptors getTableDescriptors() {
353       return new TableDescriptors() {
354         @Override
355         public HTableDescriptor remove(TableName tablename) throws IOException {
356           // TODO Auto-generated method stub
357           return null;
358         }
359 
360         @Override
361         public Map<String, HTableDescriptor> getAll() throws IOException {
362           // TODO Auto-generated method stub
363           return null;
364         }
365 
366         @Override
367         public HTableDescriptor get(TableName tablename)
368         throws IOException {
369           return createHTableDescriptor();
370         }
371 
372         @Override
373         public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
374           return null;
375         }
376 
377         @Override
378         public void add(HTableDescriptor htd) throws IOException {
379           // TODO Auto-generated method stub
380 
381         }
382         @Override
383         public void setCacheOn() throws IOException {
384         }
385 
386         @Override
387         public void setCacheOff() throws IOException {
388         }
389       };
390     }
391 
392     @Override
393     public boolean isServerShutdownHandlerEnabled() {
394       return true;
395     }
396 
397     @Override
398     public boolean registerService(Service instance) {
399       return false;
400     }
401 
402     @Override
403     public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
404       //To change body of implemented methods use File | Settings | File Templates.
405     }
406 
407     @Override
408     public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
409       //To change body of implemented methods use File | Settings | File Templates.
410     }
411 
412     @Override
413     public void deleteNamespace(String name) throws IOException {
414       //To change body of implemented methods use File | Settings | File Templates.
415     }
416 
417     @Override
418     public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
419       return null;  //To change body of implemented methods use File | Settings | File Templates.
420     }
421 
422     @Override
423     public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
424       return null;  //To change body of implemented methods use File | Settings | File Templates.
425     }
426 
427     @Override
428     public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
429         throws IOException {
430       return false;  //To change body of implemented methods use File | Settings | File Templates.
431     }
432 
433     @Override
434     public List<ProcedureInfo> listProcedures() throws IOException {
435       return null;  //To change body of implemented methods use File | Settings | File Templates.
436     }
437 
438     @Override
439     public Pair<Long, String> backupTables(
440         final BackupType type,
441         final List<TableName> tableList,
442         final String targetRootDir, final int workers,
443         final long bandwidth) throws IOException {
444       return null;
445     }
446 
447 
448     @Override
449     public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
450       return null;  //To change body of implemented methods use File | Settings | File Templates.
451     }
452 
453     @Override
454     public List<TableName> listTableNamesByNamespace(String name) throws IOException {
455       return null;
456     }
457 
458     @Override
459     public long deleteTable(
460         final TableName tableName,
461         final long nonceGroup,
462         final long nonce) throws IOException {
463       return -1;
464     }
465     @Override
466     public LoadBalancer getLoadBalancer() {
467       return null;
468     }
469 
470     public void truncateTable(
471         final TableName tableName,
472         final boolean preserveSplits,
473         final long nonceGroup,
474         final long nonce) throws IOException {
475     }
476 
477     @Override
478     public void modifyTable(
479         final TableName tableName,
480         final HTableDescriptor descriptor,
481         final long nonceGroup,
482         final long nonce) throws IOException {
483     }
484 
485     @Override
486     public long enableTable(
487         final TableName tableName,
488         final long nonceGroup,
489         final long nonce) throws IOException {
490       return -1;
491     }
492 
493     @Override
494     public long disableTable(
495         TableName tableName,
496         final long nonceGroup,
497         final long nonce) throws IOException {
498       return -1;
499     }
500 
501     @Override
502     public void addColumn(
503         final TableName tableName,
504         final HColumnDescriptor columnDescriptor,
505         final long nonceGroup,
506         final long nonce) throws IOException { }
507 
508     @Override
509     public void modifyColumn(
510         final TableName tableName,
511         final HColumnDescriptor descriptor,
512         final long nonceGroup,
513         final long nonce) throws IOException { }
514 
515     @Override
516     public void deleteColumn(
517         final TableName tableName,
518         final byte[] columnName,
519         final long nonceGroup,
520         final long nonce) throws IOException { }
521 
522     @Override
523     public TableLockManager getTableLockManager() {
524       return null;
525     }
526 
527     @Override
528     public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
529         boolean forcible) throws IOException {
530     }
531 
532     @Override
533     public boolean isInitialized() {
534       // Auto-generated method stub
535       return false;
536     }
537 
538     @Override
539     public boolean isNamespaceManagerInitialized() {
540       return false;
541     }
542 
543     @Override
544     public boolean isInMaintenanceMode() {
545       return false;
546     }
547 
548     @Override
549     public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
550       // Auto-generated method stub
551       return 0;
552     }
553 
554     @Override
555     public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
556       // Auto-generated method stub
557       return 0;
558     }
559 
560     @Override
561     public TableStateManager getTableStateManager() {
562       // TODO Auto-generated method stub
563       return null;
564     }
565   }
566 
567   @Test
568   public void testCleanParent() throws IOException, InterruptedException {
569     HBaseTestingUtility htu = new HBaseTestingUtility();
570     setRootDirAndCleanIt(htu, "testCleanParent");
571     Server server = new MockServer(htu);
572     try {
573       MasterServices services = new MockMasterServices(server);
574       CatalogJanitor janitor = new CatalogJanitor(server, services);
575       // Create regions.
576       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
577       htd.addFamily(new HColumnDescriptor("f"));
578       HRegionInfo parent =
579         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
580             Bytes.toBytes("eee"));
581       HRegionInfo splita =
582         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
583             Bytes.toBytes("ccc"));
584       HRegionInfo splitb =
585         new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
586             Bytes.toBytes("eee"));
587       // Test that when both daughter regions are in place, that we do not
588       // remove the parent.
589       Result r = createResult(parent, splita, splitb);
590       // Add a reference under splitA directory so we don't clear out the parent.
591       Path rootdir = services.getMasterFileSystem().getRootDir();
592       Path tabledir =
593         FSUtils.getTableDir(rootdir, htd.getTableName());
594       Path storedir = HStore.getStoreHomedir(tabledir, splita,
595           htd.getColumnFamilies()[0].getName());
596       Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
597       long now = System.currentTimeMillis();
598       // Reference name has this format: StoreFile#REF_NAME_PARSER
599       Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
600       FileSystem fs = services.getMasterFileSystem().getFileSystem();
601       Path path = ref.write(fs, p);
602       assertTrue(fs.exists(path));
603       assertFalse(janitor.cleanParent(parent, r));
604       // Remove the reference file and try again.
605       assertTrue(fs.delete(p, true));
606       assertTrue(janitor.cleanParent(parent, r));
607     } finally {
608       server.stop("shutdown");
609     }
610   }
611 
612   /**
613    * Make sure parent gets cleaned up even if daughter is cleaned up before it.
614    * @throws IOException
615    * @throws InterruptedException
616    */
617   @Test
618   public void testParentCleanedEvenIfDaughterGoneFirst()
619   throws IOException, InterruptedException {
620     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
621       "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
622   }
623 
624   /**
625    * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it.
626    * @throws IOException
627    * @throws InterruptedException
628    */
629   @Test
630   public void testLastParentCleanedEvenIfDaughterGoneFirst()
631   throws IOException, InterruptedException {
632     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
633       "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
634   }
635 
636   /**
637    * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it.
638    *
639    * @param rootDir the test case name, used as the HBase testing utility root
640    * @param lastEndKey the end key of the split parent
641    * @throws IOException
642    * @throws InterruptedException
643    */
644   private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
645   final String rootDir, final byte[] lastEndKey)
646   throws IOException, InterruptedException {
647     HBaseTestingUtility htu = new HBaseTestingUtility();
648     setRootDirAndCleanIt(htu, rootDir);
649     Server server = new MockServer(htu);
650     MasterServices services = new MockMasterServices(server);
651     CatalogJanitor janitor = new CatalogJanitor(server, services);
652     final HTableDescriptor htd = createHTableDescriptor();
653 
654     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
655 
656     // Parent
657     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
658       lastEndKey);
659     // Sleep a second else the encoded name on these regions comes out
660     // same for all with same start key and made in same second.
661     Thread.sleep(1001);
662 
663     // Daughter a
664     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
665       Bytes.toBytes("ccc"));
666     Thread.sleep(1001);
667     // Make daughters of daughter a; splitaa and splitab.
668     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
669       Bytes.toBytes("bbb"));
670     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
671       Bytes.toBytes("ccc"));
672 
673     // Daughter b
674     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
675       lastEndKey);
676     Thread.sleep(1001);
677     // Make Daughters of daughterb; splitba and splitbb.
678     HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
679       Bytes.toBytes("ddd"));
680     HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
681     lastEndKey);
682 
683     // First test that our Comparator works right up in CatalogJanitor.
684     // Just fo kicks.
685     SortedMap<HRegionInfo, Result> regions =
686       new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
687     // Now make sure that this regions map sorts as we expect it to.
688     regions.put(parent, createResult(parent, splita, splitb));
689     regions.put(splitb, createResult(splitb, splitba, splitbb));
690     regions.put(splita, createResult(splita, splitaa, splitab));
691     // Assert its properly sorted.
692     int index = 0;
693     for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
694       if (index == 0) {
695         assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
696       } else if (index == 1) {
697         assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
698       } else if (index == 2) {
699         assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
700       }
701       index++;
702     }
703 
704     // Now play around with the cleanParent function.  Create a ref from splita
705     // up to the parent.
706     Path splitaRef =
707       createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
708     // Make sure actual super parent sticks around because splita has a ref.
709     assertFalse(janitor.cleanParent(parent, regions.get(parent)));
710 
711     //splitba, and split bb, do not have dirs in fs.  That means that if
712     // we test splitb, it should get cleaned up.
713     assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
714 
715     // Now remove ref from splita to parent... so parent can be let go and so
716     // the daughter splita can be split (can't split if still references).
717     // BUT make the timing such that the daughter gets cleaned up before we
718     // can get a chance to let go of the parent.
719     FileSystem fs = FileSystem.get(htu.getConfiguration());
720     assertTrue(fs.delete(splitaRef, true));
721     // Create the refs from daughters of splita.
722     Path splitaaRef =
723       createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
724     Path splitabRef =
725       createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
726 
727     // Test splita.  It should stick around because references from splitab, etc.
728     assertFalse(janitor.cleanParent(splita, regions.get(splita)));
729 
730     // Now clean up parent daughter first.  Remove references from its daughters.
731     assertTrue(fs.delete(splitaaRef, true));
732     assertTrue(fs.delete(splitabRef, true));
733     assertTrue(janitor.cleanParent(splita, regions.get(splita)));
734 
735     // Super parent should get cleaned up now both splita and splitb are gone.
736     assertTrue(janitor.cleanParent(parent, regions.get(parent)));
737 
738     services.stop("test finished");
739     janitor.cancel(true);
740   }
741 
742   /**
743    * CatalogJanitor.scan() should not clean parent regions if their own
744    * parents are still referencing them. This ensures that grandfather regions
745    * do not point to deleted parent regions.
746    */
747   @Test
748   public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
749     HBaseTestingUtility htu = new HBaseTestingUtility();
750     setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
751     Server server = new MockServer(htu);
752     MasterServices services = new MockMasterServices(server);
753 
754     final HTableDescriptor htd = createHTableDescriptor();
755 
756     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
757 
758     // Parent
759     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
760       new byte[0], true);
761     // Sleep a second else the encoded name on these regions comes out
762     // same for all with same start key and made in same second.
763     Thread.sleep(1001);
764 
765     // Daughter a
766     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
767       Bytes.toBytes("ccc"), true);
768     Thread.sleep(1001);
769     // Make daughters of daughter a; splitaa and splitab.
770     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
771       Bytes.toBytes("bbb"), false);
772     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
773       Bytes.toBytes("ccc"), false);
774 
775     // Daughter b
776     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
777         new byte[0]);
778     Thread.sleep(1001);
779 
780     final Map<HRegionInfo, Result> splitParents =
781         new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
782     splitParents.put(parent, createResult(parent, splita, splitb));
783     splita.setOffline(true); //simulate that splita goes offline when it is split
784     splitParents.put(splita, createResult(splita, splitaa,splitab));
785 
786     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
787     CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
788     doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
789             10, mergedRegions, splitParents)).when(janitor)
790         .getMergedRegionsAndSplitParents();
791 
792     //create ref from splita to parent
793     Path splitaRef =
794         createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
795 
796     //parent and A should not be removed
797     assertEquals(0, janitor.scan());
798 
799     //now delete the ref
800     FileSystem fs = FileSystem.get(htu.getConfiguration());
801     assertTrue(fs.delete(splitaRef, true));
802 
803     //now, both parent, and splita can be deleted
804     assertEquals(2, janitor.scan());
805 
806     services.stop("test finished");
807     janitor.cancel(true);
808   }
809 
810   /**
811    * Test that we correctly archive all the storefiles when a region is deleted
812    * @throws Exception
813    */
814   @Test
815   public void testSplitParentFirstComparator() {
816     SplitParentFirstComparator comp = new SplitParentFirstComparator();
817     final HTableDescriptor htd = createHTableDescriptor();
818 
819     /*  Region splits:
820      *
821      *  rootRegion --- firstRegion --- firstRegiona
822      *              |               |- firstRegionb
823      *              |
824      *              |- lastRegion --- lastRegiona  --- lastRegionaa
825      *                             |                |- lastRegionab
826      *                             |- lastRegionb
827      *
828      *  rootRegion   :   []  - []
829      *  firstRegion  :   []  - bbb
830      *  lastRegion   :   bbb - []
831      *  firstRegiona :   []  - aaa
832      *  firstRegionb :   aaa - bbb
833      *  lastRegiona  :   bbb - ddd
834      *  lastRegionb  :   ddd - []
835      */
836 
837     // root region
838     HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
839       HConstants.EMPTY_START_ROW,
840       HConstants.EMPTY_END_ROW, true);
841     HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
842       HConstants.EMPTY_START_ROW,
843       Bytes.toBytes("bbb"), true);
844     HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
845       Bytes.toBytes("bbb"),
846       HConstants.EMPTY_END_ROW, true);
847 
848     assertTrue(comp.compare(rootRegion, rootRegion) == 0);
849     assertTrue(comp.compare(firstRegion, firstRegion) == 0);
850     assertTrue(comp.compare(lastRegion, lastRegion) == 0);
851     assertTrue(comp.compare(rootRegion, firstRegion) < 0);
852     assertTrue(comp.compare(rootRegion, lastRegion) < 0);
853     assertTrue(comp.compare(firstRegion, lastRegion) < 0);
854 
855     //first region split into a, b
856     HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
857       HConstants.EMPTY_START_ROW,
858       Bytes.toBytes("aaa"), true);
859     HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
860         Bytes.toBytes("aaa"),
861       Bytes.toBytes("bbb"), true);
862     //last region split into a, b
863     HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
864       Bytes.toBytes("bbb"),
865       Bytes.toBytes("ddd"), true);
866     HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
867       Bytes.toBytes("ddd"),
868       HConstants.EMPTY_END_ROW, true);
869 
870     assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
871     assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
872     assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
873     assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
874     assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
875     assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
876     assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
877 
878     assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
879     assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
880     assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
881     assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
882     assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
883     assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
884     assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
885 
886     assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
887     assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
888     assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
889     assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
890 
891     HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
892       Bytes.toBytes("bbb"),
893       Bytes.toBytes("ccc"), false);
894     HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
895       Bytes.toBytes("ccc"),
896       Bytes.toBytes("ddd"), false);
897 
898     assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
899     assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
900     assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
901 
902   }
903 
904   @Test
905   public void testArchiveOldRegion() throws Exception {
906     String table = "table";
907     HBaseTestingUtility htu = new HBaseTestingUtility();
908     setRootDirAndCleanIt(htu, "testCleanParent");
909     Server server = new MockServer(htu);
910     MasterServices services = new MockMasterServices(server);
911 
912     // create the janitor
913     CatalogJanitor janitor = new CatalogJanitor(server, services);
914 
915     // Create regions.
916     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
917     htd.addFamily(new HColumnDescriptor("f"));
918     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
919         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
920     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
921         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
922     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
923         Bytes.toBytes("ccc"),
924         Bytes.toBytes("eee"));
925 
926     // Test that when both daughter regions are in place, that we do not
927     // remove the parent.
928     Result parentMetaRow = createResult(parent, splita, splitb);
929     FileSystem fs = FileSystem.get(htu.getConfiguration());
930     Path rootdir = services.getMasterFileSystem().getRootDir();
931     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
932     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
933     // the single test passes, but when the full suite is run, things get borked).
934     FSUtils.setRootDir(fs.getConf(), rootdir);
935     Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
936     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
937     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
938       tabledir, htd.getColumnFamilies()[0].getName());
939     LOG.debug("Table dir:" + tabledir);
940     LOG.debug("Store dir:" + storedir);
941     LOG.debug("Store archive dir:" + storeArchive);
942 
943     // add a couple of store files that we can check for
944     FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
945     // get the current store files for comparison
946     FileStatus[] storeFiles = fs.listStatus(storedir);
947     int index = 0;
948     for (FileStatus file : storeFiles) {
949       LOG.debug("Have store file:" + file.getPath());
950       assertEquals("Got unexpected store file", mockFiles[index].getPath(),
951         storeFiles[index].getPath());
952       index++;
953     }
954 
955     // do the cleaning of the parent
956     assertTrue(janitor.cleanParent(parent, parentMetaRow));
957     LOG.debug("Finished cleanup of parent region");
958 
959     // and now check to make sure that the files have actually been archived
960     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
961     logFiles("archived files", storeFiles);
962     logFiles("archived files", archivedStoreFiles);
963 
964     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
965 
966     // cleanup
967     FSUtils.delete(fs, rootdir, true);
968     services.stop("Test finished");
969     server.stop("Test finished");
970     janitor.cancel(true);
971   }
972 
973   /**
974    * @param description description of the files for logging
975    * @param storeFiles the status of the files to log
976    */
977   private void logFiles(String description, FileStatus[] storeFiles) {
978     LOG.debug("Current " + description + ": ");
979     for (FileStatus file : storeFiles) {
980       LOG.debug(file.getPath());
981     }
982   }
983 
984   /**
985    * Test that if a store file with the same name is present as those already backed up cause the
986    * already archived files to be timestamped backup
987    */
988   @Test
989   public void testDuplicateHFileResolution() throws Exception {
990     String table = "table";
991     HBaseTestingUtility htu = new HBaseTestingUtility();
992     setRootDirAndCleanIt(htu, "testCleanParent");
993     Server server = new MockServer(htu);
994     MasterServices services = new MockMasterServices(server);
995 
996     // create the janitor
997 
998     CatalogJanitor janitor = new CatalogJanitor(server, services);
999 
1000     // Create regions.
1001     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
1002     htd.addFamily(new HColumnDescriptor("f"));
1003     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
1004         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
1005     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
1006         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
1007     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
1008         Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
1009     // Test that when both daughter regions are in place, that we do not
1010     // remove the parent.
1011     Result r = createResult(parent, splita, splitb);
1012 
1013     FileSystem fs = FileSystem.get(htu.getConfiguration());
1014 
1015     Path rootdir = services.getMasterFileSystem().getRootDir();
1016     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
1017     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
1018     // the single test passes, but when the full suite is run, things get borked).
1019     FSUtils.setRootDir(fs.getConf(), rootdir);
1020     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1021     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
1022     System.out.println("Old root:" + rootdir);
1023     System.out.println("Old table:" + tabledir);
1024     System.out.println("Old store:" + storedir);
1025 
1026     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
1027       tabledir, htd.getColumnFamilies()[0].getName());
1028     System.out.println("Old archive:" + storeArchive);
1029 
1030     // enable archiving, make sure that files get archived
1031     addMockStoreFiles(2, services, storedir);
1032     // get the current store files for comparison
1033     FileStatus[] storeFiles = fs.listStatus(storedir);
1034     // do the cleaning of the parent
1035     assertTrue(janitor.cleanParent(parent, r));
1036 
1037     // and now check to make sure that the files have actually been archived
1038     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
1039     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
1040 
1041     // now add store files with the same names as before to check backup
1042     // enable archiving, make sure that files get archived
1043     addMockStoreFiles(2, services, storedir);
1044 
1045     // do the cleaning of the parent
1046     assertTrue(janitor.cleanParent(parent, r));
1047 
1048     // and now check to make sure that the files have actually been archived
1049     archivedStoreFiles = fs.listStatus(storeArchive);
1050     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1051 
1052     // cleanup
1053     services.stop("Test finished");
1054     server.stop("shutdown");
1055     janitor.cancel(true);
1056   }
1057 
1058   private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1059       throws IOException {
1060     // get the existing store files
1061     FileSystem fs = services.getMasterFileSystem().getFileSystem();
1062     fs.mkdirs(storedir);
1063     // create the store files in the parent
1064     for (int i = 0; i < count; i++) {
1065       Path storeFile = new Path(storedir, "_store" + i);
1066       FSDataOutputStream dos = fs.create(storeFile, true);
1067       dos.writeBytes("Some data: " + i);
1068       dos.close();
1069     }
1070     LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1071     // make sure the mock store files are there
1072     FileStatus[] storeFiles = fs.listStatus(storedir);
1073     assertEquals("Didn't have expected store files", count, storeFiles.length);
1074     return storeFiles;
1075   }
1076 
1077   private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1078       final String subdir)
1079   throws IOException {
1080     Path testdir = htu.getDataTestDir(subdir);
1081     FileSystem fs = FileSystem.get(htu.getConfiguration());
1082     if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1083     FSUtils.setRootDir(htu.getConfiguration(), testdir);
1084     return FSUtils.getRootDir(htu.getConfiguration()).toString();
1085   }
1086 
1087   /**
1088    * @param services Master services instance.
1089    * @param htd
1090    * @param parent
1091    * @param daughter
1092    * @param midkey
1093    * @param top True if we are to write a 'top' reference.
1094    * @return Path to reference we created.
1095    * @throws IOException
1096    */
1097   private Path createReferences(final MasterServices services,
1098       final HTableDescriptor htd, final HRegionInfo parent,
1099       final HRegionInfo daughter, final byte [] midkey, final boolean top)
1100   throws IOException {
1101     Path rootdir = services.getMasterFileSystem().getRootDir();
1102     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1103     Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1104       htd.getColumnFamilies()[0].getName());
1105     Reference ref =
1106       top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1107     long now = System.currentTimeMillis();
1108     // Reference name has this format: StoreFile#REF_NAME_PARSER
1109     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1110     FileSystem fs = services.getMasterFileSystem().getFileSystem();
1111     ref.write(fs, p);
1112     return p;
1113   }
1114 
1115   private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1116       final HRegionInfo b)
1117   throws IOException {
1118     return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1119   }
1120 
1121   private HTableDescriptor createHTableDescriptor() {
1122     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1123     htd.addFamily(new HColumnDescriptor("f"));
1124     return htd;
1125   }
1126 
1127   private MultiResponse buildMultiResponse(MultiRequest req) {
1128     MultiResponse.Builder builder = MultiResponse.newBuilder();
1129     RegionActionResult.Builder regionActionResultBuilder =
1130         RegionActionResult.newBuilder();
1131     ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1132     for (RegionAction regionAction: req.getRegionActionList()) {
1133       regionActionResultBuilder.clear();
1134       for (ClientProtos.Action action: regionAction.getActionList()) {
1135         roeBuilder.clear();
1136         roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1137         roeBuilder.setIndex(action.getIndex());
1138         regionActionResultBuilder.addResultOrException(roeBuilder.build());
1139       }
1140       builder.addRegionActionResult(regionActionResultBuilder.build());
1141     }
1142     return builder.build();
1143   }
1144 
1145 }
1146