View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertTrue;
25  import static org.mockito.Mockito.doReturn;
26  import static org.mockito.Mockito.spy;
27  
28  import java.io.IOException;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.SortedMap;
32  import java.util.TreeMap;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.NamespaceDescriptor;
48  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
49  import org.apache.hadoop.hbase.Server;
50  import org.apache.hadoop.hbase.ServerName;
51  import org.apache.hadoop.hbase.TableDescriptors;
52  import org.apache.hadoop.hbase.catalog.CatalogTracker;
53  import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
54  import org.apache.hadoop.hbase.client.HConnection;
55  import org.apache.hadoop.hbase.client.HConnectionManager;
56  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.executor.ExecutorService;
59  import org.apache.hadoop.hbase.io.Reference;
60  import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
61  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
62  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
63  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
64  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
65  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
66  import org.apache.hadoop.hbase.regionserver.HStore;
67  import org.apache.hadoop.hbase.testclassification.SmallTests;
68  import org.apache.hadoop.hbase.util.Bytes;
69  import org.apache.hadoop.hbase.util.FSUtils;
70  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
71  import org.apache.hadoop.hbase.util.Triple;
72  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
73  import org.junit.Test;
74  import org.junit.experimental.categories.Category;
75  import org.mockito.Mockito;
76  
77  import com.google.protobuf.RpcController;
78  import com.google.protobuf.Service;
79  import com.google.protobuf.ServiceException;
80  
81  @Category(SmallTests.class)
82  public class TestCatalogJanitor {
83    private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
84  
85    /**
86     * Pseudo server for below tests.
87     * Be sure to call stop on the way out else could leave some mess around.
88     */
89    class MockServer implements Server {
90      private final HConnection connection;
91      private final Configuration c;
92      private final CatalogTracker ct;
93  
94      MockServer(final HBaseTestingUtility htu)
95      throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
96        this.c = htu.getConfiguration();
97        ClientProtos.ClientService.BlockingInterface ri =
98          Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
99        MutateResponse.Builder builder = MutateResponse.newBuilder();
100       builder.setProcessed(true);
101       try {
102         Mockito.when(ri.mutate(
103           (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
104             thenReturn(builder.build());
105       } catch (ServiceException se) {
106         throw ProtobufUtil.getRemoteException(se);
107       }
108       // Mock an HConnection and a AdminProtocol implementation.  Have the
109       // HConnection return the HRI.  Have the HRI return a few mocked up responses
110       // to make our test work.
111       this.connection =
112         HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
113           Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
114             ServerName.valueOf("example.org,12345,6789"),
115           HRegionInfo.FIRST_META_REGIONINFO);
116       // Set hbase.rootdir into test dir.
117       FileSystem fs = FileSystem.get(this.c);
118       Path rootdir = FSUtils.getRootDir(this.c);
119       FSUtils.setRootDir(this.c, rootdir);
120       this.ct = Mockito.mock(CatalogTracker.class);
121       AdminProtos.AdminService.BlockingInterface hri =
122         Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
123       Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
124       Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
125     }
126 
127     @Override
128     public CatalogTracker getCatalogTracker() {
129       return this.ct;
130     }
131 
132     @Override
133     public Configuration getConfiguration() {
134       return this.c;
135     }
136 
137     @Override
138     public ServerName getServerName() {
139       return ServerName.valueOf("mockserver.example.org", 1234, -1L);
140     }
141 
142     @Override
143     public ZooKeeperWatcher getZooKeeper() {
144       return null;
145     }
146 
147     @Override
148     public void abort(String why, Throwable e) {
149       //no-op
150     }
151 
152     @Override
153     public boolean isAborted() {
154       return false;
155     }
156 
157     @Override
158     public boolean isStopped() {
159       return false;
160     }
161 
162     @Override
163     public void stop(String why) {
164       if (this.ct != null) {
165         this.ct.stop();
166       }
167       if (this.connection != null) {
168         HConnectionManager.deleteConnection(this.connection.getConfiguration());
169       }
170     }
171   }
172 
173   /**
174    * Mock MasterServices for tests below.
175    */
176   class MockMasterServices implements MasterServices {
177     private final MasterFileSystem mfs;
178     private final AssignmentManager asm;
179 
180     MockMasterServices(final Server server) throws IOException {
181       this.mfs = new MasterFileSystem(server, this, false);
182       this.asm = Mockito.mock(AssignmentManager.class);
183     }
184 
185     @Override
186     public void checkTableModifiable(TableName tableName) throws IOException {
187       //no-op
188     }
189 
190     @Override
191     public void createTable(HTableDescriptor desc, byte[][] splitKeys)
192         throws IOException {
193       // no-op
194     }
195 
196     @Override
197     public AssignmentManager getAssignmentManager() {
198       return this.asm;
199     }
200 
201     @Override
202     public ExecutorService getExecutorService() {
203       return null;
204     }
205 
206     @Override
207     public MasterFileSystem getMasterFileSystem() {
208       return this.mfs;
209     }
210 
211     @Override
212     public MasterCoprocessorHost getCoprocessorHost() {
213       return null;
214     }
215 
216     @Override
217     public ServerManager getServerManager() {
218       return null;
219     }
220 
221     @Override
222     public ZooKeeperWatcher getZooKeeper() {
223       return null;
224     }
225 
226     @Override
227     public CatalogTracker getCatalogTracker() {
228       return null;
229     }
230 
231     @Override
232     public Configuration getConfiguration() {
233       return mfs.conf;
234     }
235 
236     @Override
237     public ServerName getServerName() {
238       return null;
239     }
240 
241     @Override
242     public void abort(String why, Throwable e) {
243       //no-op
244     }
245 
246     @Override
247     public boolean isAborted() {
248       return false;
249     }
250 
251     private boolean stopped = false;
252 
253     @Override
254     public void stop(String why) {
255       stopped = true;
256     }
257 
258     @Override
259     public boolean isStopped() {
260       return stopped;
261     }
262 
263     @Override
264     public TableDescriptors getTableDescriptors() {
265       return new TableDescriptors() {
266         @Override
267         public HTableDescriptor remove(TableName tablename) throws IOException {
268           // TODO Auto-generated method stub
269           return null;
270         }
271 
272         @Override
273         public Map<String, HTableDescriptor> getAll() throws IOException {
274           // TODO Auto-generated method stub
275           return null;
276         }
277 
278         @Override
279         public HTableDescriptor get(TableName tablename)
280         throws IOException {
281           return createHTableDescriptor();
282         }
283 
284         @Override
285         public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
286           return null;
287         }
288 
289         @Override
290         public void add(HTableDescriptor htd) throws IOException {
291           // TODO Auto-generated method stub
292 
293         }
294 
295         @Override
296         public void setCacheOn() throws IOException {
297         }
298 
299         @Override
300         public void setCacheOff() throws IOException {
301         }
302       };
303     }
304 
305     @Override
306     public boolean isServerShutdownHandlerEnabled() {
307       return true;
308     }
309 
310     @Override
311     public boolean registerService(Service instance) {
312       return false;
313     }
314 
315     @Override
316     public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
317       //To change body of implemented methods use File | Settings | File Templates.
318     }
319 
320     @Override
321     public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
322       //To change body of implemented methods use File | Settings | File Templates.
323     }
324 
325     @Override
326     public void deleteNamespace(String name) throws IOException {
327       //To change body of implemented methods use File | Settings | File Templates.
328     }
329 
330     @Override
331     public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
332       return null;  //To change body of implemented methods use File | Settings | File Templates.
333     }
334 
335     @Override
336     public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
337       return null;  //To change body of implemented methods use File | Settings | File Templates.
338     }
339 
340     @Override
341     public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
342       return null;  //To change body of implemented methods use File | Settings | File Templates.
343     }
344 
345     @Override
346     public List<TableName> listTableNamesByNamespace(String name) throws IOException {
347       return null;
348     }
349 
350     @Override
351     public void deleteTable(TableName tableName) throws IOException { }
352 
353     @Override
354     public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException { }
355 
356 
357     @Override
358     public void modifyTable(TableName tableName, HTableDescriptor descriptor)
359         throws IOException { }
360 
361     @Override
362     public void enableTable(TableName tableName) throws IOException { }
363 
364     @Override
365     public void disableTable(TableName tableName) throws IOException { }
366 
367     @Override
368     public void addColumn(TableName tableName, HColumnDescriptor column)
369         throws IOException { }
370 
371     @Override
372     public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
373         throws IOException { }
374 
375     @Override
376     public void deleteColumn(TableName tableName, byte[] columnName)
377         throws IOException { }
378 
379     @Override
380     public TableLockManager getTableLockManager() {
381       return null;
382     }
383 
384     @Override
385     public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
386         boolean forcible) throws IOException {
387     }
388 
389     @Override
390     public boolean isInitialized() {
391       // Auto-generated method stub
392       return false;
393     }
394   }
395 
396   @Test
397   public void testCleanParent() throws IOException, InterruptedException {
398     HBaseTestingUtility htu = new HBaseTestingUtility();
399     setRootDirAndCleanIt(htu, "testCleanParent");
400     Server server = new MockServer(htu);
401     try {
402       MasterServices services = new MockMasterServices(server);
403       CatalogJanitor janitor = new CatalogJanitor(server, services);
404       // Create regions.
405       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
406       htd.addFamily(new HColumnDescriptor("f"));
407       HRegionInfo parent =
408         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
409             Bytes.toBytes("eee"));
410       HRegionInfo splita =
411         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
412             Bytes.toBytes("ccc"));
413       HRegionInfo splitb =
414         new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
415             Bytes.toBytes("eee"));
416       // Test that when both daughter regions are in place, that we do not
417       // remove the parent.
418       Result r = createResult(parent, splita, splitb);
419       // Add a reference under splitA directory so we don't clear out the parent.
420       Path rootdir = services.getMasterFileSystem().getRootDir();
421       Path tabledir =
422         FSUtils.getTableDir(rootdir, htd.getTableName());
423       Path storedir = HStore.getStoreHomedir(tabledir, splita,
424           htd.getColumnFamilies()[0].getName());
425       Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
426       long now = System.currentTimeMillis();
427       // Reference name has this format: StoreFile#REF_NAME_PARSER
428       Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
429       FileSystem fs = services.getMasterFileSystem().getFileSystem();
430       Path path = ref.write(fs, p);
431       assertTrue(fs.exists(path));
432       assertFalse(janitor.cleanParent(parent, r));
433       // Remove the reference file and try again.
434       assertTrue(fs.delete(p, true));
435       assertTrue(janitor.cleanParent(parent, r));
436     } finally {
437       server.stop("shutdown");
438     }
439   }
440 
441   /**
442    * Make sure parent gets cleaned up even if daughter is cleaned up before it.
443    * @throws IOException
444    * @throws InterruptedException
445    */
446   @Test
447   public void testParentCleanedEvenIfDaughterGoneFirst()
448   throws IOException, InterruptedException {
449     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
450       "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
451   }
452 
453   /**
454    * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it.
455    * @throws IOException
456    * @throws InterruptedException
457    */
458   @Test
459   public void testLastParentCleanedEvenIfDaughterGoneFirst()
460   throws IOException, InterruptedException {
461     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
462       "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
463   }
464 
465   /**
466    * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it.
467    *
468    * @param rootDir the test case name, used as the HBase testing utility root
469    * @param lastEndKey the end key of the split parent
470    * @throws IOException
471    * @throws InterruptedException
472    */
473   private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
474   final String rootDir, final byte[] lastEndKey)
475   throws IOException, InterruptedException {
476     HBaseTestingUtility htu = new HBaseTestingUtility();
477     setRootDirAndCleanIt(htu, rootDir);
478     Server server = new MockServer(htu);
479     MasterServices services = new MockMasterServices(server);
480     CatalogJanitor janitor = new CatalogJanitor(server, services);
481     final HTableDescriptor htd = createHTableDescriptor();
482 
483     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
484 
485     // Parent
486     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
487       lastEndKey);
488     // Sleep a second else the encoded name on these regions comes out
489     // same for all with same start key and made in same second.
490     Thread.sleep(1001);
491 
492     // Daughter a
493     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
494       Bytes.toBytes("ccc"));
495     Thread.sleep(1001);
496     // Make daughters of daughter a; splitaa and splitab.
497     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
498       Bytes.toBytes("bbb"));
499     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
500       Bytes.toBytes("ccc"));
501 
502     // Daughter b
503     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
504       lastEndKey);
505     Thread.sleep(1001);
506     // Make Daughters of daughterb; splitba and splitbb.
507     HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
508       Bytes.toBytes("ddd"));
509     HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
510     lastEndKey);
511 
512     // First test that our Comparator works right up in CatalogJanitor.
513     // Just fo kicks.
514     SortedMap<HRegionInfo, Result> regions =
515       new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
516     // Now make sure that this regions map sorts as we expect it to.
517     regions.put(parent, createResult(parent, splita, splitb));
518     regions.put(splitb, createResult(splitb, splitba, splitbb));
519     regions.put(splita, createResult(splita, splitaa, splitab));
520     // Assert its properly sorted.
521     int index = 0;
522     for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
523       if (index == 0) {
524         assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
525       } else if (index == 1) {
526         assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
527       } else if (index == 2) {
528         assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
529       }
530       index++;
531     }
532 
533     // Now play around with the cleanParent function.  Create a ref from splita
534     // up to the parent.
535     Path splitaRef =
536       createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
537     // Make sure actual super parent sticks around because splita has a ref.
538     assertFalse(janitor.cleanParent(parent, regions.get(parent)));
539 
540     //splitba, and split bb, do not have dirs in fs.  That means that if
541     // we test splitb, it should get cleaned up.
542     assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
543 
544     // Now remove ref from splita to parent... so parent can be let go and so
545     // the daughter splita can be split (can't split if still references).
546     // BUT make the timing such that the daughter gets cleaned up before we
547     // can get a chance to let go of the parent.
548     FileSystem fs = FileSystem.get(htu.getConfiguration());
549     assertTrue(fs.delete(splitaRef, true));
550     // Create the refs from daughters of splita.
551     Path splitaaRef =
552       createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
553     Path splitabRef =
554       createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
555 
556     // Test splita.  It should stick around because references from splitab, etc.
557     assertFalse(janitor.cleanParent(splita, regions.get(splita)));
558 
559     // Now clean up parent daughter first.  Remove references from its daughters.
560     assertTrue(fs.delete(splitaaRef, true));
561     assertTrue(fs.delete(splitabRef, true));
562     assertTrue(janitor.cleanParent(splita, regions.get(splita)));
563 
564     // Super parent should get cleaned up now both splita and splitb are gone.
565     assertTrue(janitor.cleanParent(parent, regions.get(parent)));
566 
567     services.stop("test finished");
568     janitor.join();
569   }
570 
571   /**
572    * CatalogJanitor.scan() should not clean parent regions if their own
573    * parents are still referencing them. This ensures that grandfather regions
574    * do not point to deleted parent regions.
575    */
576   @Test
577   public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
578     HBaseTestingUtility htu = new HBaseTestingUtility();
579     setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
580     Server server = new MockServer(htu);
581     MasterServices services = new MockMasterServices(server);
582 
583     final HTableDescriptor htd = createHTableDescriptor();
584 
585     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
586 
587     // Parent
588     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
589       new byte[0], true);
590     // Sleep a second else the encoded name on these regions comes out
591     // same for all with same start key and made in same second.
592     Thread.sleep(1001);
593 
594     // Daughter a
595     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
596       Bytes.toBytes("ccc"), true);
597     Thread.sleep(1001);
598     // Make daughters of daughter a; splitaa and splitab.
599     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
600       Bytes.toBytes("bbb"), false);
601     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
602       Bytes.toBytes("ccc"), false);
603 
604     // Daughter b
605     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
606         new byte[0]);
607     Thread.sleep(1001);
608 
609     final Map<HRegionInfo, Result> splitParents =
610         new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
611     splitParents.put(parent, createResult(parent, splita, splitb));
612     splita.setOffline(true); //simulate that splita goes offline when it is split
613     splitParents.put(splita, createResult(splita, splitaa,splitab));
614 
615     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
616     CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
617     doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
618             10, mergedRegions, splitParents)).when(janitor)
619         .getMergedRegionsAndSplitParents();
620 
621     //create ref from splita to parent
622     Path splitaRef =
623         createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
624 
625     //parent and A should not be removed
626     assertEquals(0, janitor.scan());
627 
628     //now delete the ref
629     FileSystem fs = FileSystem.get(htu.getConfiguration());
630     assertTrue(fs.delete(splitaRef, true));
631 
632     //now, both parent, and splita can be deleted
633     assertEquals(2, janitor.scan());
634 
635     services.stop("test finished");
636     janitor.join();
637   }
638 
639   /**
640    * Test that we correctly archive all the storefiles when a region is deleted
641    * @throws Exception
642    */
643   @Test
644   public void testSplitParentFirstComparator() {
645     SplitParentFirstComparator comp = new SplitParentFirstComparator();
646     final HTableDescriptor htd = createHTableDescriptor();
647 
648     /*  Region splits:
649      *
650      *  rootRegion --- firstRegion --- firstRegiona
651      *              |               |- firstRegionb
652      *              |
653      *              |- lastRegion --- lastRegiona  --- lastRegionaa
654      *                             |                |- lastRegionab
655      *                             |- lastRegionb
656      *
657      *  rootRegion   :   []  - []
658      *  firstRegion  :   []  - bbb
659      *  lastRegion   :   bbb - []
660      *  firstRegiona :   []  - aaa
661      *  firstRegionb :   aaa - bbb
662      *  lastRegiona  :   bbb - ddd
663      *  lastRegionb  :   ddd - []
664      */
665 
666     // root region
667     HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
668       HConstants.EMPTY_START_ROW,
669       HConstants.EMPTY_END_ROW, true);
670     HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
671       HConstants.EMPTY_START_ROW,
672       Bytes.toBytes("bbb"), true);
673     HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
674       Bytes.toBytes("bbb"),
675       HConstants.EMPTY_END_ROW, true);
676 
677     assertTrue(comp.compare(rootRegion, rootRegion) == 0);
678     assertTrue(comp.compare(firstRegion, firstRegion) == 0);
679     assertTrue(comp.compare(lastRegion, lastRegion) == 0);
680     assertTrue(comp.compare(rootRegion, firstRegion) < 0);
681     assertTrue(comp.compare(rootRegion, lastRegion) < 0);
682     assertTrue(comp.compare(firstRegion, lastRegion) < 0);
683 
684     //first region split into a, b
685     HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
686       HConstants.EMPTY_START_ROW,
687       Bytes.toBytes("aaa"), true);
688     HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
689         Bytes.toBytes("aaa"),
690       Bytes.toBytes("bbb"), true);
691     //last region split into a, b
692     HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
693       Bytes.toBytes("bbb"),
694       Bytes.toBytes("ddd"), true);
695     HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
696       Bytes.toBytes("ddd"),
697       HConstants.EMPTY_END_ROW, true);
698 
699     assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
700     assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
701     assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
702     assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
703     assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
704     assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
705     assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
706 
707     assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
708     assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
709     assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
710     assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
711     assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
712     assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
713     assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
714 
715     assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
716     assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
717     assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
718     assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
719 
720     HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
721       Bytes.toBytes("bbb"),
722       Bytes.toBytes("ccc"), false);
723     HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
724       Bytes.toBytes("ccc"),
725       Bytes.toBytes("ddd"), false);
726 
727     assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
728     assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
729     assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
730 
731   }
732 
733   @Test
734   public void testArchiveOldRegion() throws Exception {
735     String table = "table";
736     HBaseTestingUtility htu = new HBaseTestingUtility();
737     setRootDirAndCleanIt(htu, "testCleanParent");
738     Server server = new MockServer(htu);
739     MasterServices services = new MockMasterServices(server);
740 
741     // create the janitor
742     CatalogJanitor janitor = new CatalogJanitor(server, services);
743 
744     // Create regions.
745     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
746     htd.addFamily(new HColumnDescriptor("f"));
747     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
748         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
749     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
750         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
751     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
752         Bytes.toBytes("ccc"),
753         Bytes.toBytes("eee"));
754 
755     // Test that when both daughter regions are in place, that we do not
756     // remove the parent.
757     Result parentMetaRow = createResult(parent, splita, splitb);
758     FileSystem fs = FileSystem.get(htu.getConfiguration());
759     Path rootdir = services.getMasterFileSystem().getRootDir();
760     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
761     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
762     // the single test passes, but when the full suite is run, things get borked).
763     FSUtils.setRootDir(fs.getConf(), rootdir);
764     Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
765     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
766     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
767       tabledir, htd.getColumnFamilies()[0].getName());
768     LOG.debug("Table dir:" + tabledir);
769     LOG.debug("Store dir:" + storedir);
770     LOG.debug("Store archive dir:" + storeArchive);
771 
772     // add a couple of store files that we can check for
773     FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
774     // get the current store files for comparison
775     FileStatus[] storeFiles = fs.listStatus(storedir);
776     int index = 0;
777     for (FileStatus file : storeFiles) {
778       LOG.debug("Have store file:" + file.getPath());
779       assertEquals("Got unexpected store file", mockFiles[index].getPath(),
780         storeFiles[index].getPath());
781       index++;
782     }
783 
784     // do the cleaning of the parent
785     assertTrue(janitor.cleanParent(parent, parentMetaRow));
786     LOG.debug("Finished cleanup of parent region");
787 
788     // and now check to make sure that the files have actually been archived
789     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
790     logFiles("archived files", storeFiles);
791     logFiles("archived files", archivedStoreFiles);
792 
793     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
794 
795     // cleanup
796     FSUtils.delete(fs, rootdir, true);
797     services.stop("Test finished");
798     server.stop("Test finished");
799     janitor.join();
800   }
801 
802   /**
803    * @param description description of the files for logging
804    * @param storeFiles the status of the files to log
805    */
806   private void logFiles(String description, FileStatus[] storeFiles) {
807     LOG.debug("Current " + description + ": ");
808     for (FileStatus file : storeFiles) {
809       LOG.debug(file.getPath());
810     }
811   }
812 
813   /**
814    * Test that if a store file with the same name is present as those already backed up cause the
815    * already archived files to be timestamped backup
816    */
817   @Test
818   public void testDuplicateHFileResolution() throws Exception {
819     String table = "table";
820     HBaseTestingUtility htu = new HBaseTestingUtility();
821     setRootDirAndCleanIt(htu, "testCleanParent");
822     Server server = new MockServer(htu);
823     MasterServices services = new MockMasterServices(server);
824 
825     // create the janitor
826     CatalogJanitor janitor = new CatalogJanitor(server, services);
827 
828     // Create regions.
829     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
830     htd.addFamily(new HColumnDescriptor("f"));
831     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
832         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
833     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
834         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
835     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
836         Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
837     // Test that when both daughter regions are in place, that we do not
838     // remove the parent.
839     Result r = createResult(parent, splita, splitb);
840 
841     FileSystem fs = FileSystem.get(htu.getConfiguration());
842 
843     Path rootdir = services.getMasterFileSystem().getRootDir();
844     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
845     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
846     // the single test passes, but when the full suite is run, things get borked).
847     FSUtils.setRootDir(fs.getConf(), rootdir);
848     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
849     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
850     System.out.println("Old root:" + rootdir);
851     System.out.println("Old table:" + tabledir);
852     System.out.println("Old store:" + storedir);
853 
854     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
855       tabledir, htd.getColumnFamilies()[0].getName());
856     System.out.println("Old archive:" + storeArchive);
857 
858     // enable archiving, make sure that files get archived
859     addMockStoreFiles(2, services, storedir);
860     // get the current store files for comparison
861     FileStatus[] storeFiles = fs.listStatus(storedir);
862     // do the cleaning of the parent
863     assertTrue(janitor.cleanParent(parent, r));
864 
865     // and now check to make sure that the files have actually been archived
866     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
867     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
868 
869     // now add store files with the same names as before to check backup
870     // enable archiving, make sure that files get archived
871     addMockStoreFiles(2, services, storedir);
872 
873     // do the cleaning of the parent
874     assertTrue(janitor.cleanParent(parent, r));
875 
876     // and now check to make sure that the files have actually been archived
877     archivedStoreFiles = fs.listStatus(storeArchive);
878     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
879 
880     // cleanup
881     services.stop("Test finished");
882     server.stop("shutdown");
883     janitor.join();
884   }
885 
886   private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
887       throws IOException {
888     // get the existing store files
889     FileSystem fs = services.getMasterFileSystem().getFileSystem();
890     fs.mkdirs(storedir);
891     // create the store files in the parent
892     for (int i = 0; i < count; i++) {
893       Path storeFile = new Path(storedir, "_store" + i);
894       FSDataOutputStream dos = fs.create(storeFile, true);
895       dos.writeBytes("Some data: " + i);
896       dos.close();
897     }
898     LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
899     // make sure the mock store files are there
900     FileStatus[] storeFiles = fs.listStatus(storedir);
901     assertEquals("Didn't have expected store files", count, storeFiles.length);
902     return storeFiles;
903   }
904 
905   private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
906       final String subdir)
907   throws IOException {
908     Path testdir = htu.getDataTestDir(subdir);
909     FileSystem fs = FileSystem.get(htu.getConfiguration());
910     if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
911     FSUtils.setRootDir(htu.getConfiguration(), testdir);
912     return FSUtils.getRootDir(htu.getConfiguration()).toString();
913   }
914 
915   /**
916    * @param services Master services instance.
917    * @param htd
918    * @param parent
919    * @param daughter
920    * @param midkey
921    * @param top True if we are to write a 'top' reference.
922    * @return Path to reference we created.
923    * @throws IOException
924    */
925   private Path createReferences(final MasterServices services,
926       final HTableDescriptor htd, final HRegionInfo parent,
927       final HRegionInfo daughter, final byte [] midkey, final boolean top)
928   throws IOException {
929     Path rootdir = services.getMasterFileSystem().getRootDir();
930     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
931     Path storedir = HStore.getStoreHomedir(tabledir, daughter,
932       htd.getColumnFamilies()[0].getName());
933     Reference ref =
934       top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
935     long now = System.currentTimeMillis();
936     // Reference name has this format: StoreFile#REF_NAME_PARSER
937     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
938     FileSystem fs = services.getMasterFileSystem().getFileSystem();
939     ref.write(fs, p);
940     return p;
941   }
942 
943   private Result createResult(final HRegionInfo parent, final HRegionInfo a,
944       final HRegionInfo b)
945   throws IOException {
946     return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
947   }
948 
949   private HTableDescriptor createHTableDescriptor() {
950     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
951     htd.addFamily(new HColumnDescriptor("f"));
952     return htd;
953   }
954 
955 }
956