View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.Server;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.catalog.CatalogTracker;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.io.HFileLink;
37  import org.apache.hadoop.hbase.testclassification.SmallTests;
38  import org.apache.hadoop.hbase.util.FSUtils;
39  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
40  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test the HFileLink Cleaner.
46   * HFiles with links cannot be deleted until a link is present.
47   */
48  @Category(SmallTests.class)
49  public class TestHFileLinkCleaner {
50  
51    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52  
53    @Test
54    public void testHFileLinkCleaning() throws Exception {
55      Configuration conf = TEST_UTIL.getConfiguration();
56      FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
57      conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
58      Path rootDir = FSUtils.getRootDir(conf);
59      FileSystem fs = FileSystem.get(conf);
60  
61      final TableName tableName = TableName.valueOf("test-table");
62      final TableName tableLinkName = TableName.valueOf("test-link");
63      final String hfileName = "1234567890";
64      final String familyName = "cf";
65  
66      HRegionInfo hri = new HRegionInfo(tableName);
67      HRegionInfo hriLink = new HRegionInfo(tableLinkName);
68  
69      Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
70      Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
71            tableName, hri.getEncodedName(), familyName);
72      Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
73            tableLinkName, hriLink.getEncodedName(), familyName);
74  
75      // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
76      Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
77      fs.mkdirs(familyPath);
78      Path hfilePath = new Path(familyPath, hfileName);
79      fs.createNewFile(hfilePath);
80  
81      // Create link to hfile
82      Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
83                                          hriLink.getEncodedName(), familyName);
84      fs.mkdirs(familyLinkPath);
85      HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
86      Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
87      assertTrue(fs.exists(linkBackRefDir));
88      FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
89      assertEquals(1, backRefs.length);
90      Path linkBackRef = backRefs[0].getPath();
91  
92      // Initialize cleaner
93      final long ttl = 1000;
94      conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
95      Server server = new DummyServer();
96      HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
97  
98      // Link backref cannot be removed
99      cleaner.chore();
100     assertTrue(fs.exists(linkBackRef));
101     assertTrue(fs.exists(hfilePath));
102 
103     // Link backref can be removed
104     fs.rename(FSUtils.getTableDir(rootDir, tableLinkName),
105         FSUtils.getTableDir(archiveDir, tableLinkName));
106     cleaner.chore();
107     assertFalse("Link should be deleted", fs.exists(linkBackRef));
108 
109     // HFile can be removed
110     Thread.sleep(ttl * 2);
111     cleaner.chore();
112     assertFalse("HFile should be deleted", fs.exists(hfilePath));
113 
114     // Remove everything
115     for (int i = 0; i < 4; ++i) {
116       Thread.sleep(ttl * 2);
117       cleaner.chore();
118     }
119     assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName)));
120     assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName)));
121 
122     cleaner.interrupt();
123   }
124 
125   private static Path getFamilyDirPath (final Path rootDir, final TableName table,
126     final String region, final String family) {
127     return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family);
128   }
129 
130   static class DummyServer implements Server {
131 
132     @Override
133     public Configuration getConfiguration() {
134       return TEST_UTIL.getConfiguration();
135     }
136 
137     @Override
138     public ZooKeeperWatcher getZooKeeper() {
139       try {
140         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
141       } catch (IOException e) {
142         e.printStackTrace();
143       }
144       return null;
145     }
146 
147     @Override
148     public CatalogTracker getCatalogTracker() {
149       return null;
150     }
151 
152     @Override
153     public ServerName getServerName() {
154       return ServerName.valueOf("regionserver,60020,000000");
155     }
156 
157     @Override
158     public void abort(String why, Throwable e) {}
159 
160     @Override
161     public boolean isAborted() {
162       return false;
163     }
164 
165     @Override
166     public void stop(String why) {}
167 
168     @Override
169     public boolean isStopped() {
170       return false;
171     }
172   }
173 }