View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.net.URI;
28  import java.util.Collection;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FSDataInputStream;
34  import org.apache.hadoop.fs.FSDataOutputStream;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.permission.FsPermission;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.testclassification.SmallTests;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.util.Progressable;
45  
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  @Category(SmallTests.class)
50  public class TestHRegionFileSystem {
51    private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52    private static final Log LOG = LogFactory.getLog(TestHRegionFileSystem.class);
53  
54    @Test
55    public void testOnDiskRegionCreation() throws IOException {
56      Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testOnDiskRegionCreation");
57      FileSystem fs = TEST_UTIL.getTestFileSystem();
58      Configuration conf = TEST_UTIL.getConfiguration();
59  
60      // Create a Region
61      HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
62      HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
63          FSUtils.getTableDir(rootDir, hri.getTable()), hri);
64  
65      // Verify if the region is on disk
66      Path regionDir = regionFs.getRegionDir();
67      assertTrue("The region folder should be created", fs.exists(regionDir));
68  
69      // Verify the .regioninfo
70      HRegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
71      assertEquals(hri, hriVerify);
72  
73      // Open the region
74      regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
75          FSUtils.getTableDir(rootDir, hri.getTable()), hri, false);
76      assertEquals(regionDir, regionFs.getRegionDir());
77  
78      // Delete the region
79      HRegionFileSystem.deleteRegionFromFileSystem(conf, fs,
80          FSUtils.getTableDir(rootDir, hri.getTable()), hri);
81      assertFalse("The region folder should be removed", fs.exists(regionDir));
82  
83      fs.delete(rootDir, true);
84    }
85  
86    @Test
87    public void testNonIdempotentOpsWithRetries() throws IOException {
88      Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testOnDiskRegionCreation");
89      FileSystem fs = TEST_UTIL.getTestFileSystem();
90      Configuration conf = TEST_UTIL.getConfiguration();
91  
92      // Create a Region
93      HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
94      HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri);
95      assertTrue(fs.exists(regionFs.getRegionDir()));
96  
97      regionFs = new HRegionFileSystem(conf, new MockFileSystemForCreate(),
98          null, null);
99      // HRegionFileSystem.createRegionOnFileSystem(conf, new MockFileSystemForCreate(), rootDir,
100     // hri);
101     boolean result = regionFs.createDir(new Path("/foo/bar"));
102     assertTrue("Couldn't create the directory", result);
103 
104 
105     regionFs = new HRegionFileSystem(conf, new MockFileSystem(), null, null);
106     result = regionFs.rename(new Path("/foo/bar"), new Path("/foo/bar2"));
107     assertTrue("Couldn't rename the directory", result);
108 
109     regionFs = new HRegionFileSystem(conf, new MockFileSystem(), null, null);
110     result = regionFs.deleteDir(new Path("/foo/bar"));
111     assertTrue("Couldn't delete the directory", result);
112     fs.delete(rootDir, true);
113   }
114 
115   static class MockFileSystemForCreate extends MockFileSystem {
116     @Override
117     public boolean exists(Path path) {
118       return false;
119     }
120   }
121 
122   /**
123    * a mock fs which throws exception for first 3 times, and then process the call (returns the
124    * excepted result).
125    */
126   static class MockFileSystem extends FileSystem {
127     int retryCount;
128     final static int successRetryCount = 3;
129 
130     public MockFileSystem() {
131       retryCount = 0;
132     }
133 
134     @Override
135     public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
136       throw new IOException("");
137     }
138 
139     @Override
140     public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
141         short arg4, long arg5, Progressable arg6) throws IOException {
142       LOG.debug("Create, " + retryCount);
143       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
144       return null;
145     }
146 
147     @Override
148     public boolean delete(Path arg0) throws IOException {
149       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
150       return true;
151     }
152 
153     @Override
154     public boolean delete(Path arg0, boolean arg1) throws IOException {
155       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
156       return true;
157     }
158 
159     @Override
160     public FileStatus getFileStatus(Path arg0) throws IOException {
161       FileStatus fs = new FileStatus();
162       return fs;
163     }
164 
165     @Override
166     public boolean exists(Path path) {
167       return true;
168     }
169 
170     @Override
171     public URI getUri() {
172       throw new RuntimeException("Something bad happen");
173     }
174 
175     @Override
176     public Path getWorkingDirectory() {
177       throw new RuntimeException("Something bad happen");
178     }
179 
180     @Override
181     public FileStatus[] listStatus(Path arg0) throws IOException {
182       throw new IOException("Something bad happen");
183     }
184 
185     @Override
186     public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
187       LOG.debug("mkdirs, " + retryCount);
188       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
189       return true;
190     }
191 
192     @Override
193     public FSDataInputStream open(Path arg0, int arg1) throws IOException {
194       throw new IOException("Something bad happen");
195     }
196 
197     @Override
198     public boolean rename(Path arg0, Path arg1) throws IOException {
199       LOG.debug("rename, " + retryCount);
200       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
201       return true;
202     }
203 
204     @Override
205     public void setWorkingDirectory(Path arg0) {
206       throw new RuntimeException("Something bad happen");
207     }
208   }
209 
210   @Test
211   public void testTempAndCommit() throws IOException {
212     Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testTempAndCommit");
213     FileSystem fs = TEST_UTIL.getTestFileSystem();
214     Configuration conf = TEST_UTIL.getConfiguration();
215 
216     // Create a Region
217     String familyName = "cf";
218     HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
219     HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri);
220 
221     // New region, no store files
222     Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
223     assertEquals(0, storeFiles != null ? storeFiles.size() : 0);
224 
225     // Create a new file in temp (no files in the family)
226     Path buildPath = regionFs.createTempName();
227     fs.createNewFile(buildPath);
228     storeFiles = regionFs.getStoreFiles(familyName);
229     assertEquals(0, storeFiles != null ? storeFiles.size() : 0);
230 
231     // commit the file
232     Path dstPath = regionFs.commitStoreFile(familyName, buildPath);
233     storeFiles = regionFs.getStoreFiles(familyName);
234     assertEquals(0, storeFiles != null ? storeFiles.size() : 0);
235     assertFalse(fs.exists(buildPath));
236 
237     fs.delete(rootDir, true);
238   }
239 }