View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.io.FileNotFoundException;
28  import java.io.IOException;
29  import java.util.Arrays;
30  import java.util.Comparator;
31  import java.util.Map;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.testclassification.MediumTests;
45  import org.apache.hadoop.hbase.TableDescriptors;
46  import org.apache.hadoop.hbase.TableExistsException;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  
50  
51  /**
52   * Tests for {@link FSTableDescriptors}.
53   */
54  // Do not support to be executed in he same JVM as other tests
55  @Category(MediumTests.class)
56  public class TestFSTableDescriptors {
57    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
58    private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
59  
60    @Test (expected=IllegalArgumentException.class)
61    public void testRegexAgainstOldStyleTableInfo() {
62      Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
63      int i = FSTableDescriptors.getTableInfoSequenceId(p);
64      assertEquals(0, i);
65      // Assert it won't eat garbage -- that it fails
66      p = new Path("/tmp", "abc");
67      FSTableDescriptors.getTableInfoSequenceId(p);
68    }
69  
70    @Test
71    public void testCreateAndUpdate() throws IOException {
72      Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
73      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
74      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
75      FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
76      assertTrue(fstd.createTableDescriptor(htd));
77      assertFalse(fstd.createTableDescriptor(htd));
78      FileStatus [] statuses = fs.listStatus(testdir);
79      assertTrue("statuses.length="+statuses.length, statuses.length == 1);
80      for (int i = 0; i < 10; i++) {
81        fstd.updateTableDescriptor(htd);
82      }
83      statuses = fs.listStatus(testdir);
84      assertTrue(statuses.length == 1);
85      Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
86      statuses = fs.listStatus(tmpTableDir);
87      assertTrue(statuses.length == 0);
88    }
89  
90    @Test
91    public void testSequenceIdAdvancesOnTableInfo() throws IOException {
92      Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
93      HTableDescriptor htd = new HTableDescriptor(
94          TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
95      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
96      FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
97      Path p0 = fstd.updateTableDescriptor(htd);
98      int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
99      Path p1 = fstd.updateTableDescriptor(htd);
100     // Assert we cleaned up the old file.
101     assertTrue(!fs.exists(p0));
102     int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
103     assertTrue(i1 == i0 + 1);
104     Path p2 = fstd.updateTableDescriptor(htd);
105     // Assert we cleaned up the old file.
106     assertTrue(!fs.exists(p1));
107     int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
108     assertTrue(i2 == i1 + 1);
109   }
110 
111   @Test
112   public void testFormatTableInfoSequenceId() {
113     Path p0 = assertWriteAndReadSequenceId(0);
114     // Assert p0 has format we expect.
115     StringBuilder sb = new StringBuilder();
116     for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
117       sb.append("0");
118     }
119     assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
120       p0.getName());
121     // Check a few more.
122     Path p2 = assertWriteAndReadSequenceId(2);
123     Path p10000 = assertWriteAndReadSequenceId(10000);
124     // Get a .tablinfo that has no sequenceid suffix.
125     Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
126     FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
127     FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
128     FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
129     FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
130     Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
131     assertTrue(comparator.compare(fs, fs0) > 0);
132     assertTrue(comparator.compare(fs0, fs2) > 0);
133     assertTrue(comparator.compare(fs2, fs10000) > 0);
134   }
135 
136   private Path assertWriteAndReadSequenceId(final int i) {
137     Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
138     int ii = FSTableDescriptors.getTableInfoSequenceId(p);
139     assertEquals(i, ii);
140     return p;
141   }
142 
143   @Test
144   public void testRemoves() throws IOException {
145     final String name = "testRemoves";
146     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
147     // Cleanup old tests if any detrius laying around.
148     Path rootdir = new Path(UTIL.getDataTestDir(), name);
149     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
150     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
151     htds.add(htd);
152     assertNotNull(htds.remove(htd.getTableName()));
153     assertNull(htds.remove(htd.getTableName()));
154   }
155 
156   @Test public void testReadingHTDFromFS() throws IOException {
157     final String name = "testReadingHTDFromFS";
158     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
159     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
160     Path rootdir = UTIL.getDataTestDir(name);
161     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
162     fstd.createTableDescriptor(htd);
163     HTableDescriptor htd2 =
164       FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
165     assertTrue(htd.equals(htd2));
166   }
167 
168   @Test public void testHTableDescriptors()
169   throws IOException, InterruptedException {
170     final String name = "testHTableDescriptors";
171     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
172     // Cleanup old tests if any debris laying around.
173     Path rootdir = new Path(UTIL.getDataTestDir(), name);
174     FSTableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir) {
175       @Override
176       public HTableDescriptor get(TableName tablename)
177           throws TableExistsException, FileNotFoundException, IOException {
178         LOG.info(tablename + ", cachehits=" + this.cachehits);
179         return super.get(tablename);
180       }
181     };
182     final int count = 10;
183     // Write out table infos.
184     for (int i = 0; i < count; i++) {
185       HTableDescriptor htd = new HTableDescriptor(name + i);
186       htds.createTableDescriptor(htd);
187     }
188 
189     for (int i = 0; i < count; i++) {
190       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
191     }
192     for (int i = 0; i < count; i++) {
193       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
194     }
195     // Update the table infos
196     for (int i = 0; i < count; i++) {
197       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
198       htd.addFamily(new HColumnDescriptor("" + i));
199       htds.updateTableDescriptor(htd);
200     }
201     // Wait a while so mod time we write is for sure different.
202     Thread.sleep(100);
203     for (int i = 0; i < count; i++) {
204       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
205     }
206     for (int i = 0; i < count; i++) {
207       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
208     }
209     assertEquals(count * 4, htds.invocations);
210     assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
211       htds.cachehits >= (count * 2));
212   }
213   @Test
214   public void testHTableDescriptorsNoCache()
215     throws IOException, InterruptedException {
216     final String name = "testHTableDescriptorsNoCache";
217     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
218     // Cleanup old tests if any debris laying around.
219     Path rootdir = new Path(UTIL.getDataTestDir(), name);
220     FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir,
221       false, false);
222     final int count = 10;
223     // Write out table infos.
224     for (int i = 0; i < count; i++) {
225       HTableDescriptor htd = new HTableDescriptor(name + i);
226       htds.createTableDescriptor(htd);
227     }
228 
229     for (int i = 0; i < count; i++) {
230       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
231     }
232     for (int i = 0; i < count; i++) {
233       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
234     }
235     // Update the table infos
236     for (int i = 0; i < count; i++) {
237       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
238       htd.addFamily(new HColumnDescriptor("" + i));
239       htds.updateTableDescriptor(htd);
240     }
241     // Wait a while so mod time we write is for sure different.
242     Thread.sleep(100);
243     for (int i = 0; i < count; i++) {
244       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
245     }
246     for (int i = 0; i < count; i++) {
247       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
248     }
249     assertEquals(count * 4, htds.invocations);
250     assertTrue("expected=0, actual=" + htds.cachehits,
251                htds.cachehits == 0);
252   }
253 
254   @Test
255   public void testGetAll()
256     throws IOException, InterruptedException {
257     final String name = "testGetAll";
258     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
259     // Cleanup old tests if any debris laying around.
260     Path rootdir = new Path(UTIL.getDataTestDir(), name);
261     FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
262     final int count = 4;
263     // Write out table infos.
264     for (int i = 0; i < count; i++) {
265       HTableDescriptor htd = new HTableDescriptor(name + i);
266       htds.createTableDescriptor(htd);
267     }
268     // add hbase:meta
269     HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
270     htds.createTableDescriptor(htd);
271 
272     assertTrue(htds.getAll().size() == count + 1);
273 
274   }
275 
276   @Test
277   public void testCacheConsistency()
278     throws IOException, InterruptedException {
279     final String name = "testCacheConsistency";
280     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
281     // Cleanup old tests if any debris laying around.
282     Path rootdir = new Path(UTIL.getDataTestDir(), name);
283     FSTableDescriptors chtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
284     FSTableDescriptors nonchtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs,
285       rootdir, false, false);
286 
287     final int count = 10;
288     // Write out table infos via non-cached FSTableDescriptors
289     for (int i = 0; i < count; i++) {
290       HTableDescriptor htd = new HTableDescriptor(name + i);
291       nonchtds.createTableDescriptor(htd);
292     }
293 
294     // Calls to getAll() won't increase the cache counter, do per table.
295     for (int i = 0; i < count; i++) {
296       assertTrue(chtds.get(TableName.valueOf(name + i)) !=  null);
297     }
298 
299     assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
300 
301     // add a new entry for hbase:meta
302     HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
303     nonchtds.createTableDescriptor(htd);
304 
305     // hbase:meta will only increase the cachehit by 1
306     assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
307 
308     for (Map.Entry entry: nonchtds.getAll().entrySet()) {
309       String t = (String) entry.getKey();
310       HTableDescriptor nchtd = (HTableDescriptor) entry.getValue();
311       assertTrue("expected " + htd.toString() +
312                    " got: " + chtds.get(TableName.valueOf(t)).toString(),
313                  (nchtd.equals(chtds.get(TableName.valueOf(t)))));
314     }
315   }
316 
317   @Test
318   public void testNoSuchTable() throws IOException {
319     final String name = "testNoSuchTable";
320     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
321     // Cleanup old tests if any detrius laying around.
322     Path rootdir = new Path(UTIL.getDataTestDir(), name);
323     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
324     assertNull("There shouldn't be any HTD for this table",
325       htds.get(TableName.valueOf("NoSuchTable")));
326   }
327 
328   @Test
329   public void testUpdates() throws IOException {
330     final String name = "testUpdates";
331     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
332     // Cleanup old tests if any detrius laying around.
333     Path rootdir = new Path(UTIL.getDataTestDir(), name);
334     TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
335     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
336     htds.add(htd);
337     htds.add(htd);
338     htds.add(htd);
339   }
340 
341   @Test
342   public void testTableInfoFileStatusComparator() {
343     FileStatus bare =
344       new FileStatus(0, false, 0, 0, -1,
345         new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
346     FileStatus future =
347       new FileStatus(0, false, 0, 0, -1,
348         new Path("/tmp/tablinfo." + System.currentTimeMillis()));
349     FileStatus farFuture =
350       new FileStatus(0, false, 0, 0, -1,
351         new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
352     FileStatus [] alist = {bare, future, farFuture};
353     FileStatus [] blist = {bare, farFuture, future};
354     FileStatus [] clist = {farFuture, bare, future};
355     Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
356     Arrays.sort(alist, c);
357     Arrays.sort(blist, c);
358     Arrays.sort(clist, c);
359     // Now assert all sorted same in way we want.
360     for (int i = 0; i < alist.length; i++) {
361       assertTrue(alist[i].equals(blist[i]));
362       assertTrue(blist[i].equals(clist[i]));
363       assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
364     }
365   }
366 
367   @Test
368   public void testReadingInvalidDirectoryFromFS() throws IOException {
369     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
370     try {
371       // .tmp dir is an invalid table name
372       new FSTableDescriptors(UTIL.getConfiguration(), fs,
373         FSUtils.getRootDir(UTIL.getConfiguration()))
374           .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
375       fail("Shouldn't be able to read a table descriptor for the archive directory.");
376     } catch (Exception e) {
377       LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
378           + e.getMessage());
379     }
380   }
381 
382   @Test
383   public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
384     Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
385     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
386         "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
387     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
388     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
389     assertTrue(fstd.createTableDescriptor(htd));
390     assertFalse(fstd.createTableDescriptor(htd));
391     htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
392     assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
393     Path tableDir = fstd.getTableDir(htd.getTableName());
394     Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
395     FileStatus[] statuses = fs.listStatus(tmpTableDir);
396     assertTrue(statuses.length == 0);
397 
398     assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
399   }
400 
401   private static class FSTableDescriptorsTest
402       extends FSTableDescriptors {
403 
404     public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir)
405     throws IOException {
406       this(conf, fs, rootdir, false, true);
407     }
408 
409     public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir,
410       boolean fsreadonly, boolean usecache)
411     throws IOException {
412       super(conf, fs, rootdir, fsreadonly, usecache);
413     }
414 
415     @Override
416     public HTableDescriptor get(TableName tablename)
417     throws TableExistsException, FileNotFoundException, IOException {
418       LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
419                    " HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
420       return super.get(tablename);
421     }
422   }
423 }
424