View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.Set;
30  import java.util.concurrent.Callable;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.master.HMaster;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
45  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46  import com.google.common.collect.Sets;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Ignore;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  @Category(MediumTests.class)
56  public class TestNamespace {
57    protected static final Log LOG = LogFactory.getLog(TestNamespace.class);
58    private static HMaster master;
59    protected final static int NUM_SLAVES_BASE = 4;
60    private static HBaseTestingUtility TEST_UTIL;
61    protected static HBaseAdmin admin;
62    protected static HBaseCluster cluster;
63    private static ZKNamespaceManager zkNamespaceManager;
64    private String prefix = "TestNamespace";
65  
66  
67    @BeforeClass
68    public static void setUp() throws Exception {
69      TEST_UTIL = new HBaseTestingUtility();
70      TEST_UTIL.getConfiguration().setInt("hbase.namespacejanitor.interval", 5000);
71      TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
72      admin = TEST_UTIL.getHBaseAdmin();
73      cluster = TEST_UTIL.getHBaseCluster();
74      master = ((MiniHBaseCluster)cluster).getMaster();
75      zkNamespaceManager =
76          new ZKNamespaceManager(master.getZooKeeperWatcher());
77      zkNamespaceManager.start();
78      LOG.info("Done initializing cluster");
79    }
80  
81    @AfterClass
82    public static void tearDown() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    @Before
87    public void beforeMethod() throws IOException {
88      for (HTableDescriptor desc : admin.listTables(prefix+".*")) {
89        admin.disableTable(desc.getTableName());
90        admin.deleteTable(desc.getTableName());
91      }
92      for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) {
93        if (ns.getName().startsWith(prefix)) {
94          admin.deleteNamespace(ns.getName());
95        }
96      }
97    }
98  
99    @Test
100   public void verifyReservedNS() throws IOException {
101     //verify existence of reserved namespaces
102     NamespaceDescriptor ns =
103         admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
104     assertNotNull(ns);
105     assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
106     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
107 
108     ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
109     assertNotNull(ns);
110     assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
111     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR));
112 
113     assertEquals(2, admin.listNamespaceDescriptors().length);
114 
115     //verify existence of system tables
116     Set<TableName> systemTables = Sets.newHashSet(
117         TableName.META_TABLE_NAME,
118         TableName.NAMESPACE_TABLE_NAME);
119     HTableDescriptor[] descs =
120         admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
121     assertEquals(systemTables.size(), descs.length);
122     for (HTableDescriptor desc : descs) {
123       assertTrue(systemTables.contains(desc.getTableName()));
124     }
125     //verify system tables aren't listed
126     assertEquals(0, admin.listTables().length);
127 
128     //Try creating default and system namespaces.
129     boolean exceptionCaught = false;
130     try {
131       admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
132     } catch (IOException exp) {
133       LOG.warn(exp);
134       exceptionCaught = true;
135     } finally {
136       assertTrue(exceptionCaught);
137     }
138 
139     exceptionCaught = false;
140     try {
141       admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
142     } catch (IOException exp) {
143       LOG.warn(exp);
144       exceptionCaught = true;
145     } finally {
146       assertTrue(exceptionCaught);
147     }
148   }
149 
150   @Test
151   public void testDeleteReservedNS() throws Exception {
152     boolean exceptionCaught = false;
153     try {
154       admin.deleteNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
155     } catch (IOException exp) {
156       LOG.warn(exp);
157       exceptionCaught = true;
158     } finally {
159       assertTrue(exceptionCaught);
160     }
161 
162     try {
163       admin.deleteNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
164     } catch (IOException exp) {
165       LOG.warn(exp);
166       exceptionCaught = true;
167     } finally {
168       assertTrue(exceptionCaught);
169     }
170   }
171 
172   @Test
173   public void createRemoveTest() throws Exception {
174     String testName = "createRemoveTest";
175     String nsName = prefix+"_"+testName;
176     LOG.info(testName);
177 
178     //create namespace and verify
179     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
180     assertEquals(3, admin.listNamespaceDescriptors().length);
181     TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
182       @Override
183       public boolean evaluate() throws Exception {
184         return zkNamespaceManager.list().size() == 3;
185       }
186     });
187     assertNotNull(zkNamespaceManager.get(nsName));
188     //remove namespace and verify
189     admin.deleteNamespace(nsName);
190     assertEquals(2, admin.listNamespaceDescriptors().length);
191     assertEquals(2, zkNamespaceManager.list().size());
192     assertNull(zkNamespaceManager.get(nsName));
193   }
194 
195   @Test
196   public void createDoubleTest() throws IOException, InterruptedException {
197     String testName = "createDoubleTest";
198     String nsName = prefix+"_"+testName;
199     LOG.info(testName);
200 
201     byte[] tableName = Bytes.toBytes("my_table");
202     byte[] tableNameFoo = Bytes.toBytes(nsName+":my_table");
203     //create namespace and verify
204     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
205     TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
206     TEST_UTIL.createTable(tableNameFoo,Bytes.toBytes(nsName));
207     assertEquals(2, admin.listTables().length);
208     assertNotNull(admin
209         .getTableDescriptor(tableName));
210     assertNotNull(admin
211         .getTableDescriptor(tableNameFoo));
212     //remove namespace and verify
213     admin.disableTable(tableName);
214     admin.deleteTable(tableName);
215     assertEquals(1, admin.listTables().length);
216   }
217 
218   @Test
219   public void createTableTest() throws IOException, InterruptedException {
220     String testName = "createTableTest";
221     String nsName = prefix+"_"+testName;
222     LOG.info(testName);
223 
224     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName+":my_table"));
225     HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
226     desc.addFamily(colDesc);
227     try {
228       admin.createTable(desc);
229       fail("Expected no namespace exists exception");
230     } catch (NamespaceNotFoundException ex) {
231     }
232     //create table and in new namespace
233     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
234     admin.createTable(desc);
235     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
236     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
237     assertTrue(fs.exists(
238         new Path(master.getMasterFileSystem().getRootDir(),
239             new Path(HConstants.BASE_NAMESPACE_DIR,
240                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
241     assertEquals(1, admin.listTables().length);
242 
243     //verify non-empty namespace can't be removed
244     try {
245       admin.deleteNamespace(nsName);
246       fail("Expected non-empty namespace constraint exception");
247     } catch (Exception ex) {
248       LOG.info("Caught expected exception: " + ex);
249     }
250 
251     //sanity check try to write and read from table
252     HTable table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
253     Put p = new Put(Bytes.toBytes("row1"));
254     p.add(Bytes.toBytes("my_cf"),Bytes.toBytes("my_col"),Bytes.toBytes("value1"));
255     table.put(p);
256     //flush and read from disk to make sure directory changes are working
257     admin.flush(desc.getTableName().getName());
258     Get g = new Get(Bytes.toBytes("row1"));
259     assertTrue(table.exists(g));
260 
261     //normal case of removing namespace
262     TEST_UTIL.deleteTable(desc.getTableName());
263     admin.deleteNamespace(nsName);
264   }
265 
266   @Test
267   public void createTableInDefaultNamespace() throws Exception {
268     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table"));
269     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
270     desc.addFamily(colDesc);
271     admin.createTable(desc);
272     assertTrue(admin.listTables().length == 1);
273     admin.disableTable(desc.getTableName());
274     admin.deleteTable(desc.getTableName());
275   }
276 
277   @Test
278   public void createTableInSystemNamespace() throws Exception {
279     String tableName = "hbase:createTableInSystemNamespace";
280     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
281     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
282     desc.addFamily(colDesc);
283     admin.createTable(desc);
284     assertEquals(0, admin.listTables().length);
285     assertTrue(admin.tableExists(Bytes.toBytes(tableName)));
286     admin.disableTable(desc.getTableName());
287     admin.deleteTable(desc.getTableName());
288   }
289 
290   @Ignore @Test
291   public void testNamespaceJanitor() throws Exception {
292     FileSystem fs = TEST_UTIL.getTestFileSystem();
293 
294     int fsCount = fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
295         HConstants.BASE_NAMESPACE_DIR)).length;
296     Path fakeNSPath =
297         FSUtils.getNamespaceDir(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), "foo");
298     assertTrue(fs.mkdirs(fakeNSPath));
299 
300     String fakeZnode = ZKUtil.joinZNode(ZooKeeperWatcher.namespaceZNode, "foo");
301     int zkCount = ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
302         ZooKeeperWatcher.namespaceZNode).size();
303     ZKUtil.createWithParents(TEST_UTIL.getZooKeeperWatcher(), fakeZnode);
304     Thread.sleep(10000);
305 
306     //verify namespace count is the same and orphan is removed
307     assertFalse(fs.exists(fakeNSPath));
308     assertEquals(fsCount, fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
309             HConstants.BASE_NAMESPACE_DIR)).length);
310 
311     assertEquals(-1, ZKUtil.checkExists(TEST_UTIL.getZooKeeperWatcher(), fakeZnode));
312     assertEquals(zkCount,
313         ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
314             ZooKeeperWatcher.namespaceZNode).size());
315   }
316 
317   @Test(timeout = 60000)
318   public void testNamespaceOperations() throws IOException {
319     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
320     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns2").build());
321 
322     // create namespace that already exists
323     runWithExpectedException(new Callable<Void>() {
324       @Override
325       public Void call() throws Exception {
326         admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
327         return null;
328       }
329     }, NamespaceExistException.class);
330 
331     // create a table in non-existing namespace
332     runWithExpectedException(new Callable<Void>() {
333       @Override
334       public Void call() throws Exception {
335         HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("non_existing_namespace", "table1"));
336         htd.addFamily(new HColumnDescriptor("family1"));
337         admin.createTable(htd);
338         return null;
339       }
340     }, NamespaceNotFoundException.class);
341 
342     // get descriptor for existing namespace
343     admin.getNamespaceDescriptor(prefix + "ns1");
344 
345     // get descriptor for non-existing namespace
346     runWithExpectedException(new Callable<NamespaceDescriptor>() {
347       @Override
348       public NamespaceDescriptor call() throws Exception {
349         return admin.getNamespaceDescriptor("non_existing_namespace");
350       }
351     }, NamespaceNotFoundException.class);
352 
353     // delete descriptor for existing namespace
354     admin.deleteNamespace(prefix + "ns2");
355 
356     // delete descriptor for non-existing namespace
357     runWithExpectedException(new Callable<Void>() {
358       @Override
359       public Void call() throws Exception {
360         admin.deleteNamespace("non_existing_namespace");
361         return null;
362       }
363     }, NamespaceNotFoundException.class);
364 
365     // modify namespace descriptor for existing namespace
366     NamespaceDescriptor ns1 = admin.getNamespaceDescriptor(prefix + "ns1");
367     ns1.setConfiguration("foo", "bar");
368     admin.modifyNamespace(ns1);
369 
370     // modify namespace descriptor for non-existing namespace
371     runWithExpectedException(new Callable<Void>() {
372       @Override
373       public Void call() throws Exception {
374         admin.modifyNamespace(NamespaceDescriptor.create("non_existing_namespace").build());
375         return null;
376       }
377     }, NamespaceNotFoundException.class);
378 
379     // get table descriptors for existing namespace
380     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(prefix + "ns1", "table1"));
381     htd.addFamily(new HColumnDescriptor("family1"));
382     admin.createTable(htd);
383     HTableDescriptor[] htds = admin.listTableDescriptorsByNamespace(prefix + "ns1");
384     assertNotNull("Should have not returned null", htds);
385     assertEquals("Should have returned non-empty array", 1, htds.length);
386 
387     // get table descriptors for non-existing namespace
388     runWithExpectedException(new Callable<Void>() {
389       @Override
390       public Void call() throws Exception {
391         admin.listTableDescriptorsByNamespace("non_existing_namespace");
392         return null;
393       }
394     }, NamespaceNotFoundException.class);
395 
396     // get table names for existing namespace
397     TableName[] tableNames = admin.listTableNamesByNamespace(prefix + "ns1");
398     assertNotNull("Should have not returned null", tableNames);
399     assertEquals("Should have returned non-empty array", 1, tableNames.length);
400 
401     // get table names for non-existing namespace
402     runWithExpectedException(new Callable<Void>() {
403       @Override
404       public Void call() throws Exception {
405         admin.listTableNamesByNamespace("non_existing_namespace");
406         return null;
407       }
408     }, NamespaceNotFoundException.class);
409 
410   }
411 
412   private static <V, E> void runWithExpectedException(Callable<V> callable, Class<E> exceptionClass) {
413     try {
414       callable.call();
415     } catch(Exception ex) {
416       Assert.assertEquals(exceptionClass, ex.getClass());
417       return;
418     }
419     fail("Should have thrown exception " + exceptionClass);
420   }
421 
422 }