1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.InvocationTargetException;
30 import java.lang.reflect.Method;
31 import java.util.List;
32 import java.util.Map;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HBaseAdmin;
39 import org.apache.hadoop.hbase.client.HConnection;
40 import org.apache.hadoop.hbase.client.HConnectionManager;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.ResultScanner;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.LoadBalancer;
48 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
49 import org.apache.hadoop.hbase.testclassification.LargeTests;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.apache.hadoop.hbase.util.Threads;
53 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
54 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
55 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
56 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
57 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
58 import org.apache.zookeeper.CreateMode;
59 import org.apache.zookeeper.KeeperException;
60 import org.apache.zookeeper.ZooDefs;
61 import org.apache.zookeeper.ZooKeeper;
62 import org.apache.zookeeper.ZooKeeper.States;
63 import org.apache.zookeeper.data.ACL;
64 import org.apache.zookeeper.data.Stat;
65 import org.junit.After;
66 import org.junit.AfterClass;
67 import org.junit.Assert;
68 import org.junit.Before;
69 import org.junit.BeforeClass;
70 import org.junit.Test;
71 import org.junit.experimental.categories.Category;
72
73
74
75 @Category(LargeTests.class)
76 public class TestZooKeeper {
77 private final Log LOG = LogFactory.getLog(this.getClass());
78
79 private final static HBaseTestingUtility
80 TEST_UTIL = new HBaseTestingUtility();
81
82
83
84
85 @BeforeClass
86 public static void setUpBeforeClass() throws Exception {
87
88 Configuration conf = TEST_UTIL.getConfiguration();
89 TEST_UTIL.startMiniDFSCluster(2);
90 TEST_UTIL.startMiniZKCluster();
91 conf.setBoolean("dfs.support.append", true);
92 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
93 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
94 LoadBalancer.class);
95 }
96
97
98
99
100 @AfterClass
101 public static void tearDownAfterClass() throws Exception {
102 TEST_UTIL.shutdownMiniCluster();
103 }
104
105
106
107
108 @Before
109 public void setUp() throws Exception {
110 TEST_UTIL.startMiniHBaseCluster(1, 2);
111 }
112
113 @After
114 public void after() throws Exception {
115 try {
116 TEST_UTIL.shutdownMiniHBaseCluster();
117 } finally {
118 TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
119 ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
120 }
121 }
122
123 private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
124 throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
125 Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
126 getterZK.setAccessible(true);
127 return (ZooKeeperWatcher) getterZK.invoke(c);
128 }
129
130
131
132
133
134
135
136
137
138 public void testClientSessionExpired() throws Exception {
139 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
140
141
142 c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
143
144 HConnection connection = HConnectionManager.getConnection(c);
145
146 ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
147 LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
148 connectionZK.hashCode()));
149 LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
150 connectionZK.getRecoverableZooKeeper().hashCode()));
151 LOG.info("session="+Long.toHexString(
152 connectionZK.getRecoverableZooKeeper().getSessionId()));
153
154 TEST_UTIL.expireSession(connectionZK);
155
156 LOG.info("Before using zkw state=" +
157 connectionZK.getRecoverableZooKeeper().getState());
158
159 try {
160 connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
161 "/1/1", false);
162 } catch (KeeperException ignored) {
163 }
164
165
166 States state = connectionZK.getRecoverableZooKeeper().getState();
167 LOG.info("After using zkw state=" + state);
168 LOG.info("session="+Long.toHexString(
169 connectionZK.getRecoverableZooKeeper().getSessionId()));
170
171
172 final long limit1 = System.currentTimeMillis() + 3000;
173 while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
174 state = connectionZK.getRecoverableZooKeeper().getState();
175 }
176 LOG.info("After using zkw loop=" + state);
177 LOG.info("ZooKeeper should have timed out");
178 LOG.info("session="+Long.toHexString(
179 connectionZK.getRecoverableZooKeeper().getSessionId()));
180
181
182
183
184
185
186
187 ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
188
189 States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
190 LOG.info("After new get state=" +state2);
191
192
193
194 final long limit2 = System.currentTimeMillis() + 3000;
195 while (System.currentTimeMillis() < limit2 &&
196 state2 != States.CONNECTED && state2 != States.CONNECTING) {
197
198 newConnectionZK = getZooKeeperWatcher(connection);
199 state2 = newConnectionZK.getRecoverableZooKeeper().getState();
200 }
201 LOG.info("After new get state loop=" + state2);
202
203 Assert.assertTrue(
204 state2 == States.CONNECTED || state2 == States.CONNECTING);
205
206 connection.close();
207 }
208
209 @Test (timeout = 60000)
210 public void testRegionServerSessionExpired() throws Exception {
211 LOG.info("Starting testRegionServerSessionExpired");
212 int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
213 TEST_UTIL.expireRegionServerSession(metaIndex);
214 testSanity("testRegionServerSessionExpired");
215 }
216
217
218
219
220 public void testMasterSessionExpired() throws Exception {
221 LOG.info("Starting testMasterSessionExpired");
222 TEST_UTIL.expireMasterSession();
223 testSanity("testMasterSessionExpired");
224 }
225
226
227
228
229
230
231 @Test(timeout = 60000)
232 public void testMasterZKSessionRecoveryFailure() throws Exception {
233 LOG.info("Starting testMasterZKSessionRecoveryFailure");
234 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
235 HMaster m = cluster.getMaster();
236 m.abort("Test recovery from zk session expired",
237 new KeeperException.SessionExpiredException());
238 assertFalse(m.isStopped());
239 testSanity("testMasterZKSessionRecoveryFailure");
240 }
241
242
243
244
245
246 private void testSanity(final String testName) throws Exception{
247 String tableName = testName + "_" + System.currentTimeMillis();
248 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
249 HColumnDescriptor family = new HColumnDescriptor("fam");
250 desc.addFamily(family);
251 LOG.info("Creating table " + tableName);
252 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
253 try {
254 admin.createTable(desc);
255 } finally {
256 admin.close();
257 }
258
259 HTable table =
260 new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
261 Put put = new Put(Bytes.toBytes("testrow"));
262 put.add(Bytes.toBytes("fam"),
263 Bytes.toBytes("col"), Bytes.toBytes("testdata"));
264 LOG.info("Putting table " + tableName);
265 table.put(put);
266 table.close();
267 }
268
269 @Test
270 public void testMultipleZK()
271 throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
272 HTable localMeta =
273 new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
274 Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
275 otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
276 HTable ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
277
278
279 final byte [] row = new byte [] {'r'};
280 localMeta.exists(new Get(row));
281 ipMeta.exists(new Get(row));
282
283
284 ZooKeeperWatcher z1 =
285 getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
286 ZooKeeperWatcher z2 =
287 getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
288 assertFalse(z1 == z2);
289 assertFalse(z1.getQuorum().equals(z2.getQuorum()));
290
291 localMeta.close();
292 ipMeta.close();
293 }
294
295
296
297
298
299 @Test
300 public void testCreateWithParents() throws Exception {
301 ZooKeeperWatcher zkw =
302 new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
303 TestZooKeeper.class.getName(), null);
304 byte[] expectedData = new byte[] { 1, 2, 3 };
305 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
306 byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
307 assertTrue(Bytes.equals(expectedData, data));
308 ZKUtil.deleteNodeRecursively(zkw, "/l1");
309
310 ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
311 data = ZKUtil.getData(zkw, "/testCreateWithParents");
312 assertTrue(Bytes.equals(expectedData, data));
313 ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
314 }
315
316
317
318
319
320
321 @Test
322 public void testZNodeDeletes() throws Exception {
323 ZooKeeperWatcher zkw = new ZooKeeperWatcher(
324 new Configuration(TEST_UTIL.getConfiguration()),
325 TestZooKeeper.class.getName(), null);
326 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
327 try {
328 ZKUtil.deleteNode(zkw, "/l1/l2");
329 fail("We should not be able to delete if znode has childs");
330 } catch (KeeperException ex) {
331 assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
332 }
333 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
334
335 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
336
337
338 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
339
340 ZKUtil.deleteNode(zkw, "/l1");
341 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
342 }
343
344 @Test
345 public void testClusterKey() throws Exception {
346 testKey("server", "2181", "hbase");
347 testKey("server1,server2,server3", "2181", "hbase");
348 try {
349 ZKUtil.transformClusterKey("2181:hbase");
350 } catch (IOException ex) {
351
352 }
353 }
354
355 private void testKey(String ensemble, String port, String znode)
356 throws IOException {
357 Configuration conf = new Configuration();
358 String key = ensemble+":"+port+":"+znode;
359 String[] parts = ZKUtil.transformClusterKey(key);
360 assertEquals(ensemble, parts[0]);
361 assertEquals(port, parts[1]);
362 assertEquals(znode, parts[2]);
363 ZKUtil.applyClusterKeyToConf(conf, key);
364 assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM));
365 assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
366 assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
367 String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf);
368 assertEquals(key, reconstructedKey);
369 }
370
371
372
373
374
375
376
377
378 @Test
379 public void testCreateSilentIsReallySilent() throws InterruptedException,
380 KeeperException, IOException {
381 Configuration c = TEST_UTIL.getConfiguration();
382
383 String aclZnode = "/aclRoot";
384 String quorumServers = ZKConfig.getZKQuorumServersString(c);
385 int sessionTimeout = 5 * 1000;
386 ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
387 zk.addAuthInfo("digest", "hbase:rox".getBytes());
388
389
390
391 ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
392 "testCreateSilentIsReallySilent", null);
393
394
395 Stat s = null;
396 List<ACL> oldACL = null;
397 while (true) {
398 try {
399 s = new Stat();
400 oldACL = zk.getACL("/", s);
401 break;
402 } catch (KeeperException e) {
403 switch (e.code()) {
404 case CONNECTIONLOSS:
405 case SESSIONEXPIRED:
406 case OPERATIONTIMEOUT:
407 LOG.warn("Possibly transient ZooKeeper exception", e);
408 Threads.sleep(100);
409 break;
410 default:
411 throw e;
412 }
413 }
414 }
415
416
417
418 while (true) {
419 try {
420 zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
421 break;
422 } catch (KeeperException e) {
423 switch (e.code()) {
424 case CONNECTIONLOSS:
425 case SESSIONEXPIRED:
426 case OPERATIONTIMEOUT:
427 LOG.warn("Possibly transient ZooKeeper exception: " + e);
428 Threads.sleep(100);
429 break;
430 default:
431 throw e;
432 }
433 }
434 }
435
436 while (true) {
437 try {
438 zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
439 break;
440 } catch (KeeperException e) {
441 switch (e.code()) {
442 case CONNECTIONLOSS:
443 case SESSIONEXPIRED:
444 case OPERATIONTIMEOUT:
445 LOG.warn("Possibly transient ZooKeeper exception: " + e);
446 Threads.sleep(100);
447 break;
448 default:
449 throw e;
450 }
451 }
452 }
453 zk.close();
454 ZKUtil.createAndFailSilent(zk2, aclZnode);
455
456
457 ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
458 zk3.addAuthInfo("digest", "hbase:rox".getBytes());
459 try {
460 zk3.setACL("/", oldACL, -1);
461 } finally {
462 zk3.close();
463 }
464 }
465
466
467
468
469
470 @Test
471 @SuppressWarnings("deprecation")
472 public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
473 throws Exception {
474 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
475 "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
476 ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
477 }
478
479
480
481
482
483
484 @Test
485 public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
486 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
487 cluster.startRegionServer();
488 cluster.waitForActiveAndReadyMaster(10000);
489 HMaster m = cluster.getMaster();
490 ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
491 int expectedNumOfListeners = zkw.getNumberOfListeners();
492
493 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
494 try {
495 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
496 Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
497 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
498 String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
499 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
500 htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
501 admin.createTable(htd, SPLIT_KEYS);
502 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
503 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
504 m.getZooKeeperWatcher().close();
505 MockLoadBalancer.retainAssignCalled = false;
506 m.abort("Test recovery from zk session expired",
507 new KeeperException.SessionExpiredException());
508 assertFalse(m.isStopped());
509
510
511 assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
512
513
514 cluster.waitForActiveAndReadyMaster(10000);
515 assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
516 } finally {
517 admin.close();
518 }
519 }
520
521
522
523
524
525 @Test(timeout = 240000)
526 public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
527 KeeperException, InterruptedException {
528 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
529 cluster.startRegionServer();
530 HMaster m = cluster.getMaster();
531
532 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
533 HTable table = null;
534 try {
535 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
536 Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
537
538 String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
539 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
540 HColumnDescriptor hcd = new HColumnDescriptor("col");
541 htd.addFamily(hcd);
542 admin.createTable(htd, SPLIT_KEYS);
543 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
544 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
545 table = new HTable(TEST_UTIL.getConfiguration(), tableName);
546 Put p;
547 int numberOfPuts;
548 for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
549 p = new Put(Bytes.toBytes(numberOfPuts));
550 p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
551 table.put(p);
552 }
553 m.getZooKeeperWatcher().close();
554 m.abort("Test recovery from zk session expired",
555 new KeeperException.SessionExpiredException());
556 assertFalse(m.isStopped());
557 cluster.getRegionServer(0).abort("Aborting");
558
559
560 Scan scan = new Scan();
561 int numberOfRows = 0;
562 ResultScanner scanner = table.getScanner(scan);
563 Result[] result = scanner.next(1);
564 while (result != null && result.length > 0) {
565 numberOfRows++;
566 result = scanner.next(1);
567 }
568 assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
569 numberOfRows);
570 } finally {
571 if (table != null) table.close();
572 admin.close();
573 }
574 }
575
576 static class MockLoadBalancer extends SimpleLoadBalancer {
577 static boolean retainAssignCalled = false;
578
579 @Override
580 public Map<ServerName, List<HRegionInfo>> retainAssignment(
581 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
582 retainAssignCalled = true;
583 return super.retainAssignment(regions, servers);
584 }
585 }
586
587 }
588