1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.SplitLogCounters.tot_mgr_wait_for_zk_delete;
22 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_final_transition_failed;
23 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_preempt_task;
24 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_acquired;
25 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_done;
26 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_err;
27 import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_resigned;
28 import static org.junit.Assert.*;
29 import java.io.IOException;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.HashSet;
33 import java.util.Iterator;
34 import java.util.LinkedList;
35 import java.util.List;
36 import java.util.NavigableSet;
37 import java.util.Set;
38 import java.util.TreeSet;
39 import java.util.concurrent.ExecutorService;
40 import java.util.concurrent.Executors;
41 import java.util.concurrent.Future;
42 import java.util.concurrent.TimeUnit;
43 import java.util.concurrent.TimeoutException;
44 import java.util.concurrent.atomic.AtomicLong;
45
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.apache.hadoop.conf.Configuration;
49 import org.apache.hadoop.fs.FSDataOutputStream;
50 import org.apache.hadoop.fs.FileStatus;
51 import org.apache.hadoop.fs.FileSystem;
52 import org.apache.hadoop.fs.Path;
53 import org.apache.hadoop.hbase.HColumnDescriptor;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.HBaseConfiguration;
56 import org.apache.hadoop.hbase.HBaseTestingUtility;
57 import org.apache.hadoop.hbase.HConstants;
58 import org.apache.hadoop.hbase.HRegionInfo;
59 import org.apache.hadoop.hbase.HTableDescriptor;
60 import org.apache.hadoop.hbase.KeyValue;
61 import org.apache.hadoop.hbase.testclassification.LargeTests;
62 import org.apache.hadoop.hbase.MiniHBaseCluster;
63 import org.apache.hadoop.hbase.NamespaceDescriptor;
64 import org.apache.hadoop.hbase.ServerName;
65 import org.apache.hadoop.hbase.SplitLogCounters;
66 import org.apache.hadoop.hbase.Waiter;
67 import org.apache.hadoop.hbase.client.Delete;
68 import org.apache.hadoop.hbase.client.Get;
69 import org.apache.hadoop.hbase.client.HConnectionManager;
70 import org.apache.hadoop.hbase.client.HTable;
71 import org.apache.hadoop.hbase.client.Increment;
72 import org.apache.hadoop.hbase.client.NonceGenerator;
73 import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator;
74 import org.apache.hadoop.hbase.client.Put;
75 import org.apache.hadoop.hbase.client.Result;
76 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
77 import org.apache.hadoop.hbase.exceptions.OperationConflictException;
78 import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
79 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
80 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
81 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
82 import org.apache.hadoop.hbase.regionserver.HRegion;
83 import org.apache.hadoop.hbase.regionserver.HRegionServer;
84 import org.apache.hadoop.hbase.regionserver.wal.HLog;
85 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
86 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
87 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
88 import org.apache.hadoop.hbase.util.Bytes;
89 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
90 import org.apache.hadoop.hbase.util.FSUtils;
91 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
92 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
93 import org.apache.hadoop.hbase.util.Threads;
94 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
95 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
96 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
97 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
98 import org.apache.hadoop.hdfs.MiniDFSCluster;
99 import org.apache.zookeeper.KeeperException;
100 import org.junit.After;
101 import org.junit.AfterClass;
102 import org.junit.Assert;
103 import org.junit.Before;
104 import org.junit.BeforeClass;
105 import org.junit.Test;
106 import org.junit.experimental.categories.Category;
107
108 @Category(LargeTests.class)
109 public class TestDistributedLogSplitting {
110 private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class);
111 static {
112
113
114
115
116
117
118 System.setProperty("hbase.tests.use.shortcircuit.reads", "false");
119
120 }
121
122
123 static final int NUM_MASTERS = 2;
124 static final int NUM_RS = 6;
125
126 MiniHBaseCluster cluster;
127 HMaster master;
128 Configuration conf;
129 static Configuration originalConf;
130 static HBaseTestingUtility TEST_UTIL;
131 static MiniDFSCluster dfsCluster;
132 static MiniZooKeeperCluster zkCluster;
133
134 @BeforeClass
135 public static void setup() throws Exception {
136 TEST_UTIL = new HBaseTestingUtility(HBaseConfiguration.create());
137 dfsCluster = TEST_UTIL.startMiniDFSCluster(1);
138 zkCluster = TEST_UTIL.startMiniZKCluster();
139 originalConf = TEST_UTIL.getConfiguration();
140 }
141
142 @AfterClass
143 public static void tearDown() throws IOException {
144 TEST_UTIL.shutdownMiniZKCluster();
145 TEST_UTIL.shutdownMiniDFSCluster();
146 }
147
148 private void startCluster(int num_rs) throws Exception {
149 SplitLogCounters.resetCounters();
150 LOG.info("Starting cluster");
151 conf.getLong("hbase.splitlog.max.resubmit", 0);
152
153 conf.setInt("zookeeper.recovery.retry", 0);
154 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
155 conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 100.0);
156 conf.setInt("hbase.regionserver.wal.max.splitters", 3);
157 conf.setInt("hfile.format.version", 3);
158 TEST_UTIL = new HBaseTestingUtility(conf);
159 TEST_UTIL.setDFSCluster(dfsCluster);
160 TEST_UTIL.setZkCluster(zkCluster);
161 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, num_rs);
162 cluster = TEST_UTIL.getHBaseCluster();
163 LOG.info("Waiting for active/ready master");
164 cluster.waitForActiveAndReadyMaster();
165 master = cluster.getMaster();
166 while (cluster.getLiveRegionServerThreads().size() < num_rs) {
167 Threads.sleep(1);
168 }
169 }
170
171 @Before
172 public void before() throws Exception {
173
174 conf = HBaseConfiguration.create(originalConf);
175 }
176
177 @After
178 public void after() throws Exception {
179 try {
180 if (TEST_UTIL.getHBaseCluster() != null) {
181 for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) {
182 mt.getMaster().abort("closing...", null);
183 }
184 }
185 TEST_UTIL.shutdownMiniHBaseCluster();
186 } finally {
187 TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
188 ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
189 }
190 }
191
192 @Test (timeout=300000)
193 public void testRecoveredEdits() throws Exception {
194 LOG.info("testRecoveredEdits");
195 conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024);
196 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
197 startCluster(NUM_RS);
198
199 final int NUM_LOG_LINES = 1000;
200 final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
201
202
203 master.balanceSwitch(false);
204 FileSystem fs = master.getMasterFileSystem().getFileSystem();
205
206 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
207
208 Path rootdir = FSUtils.getRootDir(conf);
209
210 installTable(new ZooKeeperWatcher(conf, "table-creation", null),
211 "table", "family", 40);
212 TableName table = TableName.valueOf("table");
213 List<HRegionInfo> regions = null;
214 HRegionServer hrs = null;
215 for (int i = 0; i < NUM_RS; i++) {
216 boolean foundRs = false;
217 hrs = rsts.get(i).getRegionServer();
218 regions = ProtobufUtil.getOnlineRegions(hrs);
219 for (HRegionInfo region : regions) {
220 if (region.getTable().getNameAsString().equalsIgnoreCase("table")) {
221 foundRs = true;
222 break;
223 }
224 }
225 if (foundRs) break;
226 }
227 final Path logDir = new Path(rootdir, HLogUtil.getHLogDirectoryName(hrs
228 .getServerName().toString()));
229
230 LOG.info("#regions = " + regions.size());
231 Iterator<HRegionInfo> it = regions.iterator();
232 while (it.hasNext()) {
233 HRegionInfo region = it.next();
234 if (region.getTable().getNamespaceAsString()
235 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
236 it.remove();
237 }
238 }
239 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
240
241 slm.splitLogDistributed(logDir);
242
243 int count = 0;
244 for (HRegionInfo hri : regions) {
245
246 Path tdir = FSUtils.getTableDir(rootdir, table);
247 @SuppressWarnings("deprecation")
248 Path editsdir =
249 HLogUtil.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
250 LOG.debug("checking edits dir " + editsdir);
251 FileStatus[] files = fs.listStatus(editsdir);
252 assertTrue(files.length > 1);
253 for (int i = 0; i < files.length; i++) {
254 int c = countHLog(files[i].getPath(), fs, conf);
255 count += c;
256 }
257 LOG.info(count + " edits in " + files.length + " recovered edits files.");
258 }
259
260
261 assertFalse(fs.exists(logDir));
262
263 assertEquals(NUM_LOG_LINES, count);
264 }
265
266 @Test(timeout = 300000)
267 public void testLogReplayWithNonMetaRSDown() throws Exception {
268 LOG.info("testLogReplayWithNonMetaRSDown");
269 conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024);
270 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
271 startCluster(NUM_RS);
272 final int NUM_REGIONS_TO_CREATE = 40;
273 final int NUM_LOG_LINES = 1000;
274
275
276 master.balanceSwitch(false);
277
278 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
279 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
280
281 HRegionServer hrs = findRSToKill(false, "table");
282 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
283 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
284
285
286 this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
287 ht.close();
288 zkw.close();
289 }
290
291 private static class NonceGeneratorWithDups extends PerClientRandomNonceGenerator {
292 private boolean isDups = false;
293 private LinkedList<Long> nonces = new LinkedList<Long>();
294
295 public void startDups() {
296 isDups = true;
297 }
298
299 @Override
300 public long newNonce() {
301 long nonce = isDups ? nonces.removeFirst() : super.newNonce();
302 if (!isDups) {
303 nonces.add(nonce);
304 }
305 return nonce;
306 }
307 }
308
309 @Test(timeout = 300000)
310 public void testNonceRecovery() throws Exception {
311 LOG.info("testNonceRecovery");
312 final String TABLE_NAME = "table";
313 final String FAMILY_NAME = "family";
314 final int NUM_REGIONS_TO_CREATE = 40;
315
316 conf.setLong("hbase.regionserver.hlog.blocksize", 100*1024);
317 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
318 startCluster(NUM_RS);
319 master.balanceSwitch(false);
320
321 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
322 HTable ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE);
323 NonceGeneratorWithDups ng = new NonceGeneratorWithDups();
324 NonceGenerator oldNg =
325 HConnectionManager.injectNonceGeneratorForTesting(ht.getConnection(), ng);
326
327 try {
328 List<Increment> reqs = new ArrayList<Increment>();
329 for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
330 HRegionServer hrs = rst.getRegionServer();
331 List<HRegionInfo> hris = ProtobufUtil.getOnlineRegions(hrs);
332 for (HRegionInfo hri : hris) {
333 if (TABLE_NAME.equalsIgnoreCase(hri.getTable().getNameAsString())) {
334 byte[] key = hri.getStartKey();
335 if (key == null || key.length == 0) {
336 key = Bytes.copy(hri.getEndKey());
337 --(key[key.length - 1]);
338 }
339 Increment incr = new Increment(key);
340 incr.addColumn(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes("q"), 1);
341 ht.increment(incr);
342 reqs.add(incr);
343 }
344 }
345 }
346
347 HRegionServer hrs = findRSToKill(false, "table");
348 abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
349 ng.startDups();
350 for (Increment incr : reqs) {
351 try {
352 ht.increment(incr);
353 fail("should have thrown");
354 } catch (OperationConflictException ope) {
355 LOG.debug("Caught as expected: " + ope.getMessage());
356 }
357 }
358 } finally {
359 HConnectionManager.injectNonceGeneratorForTesting(ht.getConnection(), oldNg);
360 ht.close();
361 zkw.close();
362 }
363 }
364
365 @Test(timeout = 300000)
366 public void testLogReplayWithMetaRSDown() throws Exception {
367 LOG.info("testRecoveredEditsReplayWithMetaRSDown");
368 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
369 startCluster(NUM_RS);
370 final int NUM_REGIONS_TO_CREATE = 40;
371 final int NUM_LOG_LINES = 1000;
372
373
374 master.balanceSwitch(false);
375
376 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
377 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
378
379 HRegionServer hrs = findRSToKill(true, "table");
380 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
381 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
382
383 this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
384 ht.close();
385 zkw.close();
386 }
387
388 private void abortRSAndVerifyRecovery(HRegionServer hrs, HTable ht, final ZooKeeperWatcher zkw,
389 final int numRegions, final int numofLines) throws Exception {
390
391 abortRSAndWaitForRecovery(hrs, zkw, numRegions);
392 assertEquals(numofLines, TEST_UTIL.countRows(ht));
393 }
394
395 private void abortRSAndWaitForRecovery(HRegionServer hrs, final ZooKeeperWatcher zkw,
396 final int numRegions) throws Exception {
397 final MiniHBaseCluster tmpCluster = this.cluster;
398
399
400 LOG.info("Aborting region server: " + hrs.getServerName());
401 hrs.abort("testing");
402
403
404 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
405 @Override
406 public boolean evaluate() throws Exception {
407 return (tmpCluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
408 }
409 });
410
411
412 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
413 @Override
414 public boolean evaluate() throws Exception {
415 return (getAllOnlineRegions(tmpCluster).size() >= (numRegions + 1));
416 }
417 });
418
419
420 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
421 @Override
422 public boolean evaluate() throws Exception {
423 List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
424 zkw.recoveringRegionsZNode, false);
425 return (recoveringRegions != null && recoveringRegions.size() == 0);
426 }
427 });
428 }
429
430 @Test(timeout = 300000)
431 public void testMasterStartsUpWithLogSplittingWork() throws Exception {
432 LOG.info("testMasterStartsUpWithLogSplittingWork");
433 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
434 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
435 startCluster(NUM_RS);
436
437 final int NUM_REGIONS_TO_CREATE = 40;
438 final int NUM_LOG_LINES = 1000;
439
440
441 master.balanceSwitch(false);
442
443 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
444 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
445 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
446
447 HRegionServer hrs = findRSToKill(false, "table");
448 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
449 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
450
451
452 abortMaster(cluster);
453
454
455 LOG.info("Aborting region server: " + hrs.getServerName());
456 hrs.abort("testing");
457
458
459 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
460 @Override
461 public boolean evaluate() throws Exception {
462 return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
463 }
464 });
465
466 Thread.sleep(2000);
467 LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
468
469 startMasterAndWaitUntilLogSplit(cluster);
470
471
472 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
473 @Override
474 public boolean evaluate() throws Exception {
475 return (getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
476 }
477 });
478
479 LOG.info("Current Open Regions After Master Node Starts Up:"
480 + getAllOnlineRegions(cluster).size());
481
482 assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
483
484 ht.close();
485 zkw.close();
486 }
487
488 @Test(timeout = 300000)
489 public void testMasterStartsUpWithLogReplayWork() throws Exception {
490 LOG.info("testMasterStartsUpWithLogReplayWork");
491 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
492 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
493 startCluster(NUM_RS);
494
495 final int NUM_REGIONS_TO_CREATE = 40;
496 final int NUM_LOG_LINES = 1000;
497
498
499 master.balanceSwitch(false);
500
501 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
502 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
503 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
504
505 HRegionServer hrs = findRSToKill(false, "table");
506 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
507 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
508
509
510 abortMaster(cluster);
511
512
513 LOG.info("Aborting region server: " + hrs.getServerName());
514 hrs.abort("testing");
515
516
517 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
518 @Override
519 public boolean evaluate() throws Exception {
520 return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
521 }
522 });
523
524 Thread.sleep(2000);
525 LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size());
526
527 startMasterAndWaitUntilLogSplit(cluster);
528
529
530 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
531 @Override
532 public boolean evaluate() throws Exception {
533 List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
534 zkw.recoveringRegionsZNode, false);
535 return (recoveringRegions != null && recoveringRegions.size() == 0);
536 }
537 });
538
539 LOG.info("Current Open Regions After Master Node Starts Up:"
540 + getAllOnlineRegions(cluster).size());
541
542 assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
543
544 ht.close();
545 zkw.close();
546 }
547
548
549 @Test(timeout = 300000)
550 public void testLogReplayTwoSequentialRSDown() throws Exception {
551 LOG.info("testRecoveredEditsReplayTwoSequentialRSDown");
552 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
553 startCluster(NUM_RS);
554 final int NUM_REGIONS_TO_CREATE = 40;
555 final int NUM_LOG_LINES = 1000;
556
557
558 master.balanceSwitch(false);
559
560 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
561 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
562 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
563
564 List<HRegionInfo> regions = null;
565 HRegionServer hrs1 = findRSToKill(false, "table");
566 regions = ProtobufUtil.getOnlineRegions(hrs1);
567
568 makeHLog(hrs1.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
569
570
571 LOG.info("Aborting region server: " + hrs1.getServerName());
572 hrs1.abort("testing");
573
574
575 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
576 @Override
577 public boolean evaluate() throws Exception {
578 return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
579 }
580 });
581
582
583 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
584 @Override
585 public boolean evaluate() throws Exception {
586 return (getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
587 }
588 });
589
590
591 Thread.sleep(300);
592
593 rsts = cluster.getLiveRegionServerThreads();
594 HRegionServer hrs2 = rsts.get(0).getRegionServer();
595 LOG.info("Aborting one more region server: " + hrs2.getServerName());
596 hrs2.abort("testing");
597
598
599 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
600 @Override
601 public boolean evaluate() throws Exception {
602 return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2));
603 }
604 });
605
606
607 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
608 @Override
609 public boolean evaluate() throws Exception {
610 return (getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
611 }
612 });
613
614
615 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
616 @Override
617 public boolean evaluate() throws Exception {
618 List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
619 zkw.recoveringRegionsZNode, false);
620 return (recoveringRegions != null && recoveringRegions.size() == 0);
621 }
622 });
623
624 assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
625 ht.close();
626 zkw.close();
627 }
628
629 @Test(timeout = 300000)
630 public void testMarkRegionsRecoveringInZK() throws Exception {
631 LOG.info("testMarkRegionsRecoveringInZK");
632 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
633 startCluster(NUM_RS);
634 master.balanceSwitch(false);
635 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
636 final ZooKeeperWatcher zkw = master.getZooKeeperWatcher();
637 HTable ht = installTable(zkw, "table", "family", 40);
638 final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
639
640 Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
641 HRegionInfo region = null;
642 HRegionServer hrs = null;
643 ServerName firstFailedServer = null;
644 ServerName secondFailedServer = null;
645 for (int i = 0; i < NUM_RS; i++) {
646 hrs = rsts.get(i).getRegionServer();
647 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
648 if (regions.isEmpty()) continue;
649 region = regions.get(0);
650 regionSet.add(region);
651 firstFailedServer = hrs.getServerName();
652 secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName();
653 break;
654 }
655
656 slm.markRegionsRecoveringInZK(firstFailedServer, regionSet);
657 slm.markRegionsRecoveringInZK(secondFailedServer, regionSet);
658
659 List<String> recoveringRegions = ZKUtil.listChildrenNoWatch(zkw,
660 ZKUtil.joinZNode(zkw.recoveringRegionsZNode, region.getEncodedName()));
661
662 assertEquals(recoveringRegions.size(), 2);
663
664
665 final HRegionServer tmphrs = hrs;
666 TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
667 @Override
668 public boolean evaluate() throws Exception {
669 return (tmphrs.getRecoveringRegions().size() == 0);
670 }
671 });
672 ht.close();
673 zkw.close();
674 }
675
676 @Test(timeout = 300000)
677 public void testReplayCmd() throws Exception {
678 LOG.info("testReplayCmd");
679 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
680 startCluster(NUM_RS);
681 final int NUM_REGIONS_TO_CREATE = 40;
682
683
684 master.balanceSwitch(false);
685
686 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
687 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
688 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
689
690 List<HRegionInfo> regions = null;
691 HRegionServer hrs = null;
692 for (int i = 0; i < NUM_RS; i++) {
693 boolean isCarryingMeta = false;
694 hrs = rsts.get(i).getRegionServer();
695 regions = ProtobufUtil.getOnlineRegions(hrs);
696 for (HRegionInfo region : regions) {
697 if (region.isMetaRegion()) {
698 isCarryingMeta = true;
699 break;
700 }
701 }
702 if (isCarryingMeta) {
703 continue;
704 }
705 if (regions.size() > 0) break;
706 }
707
708 this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1"));
709 String originalCheckSum = TEST_UTIL.checksumRows(ht);
710
711
712 abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
713
714 assertEquals("Data should remain after reopening of regions", originalCheckSum,
715 TEST_UTIL.checksumRows(ht));
716
717 ht.close();
718 zkw.close();
719 }
720
721 @Test(timeout = 300000)
722 public void testLogReplayForDisablingTable() throws Exception {
723 LOG.info("testLogReplayForDisablingTable");
724 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
725 startCluster(NUM_RS);
726 final int NUM_REGIONS_TO_CREATE = 40;
727 final int NUM_LOG_LINES = 1000;
728
729 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
730 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
731 HTable disablingHT = installTable(zkw, "disableTable", "family", NUM_REGIONS_TO_CREATE);
732 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE, NUM_REGIONS_TO_CREATE);
733
734
735
736 master.balanceSwitch(false);
737
738 List<HRegionInfo> regions = null;
739 HRegionServer hrs = null;
740 boolean hasRegionsForBothTables = false;
741 String tableName = null;
742 for (int i = 0; i < NUM_RS; i++) {
743 tableName = null;
744 hasRegionsForBothTables = false;
745 boolean isCarryingSystem = false;
746 hrs = rsts.get(i).getRegionServer();
747 regions = ProtobufUtil.getOnlineRegions(hrs);
748 for (HRegionInfo region : regions) {
749 if (region.getTable().isSystemTable()) {
750 isCarryingSystem = true;
751 break;
752 }
753 if (tableName != null &&
754 !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) {
755
756 hasRegionsForBothTables = true;
757 break;
758 } else if (tableName == null) {
759 tableName = region.getTable().getNameAsString();
760 }
761 }
762 if (isCarryingSystem) {
763 continue;
764 }
765 if (hasRegionsForBothTables) {
766 break;
767 }
768 }
769
770
771 Assert.assertTrue(hasRegionsForBothTables);
772
773 LOG.info("#regions = " + regions.size());
774 Iterator<HRegionInfo> it = regions.iterator();
775 while (it.hasNext()) {
776 HRegionInfo region = it.next();
777 if (region.isMetaTable()) {
778 it.remove();
779 }
780 }
781 makeHLog(hrs.getWAL(), regions, "disableTable", "family", NUM_LOG_LINES, 100, false);
782 makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
783
784 LOG.info("Disabling table\n");
785 TEST_UTIL.getHBaseAdmin().disableTable(Bytes.toBytes("disableTable"));
786
787
788 LOG.info("Aborting region server: " + hrs.getServerName());
789 hrs.abort("testing");
790
791
792 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
793 @Override
794 public boolean evaluate() throws Exception {
795 return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
796 }
797 });
798
799
800 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
801 @Override
802 public boolean evaluate() throws Exception {
803 return (getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
804 }
805 });
806
807
808 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
809 @Override
810 public boolean evaluate() throws Exception {
811 List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
812 zkw.recoveringRegionsZNode, false);
813 ServerManager serverManager = master.getServerManager();
814 return (!serverManager.areDeadServersInProgress() &&
815 recoveringRegions != null && recoveringRegions.size() == 0);
816 }
817 });
818
819 int count = 0;
820 FileSystem fs = master.getMasterFileSystem().getFileSystem();
821 Path rootdir = FSUtils.getRootDir(conf);
822 Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable"));
823 for (HRegionInfo hri : regions) {
824 @SuppressWarnings("deprecation")
825 Path editsdir =
826 HLogUtil.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
827 LOG.debug("checking edits dir " + editsdir);
828 if(!fs.exists(editsdir)) continue;
829 FileStatus[] files = fs.listStatus(editsdir);
830 if(files != null) {
831 for(FileStatus file : files) {
832 int c = countHLog(file.getPath(), fs, conf);
833 count += c;
834 LOG.info(c + " edits in " + file.getPath());
835 }
836 }
837 }
838
839 LOG.info("Verify edits in recovered.edits files");
840 assertEquals(NUM_LOG_LINES, count);
841 LOG.info("Verify replayed edits");
842 assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
843
844
845 for (HRegionInfo hri : regions) {
846 @SuppressWarnings("deprecation")
847 Path editsdir =
848 HLogUtil.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
849 fs.delete(editsdir, true);
850 }
851 disablingHT.close();
852 ht.close();
853 zkw.close();
854 }
855
856 @Test(timeout = 300000)
857 public void testDisallowWritesInRecovering() throws Exception {
858 LOG.info("testDisallowWritesInRecovering");
859 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
860 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
861 conf.setBoolean(HConstants.DISALLOW_WRITES_IN_RECOVERING, true);
862 startCluster(NUM_RS);
863 final int NUM_REGIONS_TO_CREATE = 40;
864
865
866 master.balanceSwitch(false);
867
868 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
869 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
870 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
871 final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
872
873 Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
874 HRegionInfo region = null;
875 HRegionServer hrs = null;
876 HRegionServer dstRS = null;
877 for (int i = 0; i < NUM_RS; i++) {
878 hrs = rsts.get(i).getRegionServer();
879 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
880 if (regions.isEmpty()) continue;
881 region = regions.get(0);
882 regionSet.add(region);
883 dstRS = rsts.get((i+1) % NUM_RS).getRegionServer();
884 break;
885 }
886
887 slm.markRegionsRecoveringInZK(hrs.getServerName(), regionSet);
888
889 final HRegionInfo hri = region;
890 final HRegionServer tmpRS = dstRS;
891 TEST_UTIL.getHBaseAdmin().move(region.getEncodedNameAsBytes(),
892 Bytes.toBytes(dstRS.getServerName().getServerName()));
893
894 final RegionStates regionStates =
895 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
896 TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
897 @Override
898 public boolean evaluate() throws Exception {
899 ServerName sn = regionStates.getRegionServerOfRegion(hri);
900 return (sn != null && sn.equals(tmpRS.getServerName()));
901 }
902 });
903
904 try {
905 byte[] key = region.getStartKey();
906 if (key == null || key.length == 0) {
907 key = new byte[] { 0, 0, 0, 0, 1 };
908 }
909 ht.setAutoFlush(true, true);
910 Put put = new Put(key);
911 put.add(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'});
912 ht.put(put);
913 ht.close();
914 } catch (IOException ioe) {
915 Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException);
916 RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe;
917 boolean foundRegionInRecoveryException = false;
918 for (Throwable t : re.getCauses()) {
919 if (t instanceof RegionInRecoveryException) {
920 foundRegionInRecoveryException = true;
921 break;
922 }
923 }
924 Assert.assertTrue(
925 "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(),
926 foundRegionInRecoveryException);
927 }
928
929 zkw.close();
930 }
931
932
933
934
935
936
937
938
939
940
941 @Test (timeout=300000)
942 public void testWorkerAbort() throws Exception {
943 LOG.info("testWorkerAbort");
944 startCluster(3);
945 final int NUM_LOG_LINES = 10000;
946 final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
947 FileSystem fs = master.getMasterFileSystem().getFileSystem();
948
949 final List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
950 HRegionServer hrs = findRSToKill(false, "table");
951 Path rootdir = FSUtils.getRootDir(conf);
952 final Path logDir = new Path(rootdir,
953 HLogUtil.getHLogDirectoryName(hrs.getServerName().toString()));
954
955 installTable(new ZooKeeperWatcher(conf, "table-creation", null),
956 "table", "family", 40);
957
958 makeHLog(hrs.getWAL(), ProtobufUtil.getOnlineRegions(hrs), "table", "family", NUM_LOG_LINES,
959 100);
960
961 new Thread() {
962 public void run() {
963 waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
964 for (RegionServerThread rst : rsts) {
965 rst.getRegionServer().abort("testing");
966 break;
967 }
968 }
969 }.start();
970
971 FileStatus[] logfiles = fs.listStatus(logDir);
972 TaskBatch batch = new TaskBatch();
973 slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch);
974
975 long curt = System.currentTimeMillis();
976 long waitTime = 80000;
977 long endt = curt + waitTime;
978 while (curt < endt) {
979 if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
980 tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
981 tot_wkr_preempt_task.get()) == 0) {
982 Thread.yield();
983 curt = System.currentTimeMillis();
984 } else {
985 assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
986 tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
987 tot_wkr_preempt_task.get()));
988 return;
989 }
990 }
991 fail("none of the following counters went up in " + waitTime +
992 " milliseconds - " +
993 "tot_wkr_task_resigned, tot_wkr_task_err, " +
994 "tot_wkr_final_transition_failed, tot_wkr_task_done, " +
995 "tot_wkr_preempt_task");
996 }
997
998 @Test (timeout=300000)
999 public void testThreeRSAbort() throws Exception {
1000 LOG.info("testThreeRSAbort");
1001 final int NUM_REGIONS_TO_CREATE = 40;
1002 final int NUM_ROWS_PER_REGION = 100;
1003
1004 startCluster(NUM_RS);
1005
1006 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf,
1007 "distributed log splitting test", null);
1008
1009 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
1010 populateDataInTable(NUM_ROWS_PER_REGION, "family");
1011
1012
1013 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1014 assertEquals(NUM_RS, rsts.size());
1015 rsts.get(0).getRegionServer().abort("testing");
1016 rsts.get(1).getRegionServer().abort("testing");
1017 rsts.get(2).getRegionServer().abort("testing");
1018
1019 long start = EnvironmentEdgeManager.currentTimeMillis();
1020 while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
1021 if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
1022 assertTrue(false);
1023 }
1024 Thread.sleep(200);
1025 }
1026
1027 start = EnvironmentEdgeManager.currentTimeMillis();
1028 while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 1)) {
1029 if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) {
1030 assertTrue("Timedout", false);
1031 }
1032 Thread.sleep(200);
1033 }
1034
1035
1036 TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
1037 @Override
1038 public boolean evaluate() throws Exception {
1039 List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
1040 zkw.recoveringRegionsZNode, false);
1041 return (recoveringRegions != null && recoveringRegions.size() == 0);
1042 }
1043 });
1044
1045 assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
1046 TEST_UTIL.countRows(ht));
1047 ht.close();
1048 zkw.close();
1049 }
1050
1051
1052
1053 @Test(timeout=30000)
1054 public void testDelayedDeleteOnFailure() throws Exception {
1055 LOG.info("testDelayedDeleteOnFailure");
1056 startCluster(1);
1057 final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
1058 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
1059 final Path logDir = new Path(FSUtils.getRootDir(conf), "x");
1060 fs.mkdirs(logDir);
1061 ExecutorService executor = null;
1062 try {
1063 final Path corruptedLogFile = new Path(logDir, "x");
1064 FSDataOutputStream out;
1065 out = fs.create(corruptedLogFile);
1066 out.write(0);
1067 out.write(Bytes.toBytes("corrupted bytes"));
1068 out.close();
1069 slm.ignoreZKDeleteForTesting = true;
1070 executor = Executors.newSingleThreadExecutor();
1071 Runnable runnable = new Runnable() {
1072 @Override
1073 public void run() {
1074 try {
1075
1076
1077
1078 slm.splitLogDistributed(logDir);
1079 } catch (IOException ioe) {
1080 try {
1081 assertTrue(fs.exists(corruptedLogFile));
1082
1083
1084
1085 slm.splitLogDistributed(logDir);
1086 } catch (IOException e) {
1087 assertTrue(Thread.currentThread().isInterrupted());
1088 return;
1089 }
1090 fail("did not get the expected IOException from the 2nd call");
1091 }
1092 fail("did not get the expected IOException from the 1st call");
1093 }
1094 };
1095 Future<?> result = executor.submit(runnable);
1096 try {
1097 result.get(2000, TimeUnit.MILLISECONDS);
1098 } catch (TimeoutException te) {
1099
1100 }
1101 waitForCounter(tot_mgr_wait_for_zk_delete, 0, 1, 10000);
1102 executor.shutdownNow();
1103 executor = null;
1104
1105
1106 result.get();
1107 } finally {
1108 if (executor != null) {
1109
1110
1111 executor.shutdownNow();
1112 }
1113 fs.delete(logDir, true);
1114 }
1115 }
1116
1117 @Test(timeout = 300000)
1118 public void testMetaRecoveryInZK() throws Exception {
1119 LOG.info("testMetaRecoveryInZK");
1120 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
1121 startCluster(NUM_RS);
1122
1123
1124
1125 master.balanceSwitch(false);
1126 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
1127 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1128
1129
1130 HRegionServer hrs = findRSToKill(true, null);
1131 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1132
1133 LOG.info("#regions = " + regions.size());
1134 Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>();
1135 tmpRegions.add(HRegionInfo.FIRST_META_REGIONINFO);
1136 master.getMasterFileSystem().prepareLogReplay(hrs.getServerName(), tmpRegions);
1137 Set<HRegionInfo> userRegionSet = new HashSet<HRegionInfo>();
1138 userRegionSet.addAll(regions);
1139 master.getMasterFileSystem().prepareLogReplay(hrs.getServerName(), userRegionSet);
1140 boolean isMetaRegionInRecovery = false;
1141 List<String> recoveringRegions =
1142 zkw.getRecoverableZooKeeper().getChildren(zkw.recoveringRegionsZNode, false);
1143 for (String curEncodedRegionName : recoveringRegions) {
1144 if (curEncodedRegionName.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
1145 isMetaRegionInRecovery = true;
1146 break;
1147 }
1148 }
1149 assertTrue(isMetaRegionInRecovery);
1150
1151 master.getMasterFileSystem().splitMetaLog(hrs.getServerName());
1152
1153 isMetaRegionInRecovery = false;
1154 recoveringRegions =
1155 zkw.getRecoverableZooKeeper().getChildren(zkw.recoveringRegionsZNode, false);
1156 for (String curEncodedRegionName : recoveringRegions) {
1157 if (curEncodedRegionName.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
1158 isMetaRegionInRecovery = true;
1159 break;
1160 }
1161 }
1162
1163 assertFalse(isMetaRegionInRecovery);
1164 zkw.close();
1165 }
1166
1167 @Test(timeout = 300000)
1168 public void testSameVersionUpdatesRecovery() throws Exception {
1169 LOG.info("testSameVersionUpdatesRecovery");
1170 conf.setLong("hbase.regionserver.hlog.blocksize", 15 * 1024);
1171 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
1172 startCluster(NUM_RS);
1173 final AtomicLong sequenceId = new AtomicLong(100);
1174 final int NUM_REGIONS_TO_CREATE = 40;
1175 final int NUM_LOG_LINES = 1000;
1176
1177
1178 master.balanceSwitch(false);
1179
1180 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1181 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
1182 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
1183
1184 List<HRegionInfo> regions = null;
1185 HRegionServer hrs = null;
1186 for (int i = 0; i < NUM_RS; i++) {
1187 boolean isCarryingMeta = false;
1188 hrs = rsts.get(i).getRegionServer();
1189 regions = ProtobufUtil.getOnlineRegions(hrs);
1190 for (HRegionInfo region : regions) {
1191 if (region.isMetaRegion()) {
1192 isCarryingMeta = true;
1193 break;
1194 }
1195 }
1196 if (isCarryingMeta) {
1197 continue;
1198 }
1199 break;
1200 }
1201
1202 LOG.info("#regions = " + regions.size());
1203 Iterator<HRegionInfo> it = regions.iterator();
1204 while (it.hasNext()) {
1205 HRegionInfo region = it.next();
1206 if (region.isMetaTable()
1207 || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
1208 it.remove();
1209 }
1210 }
1211 if (regions.size() == 0) return;
1212 HRegionInfo curRegionInfo = regions.get(0);
1213 byte[] startRow = curRegionInfo.getStartKey();
1214 if (startRow == null || startRow.length == 0) {
1215 startRow = new byte[] { 0, 0, 0, 0, 1 };
1216 }
1217 byte[] row = Bytes.incrementBytes(startRow, 1);
1218
1219 row = Arrays.copyOfRange(row, 3, 8);
1220 long value = 0;
1221 byte[] tableName = Bytes.toBytes("table");
1222 byte[] family = Bytes.toBytes("family");
1223 byte[] qualifier = Bytes.toBytes("c1");
1224 long timeStamp = System.currentTimeMillis();
1225 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
1226 htd.addFamily(new HColumnDescriptor(family));
1227 for (int i = 0; i < NUM_LOG_LINES; i += 1) {
1228 WALEdit e = new WALEdit();
1229 value++;
1230 e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
1231 hrs.getWAL().append(curRegionInfo, TableName.valueOf(tableName), e,
1232 System.currentTimeMillis(), htd, sequenceId);
1233 }
1234 hrs.getWAL().sync();
1235 hrs.getWAL().close();
1236
1237
1238 this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
1239
1240
1241 LOG.info("Verification Starts...");
1242 Get g = new Get(row);
1243 Result r = ht.get(g);
1244 long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
1245 assertEquals(value, theStoredVal);
1246
1247
1248 LOG.info("Verification after flush...");
1249 TEST_UTIL.getHBaseAdmin().flush(tableName);
1250 r = ht.get(g);
1251 theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
1252 assertEquals(value, theStoredVal);
1253 ht.close();
1254 }
1255
1256 @Test(timeout = 300000)
1257 public void testSameVersionUpdatesRecoveryWithCompaction() throws Exception {
1258 LOG.info("testSameVersionUpdatesRecoveryWithWrites");
1259 conf.setLong("hbase.regionserver.hlog.blocksize", 15 * 1024);
1260 conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
1261 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 30 * 1024);
1262 conf.setInt("hbase.hstore.compactionThreshold", 3);
1263 startCluster(NUM_RS);
1264 final AtomicLong sequenceId = new AtomicLong(100);
1265 final int NUM_REGIONS_TO_CREATE = 40;
1266 final int NUM_LOG_LINES = 1000;
1267
1268
1269 master.balanceSwitch(false);
1270
1271 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1272 final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
1273 HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
1274
1275 List<HRegionInfo> regions = null;
1276 HRegionServer hrs = null;
1277 for (int i = 0; i < NUM_RS; i++) {
1278 boolean isCarryingMeta = false;
1279 hrs = rsts.get(i).getRegionServer();
1280 regions = ProtobufUtil.getOnlineRegions(hrs);
1281 for (HRegionInfo region : regions) {
1282 if (region.isMetaRegion()) {
1283 isCarryingMeta = true;
1284 break;
1285 }
1286 }
1287 if (isCarryingMeta) {
1288 continue;
1289 }
1290 break;
1291 }
1292
1293 LOG.info("#regions = " + regions.size());
1294 Iterator<HRegionInfo> it = regions.iterator();
1295 while (it.hasNext()) {
1296 HRegionInfo region = it.next();
1297 if (region.isMetaTable()
1298 || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
1299 it.remove();
1300 }
1301 }
1302 if (regions.size() == 0) return;
1303 HRegionInfo curRegionInfo = regions.get(0);
1304 byte[] startRow = curRegionInfo.getStartKey();
1305 if (startRow == null || startRow.length == 0) {
1306 startRow = new byte[] { 0, 0, 0, 0, 1 };
1307 }
1308 byte[] row = Bytes.incrementBytes(startRow, 1);
1309
1310 row = Arrays.copyOfRange(row, 3, 8);
1311 long value = 0;
1312 final byte[] tableName = Bytes.toBytes("table");
1313 byte[] family = Bytes.toBytes("family");
1314 byte[] qualifier = Bytes.toBytes("c1");
1315 long timeStamp = System.currentTimeMillis();
1316 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
1317 htd.addFamily(new HColumnDescriptor(family));
1318 for (int i = 0; i < NUM_LOG_LINES; i += 1) {
1319 WALEdit e = new WALEdit();
1320 value++;
1321 e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
1322 hrs.getWAL().append(curRegionInfo, TableName.valueOf(tableName), e,
1323 System.currentTimeMillis(), htd, sequenceId);
1324 }
1325 hrs.getWAL().sync();
1326 hrs.getWAL().close();
1327
1328
1329 this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
1330
1331
1332 LOG.info("Verification Starts...");
1333 Get g = new Get(row);
1334 Result r = ht.get(g);
1335 long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
1336 assertEquals(value, theStoredVal);
1337
1338
1339 LOG.info("Verification after flush...");
1340 TEST_UTIL.getHBaseAdmin().flush(tableName);
1341 TEST_UTIL.getHBaseAdmin().compact(tableName);
1342
1343
1344 TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
1345 @Override
1346 public boolean evaluate() throws Exception {
1347 return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE);
1348 }
1349 });
1350
1351 r = ht.get(g);
1352 theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
1353 assertEquals(value, theStoredVal);
1354 ht.close();
1355 }
1356
1357 HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception {
1358 return installTable(zkw, tname, fname, nrs, 0);
1359 }
1360
1361 HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs,
1362 int existingRegions) throws Exception {
1363
1364 byte [] table = Bytes.toBytes(tname);
1365 byte [] family = Bytes.toBytes(fname);
1366 LOG.info("Creating table with " + nrs + " regions");
1367 HTable ht = TEST_UTIL.createTable(table, family);
1368 int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family, nrs);
1369 assertEquals(nrs, numRegions);
1370 LOG.info("Waiting for no more RIT\n");
1371 blockUntilNoRIT(zkw, master);
1372
1373
1374 LOG.debug("Disabling table\n");
1375 TEST_UTIL.getHBaseAdmin().disableTable(table);
1376 LOG.debug("Waiting for no more RIT\n");
1377 blockUntilNoRIT(zkw, master);
1378 NavigableSet<String> regions = getAllOnlineRegions(cluster);
1379 LOG.debug("Verifying only catalog and namespace regions are assigned\n");
1380 if (regions.size() != 2) {
1381 for (String oregion : regions)
1382 LOG.debug("Region still online: " + oregion);
1383 }
1384 assertEquals(2 + existingRegions, regions.size());
1385 LOG.debug("Enabling table\n");
1386 TEST_UTIL.getHBaseAdmin().enableTable(table);
1387 LOG.debug("Waiting for no more RIT\n");
1388 blockUntilNoRIT(zkw, master);
1389 LOG.debug("Verifying there are " + numRegions + " assigned on cluster\n");
1390 regions = getAllOnlineRegions(cluster);
1391 assertEquals(numRegions + 2 + existingRegions, regions.size());
1392 return ht;
1393 }
1394
1395 void populateDataInTable(int nrows, String fname) throws Exception {
1396 byte [] family = Bytes.toBytes(fname);
1397
1398 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1399 assertEquals(NUM_RS, rsts.size());
1400
1401 for (RegionServerThread rst : rsts) {
1402 HRegionServer hrs = rst.getRegionServer();
1403 List<HRegionInfo> hris = ProtobufUtil.getOnlineRegions(hrs);
1404 for (HRegionInfo hri : hris) {
1405 if (hri.getTable().isSystemTable()) {
1406 continue;
1407 }
1408 LOG.debug("adding data to rs = " + rst.getName() +
1409 " region = "+ hri.getRegionNameAsString());
1410 HRegion region = hrs.getOnlineRegion(hri.getRegionName());
1411 assertTrue(region != null);
1412 putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family);
1413 }
1414 }
1415 }
1416
1417 public void makeHLog(HLog log, List<HRegionInfo> regions, String tname, String fname,
1418 int num_edits, int edit_size) throws IOException {
1419 makeHLog(log, regions, tname, fname, num_edits, edit_size, true);
1420 }
1421
1422 public void makeHLog(HLog log, List<HRegionInfo> regions, String tname, String fname,
1423 int num_edits, int edit_size, boolean closeLog) throws IOException {
1424 TableName fullTName = TableName.valueOf(tname);
1425
1426 regions.remove(HRegionInfo.FIRST_META_REGIONINFO);
1427
1428 final AtomicLong sequenceId = new AtomicLong(10);
1429
1430
1431 for(Iterator<HRegionInfo> iter = regions.iterator(); iter.hasNext(); ) {
1432 HRegionInfo regionInfo = iter.next();
1433 if(regionInfo.getTable().isSystemTable()) {
1434 iter.remove();
1435 }
1436 }
1437 HTableDescriptor htd = new HTableDescriptor(fullTName);
1438 byte[] family = Bytes.toBytes(fname);
1439 htd.addFamily(new HColumnDescriptor(family));
1440 byte[] value = new byte[edit_size];
1441
1442 List<HRegionInfo> hris = new ArrayList<HRegionInfo>();
1443 for (HRegionInfo region : regions) {
1444 if (!region.getTable().getNameAsString().equalsIgnoreCase(tname)) {
1445 continue;
1446 }
1447 hris.add(region);
1448 }
1449 LOG.info("Creating wal edits across " + hris.size() + " regions.");
1450 for (int i = 0; i < edit_size; i++) {
1451 value[i] = (byte) ('a' + (i % 26));
1452 }
1453 int n = hris.size();
1454 int[] counts = new int[n];
1455 if (n > 0) {
1456 for (int i = 0; i < num_edits; i += 1) {
1457 WALEdit e = new WALEdit();
1458 HRegionInfo curRegionInfo = hris.get(i % n);
1459 byte[] startRow = curRegionInfo.getStartKey();
1460 if (startRow == null || startRow.length == 0) {
1461 startRow = new byte[] { 0, 0, 0, 0, 1 };
1462 }
1463 byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
1464 row = Arrays.copyOfRange(row, 3, 8);
1465
1466
1467 byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
1468 e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
1469 log.append(curRegionInfo, fullTName, e, System.currentTimeMillis(), htd, sequenceId);
1470 counts[i % n] += 1;
1471 }
1472 }
1473 log.sync();
1474 if(closeLog) {
1475 log.close();
1476 }
1477 for (int i = 0; i < n; i++) {
1478 LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
1479 }
1480 return;
1481 }
1482
1483 private int countHLog(Path log, FileSystem fs, Configuration conf)
1484 throws IOException {
1485 int count = 0;
1486 HLog.Reader in = HLogFactory.createReader(fs, log, conf);
1487 while (in.next() != null) {
1488 count++;
1489 }
1490 return count;
1491 }
1492
1493 private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master)
1494 throws KeeperException, InterruptedException {
1495 ZKAssign.blockUntilNoRIT(zkw);
1496 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
1497 }
1498
1499 private void putData(HRegion region, byte[] startRow, int numRows, byte [] qf,
1500 byte [] ...families)
1501 throws IOException {
1502 for(int i = 0; i < numRows; i++) {
1503 Put put = new Put(Bytes.add(startRow, Bytes.toBytes(i)));
1504 for(byte [] family : families) {
1505 put.add(family, qf, null);
1506 }
1507 region.put(put);
1508 }
1509 }
1510
1511
1512
1513
1514 private void prepareData(final HTable t, final byte[] f, final byte[] column) throws IOException {
1515 t.setAutoFlush(false, true);
1516 byte[] k = new byte[3];
1517
1518
1519 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
1520 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
1521 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
1522 k[0] = b1;
1523 k[1] = b2;
1524 k[2] = b3;
1525 Put put = new Put(k);
1526 put.add(f, column, k);
1527 t.put(put);
1528 }
1529 }
1530 }
1531 t.flushCommits();
1532
1533 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
1534 k[0] = 'a';
1535 k[1] = 'a';
1536 k[2] = b3;
1537 Delete del = new Delete(k);
1538 t.delete(del);
1539 }
1540 t.flushCommits();
1541 }
1542
1543 private NavigableSet<String> getAllOnlineRegions(MiniHBaseCluster cluster)
1544 throws IOException {
1545 NavigableSet<String> online = new TreeSet<String>();
1546 for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
1547 for (HRegionInfo region : ProtobufUtil.getOnlineRegions(rst.getRegionServer())) {
1548 online.add(region.getRegionNameAsString());
1549 }
1550 }
1551 return online;
1552 }
1553
1554 private void waitForCounter(AtomicLong ctr, long oldval, long newval,
1555 long timems) {
1556 long curt = System.currentTimeMillis();
1557 long endt = curt + timems;
1558 while (curt < endt) {
1559 if (ctr.get() == oldval) {
1560 Thread.yield();
1561 curt = System.currentTimeMillis();
1562 } else {
1563 assertEquals(newval, ctr.get());
1564 return;
1565 }
1566 }
1567 assertTrue(false);
1568 }
1569
1570 private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException {
1571 for (MasterThread mt : cluster.getLiveMasterThreads()) {
1572 if (mt.getMaster().isActiveMaster()) {
1573 mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
1574 mt.join();
1575 break;
1576 }
1577 }
1578 LOG.debug("Master is aborted");
1579 }
1580
1581 private void startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
1582 throws IOException, InterruptedException {
1583 cluster.startMaster();
1584 HMaster master = cluster.getMaster();
1585 while (!master.isInitialized()) {
1586 Thread.sleep(100);
1587 }
1588 ServerManager serverManager = master.getServerManager();
1589 while (serverManager.areDeadServersInProgress()) {
1590 Thread.sleep(100);
1591 }
1592 }
1593
1594
1595
1596
1597
1598
1599
1600
1601 private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception {
1602 List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
1603 int numOfRSs = rsts.size();
1604 List<HRegionInfo> regions = null;
1605 HRegionServer hrs = null;
1606
1607 for (int i = 0; i < numOfRSs; i++) {
1608 boolean isCarryingMeta = false;
1609 boolean foundTableRegion = false;
1610 hrs = rsts.get(i).getRegionServer();
1611 regions = ProtobufUtil.getOnlineRegions(hrs);
1612 for (HRegionInfo region : regions) {
1613 if (region.isMetaRegion()) {
1614 isCarryingMeta = true;
1615 }
1616 if (tableName == null || region.getTable().getNameAsString().equals(tableName)) {
1617 foundTableRegion = true;
1618 }
1619 if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
1620 break;
1621 }
1622 }
1623 if (isCarryingMeta && hasMetaRegion) {
1624
1625 if (!foundTableRegion) {
1626 final HRegionServer destRS = hrs;
1627
1628 List<HRegionInfo> tableRegions =
1629 TEST_UTIL.getHBaseAdmin().getTableRegions(Bytes.toBytes(tableName));
1630 final HRegionInfo hri = tableRegions.get(0);
1631 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
1632 Bytes.toBytes(destRS.getServerName().getServerName()));
1633
1634 final RegionStates regionStates =
1635 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
1636 TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
1637 @Override
1638 public boolean evaluate() throws Exception {
1639 ServerName sn = regionStates.getRegionServerOfRegion(hri);
1640 return (sn != null && sn.equals(destRS.getServerName()));
1641 }
1642 });
1643 }
1644 return hrs;
1645 } else if (hasMetaRegion || isCarryingMeta) {
1646 continue;
1647 }
1648 if (foundTableRegion) break;
1649 }
1650
1651 return hrs;
1652 }
1653
1654 }