1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Matchers.anyInt;
25 import static org.mockito.Matchers.eq;
26 import static org.mockito.Mockito.doNothing;
27 import static org.mockito.Mockito.doThrow;
28 import static org.mockito.Mockito.spy;
29 import static org.mockito.Mockito.when;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.List;
34
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.Cell;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.Server;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
48 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
49 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
50 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
51 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
52 import org.apache.hadoop.hbase.regionserver.wal.HLog;
53 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.PairOfSameType;
57 import org.apache.zookeeper.KeeperException;
58 import org.junit.After;
59 import org.junit.Before;
60 import org.junit.Test;
61 import org.junit.experimental.categories.Category;
62 import org.mockito.Mockito;
63
64 import com.google.common.collect.ImmutableList;
65
66
67
68
69
70 @Category(SmallTests.class)
71 public class TestSplitTransaction {
72 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
73 private final Path testdir =
74 TEST_UTIL.getDataTestDir(this.getClass().getName());
75 private HRegion parent;
76 private HLog wal;
77 private FileSystem fs;
78 private static final byte [] STARTROW = new byte [] {'a', 'a', 'a'};
79
80 private static final byte [] ENDROW = new byte [] {'{', '{', '{'};
81 private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'};
82 private static final byte [] CF = HConstants.CATALOG_FAMILY;
83
84 private static boolean preRollBackCalled = false;
85 private static boolean postRollBackCalled = false;
86
87 @Before public void setup() throws IOException {
88 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
89 TEST_UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
90 this.fs.delete(this.testdir, true);
91 this.wal = HLogFactory.createHLog(fs, this.testdir, "logs",
92 TEST_UTIL.getConfiguration());
93
94 this.parent = createRegion(this.testdir, this.wal);
95 RegionCoprocessorHost host = new RegionCoprocessorHost(this.parent, null, TEST_UTIL.getConfiguration());
96 this.parent.setCoprocessorHost(host);
97 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
98 }
99
100 @After public void teardown() throws IOException {
101 if (this.parent != null && !this.parent.isClosed()) this.parent.close();
102 Path regionDir = this.parent.getRegionFileSystem().getRegionDir();
103 if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
104 throw new IOException("Failed delete of " + regionDir);
105 }
106 if (this.wal != null) this.wal.closeAndDelete();
107 this.fs.delete(this.testdir, true);
108 }
109
110 @Test public void testFailAfterPONR() throws IOException, KeeperException {
111 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
112 assertTrue(rowcount > 0);
113 int parentRowCount = countRows(this.parent);
114 assertEquals(rowcount, parentRowCount);
115
116
117 SplitTransaction st = prepareGOOD_SPLIT_ROW();
118 SplitTransaction spiedUponSt = spy(st);
119 Mockito
120 .doThrow(new MockedFailedDaughterOpen())
121 .when(spiedUponSt)
122 .openDaughterRegion((Server) Mockito.anyObject(),
123 (HRegion) Mockito.anyObject());
124
125
126 boolean expectedException = false;
127 Server mockServer = Mockito.mock(Server.class);
128 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
129 try {
130 spiedUponSt.execute(mockServer, null);
131 } catch (IOException e) {
132 if (e.getCause() != null &&
133 e.getCause() instanceof MockedFailedDaughterOpen) {
134 expectedException = true;
135 }
136 }
137 assertTrue(expectedException);
138
139 assertFalse(spiedUponSt.rollback(null, null));
140
141
142
143 Path tableDir = this.parent.getRegionFileSystem().getTableDir();
144 Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName());
145 Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName());
146 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir));
147 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir));
148 }
149
150
151
152
153
154 @Test public void testPrepare() throws IOException {
155 prepareGOOD_SPLIT_ROW();
156 }
157
158 private SplitTransaction prepareGOOD_SPLIT_ROW() {
159 return prepareGOOD_SPLIT_ROW(this.parent);
160 }
161
162 private SplitTransaction prepareGOOD_SPLIT_ROW(final HRegion parentRegion) {
163 SplitTransaction st = new SplitTransaction(parentRegion, GOOD_SPLIT_ROW);
164 assertTrue(st.prepare());
165 return st;
166 }
167
168
169
170
171 @Test public void testPrepareWithRegionsWithReference() throws IOException {
172 HStore storeMock = Mockito.mock(HStore.class);
173 when(storeMock.hasReferences()).thenReturn(true);
174 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
175 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
176 this.parent.stores.put(Bytes.toBytes(""), storeMock);
177
178 SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW);
179
180 assertFalse("a region should not be splittable if it has instances of store file references",
181 st.prepare());
182 }
183
184
185
186
187 @Test public void testPrepareWithBadSplitRow() throws IOException {
188
189 SplitTransaction st = new SplitTransaction(this.parent, STARTROW);
190 assertFalse(st.prepare());
191 st = new SplitTransaction(this.parent, HConstants.EMPTY_BYTE_ARRAY);
192 assertFalse(st.prepare());
193 st = new SplitTransaction(this.parent, new byte [] {'A', 'A', 'A'});
194 assertFalse(st.prepare());
195 st = new SplitTransaction(this.parent, ENDROW);
196 assertFalse(st.prepare());
197 }
198
199 @Test public void testPrepareWithClosedRegion() throws IOException {
200 this.parent.close();
201 SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW);
202 assertFalse(st.prepare());
203 }
204
205 @Test public void testWholesomeSplit() throws IOException {
206 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true);
207 assertTrue(rowcount > 0);
208 int parentRowCount = countRows(this.parent);
209 assertEquals(rowcount, parentRowCount);
210
211
212
213 CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
214 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
215
216
217 SplitTransaction st = prepareGOOD_SPLIT_ROW();
218
219
220 Server mockServer = Mockito.mock(Server.class);
221 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
222 PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
223
224 assertTrue(this.fs.exists(this.parent.getRegionFileSystem().getSplitsDir()));
225
226 assertTrue(this.parent.isClosed());
227
228
229
230 assertEquals(0, this.fs.listStatus(this.parent.getRegionFileSystem().getSplitsDir()).length);
231
232 assertTrue(Bytes.equals(this.parent.getStartKey(), daughters.getFirst().getStartKey()));
233 assertTrue(Bytes.equals(GOOD_SPLIT_ROW, daughters.getFirst().getEndKey()));
234 assertTrue(Bytes.equals(daughters.getSecond().getStartKey(), GOOD_SPLIT_ROW));
235 assertTrue(Bytes.equals(this.parent.getEndKey(), daughters.getSecond().getEndKey()));
236
237 int daughtersRowCount = 0;
238 for (HRegion openRegion: daughters) {
239 try {
240 int count = countRows(openRegion);
241 assertTrue(count > 0 && count != rowcount);
242 daughtersRowCount += count;
243 } finally {
244 HRegion.closeHRegion(openRegion);
245 }
246 }
247 assertEquals(rowcount, daughtersRowCount);
248
249 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
250 }
251
252 @Test
253 public void testCountReferencesFailsSplit() throws IOException {
254 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
255 assertTrue(rowcount > 0);
256 int parentRowCount = countRows(this.parent);
257 assertEquals(rowcount, parentRowCount);
258
259
260 HRegion spiedRegion = spy(this.parent);
261 SplitTransaction st = prepareGOOD_SPLIT_ROW(spiedRegion);
262 SplitTransaction spiedUponSt = spy(st);
263 doThrow(new IOException("Failing split. Expected reference file count isn't equal."))
264 .when(spiedUponSt).assertReferenceFileCount(anyInt(),
265 eq(new Path(this.parent.getRegionFileSystem().getTableDir(),
266 st.getSecondDaughter().getEncodedName())));
267
268
269 boolean expectedException = false;
270 Server mockServer = Mockito.mock(Server.class);
271 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
272 try {
273 spiedUponSt.execute(mockServer, null);
274 } catch (IOException e) {
275 expectedException = true;
276 }
277 assertTrue(expectedException);
278 }
279
280
281 @Test public void testRollback() throws IOException {
282 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
283 assertTrue(rowcount > 0);
284 int parentRowCount = countRows(this.parent);
285 assertEquals(rowcount, parentRowCount);
286
287
288 HRegion spiedRegion = spy(this.parent);
289 SplitTransaction st = prepareGOOD_SPLIT_ROW(spiedRegion);
290 SplitTransaction spiedUponSt = spy(st);
291 doNothing().when(spiedUponSt).assertReferenceFileCount(anyInt(),
292 eq(parent.getRegionFileSystem().getSplitsDir(st.getFirstDaughter())));
293 when(spiedRegion.createDaughterRegionFromSplits(spiedUponSt.getSecondDaughter())).
294 thenThrow(new MockedFailedDaughterCreation());
295
296 boolean expectedException = false;
297 Server mockServer = Mockito.mock(Server.class);
298 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
299 try {
300 spiedUponSt.execute(mockServer, null);
301 } catch (MockedFailedDaughterCreation e) {
302 expectedException = true;
303 }
304 assertTrue(expectedException);
305
306 assertTrue(spiedUponSt.rollback(null, null));
307
308
309 int parentRowCount2 = countRows(this.parent);
310 assertEquals(parentRowCount, parentRowCount2);
311
312
313 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getFirstDaughter())));
314 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getSecondDaughter())));
315 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
316
317
318 assertTrue(st.prepare());
319 PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
320
321 int daughtersRowCount = 0;
322 for (HRegion openRegion: daughters) {
323 try {
324 int count = countRows(openRegion);
325 assertTrue(count > 0 && count != rowcount);
326 daughtersRowCount += count;
327 } finally {
328 HRegion.closeHRegion(openRegion);
329 }
330 }
331 assertEquals(rowcount, daughtersRowCount);
332
333 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
334 assertTrue("Rollback hooks should be called.", wasRollBackHookCalled());
335 }
336
337 private boolean wasRollBackHookCalled(){
338 return (preRollBackCalled && postRollBackCalled);
339 }
340
341
342
343
344 @SuppressWarnings("serial")
345 private class MockedFailedDaughterCreation extends IOException {}
346 private class MockedFailedDaughterOpen extends IOException {}
347
348 private int countRows(final HRegion r) throws IOException {
349 int rowcount = 0;
350 InternalScanner scanner = r.getScanner(new Scan());
351 try {
352 List<Cell> kvs = new ArrayList<Cell>();
353 boolean hasNext = true;
354 while (hasNext) {
355 hasNext = scanner.next(kvs);
356 if (!kvs.isEmpty()) rowcount++;
357 }
358 } finally {
359 scanner.close();
360 }
361 return rowcount;
362 }
363
364 HRegion createRegion(final Path testdir, final HLog wal)
365 throws IOException {
366
367
368 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
369 HColumnDescriptor hcd = new HColumnDescriptor(CF);
370 htd.addFamily(hcd);
371 HRegionInfo hri = new HRegionInfo(htd.getTableName(), STARTROW, ENDROW);
372 HRegion r = HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
373 HRegion.closeHRegion(r);
374 return HRegion.openHRegion(testdir, hri, htd, wal,
375 TEST_UTIL.getConfiguration());
376 }
377
378 public static class CustomObserver extends BaseRegionObserver{
379 @Override
380 public void preRollBackSplit(
381 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
382 preRollBackCalled = true;
383 }
384
385 @Override
386 public void postRollBackSplit(
387 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
388 postRollBackCalled = true;
389 }
390 }
391
392 }
393