View Javadoc

1   /*
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication;
21  
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.List;
30  import java.util.Map;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
43  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
44  import org.apache.hadoop.hbase.testclassification.LargeTests;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
47  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48  import org.junit.AfterClass;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  @Category(LargeTests.class)
54  public class TestPerTableCFReplication {
55  
56    private static final Log LOG = LogFactory.getLog(TestPerTableCFReplication.class);
57  
58    private static Configuration conf1;
59    private static Configuration conf2;
60    private static Configuration conf3;
61  
62    private static HBaseTestingUtility utility1;
63    private static HBaseTestingUtility utility2;
64    private static HBaseTestingUtility utility3;
65    private static final long SLEEP_TIME = 500;
66    private static final int NB_RETRIES = 100;
67  
68    private static final byte[] tableName = Bytes.toBytes("test");
69    private static final byte[] tabAName = Bytes.toBytes("TA");
70    private static final byte[] tabBName = Bytes.toBytes("TB");
71    private static final byte[] tabCName = Bytes.toBytes("TC");
72    private static final byte[] famName = Bytes.toBytes("f");
73    private static final byte[] f1Name = Bytes.toBytes("f1");
74    private static final byte[] f2Name = Bytes.toBytes("f2");
75    private static final byte[] f3Name = Bytes.toBytes("f3");
76    private static final byte[] row1 = Bytes.toBytes("row1");
77    private static final byte[] row2 = Bytes.toBytes("row2");
78    private static final byte[] noRepfamName = Bytes.toBytes("norep");
79    private static final byte[] val = Bytes.toBytes("myval");
80  
81    private static HTableDescriptor table;
82    private static HTableDescriptor tabA;
83    private static HTableDescriptor tabB;
84    private static HTableDescriptor tabC;
85  
86    @BeforeClass
87    public static void setUpBeforeClass() throws Exception {
88      conf1 = HBaseConfiguration.create();
89      conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
90      // smaller block size and capacity to trigger more operations
91      // and test them
92      conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
93      conf1.setInt("replication.source.size.capacity", 1024);
94      conf1.setLong("replication.source.sleepforretries", 100);
95      conf1.setInt("hbase.regionserver.maxlogs", 10);
96      conf1.setLong("hbase.master.logcleaner.ttl", 10);
97      conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
98      conf1.setBoolean("dfs.support.append", true);
99      conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
100     conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
101         "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
102 
103     utility1 = new HBaseTestingUtility(conf1);
104     utility1.startMiniZKCluster();
105     MiniZooKeeperCluster miniZK = utility1.getZkCluster();
106     new ZooKeeperWatcher(conf1, "cluster1", null, true);
107 
108     conf2 = new Configuration(conf1);
109     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
110 
111     conf3 = new Configuration(conf1);
112     conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
113 
114     utility2 = new HBaseTestingUtility(conf2);
115     utility2.setZkCluster(miniZK);
116     new ZooKeeperWatcher(conf2, "cluster3", null, true);
117 
118     utility3 = new HBaseTestingUtility(conf3);
119     utility3.setZkCluster(miniZK);
120     new ZooKeeperWatcher(conf3, "cluster3", null, true);
121 
122     table = new HTableDescriptor(TableName.valueOf(tableName));
123     HColumnDescriptor fam = new HColumnDescriptor(famName);
124     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
125     table.addFamily(fam);
126     fam = new HColumnDescriptor(noRepfamName);
127     table.addFamily(fam);
128 
129     tabA = new HTableDescriptor(tabAName);
130     fam = new HColumnDescriptor(f1Name);
131     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
132     tabA.addFamily(fam);
133     fam = new HColumnDescriptor(f2Name);
134     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
135     tabA.addFamily(fam);
136     fam = new HColumnDescriptor(f3Name);
137     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
138     tabA.addFamily(fam);
139 
140     tabB = new HTableDescriptor(tabBName);
141     fam = new HColumnDescriptor(f1Name);
142     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
143     tabB.addFamily(fam);
144     fam = new HColumnDescriptor(f2Name);
145     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
146     tabB.addFamily(fam);
147     fam = new HColumnDescriptor(f3Name);
148     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
149     tabB.addFamily(fam);
150 
151     tabC = new HTableDescriptor(tabCName);
152     fam = new HColumnDescriptor(f1Name);
153     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
154     tabC.addFamily(fam);
155     fam = new HColumnDescriptor(f2Name);
156     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
157     tabC.addFamily(fam);
158     fam = new HColumnDescriptor(f3Name);
159     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
160     tabC.addFamily(fam);
161 
162     utility1.startMiniCluster();
163     utility2.startMiniCluster();
164     utility3.startMiniCluster();
165   }
166 
167   @AfterClass
168   public static void tearDownAfterClass() throws Exception {
169     utility3.shutdownMiniCluster();
170     utility2.shutdownMiniCluster();
171     utility1.shutdownMiniCluster();
172   }
173 
174   @Test
175   public void testParseTableCFsFromConfig() {
176     Map<String, List<String>> tabCFsMap = null;
177 
178     // 1. null or empty string, result should be null
179     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig(null);
180     assertEquals(null, tabCFsMap);
181 
182     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("");
183     assertEquals(null, tabCFsMap);
184 
185     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("   ");
186     assertEquals(null, tabCFsMap);
187 
188     // 2. single table: "tab1" / "tab2:cf1" / "tab3:cf1,cf3"
189     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("tab1");
190     assertEquals(1, tabCFsMap.size()); // only one table
191     assertTrue(tabCFsMap.containsKey("tab1"));   // its table name is "tab1"
192     assertFalse(tabCFsMap.containsKey("tab2"));  // not other table
193     assertEquals(null, tabCFsMap.get("tab1"));   // null cf-list,
194 
195     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("tab2:cf1");
196     assertEquals(1, tabCFsMap.size()); // only one table
197     assertTrue(tabCFsMap.containsKey("tab2"));   // its table name is "tab2"
198     assertFalse(tabCFsMap.containsKey("tab1"));  // not other table
199     assertEquals(1, tabCFsMap.get("tab2").size());   // cf-list contains only 1 cf
200     assertEquals("cf1", tabCFsMap.get("tab2").get(0));// the only cf is "cf1"
201 
202     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("tab3 : cf1 , cf3");
203     assertEquals(1, tabCFsMap.size()); // only one table
204     assertTrue(tabCFsMap.containsKey("tab3"));   // its table name is "tab2"
205     assertFalse(tabCFsMap.containsKey("tab1"));  // not other table
206     assertEquals(2, tabCFsMap.get("tab3").size());   // cf-list contains 2 cf
207     assertTrue(tabCFsMap.get("tab3").contains("cf1"));// contains "cf1"
208     assertTrue(tabCFsMap.get("tab3").contains("cf3"));// contains "cf3"
209 
210     // 3. multiple tables: "tab1 ; tab2:cf1 ; tab3:cf1,cf3"
211     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig("tab1 ; tab2:cf1 ; tab3:cf1,cf3");
212     // 3.1 contains 3 tables : "tab1", "tab2" and "tab3"
213     assertEquals(3, tabCFsMap.size());
214     assertTrue(tabCFsMap.containsKey("tab1"));
215     assertTrue(tabCFsMap.containsKey("tab2"));
216     assertTrue(tabCFsMap.containsKey("tab3"));
217     // 3.2 table "tab1" : null cf-list
218     assertEquals(null, tabCFsMap.get("tab1"));
219     // 3.3 table "tab2" : cf-list contains a single cf "cf1"
220     assertEquals(1, tabCFsMap.get("tab2").size());
221     assertEquals("cf1", tabCFsMap.get("tab2").get(0));
222     // 3.4 table "tab3" : cf-list contains "cf1" and "cf3"
223     assertEquals(2, tabCFsMap.get("tab3").size());
224     assertTrue(tabCFsMap.get("tab3").contains("cf1"));
225     assertTrue(tabCFsMap.get("tab3").contains("cf3"));
226 
227     // 4. contiguous or additional ";"(table delimiter) or ","(cf delimiter) can be tolerated
228     // still use the example of multiple tables: "tab1 ; tab2:cf1 ; tab3:cf1,cf3"
229     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig(
230       "tab1 ; ; tab2:cf1 ; tab3:cf1,,cf3 ;");
231     // 4.1 contains 3 tables : "tab1", "tab2" and "tab3"
232     assertEquals(3, tabCFsMap.size());
233     assertTrue(tabCFsMap.containsKey("tab1"));
234     assertTrue(tabCFsMap.containsKey("tab2"));
235     assertTrue(tabCFsMap.containsKey("tab3"));
236     // 4.2 table "tab1" : null cf-list
237     assertEquals(null, tabCFsMap.get("tab1"));
238     // 4.3 table "tab2" : cf-list contains a single cf "cf1"
239     assertEquals(1, tabCFsMap.get("tab2").size());
240     assertEquals("cf1", tabCFsMap.get("tab2").get(0));
241     // 4.4 table "tab3" : cf-list contains "cf1" and "cf3"
242     assertEquals(2, tabCFsMap.get("tab3").size());
243     assertTrue(tabCFsMap.get("tab3").contains("cf1"));
244     assertTrue(tabCFsMap.get("tab3").contains("cf3"));
245 
246     // 5. invalid format "tab1:tt:cf1 ; tab2::cf1 ; tab3:cf1,cf3"
247     //    "tab1:tt:cf1" and "tab2::cf1" are invalid and will be ignored totally
248     tabCFsMap = ReplicationPeerZKImpl.parseTableCFsFromConfig(
249       "tab1:tt:cf1 ; tab2::cf1 ; tab3:cf1,cf3");
250     // 5.1 no "tab1" and "tab2", only "tab3"
251     assertEquals(1, tabCFsMap.size()); // only one table
252     assertFalse(tabCFsMap.containsKey("tab1"));
253     assertFalse(tabCFsMap.containsKey("tab2"));
254     assertTrue(tabCFsMap.containsKey("tab3"));
255    // 5.2 table "tab3" : cf-list contains "cf1" and "cf3"
256     assertEquals(2, tabCFsMap.get("tab3").size());
257     assertTrue(tabCFsMap.get("tab3").contains("cf1"));
258     assertTrue(tabCFsMap.get("tab3").contains("cf3"));
259  }
260 
261   @Test(timeout=300000)
262   public void testPerTableCFReplication() throws Exception {
263     LOG.info("testPerTableCFReplication");
264     ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
265 
266     new HBaseAdmin(conf1).createTable(tabA);
267     new HBaseAdmin(conf1).createTable(tabB);
268     new HBaseAdmin(conf1).createTable(tabC);
269     new HBaseAdmin(conf2).createTable(tabA);
270     new HBaseAdmin(conf2).createTable(tabB);
271     new HBaseAdmin(conf2).createTable(tabC);
272     new HBaseAdmin(conf3).createTable(tabA);
273     new HBaseAdmin(conf3).createTable(tabB);
274     new HBaseAdmin(conf3).createTable(tabC);
275 
276     HTable htab1A = new HTable(conf1, tabAName);
277     HTable htab2A = new HTable(conf2, tabAName);
278     HTable htab3A = new HTable(conf3, tabAName);
279 
280     HTable htab1B = new HTable(conf1, tabBName);
281     HTable htab2B = new HTable(conf2, tabBName);
282     HTable htab3B = new HTable(conf3, tabBName);
283 
284     HTable htab1C = new HTable(conf1, tabCName);
285     HTable htab2C = new HTable(conf2, tabCName);
286     HTable htab3C = new HTable(conf3, tabCName);
287 
288     // A. add cluster2/cluster3 as peers to cluster1
289     admin1.addPeer("2", utility2.getClusterKey(), "TC;TB:f1,f3");
290     admin1.addPeer("3", utility3.getClusterKey(), "TA;TB:f1,f2");
291 
292     // A1. tableA can only replicated to cluster3
293     putAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
294     ensureRowNotReplicated(row1, f1Name, htab2A);
295     deleteAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
296 
297     putAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
298     ensureRowNotReplicated(row1, f2Name, htab2A);
299     deleteAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
300 
301     putAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
302     ensureRowNotReplicated(row1, f3Name, htab2A);
303     deleteAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
304 
305     // A2. cf 'f1' of tableB can replicated to both cluster2 and cluster3
306     putAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
307     deleteAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
308 
309     //  cf 'f2' of tableB can only replicated to cluster3
310     putAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
311     ensureRowNotReplicated(row1, f2Name, htab2B);
312     deleteAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
313 
314     //  cf 'f3' of tableB can only replicated to cluster2
315     putAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
316     ensureRowNotReplicated(row1, f3Name, htab3B);
317     deleteAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
318 
319     // A3. tableC can only replicated to cluster2
320     putAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
321     ensureRowNotReplicated(row1, f1Name, htab3C);
322     deleteAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
323 
324     putAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
325     ensureRowNotReplicated(row1, f2Name, htab3C);
326     deleteAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
327 
328     putAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
329     ensureRowNotReplicated(row1, f3Name, htab3C);
330     deleteAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
331 
332     // B. change peers' replicable table-cf config
333     admin1.setPeerTableCFs("2", "TA:f1,f2; TC:f2,f3");
334     admin1.setPeerTableCFs("3", "TB; TC:f3");
335 
336     // B1. cf 'f1' of tableA can only replicated to cluster2
337     putAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
338     ensureRowNotReplicated(row2, f1Name, htab3A);
339     deleteAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
340     //     cf 'f2' of tableA can only replicated to cluster2
341     putAndWaitWithFamily(row2, f2Name, htab1A, htab2A);
342     ensureRowNotReplicated(row2, f2Name, htab3A);
343     deleteAndWaitWithFamily(row2, f2Name, htab1A, htab2A);
344     //     cf 'f3' of tableA isn't replicable to either cluster2 or cluster3
345     putAndWaitWithFamily(row2, f3Name, htab1A);
346     ensureRowNotReplicated(row2, f3Name, htab2A, htab3A);
347     deleteAndWaitWithFamily(row2, f3Name, htab1A);
348 
349     // B2. tableB can only replicated to cluster3
350     putAndWaitWithFamily(row2, f1Name, htab1B, htab3B);
351     ensureRowNotReplicated(row2, f1Name, htab2B);
352     deleteAndWaitWithFamily(row2, f1Name, htab1B, htab3B);
353 
354     putAndWaitWithFamily(row2, f2Name, htab1B, htab3B);
355     ensureRowNotReplicated(row2, f2Name, htab2B);
356     deleteAndWaitWithFamily(row2, f2Name, htab1B, htab3B);
357 
358     putAndWaitWithFamily(row2, f3Name, htab1B, htab3B);
359     ensureRowNotReplicated(row2, f3Name, htab2B);
360     deleteAndWaitWithFamily(row2, f3Name, htab1B, htab3B);
361 
362     // B3. cf 'f1' of tableC non-replicable to either cluster
363     putAndWaitWithFamily(row2, f1Name, htab1C);
364     ensureRowNotReplicated(row2, f1Name, htab2C, htab3C);
365     deleteAndWaitWithFamily(row2, f1Name, htab1C);
366     //     cf 'f2' of tableC can only replicated to cluster2
367     putAndWaitWithFamily(row2, f2Name, htab1C, htab2C);
368     ensureRowNotReplicated(row2, f2Name, htab3C);
369     deleteAndWaitWithFamily(row2, f2Name, htab1C, htab2C);
370     //     cf 'f3' of tableC can replicated to cluster2 and cluster3
371     putAndWaitWithFamily(row2, f3Name, htab1C, htab2C, htab3C);
372     deleteAndWaitWithFamily(row2, f3Name, htab1C, htab2C, htab3C);
373  }
374 
375   private void ensureRowNotReplicated(byte[] row, byte[] fam, HTable... tables) throws IOException {
376     Get get = new Get(row);
377     get.addFamily(fam);
378     for (HTable table : tables) {
379       Result res = table.get(get);
380       assertEquals(0, res.size());
381     }
382   }
383 
384   private void deleteAndWaitWithFamily(byte[] row, byte[] fam,
385       HTable source, HTable... targets)
386     throws Exception {
387     Delete del = new Delete(row);
388     del.deleteFamily(fam);
389     source.delete(del);
390 
391     Get get = new Get(row);
392     get.addFamily(fam);
393     for (int i = 0; i < NB_RETRIES; i++) {
394       if (i==NB_RETRIES-1) {
395         fail("Waited too much time for del replication");
396       }
397       boolean removedFromAll = true;
398       for (HTable target : targets) {
399         Result res = target.get(get);
400         if (res.size() >= 1) {
401           LOG.info("Row not deleted");
402           removedFromAll = false;
403           break;
404         }
405       }
406       if (removedFromAll) {
407         break;
408       } else {
409         Thread.sleep(SLEEP_TIME);
410       }
411     }
412   }
413 
414   private void putAndWaitWithFamily(byte[] row, byte[] fam,
415       HTable source, HTable... targets)
416     throws Exception {
417     Put put = new Put(row);
418     put.add(fam, row, val);
419     source.put(put);
420 
421     Get get = new Get(row);
422     get.addFamily(fam);
423     for (int i = 0; i < NB_RETRIES; i++) {
424       if (i==NB_RETRIES-1) {
425         fail("Waited too much time for put replication");
426       }
427       boolean replicatedToAll = true;
428       for (HTable target : targets) {
429         Result res = target.get(get);
430         if (res.size() == 0) {
431           LOG.info("Row not available");
432           replicatedToAll = false;
433           break;
434         } else {
435           assertEquals(res.size(), 1);
436           assertArrayEquals(res.value(), val);
437         }
438       }
439       if (replicatedToAll) {
440         break;
441       } else {
442         Thread.sleep(SLEEP_TIME);
443       }
444     }
445   }
446 }