View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.util.List;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.TableName;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.testclassification.LargeTests;
35  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.After;
40  import org.junit.AfterClass;
41  import org.junit.Assert;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * Test to verify that the cloned table is independent of the table from which it was cloned
49   */
50  @Category(LargeTests.class)
51  public class TestSnapshotCloneIndependence {
52    private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
53  
54    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55  
56    private static final int NUM_RS = 2;
57    private static final String STRING_TABLE_NAME = "test";
58    private static final String TEST_FAM_STR = "fam";
59    private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
60    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
61  
62    /**
63     * Setup the config for the cluster and start it
64     * @throws Exception on failure
65     */
66    @BeforeClass
67    public static void setupCluster() throws Exception {
68      setupConf(UTIL.getConfiguration());
69      UTIL.startMiniCluster(NUM_RS);
70    }
71  
72    private static void setupConf(Configuration conf) {
73      // enable snapshot support
74      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
75      // disable the ui
76      conf.setInt("hbase.regionsever.info.port", -1);
77      // change the flush size to a small amount, regulating number of store files
78      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
79      // so make sure we get a compaction when doing a load, but keep around
80      // some files in the store
81      conf.setInt("hbase.hstore.compaction.min", 10);
82      conf.setInt("hbase.hstore.compactionThreshold", 10);
83      // block writes if we get to 12 store files
84      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
85      conf.setInt("hbase.regionserver.msginterval", 100);
86      conf.setBoolean("hbase.master.enabletable.roundrobin", true);
87      // Avoid potentially aggressive splitting which would cause snapshot to fail
88      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
89        ConstantSizeRegionSplitPolicy.class.getName());
90    }
91  
92    @Before
93    public void setup() throws Exception {
94      UTIL.createTable(TABLE_NAME, TEST_FAM);
95    }
96  
97    @After
98    public void tearDown() throws Exception {
99      UTIL.deleteTable(TABLE_NAME);
100     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
101     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
102   }
103 
104   @AfterClass
105   public static void cleanupTest() throws Exception {
106     try {
107       UTIL.shutdownMiniCluster();
108     } catch (Exception e) {
109       LOG.warn("failure shutting down cluster", e);
110     }
111   }
112 
113   /**
114    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
115    * it is taken as an online snapshot.
116    */
117   @Test (timeout=300000)
118   public void testOnlineSnapshotAppendIndependent() throws Exception {
119     runTestSnapshotAppendIndependent(true);
120   }
121 
122   /**
123    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
124    * it is taken as an offline snapshot.
125    */
126   @Test (timeout=300000)
127   public void testOfflineSnapshotAppendIndependent() throws Exception {
128     runTestSnapshotAppendIndependent(false);
129   }
130 
131   /**
132    * Verify that adding metadata to the cloned table will not affect the original, and vice-versa
133    * when it is taken as an online snapshot.
134    */
135   @Test (timeout=300000)
136   public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
137     runTestSnapshotMetadataChangesIndependent(true);
138   }
139 
140   /**
141    * Verify that adding netadata to the cloned table will not affect the original, and vice-versa
142    * when is taken as an online snapshot.
143    */
144   @Test (timeout=300000)
145   public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
146     runTestSnapshotMetadataChangesIndependent(false);
147   }
148 
149   /**
150    * Verify that region operations, in this case splitting a region, are independent between the
151    * cloned table and the original.
152    */
153   @Test (timeout=300000)
154   public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
155     runTestRegionOperationsIndependent(false);
156   }
157 
158   /**
159    * Verify that region operations, in this case splitting a region, are independent between the
160    * cloned table and the original.
161    */
162   @Test (timeout=300000)
163   public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
164     runTestRegionOperationsIndependent(true);
165   }
166 
167   private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
168     for (int i = 0; i < 200; i++) {
169       try {
170         Thread.sleep(50);
171       } catch (InterruptedException e) {
172         // Restore the interrupted status
173         Thread.currentThread().interrupt();
174       }
175       if (t.getRegionLocations().size() > originalCount) {
176         return;
177       }
178     }
179     throw new Exception("Split did not increase the number of regions");
180   }
181 
182   /*
183    * Take a snapshot of a table, add data, and verify that this only
184    * affects one table
185    * @param online - Whether the table is online or not during the snapshot
186    */
187   private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
188     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
189     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
190 
191     HBaseAdmin admin = UTIL.getHBaseAdmin();
192     final long startTime = System.currentTimeMillis();
193     final TableName localTableName =
194         TableName.valueOf(STRING_TABLE_NAME + startTime);
195 
196     HTable original = UTIL.createTable(localTableName, TEST_FAM);
197     try {
198 
199       UTIL.loadTable(original, TEST_FAM);
200       final int origTableRowCount = UTIL.countRows(original);
201 
202       // Take a snapshot
203       final String snapshotNameAsString = "snapshot_" + localTableName;
204       byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
205 
206       SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
207         snapshotNameAsString, rootDir, fs, online);
208 
209       if (!online) {
210         admin.enableTable(localTableName);
211       }
212       byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
213       admin.cloneSnapshot(snapshotName, cloneTableName);
214 
215       HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
216 
217       try {
218         final int clonedTableRowCount = UTIL.countRows(clonedTable);
219 
220         Assert.assertEquals(
221           "The line counts of original and cloned tables do not match after clone. ",
222           origTableRowCount, clonedTableRowCount);
223 
224         // Attempt to add data to the test
225         final String rowKey = "new-row-" + System.currentTimeMillis();
226 
227         Put p = new Put(Bytes.toBytes(rowKey));
228         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
229         original.put(p);
230         original.flushCommits();
231 
232         // Verify that it is not present in the original table
233         Assert.assertEquals("The row count of the original table was not modified by the put",
234           origTableRowCount + 1, UTIL.countRows(original));
235         Assert.assertEquals(
236           "The row count of the cloned table changed as a result of addition to the original",
237           clonedTableRowCount, UTIL.countRows(clonedTable));
238 
239         p = new Put(Bytes.toBytes(rowKey));
240         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
241         clonedTable.put(p);
242         clonedTable.flushCommits();
243 
244         // Verify that the new family is not in the restored table's description
245         Assert.assertEquals(
246           "The row count of the original table was modified by the put to the clone",
247           origTableRowCount + 1, UTIL.countRows(original));
248         Assert.assertEquals("The row count of the cloned table was not modified by the put",
249           clonedTableRowCount + 1, UTIL.countRows(clonedTable));
250       } finally {
251 
252         clonedTable.close();
253       }
254     } finally {
255 
256       original.close();
257     }
258   }
259 
260   /*
261    * Take a snapshot of a table, do a split, and verify that this only affects one table
262    * @param online - Whether the table is online or not during the snapshot
263    */
264   private void runTestRegionOperationsIndependent(boolean online) throws Exception {
265     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
266     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
267 
268     // Create a table
269     HBaseAdmin admin = UTIL.getHBaseAdmin();
270     final long startTime = System.currentTimeMillis();
271     final TableName localTableName =
272         TableName.valueOf(STRING_TABLE_NAME + startTime);
273     HTable original = UTIL.createTable(localTableName, TEST_FAM);
274     UTIL.loadTable(original, TEST_FAM);
275     final int loadedTableCount = UTIL.countRows(original);
276     System.out.println("Original table has: " + loadedTableCount + " rows");
277 
278     final String snapshotNameAsString = "snapshot_" + localTableName;
279 
280     // Create a snapshot
281     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
282       snapshotNameAsString, rootDir, fs, online);
283 
284     if (!online) {
285       admin.enableTable(localTableName);
286     }
287 
288     byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
289 
290     // Clone the snapshot
291     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
292     admin.cloneSnapshot(snapshotName, cloneTableName);
293 
294     // Verify that region information is the same pre-split
295     original.clearRegionCache();
296     List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
297 
298     final int originalRegionCount = originalTableHRegions.size();
299     final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
300     Assert.assertEquals(
301       "The number of regions in the cloned table is different than in the original table.",
302       originalRegionCount, cloneTableRegionCount);
303 
304     // Split a region on the parent table
305     admin.split(originalTableHRegions.get(0).getRegionName());
306     waitOnSplit(original, originalRegionCount);
307 
308     // Verify that the cloned table region is not split
309     final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
310     Assert.assertEquals(
311       "The number of regions in the cloned table changed though none of its regions were split.",
312       cloneTableRegionCount, cloneTableRegionCount2);
313   }
314 
315   /*
316    * Take a snapshot of a table, add metadata, and verify that this only
317    * affects one table
318    * @param online - Whether the table is online or not during the snapshot
319    */
320   private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
321     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
322     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
323 
324     // Create a table
325     HBaseAdmin admin = UTIL.getHBaseAdmin();
326     final long startTime = System.currentTimeMillis();
327     final TableName localTableName =
328         TableName.valueOf(STRING_TABLE_NAME + startTime);
329     HTable original = UTIL.createTable(localTableName, TEST_FAM);
330     UTIL.loadTable(original, TEST_FAM);
331 
332     final String snapshotNameAsString = "snapshot_" + localTableName;
333 
334     // Create a snapshot
335     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
336       snapshotNameAsString, rootDir, fs, online);
337 
338     if (!online) {
339       admin.enableTable(localTableName);
340     }
341     byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
342 
343     // Clone the snapshot
344     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
345     admin.cloneSnapshot(snapshotName, cloneTableName);
346 
347     // Add a new column family to the original table
348     byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
349     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
350 
351     admin.disableTable(localTableName);
352     admin.addColumn(localTableName, hcd);
353 
354     // Verify that it is not in the snapshot
355     admin.enableTable(localTableName);
356 
357     // get a description of the cloned table
358     // get a list of its families
359     // assert that the family is there
360     HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
361     HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
362 
363     Assert.assertTrue("The original family was not found. There is something wrong. ",
364       originalTableDescriptor.hasFamily(TEST_FAM));
365     Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
366       clonedTableDescriptor.hasFamily(TEST_FAM));
367 
368     Assert.assertTrue("The new family was not found. ",
369       originalTableDescriptor.hasFamily(TEST_FAM_2));
370     Assert.assertTrue("The new family was not found. ",
371       !clonedTableDescriptor.hasFamily(TEST_FAM_2));
372   }
373 }