View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES;
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
24  import static org.junit.Assert.assertEquals;
25  import static org.junit.Assert.assertTrue;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.testclassification.MediumTests;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.regionserver.wal.HLog;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.junit.After;
42  import org.junit.Before;
43  import org.junit.Rule;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  import org.junit.rules.TestName;
47  
48  
49  /**
50   * Test minor compactions
51   */
52  @Category(MediumTests.class)
53  public class TestMinorCompaction {
54    @Rule public TestName name = new TestName();
55    static final Log LOG = LogFactory.getLog(TestMinorCompaction.class.getName());
56    private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
57    protected Configuration conf = UTIL.getConfiguration();
58    
59    private HRegion r = null;
60    private HTableDescriptor htd = null;
61    private int compactionThreshold;
62    private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
63    final private byte[] col1, col2;
64  
65    /** constructor */
66    public TestMinorCompaction() {
67      super();
68  
69      // Set cache flush size to 1MB
70      conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
71      conf.setInt("hbase.hregion.memstore.block.multiplier", 100);
72      compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
73  
74      firstRowBytes = START_KEY_BYTES;
75      secondRowBytes = START_KEY_BYTES.clone();
76      // Increment the least significant character so we get to next row.
77      secondRowBytes[START_KEY_BYTES.length - 1]++;
78      thirdRowBytes = START_KEY_BYTES.clone();
79      thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
80      col1 = Bytes.toBytes("column1");
81      col2 = Bytes.toBytes("column2");
82    }
83  
84    @Before
85    public void setUp() throws Exception {
86      this.htd = UTIL.createTableDescriptor(name.getMethodName());
87      this.r = UTIL.createLocalHRegion(htd, null, null);
88    }
89  
90    @After
91    public void tearDown() throws Exception {
92      HLog hlog = r.getLog();
93      this.r.close();
94      hlog.closeAndDelete();
95    }
96  
97    @Test
98    public void testMinorCompactionWithDeleteRow() throws Exception {
99      Delete deleteRow = new Delete(secondRowBytes);
100     testMinorCompactionWithDelete(deleteRow);
101   }
102 
103   @Test
104   public void testMinorCompactionWithDeleteColumn1() throws Exception {
105     Delete dc = new Delete(secondRowBytes);
106     /* delete all timestamps in the column */
107     dc.deleteColumns(fam2, col2);
108     testMinorCompactionWithDelete(dc);
109   }
110 
111   @Test
112   public void testMinorCompactionWithDeleteColumn2() throws Exception {
113     Delete dc = new Delete(secondRowBytes);
114     dc.deleteColumn(fam2, col2);
115     /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
116      * we only delete the latest version. One might expect to see only
117      * versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
118      * This is okay as well. Since there was no compaction done before the
119      * delete, version 0 seems to stay on.
120      */
121     testMinorCompactionWithDelete(dc, 3);
122   }
123 
124   @Test
125   public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
126     Delete deleteCF = new Delete(secondRowBytes);
127     deleteCF.deleteFamily(fam2);
128     testMinorCompactionWithDelete(deleteCF);
129   }
130 
131   @Test
132   public void testMinorCompactionWithDeleteVersion1() throws Exception {
133     Delete deleteVersion = new Delete(secondRowBytes);
134     deleteVersion.deleteColumns(fam2, col2, 2);
135     /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
136      * We delete versions 0 ... 2. So, we still have one remaining.
137      */
138     testMinorCompactionWithDelete(deleteVersion, 1);
139   }
140 
141   @Test
142   public void testMinorCompactionWithDeleteVersion2() throws Exception {
143     Delete deleteVersion = new Delete(secondRowBytes);
144     deleteVersion.deleteColumn(fam2, col2, 1);
145     /*
146      * the table has 4 versions: 0, 1, 2, and 3.
147      * We delete 1.
148      * Should have 3 remaining.
149      */
150     testMinorCompactionWithDelete(deleteVersion, 3);
151   }
152 
153   /*
154    * A helper function to test the minor compaction algorithm. We check that
155    * the delete markers are left behind. Takes delete as an argument, which
156    * can be any delete (row, column, columnfamliy etc), that essentially
157    * deletes row2 and column2. row1 and column1 should be undeleted
158    */
159   private void testMinorCompactionWithDelete(Delete delete) throws Exception {
160     testMinorCompactionWithDelete(delete, 0);
161   }
162 
163   private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception {
164     HRegionIncommon loader = new HRegionIncommon(r);
165     for (int i = 0; i < compactionThreshold + 1; i++) {
166       HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
167         thirdRowBytes, i);
168       HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
169         thirdRowBytes, i);
170       HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
171         thirdRowBytes, i);
172       HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
173         thirdRowBytes, i);
174       r.flushcache();
175     }
176 
177     Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
178     assertEquals(compactionThreshold, result.size());
179     result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
180     assertEquals(compactionThreshold, result.size());
181 
182     // Now add deletes to memstore and then flush it.  That will put us over
183     // the compaction threshold of 3 store files.  Compacting these store files
184     // should result in a compacted store file that has no references to the
185     // deleted row.
186     r.delete(delete);
187 
188     // Make sure that we have only deleted family2 from secondRowBytes
189     result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
190     assertEquals(expectedResultsAfterDelete, result.size());
191     // but we still have firstrow
192     result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
193     assertEquals(compactionThreshold, result.size());
194 
195     r.flushcache();
196     // should not change anything.
197     // Let us check again
198 
199     // Make sure that we have only deleted family2 from secondRowBytes
200     result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
201     assertEquals(expectedResultsAfterDelete, result.size());
202     // but we still have firstrow
203     result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
204     assertEquals(compactionThreshold, result.size());
205 
206     // do a compaction
207     Store store2 = this.r.stores.get(fam2);
208     int numFiles1 = store2.getStorefiles().size();
209     assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3
210     ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold);   // = 3
211     int numFiles2 = store2.getStorefiles().size();
212     // Check that we did compact
213     assertTrue("Number of store files should go down", numFiles1 > numFiles2);
214     // Check that it was a minor compaction.
215     assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
216 
217     // Make sure that we have only deleted family2 from secondRowBytes
218     result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
219     assertEquals(expectedResultsAfterDelete, result.size());
220     // but we still have firstrow
221     result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
222     assertEquals(compactionThreshold, result.size());
223   }
224 }