View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.HashMap;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Set;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.hbase.*;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.client.Durability;
38  import org.apache.hadoop.hbase.testclassification.SmallTests;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.junit.Rule;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  import org.junit.rules.TestName;
44  
45  @Category(SmallTests.class)
46  public class TestColumnSeeking {
47    @Rule public TestName name = new TestName();
48  
49    private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
50  
51    static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
52  
53    @SuppressWarnings("unchecked")
54    @Test
55    public void testDuplicateVersions() throws IOException {
56      String family = "Family";
57      byte[] familyBytes = Bytes.toBytes("Family");
58      TableName table = TableName.valueOf(name.getMethodName());
59  
60      HColumnDescriptor hcd =
61          new HColumnDescriptor(familyBytes).setMaxVersions(1000);
62      hcd.setMaxVersions(3);
63      HTableDescriptor htd = new HTableDescriptor(table);
64      htd.addFamily(hcd);
65      HRegionInfo info = new HRegionInfo(table, null, null, false);
66      // Set this so that the archiver writes to the temp dir as well.
67      HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
68      try {
69        List<String> rows = generateRandomWords(10, "row");
70        List<String> allColumns = generateRandomWords(10, "column");
71        List<String> values = generateRandomWords(100, "value");
72  
73        long maxTimestamp = 2;
74        double selectPercent = 0.5;
75        int numberOfTests = 5;
76        double flushPercentage = 0.2;
77        double minorPercentage = 0.2;
78        double majorPercentage = 0.2;
79        double putPercentage = 0.2;
80  
81        HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
82  
83        HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
84        ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
85  
86        for (int i = 0; i < numberOfTests; i++) {
87          kvMaps[i] = new HashMap<String, KeyValue>();
88          columnLists[i] = new ArrayList<String>();
89          for (String column : allColumns) {
90            if (Math.random() < selectPercent) {
91              columnLists[i].add(column);
92            }
93          }
94        }
95  
96        for (String value : values) {
97          for (String row : rows) {
98            Put p = new Put(Bytes.toBytes(row));
99            p.setDurability(Durability.SKIP_WAL);
100           for (String column : allColumns) {
101             for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
102               KeyValue kv =
103                   KeyValueTestUtil.create(row, family, column, timestamp, value);
104               if (Math.random() < putPercentage) {
105                 p.add(kv);
106                 allKVMap.put(kv.getKeyString(), kv);
107                 for (int i = 0; i < numberOfTests; i++) {
108                   if (columnLists[i].contains(column)) {
109                     kvMaps[i].put(kv.getKeyString(), kv);
110                   }
111                 }
112               }
113             }
114           }
115           region.put(p);
116           if (Math.random() < flushPercentage) {
117             LOG.info("Flushing... ");
118             region.flushcache();
119           }
120 
121           if (Math.random() < minorPercentage) {
122             LOG.info("Minor compacting... ");
123             region.compactStores(false);
124           }
125 
126           if (Math.random() < majorPercentage) {
127             LOG.info("Major compacting... ");
128             region.compactStores(true);
129           }
130         }
131       }
132 
133       for (int i = 0; i < numberOfTests + 1; i++) {
134         Collection<KeyValue> kvSet;
135         Scan scan = new Scan();
136         scan.setMaxVersions();
137         if (i < numberOfTests) {
138           if (columnLists[i].size() == 0) continue; // HBASE-7700
139           kvSet = kvMaps[i].values();
140           for (String column : columnLists[i]) {
141             scan.addColumn(familyBytes, Bytes.toBytes(column));
142           }
143           LOG.info("ExplicitColumns scanner");
144           LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
145               + kvSet.size());
146         } else {
147           kvSet = allKVMap.values();
148           LOG.info("Wildcard scanner");
149           LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
150 
151         }
152         InternalScanner scanner = region.getScanner(scan);
153         List<Cell> results = new ArrayList<Cell>();
154         while (scanner.next(results))
155           ;
156         assertEquals(kvSet.size(), results.size());
157         assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
158       }
159     } finally {
160       HRegion.closeHRegion(region);
161     }
162 
163     HRegion.closeHRegion(region);
164   }
165 
166   @SuppressWarnings("unchecked")
167   @Test
168   public void testReseeking() throws IOException {
169     String family = "Family";
170     byte[] familyBytes = Bytes.toBytes("Family");
171     TableName table = TableName.valueOf(name.getMethodName());
172 
173     HTableDescriptor htd = new HTableDescriptor(table);
174     HColumnDescriptor hcd = new HColumnDescriptor(family);
175     hcd.setMaxVersions(3);
176     htd.addFamily(hcd);
177 
178     HRegionInfo info = new HRegionInfo(table, null, null, false);
179     HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
180 
181     List<String> rows = generateRandomWords(10, "row");
182     List<String> allColumns = generateRandomWords(100, "column");
183 
184     long maxTimestamp = 2;
185     double selectPercent = 0.5;
186     int numberOfTests = 5;
187     double flushPercentage = 0.2;
188     double minorPercentage = 0.2;
189     double majorPercentage = 0.2;
190     double putPercentage = 0.2;
191 
192     HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
193 
194     HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
195     ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
196     String valueString = "Value";
197 
198     for (int i = 0; i < numberOfTests; i++) {
199       kvMaps[i] = new HashMap<String, KeyValue>();
200       columnLists[i] = new ArrayList<String>();
201       for (String column : allColumns) {
202         if (Math.random() < selectPercent) {
203           columnLists[i].add(column);
204         }
205       }
206     }
207 
208     for (String row : rows) {
209       Put p = new Put(Bytes.toBytes(row));
210       p.setDurability(Durability.SKIP_WAL);
211       for (String column : allColumns) {
212         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
213           KeyValue kv =
214               KeyValueTestUtil.create(row, family, column, timestamp,
215                   valueString);
216           if (Math.random() < putPercentage) {
217             p.add(kv);
218             allKVMap.put(kv.getKeyString(), kv);
219             for (int i = 0; i < numberOfTests; i++) {
220               if (columnLists[i].contains(column)) {
221                 kvMaps[i].put(kv.getKeyString(), kv);
222               }
223             }
224           }
225 
226         }
227       }
228       region.put(p);
229       if (Math.random() < flushPercentage) {
230         LOG.info("Flushing... ");
231         region.flushcache();
232       }
233 
234       if (Math.random() < minorPercentage) {
235         LOG.info("Minor compacting... ");
236         region.compactStores(false);
237       }
238 
239       if (Math.random() < majorPercentage) {
240         LOG.info("Major compacting... ");
241         region.compactStores(true);
242       }
243     }
244 
245     for (int i = 0; i < numberOfTests + 1; i++) {
246       Collection<KeyValue> kvSet;
247       Scan scan = new Scan();
248       scan.setMaxVersions();
249       if (i < numberOfTests) {
250         if (columnLists[i].size() == 0) continue; // HBASE-7700
251         kvSet = kvMaps[i].values();
252         for (String column : columnLists[i]) {
253           scan.addColumn(familyBytes, Bytes.toBytes(column));
254         }
255         LOG.info("ExplicitColumns scanner");
256         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
257             + kvSet.size());
258       } else {
259         kvSet = allKVMap.values();
260         LOG.info("Wildcard scanner");
261         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
262 
263       }
264       InternalScanner scanner = region.getScanner(scan);
265       List<Cell> results = new ArrayList<Cell>();
266       while (scanner.next(results))
267         ;
268       assertEquals(kvSet.size(), results.size());
269       assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
270     }
271 
272     HRegion.closeHRegion(region);
273   }
274 
275   List<String> generateRandomWords(int numberOfWords, String suffix) {
276     Set<String> wordSet = new HashSet<String>();
277     for (int i = 0; i < numberOfWords; i++) {
278       int lengthOfWords = (int) (Math.random() * 5) + 1;
279       char[] wordChar = new char[lengthOfWords];
280       for (int j = 0; j < wordChar.length; j++) {
281         wordChar[j] = (char) (Math.random() * 26 + 97);
282       }
283       String word;
284       if (suffix == null) {
285         word = new String(wordChar);
286       } else {
287         word = new String(wordChar) + suffix;
288       }
289       wordSet.add(word);
290     }
291     List<String> wordList = new ArrayList<String>(wordSet);
292     return wordList;
293   }
294 
295 }
296