1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.List;
28 import java.util.NavigableSet;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.Cell;
37 import org.apache.hadoop.hbase.CellUtil;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.KeyValueTestUtil;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.io.compress.Compression;
45 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
46 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
47 import org.apache.hadoop.hbase.testclassification.SmallTests;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.junit.Before;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52 import org.junit.runner.RunWith;
53 import org.junit.runners.Parameterized;
54 import org.junit.runners.Parameterized.Parameters;
55
56 import static org.junit.Assert.*;
57
58
59
60
61
62 @RunWith(Parameterized.class)
63 @Category(SmallTests.class)
64 public class TestScanWithBloomError {
65
66 private static final Log LOG =
67 LogFactory.getLog(TestScanWithBloomError.class);
68
69 private static final String TABLE_NAME = "ScanWithBloomError";
70 private static final String FAMILY = "myCF";
71 private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
72 private static final String ROW = "theRow";
73 private static final String QUALIFIER_PREFIX = "qual";
74 private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
75 private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
76 private HRegion region;
77 private BloomType bloomType;
78 private FileSystem fs;
79 private Configuration conf;
80
81 private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
82
83 @Parameters
84 public static final Collection<Object[]> parameters() {
85 List<Object[]> configurations = new ArrayList<Object[]>();
86 for (BloomType bloomType : BloomType.values()) {
87 configurations.add(new Object[] { bloomType });
88 }
89 return configurations;
90 }
91
92 public TestScanWithBloomError(BloomType bloomType) {
93 this.bloomType = bloomType;
94 }
95
96 @Before
97 public void setUp() throws IOException{
98 conf = TEST_UTIL.getConfiguration();
99 fs = FileSystem.get(conf);
100 }
101
102 @Test
103 public void testThreeStoreFiles() throws IOException {
104 region = TEST_UTIL.createTestRegion(TABLE_NAME,
105 new HColumnDescriptor(FAMILY)
106 .setCompressionType(Compression.Algorithm.GZ)
107 .setBloomFilterType(bloomType)
108 .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
109 createStoreFile(new int[] {1, 2, 6});
110 createStoreFile(new int[] {1, 2, 3, 7});
111 createStoreFile(new int[] {1, 9});
112 scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
113
114 HRegion.closeHRegion(region);
115 }
116
117 private void scanColSet(int[] colSet, int[] expectedResultCols)
118 throws IOException {
119 LOG.info("Scanning column set: " + Arrays.toString(colSet));
120 Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
121 addColumnSetToScan(scan, colSet);
122 RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
123 KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
124 assertEquals(0, storeHeap.getHeap().size());
125 StoreScanner storeScanner =
126 (StoreScanner) storeHeap.getCurrentForTesting();
127 @SuppressWarnings({ "unchecked", "rawtypes" })
128 List<StoreFileScanner> scanners = (List<StoreFileScanner>)
129 (List) storeScanner.getAllScannersForTesting();
130
131
132 Collections.sort(scanners, new Comparator<StoreFileScanner>() {
133 @Override
134 public int compare(StoreFileScanner s1, StoreFileScanner s2) {
135 Path p1 = s1.getReaderForTesting().getHFileReader().getPath();
136 Path p2 = s2.getReaderForTesting().getHFileReader().getPath();
137 long t1, t2;
138 try {
139 t1 = fs.getFileStatus(p1).getModificationTime();
140 t2 = fs.getFileStatus(p2).getModificationTime();
141 } catch (IOException ex) {
142 throw new RuntimeException(ex);
143 }
144 return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
145 }
146 });
147
148 StoreFile.Reader lastStoreFileReader = null;
149 for (StoreFileScanner sfScanner : scanners)
150 lastStoreFileReader = sfScanner.getReaderForTesting();
151
152 new HFilePrettyPrinter(conf).run(new String[]{ "-m", "-p", "-f",
153 lastStoreFileReader.getHFileReader().getPath().toString()});
154
155
156
157 LOG.info("Disabling Bloom filter for: "
158 + lastStoreFileReader.getHFileReader().getName());
159 lastStoreFileReader.disableBloomFilterForTesting();
160
161 List<Cell> allResults = new ArrayList<Cell>();
162
163 {
164 List<Cell> results = new ArrayList<Cell>();
165 while (scanner.next(results) || results.size() > 0) {
166 allResults.addAll(results);
167 results.clear();
168 }
169 }
170
171 List<Integer> actualIds = new ArrayList<Integer>();
172 for (Cell kv : allResults) {
173 String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
174 assertTrue(qual.startsWith(QUALIFIER_PREFIX));
175 actualIds.add(Integer.valueOf(qual.substring(
176 QUALIFIER_PREFIX.length())));
177 }
178 List<Integer> expectedIds = new ArrayList<Integer>();
179 for (int expectedId : expectedResultCols)
180 expectedIds.add(expectedId);
181
182 LOG.info("Column ids returned: " + actualIds + ", expected: "
183 + expectedIds);
184 assertEquals(expectedIds.toString(), actualIds.toString());
185 }
186
187 private void addColumnSetToScan(Scan scan, int[] colIds) {
188 for (int colId : colIds) {
189 scan.addColumn(FAMILY_BYTES,
190 Bytes.toBytes(qualFromId(colId)));
191 }
192 }
193
194 private String qualFromId(int colId) {
195 return QUALIFIER_PREFIX + colId;
196 }
197
198 private void createStoreFile(int[] colIds)
199 throws IOException {
200 Put p = new Put(ROW_BYTES);
201 for (int colId : colIds) {
202 long ts = Long.MAX_VALUE;
203 String qual = qualFromId(colId);
204 allColIds.add(colId);
205 KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
206 qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
207 p.add(kv);
208 }
209 region.put(p);
210 region.flushcache();
211 }
212
213
214 }
215