1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import org.apache.commons.io.IOUtils;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FileUtil;
24 import org.apache.hadoop.fs.LocalFileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.testclassification.LargeTests;
28 import org.apache.hadoop.hbase.client.HTable;
29 import org.apache.hadoop.hbase.client.Put;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32 import org.apache.hadoop.mapreduce.Job;
33 import org.apache.hadoop.util.GenericOptionsParser;
34 import org.junit.AfterClass;
35 import org.junit.BeforeClass;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39 import java.io.*;
40
41 import static org.junit.Assert.assertTrue;
42 import static org.junit.Assert.assertEquals;
43 import static org.junit.Assert.fail;
44
45 @Category(LargeTests.class)
46 public class TestCellCounter {
47 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48 private static final byte[] ROW1 = Bytes.toBytes("row1");
49 private static final byte[] ROW2 = Bytes.toBytes("row2");
50 private static final String FAMILY_A_STRING = "a";
51 private static final String FAMILY_B_STRING = "b";
52 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54 private static final byte[] QUALIFIER = Bytes.toBytes("q");
55
56 private static Path FQ_OUTPUT_DIR;
57 private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58 + "output";
59 private static long now = System.currentTimeMillis();
60
61 @BeforeClass
62 public static void beforeClass() throws Exception {
63 UTIL.startMiniCluster();
64 UTIL.startMiniMapReduceCluster();
65 FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66 FileUtil.fullyDelete(new File(OUTPUT_DIR));
67 }
68
69 @AfterClass
70 public static void afterClass() throws Exception {
71 UTIL.shutdownMiniMapReduceCluster();
72 UTIL.shutdownMiniCluster();
73 }
74
75
76
77
78
79 @Test (timeout=300000)
80 public void testCellCounter() throws Exception {
81 String sourceTable = "sourceTable";
82 byte[][] families = { FAMILY_A, FAMILY_B };
83 HTable t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
84 try{
85 Put p = new Put(ROW1);
86 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
87 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
88 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
89 t.put(p);
90 p = new Put(ROW2);
91 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
92 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
93 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
94 t.put(p);
95 String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
96 runCount(args);
97 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
98 "part-r-00000");
99 String data = IOUtils.toString(inputStream);
100 inputStream.close();
101 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
102 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
103 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
104 assertTrue(data.contains("b;q" + "\t" + "1"));
105 assertTrue(data.contains("a;q" + "\t" + "1"));
106 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
107 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
108 }finally{
109 t.close();
110 FileUtil.fullyDelete(new File(OUTPUT_DIR));
111 }
112
113 }
114
115
116
117
118 @Test (timeout=300000)
119 public void testCellCounterStartTimeRange() throws Exception {
120 String sourceTable = "testCellCounterStartTimeRange";
121 byte[][] families = { FAMILY_A, FAMILY_B };
122 HTable t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
123 try{
124 Put p = new Put(ROW1);
125 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
126 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
127 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
128 t.put(p);
129 p = new Put(ROW2);
130 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
131 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
132 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
133 t.put(p);
134 String[] args = {
135 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--starttime=" + now,
136 "--endtime=" + now + 2 };
137 runCount(args);
138 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
139 "part-r-00000");
140 String data = IOUtils.toString(inputStream);
141 inputStream.close();
142 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
143 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
144 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
145 assertTrue(data.contains("b;q" + "\t" + "1"));
146 assertTrue(data.contains("a;q" + "\t" + "1"));
147 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
148 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
149 }finally{
150 t.close();
151 FileUtil.fullyDelete(new File(OUTPUT_DIR));
152 }
153 }
154
155
156
157
158 @Test (timeout=300000)
159 public void testCellCounteEndTimeRange() throws Exception {
160 String sourceTable = "testCellCounterEndTimeRange";
161 byte[][] families = { FAMILY_A, FAMILY_B };
162 HTable t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
163 try{
164 Put p = new Put(ROW1);
165 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
166 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
167 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
168 t.put(p);
169 p = new Put(ROW2);
170 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
171 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
172 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
173 t.put(p);
174 String[] args = {
175 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--endtime=" + now + 1 };
176 runCount(args);
177 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
178 "part-r-00000");
179 String data = IOUtils.toString(inputStream);
180 inputStream.close();
181 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
182 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
183 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
184 assertTrue(data.contains("b;q" + "\t" + "1"));
185 assertTrue(data.contains("a;q" + "\t" + "1"));
186 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
187 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
188 }finally{
189 t.close();
190 FileUtil.fullyDelete(new File(OUTPUT_DIR));
191 }
192 }
193
194
195
196
197 @Test (timeout=300000)
198 public void testCellCounteOutOfTimeRange() throws Exception {
199 String sourceTable = "testCellCounterOutTimeRange";
200 byte[][] families = { FAMILY_A, FAMILY_B };
201 HTable t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
202 try{
203 Put p = new Put(ROW1);
204 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
205 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
206 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
207 t.put(p);
208 p = new Put(ROW2);
209 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
210 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
211 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
212 t.put(p);
213 String[] args = {
214 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "--starttime=" + now + 1,
215 "--endtime=" + now + 2 };
216
217 runCount(args);
218 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
219 "part-r-00000");
220 String data = IOUtils.toString(inputStream);
221 inputStream.close();
222
223 assertTrue(data.isEmpty());
224 }finally{
225 t.close();
226 FileUtil.fullyDelete(new File(OUTPUT_DIR));
227 }
228 }
229
230
231 private boolean runCount(String[] args) throws IOException, InterruptedException,
232 ClassNotFoundException {
233
234
235 GenericOptionsParser opts = new GenericOptionsParser(
236 new Configuration(UTIL.getConfiguration()), args);
237 Configuration configuration = opts.getConfiguration();
238 args = opts.getRemainingArgs();
239 Job job = CellCounter.createSubmittableJob(configuration, args);
240 job.waitForCompletion(false);
241 return job.isSuccessful();
242 }
243
244
245
246
247 @Test (timeout=300000)
248 public void testCellCounterMain() throws Exception {
249
250 PrintStream oldPrintStream = System.err;
251 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
252 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
253 System.setSecurityManager(newSecurityManager);
254 ByteArrayOutputStream data = new ByteArrayOutputStream();
255 String[] args = {};
256 System.setErr(new PrintStream(data));
257 try {
258 System.setErr(new PrintStream(data));
259
260 try {
261 CellCounter.main(args);
262 fail("should be SecurityException");
263 } catch (SecurityException e) {
264 assertEquals(-1, newSecurityManager.getExitCode());
265 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
266
267 assertTrue(data.toString().contains("Usage:"));
268 }
269
270 } finally {
271 System.setErr(oldPrintStream);
272 System.setSecurityManager(SECURITY_MANAGER);
273 }
274 }
275
276 @Test(timeout = 300000)
277 public void testCellCounterForCompleteTable() throws Exception {
278 String sourceTable = "testCellCounterForCompleteTable";
279 String outputPath = OUTPUT_DIR + sourceTable;
280 LocalFileSystem localFileSystem = new LocalFileSystem();
281 Path outputDir =
282 new Path(outputPath).makeQualified(localFileSystem.getUri(),
283 localFileSystem.getWorkingDirectory());
284 byte[][] families = { FAMILY_A, FAMILY_B };
285 HTable t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
286 try {
287 Put p = new Put(ROW1);
288 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
289 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
290 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
291 t.put(p);
292 p = new Put(ROW2);
293 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
294 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
295 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
296 t.put(p);
297 String[] args = { sourceTable, outputDir.toString(), ";" };
298 runCount(args);
299 FileInputStream inputStream =
300 new FileInputStream(outputPath + File.separator + "part-r-00000");
301 String data = IOUtils.toString(inputStream);
302 inputStream.close();
303 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
304 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
305 assertTrue(data.contains("Total ROWS" + "\t" + "2"));
306 assertTrue(data.contains("b;q" + "\t" + "2"));
307 assertTrue(data.contains("a;q" + "\t" + "2"));
308 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
309 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
310 assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
311 assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
312 } finally {
313 t.close();
314 FileUtil.fullyDelete(new File(outputPath));
315 }
316 }
317
318 @Test (timeout=300000)
319 public void TestCellCounterWithoutOutputDir() throws Exception {
320 PrintStream oldPrintStream = System.err;
321 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
322 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
323 System.setSecurityManager(newSecurityManager);
324 ByteArrayOutputStream data = new ByteArrayOutputStream();
325 String[] args = {"tableName"};
326 System.setErr(new PrintStream(data));
327 try {
328 System.setErr(new PrintStream(data));
329 try {
330 CellCounter.main(args);
331 fail("should be SecurityException");
332 } catch (SecurityException e) {
333 assertEquals(-1, newSecurityManager.getExitCode());
334 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
335
336 assertTrue(data.toString().contains("Usage:"));
337 }
338
339 } finally {
340 System.setErr(oldPrintStream);
341 System.setSecurityManager(SECURITY_MANAGER);
342 }
343 }
344 }