1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static java.lang.String.format;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.Arrays;
27 import java.util.Iterator;
28 import java.util.Set;
29 import java.util.TreeSet;
30 import java.util.UUID;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configurable;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.Cell;
39 import org.apache.hadoop.hbase.HBaseConfiguration;
40 import org.apache.hadoop.hbase.IntegrationTestingUtility;
41 import org.apache.hadoop.hbase.testclassification.IntegrationTests;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.KeyValue.Type;
44 import org.apache.hadoop.hbase.client.HTable;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
49 import org.apache.hadoop.util.GenericOptionsParser;
50 import org.apache.hadoop.util.Tool;
51 import org.apache.hadoop.util.ToolRunner;
52 import org.junit.AfterClass;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57
58
59
60 @Category(IntegrationTests.class)
61 public class IntegrationTestImportTsv implements Configurable, Tool {
62
63 private static final String NAME = IntegrationTestImportTsv.class.getSimpleName();
64 protected static final Log LOG = LogFactory.getLog(IntegrationTestImportTsv.class);
65
66 protected static final String simple_tsv =
67 "row1\t1\tc1\tc2\n" +
68 "row2\t1\tc1\tc2\n" +
69 "row3\t1\tc1\tc2\n" +
70 "row4\t1\tc1\tc2\n" +
71 "row5\t1\tc1\tc2\n" +
72 "row6\t1\tc1\tc2\n" +
73 "row7\t1\tc1\tc2\n" +
74 "row8\t1\tc1\tc2\n" +
75 "row9\t1\tc1\tc2\n" +
76 "row10\t1\tc1\tc2\n";
77
78 protected static final Set<KeyValue> simple_expected =
79 new TreeSet<KeyValue>(KeyValue.COMPARATOR) {
80 private static final long serialVersionUID = 1L;
81 {
82 byte[] family = Bytes.toBytes("d");
83 for (String line : simple_tsv.split("\n")) {
84 String[] row = line.split("\t");
85 byte[] key = Bytes.toBytes(row[0]);
86 long ts = Long.parseLong(row[1]);
87 byte[][] fields = { Bytes.toBytes(row[2]), Bytes.toBytes(row[3]) };
88 add(new KeyValue(key, family, fields[0], ts, Type.Put, fields[0]));
89 add(new KeyValue(key, family, fields[1], ts, Type.Put, fields[1]));
90 }
91 }
92 };
93
94
95
96 protected static IntegrationTestingUtility util = null;
97
98 public Configuration getConf() {
99 return util.getConfiguration();
100 }
101
102 public void setConf(Configuration conf) {
103 throw new IllegalArgumentException("setConf not supported");
104 }
105
106 @BeforeClass
107 public static void provisionCluster() throws Exception {
108 if (null == util) {
109 util = new IntegrationTestingUtility();
110 }
111 util.initializeCluster(1);
112 if (!util.isDistributedCluster()) {
113
114 util.startMiniMapReduceCluster();
115 }
116 }
117
118 @AfterClass
119 public static void releaseCluster() throws Exception {
120 util.restoreCluster();
121 if (!util.isDistributedCluster()) {
122 util.shutdownMiniMapReduceCluster();
123 }
124 util = null;
125 }
126
127
128
129
130
131 protected void doLoadIncrementalHFiles(Path hfiles, String tableName)
132 throws Exception {
133
134 String[] args = { hfiles.toString(), tableName };
135 LOG.info(format("Running LoadIncrememntalHFiles with args: %s", Arrays.asList(args)));
136 assertEquals("Loading HFiles failed.",
137 0, ToolRunner.run(new LoadIncrementalHFiles(new Configuration(getConf())), args));
138
139 HTable table = null;
140 Scan scan = new Scan() {{
141 setCacheBlocks(false);
142 setCaching(1000);
143 }};
144 try {
145 table = new HTable(getConf(), tableName);
146 Iterator<Result> resultsIt = table.getScanner(scan).iterator();
147 Iterator<KeyValue> expectedIt = simple_expected.iterator();
148 while (resultsIt.hasNext() && expectedIt.hasNext()) {
149 Result r = resultsIt.next();
150 for (Cell actual : r.rawCells()) {
151 assertTrue(
152 "Ran out of expected values prematurely!",
153 expectedIt.hasNext());
154 KeyValue expected = expectedIt.next();
155 assertTrue(
156 format("Scan produced surprising result. expected: <%s>, actual: %s",
157 expected, actual),
158 KeyValue.COMPARATOR.compare(expected, actual) == 0);
159 }
160 }
161 assertFalse("Did not consume all expected values.", expectedIt.hasNext());
162 assertFalse("Did not consume all scan results.", resultsIt.hasNext());
163 } finally {
164 if (null != table) table.close();
165 }
166 }
167
168
169
170
171 protected static void validateDeletedPartitionsFile(Configuration conf) throws IOException {
172 if (!conf.getBoolean(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, false))
173 return;
174
175 FileSystem fs = FileSystem.get(conf);
176 Path partitionsFile = new Path(TotalOrderPartitioner.getPartitionFile(conf));
177 assertFalse("Failed to clean up partitions file.", fs.exists(partitionsFile));
178 }
179
180 @Test
181 public void testGenerateAndLoad() throws Exception {
182 LOG.info("Running test testGenerateAndLoad.");
183 String table = NAME + "-" + UUID.randomUUID();
184 String cf = "d";
185 Path hfiles = new Path(util.getDataTestDirOnTestFS(table), "hfiles");
186
187 String[] args = {
188 format("-D%s=%s", ImportTsv.BULK_OUTPUT_CONF_KEY, hfiles),
189 format("-D%s=HBASE_ROW_KEY,HBASE_TS_KEY,%s:c1,%s:c2",
190 ImportTsv.COLUMNS_CONF_KEY, cf, cf),
191
192
193 format("-D%s=false", TestImportTsv.DELETE_AFTER_LOAD_CONF),
194 table
195 };
196
197
198 util.createTable(table, cf);
199 Tool t = TestImportTsv.doMROnTableTest(util, cf, simple_tsv, args);
200 doLoadIncrementalHFiles(hfiles, table);
201
202
203 validateDeletedPartitionsFile(t.getConf());
204
205
206 util.deleteTable(table);
207 util.cleanupDataTestDirOnTestFS(table);
208 LOG.info("testGenerateAndLoad completed successfully.");
209 }
210
211 public int run(String[] args) throws Exception {
212 if (args.length != 0) {
213 System.err.println(format("%s [genericOptions]", NAME));
214 System.err.println(" Runs ImportTsv integration tests against a distributed cluster.");
215 System.err.println();
216 GenericOptionsParser.printGenericCommandUsage(System.err);
217 return 1;
218 }
219
220
221
222 provisionCluster();
223 testGenerateAndLoad();
224 releaseCluster();
225
226 return 0;
227 }
228
229 public static void main(String[] args) throws Exception {
230 Configuration conf = HBaseConfiguration.create();
231 IntegrationTestingUtility.setUseDistributedCluster(conf);
232 util = new IntegrationTestingUtility(conf);
233
234 args = new GenericOptionsParser(conf, args).getRemainingArgs();
235 int status = new IntegrationTestImportTsv().run(args);
236 System.exit(status);
237 }
238 }