View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapred;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.File;
24  import java.io.IOException;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FileUtil;
29  import org.apache.hadoop.hbase.testclassification.LargeTests;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
34  import org.apache.hadoop.hbase.mapreduce.TestTableMapReduceBase;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.mapred.JobClient;
37  import org.apache.hadoop.mapred.JobConf;
38  import org.apache.hadoop.mapred.MapReduceBase;
39  import org.apache.hadoop.mapred.OutputCollector;
40  import org.apache.hadoop.mapred.Reporter;
41  import org.apache.hadoop.mapred.RunningJob;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test Map/Reduce job over HBase tables. The map/reduce process we're testing
46   * on our tables is simple - take every row in the table, reverse the value of
47   * a particular cell, and write it back to the table.
48   */
49  @Category(LargeTests.class)
50  @SuppressWarnings("deprecation")
51  public class TestTableMapReduce extends TestTableMapReduceBase {
52    private static final Log LOG =
53      LogFactory.getLog(TestTableMapReduce.class.getName());
54  
55    protected Log getLog() { return LOG; }
56  
57    /**
58     * Pass the given key and processed record reduce
59     */
60    static class ProcessContentsMapper extends MapReduceBase implements
61        TableMap<ImmutableBytesWritable, Put> {
62  
63      /**
64       * Pass the key, and reversed value to reduce
65       */
66      public void map(ImmutableBytesWritable key, Result value,
67        OutputCollector<ImmutableBytesWritable, Put> output,
68        Reporter reporter)
69      throws IOException {
70        output.collect(key, TestTableMapReduceBase.map(key, value));
71      }
72    }
73  
74    @Override
75    protected void runTestOnTable(HTable table) throws IOException {
76      JobConf jobConf = null;
77      try {
78        LOG.info("Before map/reduce startup");
79        jobConf = new JobConf(UTIL.getConfiguration(), TestTableMapReduce.class);
80        jobConf.setJobName("process column contents");
81        jobConf.setNumReduceTasks(1);
82        TableMapReduceUtil.initTableMapJob(Bytes.toString(table.getTableName()),
83          Bytes.toString(INPUT_FAMILY), ProcessContentsMapper.class,
84          ImmutableBytesWritable.class, Put.class, jobConf);
85        TableMapReduceUtil.initTableReduceJob(Bytes.toString(table.getTableName()),
86          IdentityTableReduce.class, jobConf);
87  
88        LOG.info("Started " + Bytes.toString(table.getTableName()));
89        RunningJob job = JobClient.runJob(jobConf);
90        assertTrue(job.isSuccessful());
91        LOG.info("After map/reduce completion");
92  
93        // verify map-reduce results
94        verify(Bytes.toString(table.getTableName()));
95      } finally {
96        if (jobConf != null) {
97          FileUtil.fullyDelete(new File(jobConf.get("hadoop.tmp.dir")));
98        }
99      }
100   }
101 }
102