20,808
社区成员
发帖
与我相关
我的任务
分享
package hbaseExec3;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
//import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class WordCountHbase extends Configured implements Tool {
public static class MyMapper extends
Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
@Override
protected void setup(Context context) throws IOException,
InterruptedException {
// TODO Auto-generated method stub
super.setup(context);
}
@Override
protected void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
System.out.println("开始执行map函数");
StringTokenizer tokenizer = new StringTokenizer(value.toString());
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
context.write(word, one);
}
}
}
public static class MyReducer extends
TableReducer<Text, IntWritable, NullWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
System.out.println("开始执行reduce函数");
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
// Put实例化,每个词存一行
Put put = new Put(Bytes.toBytes(key.toString()));
// 列族为content,列修饰符为count,列值为数目
put.add(Bytes.toBytes("content"), Bytes.toBytes("count"),
Bytes.toBytes(String.valueOf(sum)));
context.write(NullWritable.get(), put);
}
}
// 创建HBase数据表
public static void createHBaseTable(String tableName) throws Exception {
// 创建表描述
HTableDescriptor htd = new HTableDescriptor(tableName);
// 创建列族描述
HColumnDescriptor col = new HColumnDescriptor("content");
htd.addFamily(col);
// 配置HBase
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "172.16.122.212");
conf.set("hbase.zookeeper.property.clientPort", "2181");
HBaseAdmin hAdmin = new HBaseAdmin(conf);
if (hAdmin.tableExists(tableName)) {
System.out.println("该数据表已经存在,正在重新创建...");
hAdmin.disableTable(tableName);
hAdmin.deleteTable(tableName);
}
System.out.println("创建表:" + tableName);
hAdmin.createTable(htd);
}
@Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
String tableName="wordcount";
// 第一步:创建数据库表
WordCountHbase.createHBaseTable(tableName);
// 第二步:进行MapReduce处理
// 配置MapReduce
Configuration conf = new Configuration();
// 这几句话很关键
conf.set("mapred.job.tracker", "172.16.122.212:9001");
//conf.set("fs.defaultFS", "hdfs://hadoop:9000");
conf.set("hbase.zookeeper.quorum", "172.16.122.212");
conf.set("hbase.zookeeper.property.clientPort", "2181");
conf.set("mapreduce.framework.name", "yarn");
conf.set(TableOutputFormat.OUTPUT_TABLE, tableName);
Job job = new Job(conf, "WordCount-hbase");
job.setJarByClass(WordCountHbase.class);
// 设置Map和Reduce处理类
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);
// 设置输出类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// 设置输入和输出格式
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TableOutputFormat.class);
// 设置输入目录
FileInputFormat.addInputPath(job, new Path(args[0]));
//FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true)?0:1);
return 0;
}
public static void main(String[] args) throws Exception {
System.out.println("开始执行main函数");
int res = ToolRunner.run(new Configuration(), new WordCountHbase(),
args);
System.exit(res);
}
}