20,808
社区成员
发帖
与我相关
我的任务
分享
package com.vdata.check;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class MyDriver extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
String[] otherArgs = new GenericOptionsParser(getConf(), args)
.getRemainingArgs();
if (otherArgs.length < 2) {
System.out
.println("Wrong number of arguments: " + otherArgs.length);
return -1;
}
String hdfsInPath = otherArgs[0];// 存放在hdfs上面的文件路径
String hdfsOutPath = otherArgs[1];// 输出路径
Job job = Job.getInstance();
job.setJarByClass(MyWritable.class);
job.setJar("e://MyWritable.jar");
job.setJobName("qweqwe");
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(hdfsInPath));
FileOutputFormat.setOutputPath(job, new Path(hdfsOutPath));
int status=job.waitForCompletion(true)?0:1;
return status;
}
public static void main(String[] args) throws Exception {
System.setProperty("HADOOP_USER_NAME", "root");
int status = ToolRunner.run(new MyDriver(), args);
System.exit(status);
}
}
package com.vdata.check;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
public class MyWritable implements WritableComparable<MyWritable>{
private Text key;
private LongWritable num;
public MyWritable() {
this(new Text(),new LongWritable());
}
public MyWritable(Text key, LongWritable num) {
this.key = key;
this.num = num;
}
public Text getKey() {
return key;
}
public void setKey(Text key) {
this.key = key;
}
public LongWritable getNum() {
return num;
}
public void setNum(LongWritable num) {
this.num = num;
}
@Override
public void readFields(DataInput input) throws IOException {
key.readFields(input);
num.readFields(input);
}
@Override
public void write(DataOutput output) throws IOException {
key.write(output);
num.write(output);
}
@Override
public int compareTo(MyWritable obj) {
int status=0;
if (obj !=null) {
status=this.key.compareTo(obj.getKey());
}
if (status!=0) {
return status;
}
status =this.num.compareTo(obj.getNum());
return status;
}
}