20,848
社区成员




public class WritableSample extends Configured implements Tool{
public static void main(String[] args) throws Exception {
ToolRunner.run(new WritableSample(), args);
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = getConf();
Job job = new Job(conf);
job.setJarByClass(getClass());
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("out"), true);
FileInputFormat.addInputPath(job, new Path("sample.txt"));
FileOutputFormat.setOutputPath(job, new Path("out"));
job.setMapperClass(MyWritableMap.class);
job.setOutputKeyClass(MyPairWritable.class);
job.setOutputValueClass(NullWritable.class);
job.setSortComparatorClass(PairKeyComparator.class);
job.waitForCompletion(true);
return 0;
}
}
class MyWritableMap extends Mapper<LongWritable, Text, MyPairWritable, NullWritable>{
MyPairWritable pair= new MyPairWritable();
protected void map(LongWritable key, Text value, Context context) throws java.io.IOException ,InterruptedException {
String[] strs = value.toString().split(" ");
Text keyy = new Text(strs[0]);
IntWritable valuee = new IntWritable(Integer.parseInt(strs[1]));
pair.set(keyy, valuee);
context.write(pair, NullWritable.get());
};
}
class MyWritableReduce extends Reducer<MyPairWritable, NullWritable,MyPairWritable, NullWritable>{
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws java.io.IOException ,InterruptedException {
};
}
class PairKeyComparator extends WritableComparator{
public PairKeyComparator() {
super(MyPairWritable.class,true);
}
@Override
public int compare(WritableComparable a, WritableComparable b) {
MyPairWritable p1 = (MyPairWritable)a;
MyPairWritable p2 = (MyPairWritable)b;
System.out.println(p1 + "___" + p2);
if(p1.getFirst() != p2.getFirst()){
return p1.first.toString().compareTo(p2.toString());
}else if(p1.second != p2.getSecond()){
return p1.second.get() - p2.getSecond().get();
}
else return 0;
}
}
class MyPairPartitioner extends Partitioner<MyPairWritable, NullWritable>{
public int getPartition(MyPairWritable key, NullWritable value, int numPartitions) {
if(key.getFirst().toString().startsWith("aaa"))
return 1;
if(key.getFirst().toString().startsWith("bbb"))
return 2;
if(key.getFirst().toString().startsWith("ccc"))
return 3;
else return 0;
};
}
class MyPairWritable implements WritableComparable<MyPairWritable>{
Text first;
IntWritable second;
public void set(Text first, IntWritable second)
{
this.first = first;
this.second = second;
}
public Text getFirst()
{
return first;
}
public IntWritable getSecond()
{
return second;
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
first = new Text(in.readUTF());
second = new IntWritable(in.readInt());
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(first.toString());
out.writeInt(second.get());
}
@Override
public int compareTo(MyPairWritable o) {
if(this.first != o.getFirst()){
return this.first.toString().compareTo(o.toString());
}else if(this.second != o.getSecond()){
return this.second.get() - o.getSecond().get();
}
else return 0;
}
@Override
public String toString() {
// TODO Auto-generated method stub
return first.toString() + " " + second.get();
}
}