hadoop提交到远程集群直接运行出现以下信息:map 0% reduce 0% ClassNotFoundException: test2.Test2$Map

toutouping 2014-06-05 03:42:23
package test2;

public class Test2 extends Configured implements Tool
{
enum Counter
{
LINESKIP, //出错的行
}

public static class Map extends Mapper<LongWritable,Text,Text,Text>
{
public void map(LongWritable key,Text value,Context context)throws IOException,InterruptedException
{
String line=value.toString(); //读取每一行源数据

try
{
//数据处理
String[] lineSplit=line.split(" ");
String anum=lineSplit[0];
String bnum=lineSplit[1];

context.write(new Text(bnum), new Text(anum));//输出
}
catch(java.lang.ArrayIndexOutOfBoundsException e)
{
context.getCounter(Counter.LINESKIP).increment(1); //出错令计数器+1;
return;
}
}
}

public static class Reduce extends Reducer<Text,Text,Text,Text>
{
public void reduce(Text key,Iterable<Text>values,Context context)throws IOException,InterruptedException
{
String valueString;
String out="";

for(Text value:values)
{
valueString=value.toString();
out+=valueString+"|";
}
context.write(key, new Text(out));
}
}

public int run(String[] args)throws Exception
{
Configuration conf=new Configuration();
conf.set("mapred.job.tracker", "hdfs://hadoop:9001");
Job job=new Job(conf,"Test2"); //任务名
job.setJarByClass(Test2.class); //指定class

FileInputFormat.addInputPath(job, new Path(args[0])); //输入路径
FileOutputFormat.setOutputPath(job, new Path(args[1])); //输出路径

job.setMapperClass(Map.class); //调用自定义Map类作为Map任务代码
job.setReducerClass(Reduce.class); //调用自定义Reduce类作为Reduce任务代码

job.setOutputFormatClass(org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class);
job.setOutputKeyClass(Text.class); //指定输出的Key的格式
job.setOutputValueClass(IntWritable.class); //指定输出的VALUE的格式
job.waitForCompletion(true);

return job.isSuccessful()?0:1;
}

public static void main(String[] args)throws Exception
{
//运行任务
int res=ToolRunner.run(new Configuration(), new Test2(), args);
System.exit(res);
}

}

eclipse 运行hadoop 提交到远程集群直接运行出现以下信息
INFO mapred.JobClient: map 0% reduce 0%
14/06/05 15:31:24 INFO mapred.JobClient: Task Id : attempt_201406042148_0025_m_000000_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: test2.Test2$Map
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: test2.Test2$Map
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 8 more
...全文
2375 1 打赏 收藏 转发到动态 举报
写回复
用AI写文章
1 条回复
切换为时间正序
请发表友善的回复…
发表回复
w2586322 2014-06-08
  • 打赏
  • 举报
回复
引用 楼主 toutouping 的回复:
package test2; public class Test2 extends Configured implements Tool { enum Counter { LINESKIP, //出错的行 } public static class Map extends Mapper<LongWritable,Text,Text,Text> { public void map(LongWritable key,Text value,Context context)throws IOException,InterruptedException { String line=value.toString(); //读取每一行源数据 try { //数据处理 String[] lineSplit=line.split(" "); String anum=lineSplit[0]; String bnum=lineSplit[1]; context.write(new Text(bnum), new Text(anum));//输出 } catch(java.lang.ArrayIndexOutOfBoundsException e) { context.getCounter(Counter.LINESKIP).increment(1); //出错令计数器+1; return; } } } public static class Reduce extends Reducer<Text,Text,Text,Text> { public void reduce(Text key,Iterable<Text>values,Context context)throws IOException,InterruptedException { String valueString; String out=""; for(Text value:values) { valueString=value.toString(); out+=valueString+"|"; } context.write(key, new Text(out)); } } public int run(String[] args)throws Exception { Configuration conf=new Configuration(); conf.set("mapred.job.tracker", "hdfs://hadoop:9001"); Job job=new Job(conf,"Test2"); //任务名 job.setJarByClass(Test2.class); //指定class FileInputFormat.addInputPath(job, new Path(args[0])); //输入路径 FileOutputFormat.setOutputPath(job, new Path(args[1])); //输出路径 job.setMapperClass(Map.class); //调用自定义Map类作为Map任务代码 job.setReducerClass(Reduce.class); //调用自定义Reduce类作为Reduce任务代码 job.setOutputFormatClass(org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class); job.setOutputKeyClass(Text.class); //指定输出的Key的格式 job.setOutputValueClass(IntWritable.class); //指定输出的VALUE的格式 job.waitForCompletion(true); return job.isSuccessful()?0:1; } public static void main(String[] args)throws Exception { //运行任务 int res=ToolRunner.run(new Configuration(), new Test2(), args); System.exit(res); } } eclipse 运行hadoop 提交到远程集群直接运行出现以下信息 INFO mapred.JobClient: map 0% reduce 0% 14/06/05 15:31:24 INFO mapred.JobClient: Task Id : attempt_201406042148_0025_m_000000_0, Status : FAILED java.lang.RuntimeException: java.lang.ClassNotFoundException: test2.Test2$Map at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849) at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370) at org.apache.hadoop.mapred.Child$4.run(Child.java:255) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149) at org.apache.hadoop.mapred.Child.main(Child.java:249) Caused by: java.lang.ClassNotFoundException: test2.Test2$Map at java.net.URLClassLoader$1.run(URLClassLoader.java:202) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:190) at java.lang.ClassLoader.loadClass(ClassLoader.java:307) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) at java.lang.ClassLoader.loadClass(ClassLoader.java:248) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:247) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847) ... 8 more

547

社区成员

发帖
与我相关
我的任务
社区描述
Cloud Foundry是业界第一个开源PaaS云平台,它支持多种框架、语言、运行时环境、云平台及应用服务,使开发人员能够在几秒钟内进行应用程序的部署和扩展,无需担心任何基础架构的问题。
社区管理员
  • Cloud Foundry社区
加入社区
  • 近7日
  • 近30日
  • 至今
社区公告
暂无公告

试试用AI创作助手写篇文章吧