hbase入库问题！！！求高手帮忙。（hadoop mapreduce）

yangwen_2007 2012-08-14 02:37:06

小弟最近开始接触hbase，最近编写了用mapreduce的入库程序，在eclipse下运行没有问题，但用命令行运行总是提示org.hadoop.hbase.*的类找不到，我再jar包里的MENIFEST.MF文件里已经加入了需要用到的hbase包的classpath。
程序如下：
import java.io.IOException;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;

public class cdrDump {

public static class cdrMapper extends Mapper<Object,Text,NullWritable,NullWritable>
{
public static Configuration conf;
Put put = null;
HTable table = null;
private boolean wal = true;
String tableName = "cdrtable";
HTablePool pool = null;
@Override
public void setup(Context context)
{
System.out.println("create cdr table......");
try
{
// conf = context.getConfiguration();
HBaseAdmin admin = new HBaseAdmin(conf);
if(admin.tableExists(tableName))
{
admin.disableTable(tableName);
admin.deleteTable(tableName);
System.out.println("table exists......");
}
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
tableDescriptor.addFamily(new HColumnDescriptor("cdr"));
// tableDescriptor.addFamily(new HColumnDescriptor("phoneNumber"));
tableDescriptor.addFamily(new HColumnDescriptor("IP"));
tableDescriptor.addFamily(new HColumnDescriptor("IMSI"));
admin.createTable(tableDescriptor);
pool = new HTablePool(conf,1000);
table = (HTable)pool.getTable(tableName);
table.setAutoFlush(false);
// table.setWriteBufferSize(12*1024*1024);
admin.close();
}
catch(MasterNotRunningException e)
{
e.printStackTrace();
}
catch(ZooKeeperConnectionException e)
{
e.printStackTrace();
}
catch(IOException e)
{
e.printStackTrace();
}
}
@Override
public void map(Object key,Text value,Context context)throws IOException
{
String[] cutStr = value.toString().split("\t",-1);
// System.out.println(cutStr.length);
try
{
put = new Put(Bytes.toBytes(cutStr[0]));
put.add(Bytes.toBytes("cdr"), null, value.toString().getBytes());
// put.add(Bytes.toBytes("phoneNumber"), null, cutStr[4].getBytes());
put.add(Bytes.toBytes("IP"), null, cutStr[27].getBytes());
put.add(Bytes.toBytes("IMSI"), null, cutStr[6].getBytes());
if(!wal)
{

put.setWriteToWAL(wal);
}
// System.out.println("insert table......");
table.put(put);
}
catch (ArrayIndexOutOfBoundsException e)
{
e.printStackTrace();
}

}
@Override
public void cleanup(Context context)throws IOException
{
table.close();
// pool.putTable(table);
pool.close();
}
}

public static void main(String[] args)throws Exception
{
Configuration conf = HBaseConfiguration.create();
String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
if(otherArgs.length != 1)
{
System.err.println("Usage: cdrDup input");
System.exit(2);
}
Job job = new Job(conf,"cdrdump");
job.setJarByClass(cdrDump.class);
job.setMapperClass(cdrMapper.class);
job.setOutputFormatClass(NullOutputFormat.class);
job.setNumReduceTasks(0);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
long startTime = System.currentTimeMillis();
job.waitForCompletion(true);
long endTime = System.currentTimeMillis();
System.out.println("Time used: " + (endTime - startTime));
System.exit(job.waitForCompletion(true)?0:1);
}
}

命令行运行报错如下：
12/07/29 01:50:22 INFO input.FileInputFormat: Total input paths to process : 1
12/07/29 01:50:22 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
12/07/29 01:50:22 WARN snappy.LoadSnappy: Snappy native library not loaded
12/07/29 01:50:23 INFO mapred.JobClient: Running job: job_201207290035_0001
12/07/29 01:50:24 INFO mapred.JobClient: map 0% reduce 0%
12/07/29 01:50:40 INFO mapred.JobClient: Task Id : attempt_201207290035_0001_m_000000_0, Status : FAILED
Error: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.MasterNotRunningException
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:820)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:865)
at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1093)
at org.apache.hadoop.mapred.Child.main(Child.java:249)

不知道是什么原因，望高手相助，不胜感激！！！

...全文