hadoop问题:连接postgresql数据库错误
主要main函数代码如下:
public static void main(String[] args) throws IOException, ClassNotFoundException {
// TODO Auto-generated method stub
JobConf conf = new JobConf(DBAccess.class);
conf.set("mapred.job.tracker","localhost:9101");
DBConfiguration.configureDB(conf, "org.postgresql.Driver","jdbc:postgresql://localhost:5433/xiaohe","pg","");
conf.setOutputKeyClass(LongWritable.class);
conf.setOutputValueClass(Text.class);
conf.setInputFormat(DBInputFormat.class);
String [] fields = {"id", "name", "age", "department"};
DBInputFormat.setInput(conf, TeacherRecord.class, "teacher",null, "id", fields);
FileOutputFormat.setOutputPath(conf, new Path("hdfs://localhost:9100/dbout"));
conf.setMapperClass(DBAccessMapper.class);
conf.setReducerClass(IdentityReducer.class);
JobClient.runJob(conf);
}
报错如下:
Exception in thread "main" java.lang.RuntimeException: Error in configuring object
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.JobConf.getInputFormat(JobConf.java:400)
at org.apache.hadoop.mapred.JobClient.writeOldSplits(JobClient.java:810)
at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:781)
at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:730)
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1249)
at testDB.DBAccess.main(DBAccess.java:49)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 8 more
Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.postgresql.Driver
at org.apache.hadoop.mapred.lib.db.DBInputFormat.configure(DBInputFormat.java:271)
... 13 more
Caused by: java.lang.ClassNotFoundException: org.postgresql.Driver
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Unknown Source)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getConnection(DBConfiguration.java:123)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.configure(DBInputFormat.java:266)
... 13 more
网上说配环境变量也配了,还是不行啊!!
public static void main(String[] args) throws IOException, ClassNotFoundException {
// TODO Auto-generated method stub
JobConf conf = new JobConf(DBAccess.class);
conf.set("mapred.job.tracker","localhost:9101");
DBConfiguration.configureDB(conf, "org.postgresql.Driver","jdbc:postgresql://localhost:5433/xiaohe","pg","");
conf.setOutputKeyClass(LongWritable.class);
conf.setOutputValueClass(Text.class);
conf.setInputFormat(DBInputFormat.class);
String [] fields = {"id", "name", "age", "department"};
DBInputFormat.setInput(conf, TeacherRecord.class, "teacher",null, "id", fields);
FileOutputFormat.setOutputPath(conf, new Path("hdfs://localhost:9100/dbout"));
conf.setMapperClass(DBAccessMapper.class);
conf.setReducerClass(IdentityReducer.class);
JobClient.runJob(conf);
}
报错如下:
Exception in thread "main" java.lang.RuntimeException: Error in configuring object
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.JobConf.getInputFormat(JobConf.java:400)
at org.apache.hadoop.mapred.JobClient.writeOldSplits(JobClient.java:810)
at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:781)
at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:730)
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1249)
at testDB.DBAccess.main(DBAccess.java:49)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 8 more
Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.postgresql.Driver
at org.apache.hadoop.mapred.lib.db.DBInputFormat.configure(DBInputFormat.java:271)
... 13 more
Caused by: java.lang.ClassNotFoundException: org.postgresql.Driver
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Unknown Source)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getConnection(DBConfiguration.java:123)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.configure(DBInputFormat.java:266)
... 13 more
网上说配环境变量也配了,还是不行啊!!
...全文
相关推荐
Hadoop开发第四期
#### 四、配置Hive元数据DB为PostgreSQL- **背景**: - Hive默认使用嵌入式的Derby数据库作为其元数据存储。
《深入掌握PostgreSQL数据库》 - 专栏介绍和目录
本专栏系统讲解PostgreSQL数据库的全栈知识,涵盖基础入门到高级实战。内容分为语言基础与进阶、核心原理、高可用架构、性能优化、运维管理、扩展开发等多个模块。从安装配置、数据类型选择到窗口函数、CTE等高级特性,再到索引优化、执行计划分析等性能调优技巧,全面解析PostgreSQL的核心功能。同时深入探讨MVCC机制、WAL日志、并发控制等底层原理,并分享高可用架构设计、备份恢复等运维实践。适合各层次开发者系统学习PostgreSQL,掌握这一强大开源数据库的关键技术。
PostgreSQL与Hive双模数据库结构智能解析与SQL查询自动生成工具_支持PostgreSQL表结构列定义主键外键约束信息提取与Hive表结构列定义分区信息元数据抽取并利用.zip
Hive则是一个建立在Hadoop上的数据仓库工具,它提供了一种易于使用SQL方言(HiveQL)来查询存储在Hadoop文件系统中的大数据的方式。
Hadoop开发者第四期
#### 四、配置Hive元数据DB为PostgreSQL默认情况下,Hive使用嵌入式的Derby数据库作为元数据存储。
CM安装Hue测试hue数据库连接异常使用; .zip
**日志排查**:如果连接仍然失败,查看Hue的日志文件(通常在 `/var/log/hue` 目录下),找出错误信息,以便定位具体问题。


IBM云计算
