saveasnewapihadoopdatast 保存数据到hbase报空指针异常什么情况 python语言开发的

[Orange橙不孤单]

from pyspark import SparkConf, SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
import time

sc = SparkContext(master = "local[2]",appName="StreamingWordCount")
ssc = StreamingContext(sc, 5)
# HBASE表，需要提前在HBASE中建好
table = 'flume'
broker = "192.168.159.148:9094"
# kafka的topic
topic = "hbasespark"
# HBASE的zookeeper
hbaseZK = "192.168.159.148"
keyConv = "org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter"
valueConv = "org.apache.spark.examples.pythonconverters.StringListToPutConverter"
hbaseConf = {"hbase.zookeeper.quorum": hbaseZK, "hbase.mapred.outputtable": table,
"mapreduce.job.output.value.class": "org.apache.hadoop.io.Writable"}

#打印日志
def log(str):
t = time.strftime(r"%Y-%m-%d %H:%M:%S", time.localtime())
print("[%s]%s" % (t, str))

#处理RDD元素，此RDD元素需为字典类型
def fmt_data(msg_dict):
if msg_dict is not None:
t = time.strftime(r"%Y-%m-%d %H:%M:%S", time.localtime())
rowkey =t
lst = []
for d, x in msg_dict.items():
col_name = d
col_value = str(x)
col_family = 'c1'
msg_tuple = (rowkey, [rowkey, col_family, col_name, col_value])
print("rowkey:" + rowkey + "\ndata " + str(msg_tuple) + " append success")
if msg_tuple is not None:
lst.append(msg_tuple)
return lst

#处理RDD并向HBASE中写入
def connectAndWrite(data):
if not data.isEmpty():
# 接收到的RDD中的元素转为字典，收到的格式为(None,[json串])，所以map第二个元素反序列化成为字典类型
msg_list = data.map(lambda x: json.loads(x[1]))
# 打印RDD观察，类似一个以字典类型数据为元素的列表
log(msg_list.collect())
try:
# 处理RDD中元素为写入HBASE需要的格式，形成元组格式
msg_row = msg_list.map(lambda x: fmt_data(x))
# print(msg_row.flatMap(lambda x: x).map(lambda x: x).collect())
# 将RDD中所有元素中的元组扁平化，再map后往HBASE存储
msg_row.flatMap(lambda x: x).map(lambda x: x).saveAsNewAPIHadoopDataset(conf=hbaseConf, keyConverter=keyConv,valueConverter=valueConv)
print("插入数据成功")
except Exception as ex:
print(str(ex) + " 插入数据失败")


kafkaStreams = KafkaUtils.createDirectStream(ssc, [topic], kafkaParams={"metadata.broker.list": broker})
#kafkaStreams.map(lambda x:x[1]).pprint()
kafkaStreams.foreachRDD(connectAndWrite)


log('start consumer')
ssc.start()
ssc.awaitTermination()

运行之后会报
18/06/06 10:07:20 INFO python.Converter: Loaded converter: org.apache.spark.examples.pythonconverters.StringListToPutConverter
An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
: java.lang.NullPointerException

这个错误的根源在哪里啊 求指教

[gavin-yue]

conf = SparkConf().set("spark.hadoop.validateOutputSpecs", False) sc = SparkContext(master = "local[2]",appName="StreamingWordCount",conf=conf) 不验证输出参数，添加 spark.hadoop.validateOutputSpecs 参数为false，按如上conf内容进行配置