Failed to send requests for topics mykafka1 with correlation ids in [0,20]
最近写了个kafka_producer测试程序,可是每当程序运行到一定时间后就报错,重新创建新topic也不行:
Fetching topic metadata with correlation id 15 for topics [Set(mykafka1)] from broker [id:1,host:192.168.231.12,port:9092] fail
ERROR DefaultEventHandler: Failed to send requests for topics mykafka1 with correlation ids in [0,20]
producer:
object MyProducer {
def getProducerConfig(brokerAddr: String): Properties = {
val props = new Properties()
props.put("metadata.broker.list", brokerAddr)
props.put("serializer.class", classOf[ItelogEncoder[Person]].getName)
props.put("key.serializer.class", classOf[StringEncoder].getName)
// props.put("replica.fetch.max.bytes", "" + 1024 * 1024 * 6)
// props.put("message.max.bytes", "" + 1024 * 1024 * 4)
props
}
def sendMessages(topic: String, messages: List[Person], brokerAddr: String) = {
val producer = new Producer[String, Person](new ProducerConfig(getProducerConfig(brokerAddr)))
producer.send(messages.map {
new KeyedMessage[String, Person](topic, "Iteblog", _)
}: _*)
producer.close()
}
def main(args: Array[String]) {
val Array(brokerAddr, topic) = args
val sparkConf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[2]")
val ssc = new StreamingContext(sparkConf, Seconds(2))
for (i <- 0.to(10000)) {
val data = List(Person("xiaoming" + i, 23), Person("xiaohua" + i, 24), Person("xiaoliu" + i, 26))
sendMessages(topic, data, brokerAddr)
}
}
case class Person(var name: String, var age: Int)
求大神们帮帮忙,小白在此谢过