1,258
社区成员
发帖
与我相关
我的任务
分享
object PFP {
def main(args: Array[String]): Unit = {
val texts = mutable.Map(
// "T25I10D10K.txt"->List(0.005,0.004,0.003,0.002),
// "mushroom.txt"->List(0.01))
// "chess.txt"->List(0.4))
// "accidents.txt"->List(0.1))
// "T10I4D100K.txt"->List(0.005,0.004,0.003,0.002,0.001),
//"T40I10D100K.txt"->List(0.01))
"connect-4.txt"->List(0.3))
// "kddcup99.txt"->List(0.0001,0.00009,0.00008,0.00007,0.00006))
//"USCensus.txt"->List(0.5))
// "connect-4.txt"->List(0.5))
val conf =new SparkConf().setAppName("PFP_scala")
val sc =new SparkContext(conf)
texts.foreach{ text =>
val writer =new PrintWriter(new File("/root/app/scala2.10/PFP/"+text._1))
val data= sc.textFile("/usr/local/eclipsews/"+text._1)
val startTime = System.currentTimeMillis()
val transactions: RDD[Array[String]] = data.map(s => s.trim.split(' '))
val ioTime = System.currentTimeMillis() - startTime
text._2.foreach{ support =>
for(i<-0 to 0){
val time1 = System.currentTimeMillis()
val fpg = new FPGrowth()
.setMinSupport(support)
.setNumPartitions(16)
val model = fpg.run(transactions)
val process = java.lang.Runtime.getRuntime.exec("/root/app/scala2.10/PFP/checkHDFS.sh")
process.waitFor();
model.freqItemsets.saveAsTextFile("/usr/local/PFP")
val endTime = System.currentTimeMillis()
val mineTime =endTime - time1
//hehe.foreach { itemset =>println(itemset.items.mkString("[", ",", "]") + ", " + itemset.freq)}
val time = mineTime + ioTime
writer.write("database: "+text._1+" support: "+support+ " iotime: " +ioTime +" mineTime "+ mineTime+ " time: "+ time+ "\n")
}
}
writer.close()
}