1,258
社区成员
发帖
与我相关
我的任务
分享
public class LinearRegression {
public static void main(String[] args) {
// TODO Auto-generated method stub
SparkConf conf = new SparkConf().setAppName("JavaStreamingLinearRegressionWithSGDE");
JavaStreamingContext jssc=new JavaStreamingContext(conf, Durations.seconds(10L));
JavaDStream<String> data = jssc.textFileStream("/LienarRegression/lpsa.data");
JavaDStream<LabeledPoint> parsedData = data.map(line -> {
String[] parts = line.split(",");
String[] features = parts[1].split(" ");
double[] v = new double[features.length];
for (int i = 0; i < features.length - 1; i++) {
v[i] = Double.parseDouble(features[i]);
}
return new LabeledPoint(Double.parseDouble(parts[0]), Vectors.dense(v));
});
parsedData.cache();
JavaDStream<String> data_1 = jssc.textFileStream("/LienarRegression/lpsa_1.data");
// JavaDStream转JavaPairDStream (JavaPairDStream<K,Vector> data)
int numIterations = 3;
StreamingLinearAlgorithm model = new StreamingLinearRegressionWithSGD().setInitialWeights(Vectors.zeros(numIterations));
model.trainOn(parsedData);
model.latestModel();
//model.predictOnValues();
}
}
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("JavaStreamingLinearRegressionWithSGDETest");
JavaStreamingContext jssc = new JavaStreamingContext(conf , Durations.seconds(2L));
JavaDStream<String> lines= jssc.socketTextStream("***.**.**.***", 9999);
JavaDStream<LabeledPoint> parsedData = lines.map(line -> {
String[] parts = line.split(",");
String[] features = parts[1].split(" ");
double[] v = new double[features.length];
for (int i = 0; i < features.length - 1; i++) {
v[i] = Double.parseDouble(features[i]);
}
return new LabeledPoint(Double.parseDouble(parts[0]), Vectors.dense(v));
});
parsedData.cache();
JavaPairDStream<Double, Vector> temp = parsedData.mapToPair(new PairFunction<LabeledPoint, Double, Vector>() {
public Tuple2<Double, Vector> call(LabeledPoint p) {
return new Tuple2<Double, Vector>(p.label() , (Vector) p.features());
}
});
int numIterations = 3;
StreamingLinearAlgorithm model = new StreamingLinearRegressionWithSGD().setInitialWeights(Vectors.zeros(numIterations));
model.trainOn(parsedData);
model.latestModel();
model.predictOnValues(temp).print();
jssc.start();
try {
jssc.awaitTermination();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
JavaPairDStream<Double, Vector> temp = parsedData.mapToPair(new PairFunction<LabeledPoint, Double, Vector>() {
public Tuple2<Double, Vector> call(LabeledPoint p) {
return new Tuple2<Double, Vector>(p.label() , (Vector) p.features());
}
});