Change write type
This commit is contained in:
parent
2113e885b3
commit
b9adcde936
@ -11,6 +11,7 @@ import org.apache.spark.api.java.Optional;
|
|||||||
import org.apache.spark.api.java.function.Function;
|
import org.apache.spark.api.java.function.Function;
|
||||||
import org.apache.spark.api.java.function.Function2;
|
import org.apache.spark.api.java.function.Function2;
|
||||||
import org.apache.spark.sql.Dataset;
|
import org.apache.spark.sql.Dataset;
|
||||||
|
import org.apache.spark.sql.Encoders;
|
||||||
import org.apache.spark.sql.Row;
|
import org.apache.spark.sql.Row;
|
||||||
import org.apache.spark.sql.SparkSession;
|
import org.apache.spark.sql.SparkSession;
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
@ -136,6 +137,7 @@ public class Project4 {
|
|||||||
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT CONVERT(varchar(10), m.movieId) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
|
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT CONVERT(varchar(10), m.movieId) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
|
||||||
conf.q3Occupation + " AND r.rating = " + conf.q3Rating);
|
conf.q3Occupation + " AND r.rating = " + conf.q3Rating);
|
||||||
resultDF.show();
|
resultDF.show();
|
||||||
|
resultDF.map((MapFunction<Row, String>) row -> "" + row.getInteger(0), Encoders.STRING());
|
||||||
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
|
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
|
||||||
|
|
||||||
//Don't forget to stop spark session
|
//Don't forget to stop spark session
|
||||||
@ -177,6 +179,7 @@ public class Project4 {
|
|||||||
Dataset<Row> resultDF = spark.sql(
|
Dataset<Row> resultDF = spark.sql(
|
||||||
"SELECT m.title, AVG(r.rating) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.age = " + conf.q4Age + " GROUP BY m.title");
|
"SELECT m.title, AVG(r.rating) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.age = " + conf.q4Age + " GROUP BY m.title");
|
||||||
resultDF.show();
|
resultDF.show();
|
||||||
|
resultDF.map((MapFunction<Row, String>) row -> row.getString(0) + "::" + row.getDecimal(1), Encoders.STRING());
|
||||||
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
|
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
|
||||||
|
|
||||||
//Don't forget to stop spark session
|
//Don't forget to stop spark session
|
||||||
|
Loading…
Reference in New Issue
Block a user