Map text then write
This commit is contained in:
parent
074c4614de
commit
1c10b611e5
@ -135,11 +135,12 @@ public class Project4 {
|
||||
ratingDF.createOrReplaceTempView("Rating");
|
||||
|
||||
// Compute the result.
|
||||
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT CONVERT(varchar(10), m.movieId) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
|
||||
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT m.movieId FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
|
||||
conf.q3Occupation + " AND r.rating = " + conf.q3Rating);
|
||||
resultDF.show();
|
||||
resultDF.map((MapFunction<Row, String>) row -> "" + row.getInt(0), Encoders.STRING());
|
||||
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
|
||||
Dataset<String> CompressedResult = resultDF.map((MapFunction<Row, String>) row -> "" + row.getInt(0), Encoders.STRING());
|
||||
CompressedResult.show();
|
||||
CompressedResult.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
|
||||
|
||||
//Don't forget to stop spark session
|
||||
spark.stop();
|
||||
@ -179,9 +180,9 @@ public class Project4 {
|
||||
// conf.q4Age);
|
||||
Dataset<Row> resultDF = spark.sql(
|
||||
"SELECT m.title, AVG(r.rating) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.age = " + conf.q4Age + " GROUP BY m.title");
|
||||
resultDF.show();
|
||||
resultDF.map((MapFunction<Row, String>) row -> row.getString(0) + "::" + row.getDecimal(1), Encoders.STRING());
|
||||
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
|
||||
Dataset<String> CompressedResult = resultDF.map((MapFunction<Row, String>) row -> row.getString(0) + "::" + row.getDecimal(1), Encoders.STRING());
|
||||
CompressedResult.show();
|
||||
CompressedResult.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
|
||||
|
||||
//Don't forget to stop spark session
|
||||
spark.stop();
|
||||
|
Loading…
Reference in New Issue
Block a user