max/cs448
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Map text then write

Browse Source
This commit is contained in:
Max O'Cull 2019-04-20 23:26:11 -04:00
parent 074c4614de
commit 1c10b611e5
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
p4/src/main/java/cs448/Project4.java
View File

@ -135,11 +135,12 @@ public class Project4 {
ratingDF.createOrReplaceTempView("Rating");
// Compute the result.
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT CONVERT(varchar(10), m.movieId) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
Dataset<Row> resultDF = spark.sql("SELECT DISTINCT m.movieId FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.occupation = " +
conf.q3Occupation + " AND r.rating = " + conf.q3Rating);
resultDF.show();
resultDF.map((MapFunction<Row, String>) row -> "" + row.getInt(0), Encoders.STRING());
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
Dataset<String> CompressedResult = resultDF.map((MapFunction<Row, String>) row -> "" + row.getInt(0), Encoders.STRING());
CompressedResult.show();
CompressedResult.write().text(CS448Utils.resolveUri(conf.outPath, "query-3"));
//Don't forget to stop spark session
spark.stop();
@ -179,9 +180,9 @@ public class Project4 {
// conf.q4Age);
Dataset<Row> resultDF = spark.sql(
"SELECT m.title, AVG(r.rating) FROM Movie m, Rating r, User u WHERE m.movieId = r.movieId AND r.userId = u.userId AND u.age = " + conf.q4Age + " GROUP BY m.title");
resultDF.show();
resultDF.map((MapFunction<Row, String>) row -> row.getString(0) + "::" + row.getDecimal(1), Encoders.STRING());
resultDF.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
Dataset<String> CompressedResult = resultDF.map((MapFunction<Row, String>) row -> row.getString(0) + "::" + row.getDecimal(1), Encoders.STRING());
CompressedResult.show();
CompressedResult.write().text(CS448Utils.resolveUri(conf.outPath, "query-4"));
//Don't forget to stop spark session
spark.stop();