apache spark

src/main/java/kr/co/swh/lecture/opensource/apache/spark/JavaWordCount.java

@@ -0,0 +1,44 @@
+package kr.co.swh.lecture.opensource.apache.spark; 
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.sql.SparkSession;
+
+import scala.Tuple2;
+
+public final class JavaWordCount {
+	private static final Pattern SPACE = Pattern.compile(" ");
+
+	public static void main(String[] args) throws Exception {
+
+		String text = "sadfasdf sdfsdfasd fdsfsdf asdfasdf sdfasdf sadfdf";
+		
+		SparkConf conf = new SparkConf();
+		conf.setMaster("local[2]");
+//		SparkSession spark = new SparkSession(new SparkConf().setAppName("Spark WordCount").setMaster("local[2]"));
+		SparkSession spark = SparkSession
+				.builder()
+				.appName("JavaWordCount")
+//				.config(conf);
+				.getOrCreate();
+
+		JavaRDD<String> lines = spark.read().textFile(text).javaRDD();
+
+		JavaRDD<String> words = lines.flatMap(s -> Arrays.asList(SPACE.split(s)).iterator());
+
+		JavaPairRDD<String, Integer> ones = words.mapToPair(s -> new Tuple2<>(s, 1));
+
+		JavaPairRDD<String, Integer> counts = ones.reduceByKey((i1, i2) -> i1 + i2);
+
+		List<Tuple2<String, Integer>> output = counts.collect();
+		for (Tuple2<?,?> tuple : output) {
+			System.out.println(tuple._1() + ": " + tuple._2());
+		}
+		spark.stop();
+	}
+}

src/main/java/kr/co/swh/lecture/opensource/apache/spark/Test.java

@@ -0,0 +1,69 @@
+package kr.co.swh.lecture.opensource.apache.spark; 
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * <pre>
+ * kr.co.swh.lecture.opensource.apache.spark 
+ * Test.java
+ *
+ * 설명 :
+ * </pre>
+ * 
+ * @since : 2020. 11. 8.
+ * @author : tobby48
+ * @version : v1.0
+ */
+public class Test {
+
+
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+		
+		SparkSession spark = SparkSession
+				.builder()
+				.appName("JavaWordCount")
+				.getOrCreate();
+		
+		// A JSON dataset is pointed to by path.
+		// The path can be either a single text file or a directory storing text files
+		Dataset<Row> people = spark.read().json("examples/src/main/resources/people.json");
+
+		// The inferred schema can be visualized using the printSchema() method
+		people.printSchema();
+		// root
+		//  |-- age: long (nullable = true)
+		//  |-- name: string (nullable = true)
+
+		// Creates a temporary view using the DataFrame
+		people.createOrReplaceTempView("people");
+
+		// SQL statements can be run by using the sql methods provided by spark
+		Dataset<Row> namesDF = spark.sql("SELECT name FROM people WHERE age BETWEEN 13 AND 19");
+		namesDF.show();
+		// +------+
+		// |  name|
+		// +------+
+		// |Justin|
+		// +------+
+
+		// Alternatively, a DataFrame can be created for a JSON dataset represented by
+		// a Dataset<String> storing one JSON object per string.
+		List<String> jsonData = Arrays.asList(
+		        "{\"name\":\"Yin\",\"address\":{\"city\":\"Columbus\",\"state\":\"Ohio\"}}");
+		Dataset<String> anotherPeopleDataset = spark.createDataset(jsonData, Encoders.STRING());
+		Dataset<Row> anotherPeople = spark.read().json(anotherPeopleDataset);
+		anotherPeople.show();
+		// +---------------+----+
+		// |        address|name|
+		// +---------------+----+
+		// |[Columbus,Ohio]| Yin|
+		// +---------------+----+
+	}
+
+}