forked from Huydatnguyen/LSDMLab
-
Notifications
You must be signed in to change notification settings - Fork 0
/
question1-Dataframe.py
38 lines (26 loc) · 1013 Bytes
/
question1-Dataframe.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import sys
from pyspark import SparkContext
from pyspark.sql import SparkSession
import time
# Question 1 Dataframe Spark____________________________________________________________start
# start timer
start = time.time()
# start spark with 1 worker thread
sc = SparkContext("local[1]")
sc.setLogLevel("ERROR")
# init spark application
spark = SparkSession.builder.appName('SparkByExamples.com').getOrCreate()
# read all the input files into an Dataframe
df = spark.read.csv("./Machine_events/*")
# rename the Dataframe columns
newColumns = ["timeStamp","machineID","eventType","platformID","cpu","memory"]
df = df.toDF(*newColumns)
# sum of machines
sum_of_machines = df.count()
# collect the identical data into groups and count them and show the result
df.groupBy("cpu").count().show(truncate=False)
# end timer
end = time.time()
print("elapsed time: " , end-start)
# Question 1 Dataframe Spark______________________________________________________________end
input("Press Enter to continnnue...")