jm56

from pyspark import SparkConf,SparkContext
#創建sparkconf對象
conf = SparkConf().setMaster("local[*]").setAppName("test_app")
#基于sparkconf對象創建sparkContext對象
sc = SparkContext(conf=conf)
##########基本結構

#map計算
# rdd = sc.parallelize([1,2,3,4,5])
# def func(date):
# return date*10
# rdds=rdd.map(func)

#flatMap解除嵌套
# rdd = sc.parallelize(["dwad wad wdas","dwadw dfgawdfw dwad","dwadwad"])
# rdds=rdd.flatMap(lambda x : x.split(" "))

#reduceByKey分組兩兩計算
# rdd=sc.parallelize([('男',99),('女',99),('女',99),('男',99),('男',99),('男',99)])
# rdds = rdd.reduceByKey(lambda a, b: a+b)
# print(rdds.collect())

#filter過濾數據
rdd=sc.parallelize([1,2,3,4,5])
rdds = rdd.filter(lambda num: num % 2 == 0)
print(rdds.collect())

#distinct去重
rdd=sc.parallelize([1, 2, 3, 4, 5, 1])
rdds = rdd.distinct()
print(rdds.collect())

#停止spark
sc.stop()