from pyspark import SparkConf, SparkContext
conf = SparkConf().setMaster("local").setAppName("My App")
sc = SparkContext(conf = conf)
lines = sc.textFile("/var/log/syslog")
LinesError = lines.filter(lambda line: "error" in line)
LinesWarn = lines.filter(lambda line: "warn" in line)
print "----output----------------------"
print lines.count()
print LinesError.count()
print LinesWarn.count()
#通过spark的action操作函数：collect函数可以提取出所有rdd里的数据项
re = LinesWarn.collect()
i = 0
for x in re:
        print i, x
        i += 1
sc.stop()