SCALA_VERSION=2.10
SPARK_HADOOP_VERSION=2.5.0-cdh5.3.3
SPARK_HIVE=1

VERSION=$("$MVN" help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | tail -n 1)
SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null    | grep -v "INFO"    | tail -n 1)
SPARK_HIVE=$("$MVN" help:evaluate -Dexpression=project.activeProfiles -pl sql/hive $@ 2>/dev/null    | grep -v "INFO"    | fgrep --count "<id>hive</id>";    # Reset exit status to 0, otherwise the script stops here if the last grep finds nothing\
    # because we use "set -o pipefail"
    echo -n)

JAVA_HOME=/home/softwares/jdk1.8.0_60
SCALA_HOME=/home/hadoop01/softwares/scala-2.11.4

HADOOP_CONF_DIR=/home/hadoop01/softwares/hadoop-2.5.0-cdh5.3.3/etc/hadoop

SPARK_MASTER_IP=hadoop01.hpqh.com
SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8080
SPARK_WORKER_CORES=1
SPARK_WORKER_MEMORY=lg
SPARK_WORKER_PORT=7078
SPARK_WORKER_WEBUI_PORT=8081
SPARK_WORKER_INSTANCES=1

启动各个进程：
	sbin/start-master.sh
	sbin/start-slaves.sh

Rdd:transformation action

rdd1 -> 

val textfile = sc.textFile("hdfs://hadoop01.hpqh.com:8020/user/hadoop01/mapreduce/easy.txt")
val rdd1 = textfile.flatMap(word => word.split(" ")))
val rdd2 = rdd1.map(word => (word,1))
val rdd3 = rdd2.reduceByKey((a,b) => (a+b))

val result = sc.textFile("hdfs://hadoop01.hpqh.com:8020/user/hadoop01/mapreduce/easy.txt").flatMap(word => word.split(" ")).map(word => (word,1)).reduceByKey((a,b) => (a+b)).
saveasTextFile("hdfs://hadoop01.hpqh.com:8020/xxxx")

import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._ // not necessary since Spark 1.3

// Create a local StreamingContext with two working thread and batch interval of 1 second.
// The master requires 2 cores to prevent from a starvation scenario.

val conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount")
val ssc = new StreamingContext(conf, Seconds(1))
val lines = ssc.socketTextStream("localhost", 9999)
val words = lines.flatMap(_.split(" "))
import org.apache.spark.streaming.StreamingContext._
val pairs = words.map(word => (word, 1))
val wordCounts = pairs.reduceByKey(_ + _)
wordCounts.print()
ssc.start()

import org.apache.spark._
import org.apache.spark.streaming._

val ssc = new StreamingContext(sc, Seconds(5))
val lines = ssc.textFileStream("hdfs://wei:9000/input")
val words = lines.flatMap(_.split(" "))
val pairs = words.map(word => (word, 1))
val wordCounts = pairs.reduceByKey(_+_)
wordCounts.saveAsTextFiles("hdfs://wei:9000/output/spark_streaming")
ssc.start()