package com.husor.Spark/** * Created by huxiu on 2014/11/26. */import org.apache.spark.{SparkContext, SparkConf}import org.apache.spark.SparkContext._object SparkWordCount {  def main(args: Array[String]) {    println("Test is starting......")    System.setProperty("hadoop.home.dir", "d:\\winutil\\")    //val conf = new SparkConf().setAppName("WordCount")    //                          .setMaster("spark://Master:7077")    //                          .setSparkHome("SPARK_HOME")    //                          .set("spark.cores.max","2")    //val spark = new SparkContext(conf)

    //以本地模式运行WordCount程序
    val spark = new SparkContext("local","WordCount",System.getenv("SPARK_HOME")) 
    val file = spark.textFile("hdfs://Master:9000/data/test1") 

　　 //将输出结果直接输出到控制台上    //file.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_).collect().foreach(println)    val wordCounts = file.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_)

    //将输出结果直接输出到hdfs上
    wordCounts.saveAsTextFile("hdfs://Master:9000/user/huxiu/WordCountOutput") 

　　 spark.stop()

　　 println("Test is Succeed!!!") 

  } 
}

1 <property>2     <name>mapreduce.jobtracker.staging.root.dir</name>3     <value>/user</value>4 </property>

1 [Spark@Master hadoop]$ hadoop fs -mkdir /user2 14/11/26 14:00:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable3 [Spark@Master hadoop]$ hadoop fs -mkdir /user/huxiu4 14/11/26 14:04:34 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable5 [Spark@Master hadoop]$ hadoop fs -chown huxiu:huxiu /user/huxiu6 14/11/26 14:04:42 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

"C:\Program Files\Java\jdk1.7.0_67\bin\java" -Didea.launcher.port=7546 "-Didea.launcher.bin.path=D:\ScalaIDE\IntelliJ IDEA Community Edition 14.0.1\bin" -Dfile.encoding=UTF-8 -classpath "C:\Program Files\Java\jdk1.7.0_67\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jce.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jfxrt.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\resources.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\rt.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\zipfs.jar;D:\IntelliJ_IDE\WorkSpace\out\production\Test;D:\vagrant\data\Scala2.10.4\lib\scala-actors-migration.jar;D:\vagrant\data\Scala2.10.4\lib\scala-actors.jar;D:\vagrant\data\Scala2.10.4\lib\scala-library.jar;D:\vagrant\data\Scala2.10.4\lib\scala-reflect.jar;D:\vagrant\data\Scala2.10.4\lib\scala-swing.jar;D:\SparkSrc\spark-assembly-1.1.0-hadoop2.4.0.jar;D:\ScalaIDE\IntelliJ IDEA Community Edition 14.0.1\lib\idea_rt.jar" com.intellij.rt.execution.application.AppMain com.husor.Spark.SparkWordCountTest is starting......Using Spark‘s default log4j profile: org/apache/spark/log4j-defaults.properties14/11/26 14:15:44 INFO SecurityManager: Changing view acls to: huxiu,14/11/26 14:15:44 INFO SecurityManager: Changing modify acls to: huxiu,14/11/26 14:15:44 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(huxiu, ); users with modify permissions: Set(huxiu, )14/11/26 14:15:44 INFO Slf4jLogger: Slf4jLogger started14/11/26 14:15:44 INFO Remoting: Starting remoting14/11/26 14:15:44 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@huxiu-PC:54972]14/11/26 14:15:44 INFO Remoting: Remoting now listens on addresses: [akka.tcp://sparkDriver@huxiu-PC:54972]14/11/26 14:15:44 INFO Utils: Successfully started service ‘sparkDriver‘ on port 54972.14/11/26 14:15:44 INFO SparkEnv: Registering MapOutputTracker14/11/26 14:15:45 INFO SparkEnv: Registering BlockManagerMaster14/11/26 14:15:45 INFO DiskBlockManager: Created local directory at C:\Users\huxiu\AppData\Local\Temp\spark-local-20141126141545-9dad14/11/26 14:15:45 INFO Utils: Successfully started service ‘Connection manager for block manager‘ on port 54975.14/11/26 14:15:45 INFO ConnectionManager: Bound socket to port 54975 with id = ConnectionManagerId(huxiu-PC,54975)14/11/26 14:15:45 INFO MemoryStore: MemoryStore started with capacity 969.6 MB14/11/26 14:15:45 INFO BlockManagerMaster: Trying to register BlockManager14/11/26 14:15:45 INFO BlockManagerMasterActor: Registering block manager huxiu-PC:54975 with 969.6 MB RAM14/11/26 14:15:45 INFO BlockManagerMaster: Registered BlockManager14/11/26 14:15:45 INFO HttpFileServer: HTTP File server directory is C:\Users\huxiu\AppData\Local\Temp\spark-423dcd83-624e-404a-bbf6-a1190f77290f14/11/26 14:15:45 INFO HttpServer: Starting HTTP Server14/11/26 14:15:45 INFO Utils: Successfully started service ‘HTTP file server‘ on port 54976.14/11/26 14:15:45 INFO Utils: Successfully started service ‘SparkUI‘ on port 4040.14/11/26 14:15:45 INFO SparkUI: Started SparkUI at http://huxiu-PC:404014/11/26 14:15:45 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable14/11/26 14:15:45 INFO AkkaUtils: Connecting to HeartbeatReceiver: akka.tcp://sparkDriver@huxiu-PC:54972/user/HeartbeatReceiver14/11/26 14:15:46 INFO MemoryStore: ensureFreeSpace(163705) called with curMem=0, maxMem=101666783214/11/26 14:15:46 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 159.9 KB, free 969.4 MB)14/11/26 14:15:46 INFO FileInputFormat: Total input paths to process : 114/11/26 14:15:46 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id14/11/26 14:15:46 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id14/11/26 14:15:46 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap14/11/26 14:15:46 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition14/11/26 14:15:46 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id14/11/26 14:15:47 INFO SparkContext: Starting job: saveAsTextFile at SparkWordCount.scala:3314/11/26 14:15:47 INFO DAGScheduler: Registering RDD 3 (map at SparkWordCount.scala:32)14/11/26 14:15:47 INFO DAGScheduler: Got job 0 (saveAsTextFile at SparkWordCount.scala:33) with 1 output partitions (allowLocal=false)14/11/26 14:15:47 INFO DAGScheduler: Final stage: Stage 0(saveAsTextFile at SparkWordCount.scala:33)14/11/26 14:15:47 INFO DAGScheduler: Parents of final stage: List(Stage 1)14/11/26 14:15:47 INFO DAGScheduler: Missing parents: List(Stage 1)14/11/26 14:15:47 INFO DAGScheduler: Submitting Stage 1 (MappedRDD[3] at map at SparkWordCount.scala:32), which has no missing parents14/11/26 14:15:47 INFO MemoryStore: ensureFreeSpace(3424) called with curMem=163705, maxMem=101666783214/11/26 14:15:47 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 969.4 MB)14/11/26 14:15:47 INFO DAGScheduler: Submitting 1 missing tasks from Stage 1 (MappedRDD[3] at map at SparkWordCount.scala:32)14/11/26 14:15:47 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks14/11/26 14:15:47 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 0, localhost, ANY, 1174 bytes)14/11/26 14:15:47 INFO Executor: Running task 0.0 in stage 1.0 (TID 0)14/11/26 14:15:47 INFO HadoopRDD: Input split: hdfs://Master:9000/data/test1:0+2714/11/26 14:15:47 INFO Executor: Finished task 0.0 in stage 1.0 (TID 0). 1862 bytes result sent to driver14/11/26 14:15:47 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 0) in 489 ms on localhost (1/1)14/11/26 14:15:47 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 14/11/26 14:15:47 INFO DAGScheduler: Stage 1 (map at SparkWordCount.scala:32) finished in 0.500 s14/11/26 14:15:47 INFO DAGScheduler: looking for newly runnable stages14/11/26 14:15:47 INFO DAGScheduler: running: Set()14/11/26 14:15:47 INFO DAGScheduler: waiting: Set(Stage 0)14/11/26 14:15:47 INFO DAGScheduler: failed: Set()14/11/26 14:15:47 INFO DAGScheduler: Missing parents for Stage 0: List()14/11/26 14:15:47 INFO DAGScheduler: Submitting Stage 0 (MappedRDD[5] at saveAsTextFile at SparkWordCount.scala:33), which is now runnable14/11/26 14:15:47 INFO MemoryStore: ensureFreeSpace(57512) called with curMem=167129, maxMem=101666783214/11/26 14:15:47 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 56.2 KB, free 969.4 MB)14/11/26 14:15:47 INFO DAGScheduler: Submitting 1 missing tasks from Stage 0 (MappedRDD[5] at saveAsTextFile at SparkWordCount.scala:33)14/11/26 14:15:47 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks14/11/26 14:15:47 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 948 bytes)14/11/26 14:15:47 INFO Executor: Running task 0.0 in stage 0.0 (TID 1)14/11/26 14:15:47 INFO BlockFetcherIterator$BasicBlockFetcherIterator: maxBytesInFlight: 50331648, targetRequestSize: 1006632914/11/26 14:15:47 INFO BlockFetcherIterator$BasicBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks14/11/26 14:15:47 INFO BlockFetcherIterator$BasicBlockFetcherIterator: Started 0 remote fetches in 4 ms14/11/26 14:15:48 INFO FileOutputCommitter: Saved output of task ‘attempt_201411261415_0000_m_000000_1‘ to hdfs://Master:9000/user/huxiu/WordCountOutput/_temporary/0/task_201411261415_0000_m_00000014/11/26 14:15:48 INFO SparkHadoopWriter: attempt_201411261415_0000_m_000000_1: Committed14/11/26 14:15:48 INFO Executor: Finished task 0.0 in stage 0.0 (TID 1). 826 bytes result sent to driver14/11/26 14:15:48 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 1) in 847 ms on localhost (1/1)14/11/26 14:15:48 INFO DAGScheduler: Stage 0 (saveAsTextFile at SparkWordCount.scala:33) finished in 0.847 s14/11/26 14:15:48 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 14/11/26 14:15:48 INFO SparkContext: Job finished: saveAsTextFile at SparkWordCount.scala:33, took 1.469630513 s14/11/26 14:15:48 INFO SparkUI: Stopped Spark web UI at http://huxiu-PC:404014/11/26 14:15:48 INFO DAGScheduler: Stopping DAGScheduler14/11/26 14:15:49 INFO MapOutputTrackerMasterActor: MapOutputTrackerActor stopped!14/11/26 14:15:49 INFO ConnectionManager: Selector thread was interrupted!14/11/26 14:15:49 INFO ConnectionManager: ConnectionManager stopped14/11/26 14:15:49 INFO MemoryStore: MemoryStore cleared14/11/26 14:15:49 INFO BlockManager: BlockManager stopped14/11/26 14:15:49 INFO BlockManagerMaster: BlockManagerMaster stoppedTest is Succeed!!!14/11/26 14:15:49 INFO SparkContext: Successfully stopped SparkContext14/11/26 14:15:49 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.14/11/26 14:15:49 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.14/11/26 14:15:49 INFO Remoting: Remoting shut down14/11/26 14:15:49 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.Process finished with exit code 0