1 package org.admln.java7OperateSpark; 2  3 import java.util.Arrays; 4 import java.util.List; 5 import java.util.regex.Pattern; 6  7 import org.apache.spark.SparkConf; 8 import org.apache.spark.api.java.JavaPairRDD; 9 import org.apache.spark.api.java.JavaRDD;10 import org.apache.spark.api.java.JavaSparkContext;11 import org.apache.spark.api.java.function.FlatMapFunction;12 import org.apache.spark.api.java.function.Function2;13 import org.apache.spark.api.java.function.PairFunction;14 15 import scala.Tuple2;16 17 public class OperateSpark {18     //单词切分分隔符19     private static final Pattern SPACE = Pattern.compile(" ");20     21     public static void main(String[] args) {22         //初始化23         SparkConf sparkConf = new SparkConf().setAppName("JavaWordCount").setMaster("spark://hadoop:7077");24         JavaSparkContext ctx = new JavaSparkContext(sparkConf);25         26         //第二个参数是文件的最小切分27         JavaRDD<String> lines = ctx.textFile("hdfs://hadoop:8020/in/spark/javaOperateSpark/wordcount.txt");28         JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String,String>() {29             public Iterable<String> call(String s) {30                 return Arrays.asList(SPACE.split(s));31             }32         });33         34         //划成键值对35         JavaPairRDD<String,Integer> ones = words.mapToPair(new PairFunction<String,String,Integer>() {36             public Tuple2<String, Integer> call(String t) {37                 return new Tuple2<String,Integer>(t,1);38             }39         });40         41         JavaPairRDD<String,Integer> counts = ones.reduceByKey(new Function2<Integer,Integer,Integer>() {42             public Integer call(Integer v1, Integer v2) {43                 return v1 + v2;44             }45         });46         47         List<Tuple2<String,Integer>> output = counts.collect();48         for(Tuple2<?,?> tuple : output) {49             System.out.println(tuple._1() + ":" +tuple._2());50         }51         counts.saveAsTextFile("hdfs://hadoop:8020/out/spark/javaOperateSpark2/");52         ctx.stop();53     }54 }

Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.hash.HashFunction.hashInt(I)Lcom/google/common/hash/HashCode;    at org.apache.spark.util.collection.OpenHashSet.org$apache$spark$util$collection$OpenHashSet$$hashcode(OpenHashSet.scala:261)    at org.apache.spark.util.collection.OpenHashSet$mcI$sp.getPos$mcI$sp(OpenHashSet.scala:165)    at org.apache.spark.util.collection.OpenHashSet$mcI$sp.contains$mcI$sp(OpenHashSet.scala:102)    at org.apache.spark.util.SizeEstimator$$anonfun$visitArray$2.apply$mcVI$sp(SizeEstimator.scala:214)    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)    at org.apache.spark.util.SizeEstimator$.visitArray(SizeEstimator.scala:210)    at org.apache.spark.util.SizeEstimator$.visitSingleObject(SizeEstimator.scala:169)    at org.apache.spark.util.SizeEstimator$.org$apache$spark$util$SizeEstimator$$estimate(SizeEstimator.scala:161)    at org.apache.spark.util.SizeEstimator$.estimate(SizeEstimator.scala:155)    at org.apache.spark.util.collection.SizeTracker$class.takeSample(SizeTracker.scala:78)    at org.apache.spark.util.collection.SizeTracker$class.afterUpdate(SizeTracker.scala:70)    at org.apache.spark.util.collection.SizeTrackingVector.$plus$eq(SizeTrackingVector.scala:31)    at org.apache.spark.storage.MemoryStore.unrollSafely(MemoryStore.scala:249)    at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:136)    at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:114)    at org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:787)    at org.apache.spark.storage.BlockManager.putIterator(BlockManager.scala:638)    at org.apache.spark.storage.BlockManager.putSingle(BlockManager.scala:992)    at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:98)    at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:84)    at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:34)    at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:29)    at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:62)    at org.apache.spark.SparkContext.broadcast(SparkContext.scala:945)    at org.apache.spark.SparkContext.hadoopFile(SparkContext.scala:695)    at org.apache.spark.SparkContext.textFile(SparkContext.scala:540)    at org.apache.spark.api.java.JavaSparkContext.textFile(JavaSparkContext.scala:184)    at org.admln.java7OperateSpark.OperateSpark.main(OperateSpark.java:27)

Exception in thread "main" java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$AddBlockRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet;        at java.lang.ClassLoader.defineClass1(Native Method)        at java.lang.ClassLoader.defineClass(ClassLoader.java:800)        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)        at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)        ．．．　．．．        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)        at java.lang.reflect.Method.invoke(Method.java:606)        at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:358)        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

　　<dependency>        <groupId>com.google.guava</groupId>        <artifactId>guava</artifactId>        <version>14.0.1</version>    </dependency>

WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient memory

INFO SparkDeploySchedulerBackend: Granted executor ID app-20150111003236-0000/3 on hostPort hadoop:34766 with 2 cores, 512.0 MB RAM

ERROR SparkDeploySchedulerBackend: Asked to remove non-existent executor 2