cd ~#生成公钥和私钥ssh-keygen -q -t rsa -N "" -f /home/ebupt/.ssh/id_rsacd .sshcat id_rsa.pub > authorized_keyschmod go-wx authorized_keys#把文件authorized_keys复制到所有子节点的/home/ebupt/.ssh目录下scp ~/.ssh/authorized_keys ebupt@eb175:~/.ssh/scp ~/.ssh/authorized_keys ebupt@eb176:~/.ssh/

[ebupt@eb174 ~]$rm ~/.ssh/*[ebupt@eb170 ~]$scp -r ~/.ssh/ ebupt@eb174:~/.ssh/

#添加环境变量export SCALA_HOME=$HOME/scalaexport PATH=$PATH:$SCALA_HOME/bin

#add the slaveseb174eb175eb176

export SCALA_HOME=/home/ebupt/scalaexport JAVA_HOME=/home/ebupt/eb/jdk1.7.0_60export SPARK_MASTER_IP=eb174export SPARK_WORKER_MEMORY=4000m

export SPARK_MEM=1g# (Required) Set the master program‘s memoryexport SHARK_MASTER_MEM=1g# (Optional) Specify the location of Hive‘s configuration directory. By default,# Shark run scripts will point it to $SHARK_HOME/confexport HIVE_HOME=/home/ebupt/hiveexport HIVE_CONF_DIR="$HIVE_HOME/conf"# For running Shark in distributed mode, set the following:export HADOOP_HOME=/home/ebupt/hadoopexport SPARK_HOME=/home/ebupt/sparkexport MASTER=spark://eb174:7077# Only required if using Mesos:#export MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.sosource $SPARK_HOME/conf/spark-env.sh#LZO compression native libexport LD_LIBRARY_PATH=/home/ebupt/hadoop/share/hadoop/common# (Optional) Extra classpathexport SPARK_LIBRARY_PATH=/home/ebupt/hadoop/lib/native# Java options# On EC2, change the local.dir to /mnt/tmpSPARK_JAVA_OPTS=" -Dspark.local.dir=/tmp "SPARK_JAVA_OPTS+="-Dspark.kryoserializer.buffer.mb=10 "SPARK_JAVA_OPTS+="-verbose:gc -XX:-PrintGCDetails -XX:+PrintGCTimeStamps "SPARK_JAVA_OPTS+="-XX:MaxPermSize=256m "SPARK_JAVA_OPTS+="-Dspark.cores.max=12 "export SPARK_JAVA_OPTS# (Optional) Tachyon Related Configuration#export TACHYON_MASTER="" # e.g. "localhost:19998"#export TACHYON_WAREHOUSE_PATH=/sharktables # Could be any valid path nameexport SCALA_HOME=/home/ebupt/scalaexport JAVA_HOME=/home/ebupt/eb/jdk1.7.0_60

# .bash_profile# Get the aliases and functionsif [ -f ~/.bashrc ]; then        . ~/.bashrcfi# User specific environment and startup programsPATH=$PATH:$HOME/binexport PATHexport JAVA_HOME=/home/ebupt/eb/jdk1.7.0_60export PATH=$JAVA_HOME/bin:$PATHexport CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarexport HADOOP_HOME=$HOME/hadoopexport PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinexport ZOOKEEPER_HOME=$HOME/zookeeperexport PATH=$ZOOKEEPER_HOME/bin:$PATHexport HIVE_HOME=$HOME/hiveexport PATH=$HIVE_HOME/bin:$PATHexport HBASE_HOME=$HOME/hbaseexport PATH=$PATH:$HBASE_HOME/binexport MAVEN_HOME=$HOME/eb/apache-maven-3.0.5export PATH=$PATH:$MAVEN_HOME/binexport STORM_HOME=$HOME/stormexport PATH=$PATH:$STORM_HOME/storm-yarn-master/bin:$STORM_HOME/storm-0.9.0-wip21/binexport SCALA_HOME=$HOME/scalaexport PATH=$PATH:$SCALA_HOME/binexport SPARK_HOME=$HOME/sparkexport PATH=$PATH:$SPARK_HOME/binexport SHARK_HOME=$HOME/sharkexport PATH=$PATH:$SHARK_HOME/bin

scp -r /home/ebupt/eb/scala-2.11.2 ebupt@eb175:/home/ebupt/eb/scp -r /home/ebupt/eb/scala-2.11.2 ebupt@eb176:/home/ebupt/eb/scp -r /home/ebupt/eb/spark-1.0.2-bin-hadoop2 ebupt@eb175:/home/ebupt/eb/scp -r /home/ebupt/eb/spark-1.0.2-bin-hadoop2 ebupt@eb176:/home/ebupt/eb/scp -r /home/ebupt/eb/spark-0.9.1-bin-hadoop2 ebupt@eb175:/home/ebupt/eb/scp -r /home/ebupt/eb/spark-0.9.1-bin-hadoop2 ebupt@eb176:/home/ebupt/eb/scp ~/.bash_profile ebupt@eb175:~/scp ~/.bash_profile ebupt@eb176:~/

#!/bin/bashsource ~/.bash_profile
ssh eb175 > /dev/null 2>&1 << eeooffln -s /home/ebupt/eb/scala-2.11.2/ /home/ebupt/scalaln -s /home/ebupt/eb/spark-0.9.1-bin-hadoop2/ /home/ebupt/sparkln -s /home/ebupt/eb/shark-0.9.1-bin-hadoop2/ /home/ebupt/sharksource ~/.bash_profileexiteeooffecho eb175 done!
ssh eb176 > /dev/null 2>&1 << eeooffxxln -s /home/ebupt/eb/scala-2.11.2/ /home/ebupt/scalaln -s /home/ebupt/eb/spark-0.9.1-bin-hadoop2/ /home/ebupt/sparkln -s /home/ebupt/eb/shark-0.9.1-bin-hadoop2/ /home/ebupt/sharksource ~/.bash_profileexiteeooffxxecho eb176 done!

shark> select * from test;17.096: [Full GC 71198K->24382K(506816K), 0.3150970 secs]Exception in thread "main" java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$SetOwnerRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet; at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:800) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:449) at java.net.URLClassLoader.access$100(URLClassLoader.java:71) at java.net.URLClassLoader$1.run(URLClassLoader.java:361) at java.net.URLClassLoader$1.run(URLClassLoader.java:355) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:354) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at java.lang.Class.getDeclaredMethods0(Native Method) at java.lang.Class.privateGetDeclaredMethods(Class.java:2531) at java.lang.Class.privateGetPublicMethods(Class.java:2651) at java.lang.Class.privateGetPublicMethods(Class.java:2661) at java.lang.Class.getMethods(Class.java:1467) at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:426) at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:323) at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:636) at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:722) at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92) at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537) at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:334) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:241) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:141) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:576) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:521) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:146) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2397) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:89) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2431) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2413) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:368) at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296) at org.apache.hadoop.hive.ql.Context.getScratchDir(Context.java:180) at org.apache.hadoop.hive.ql.Context.getMRScratchDir(Context.java:231) at org.apache.hadoop.hive.ql.Context.getMRTmpFileURI(Context.java:288) at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getMetaData(SemanticAnalyzer.java:1274) at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getMetaData(SemanticAnalyzer.java:1059) at shark.parse.SharkSemanticAnalyzer.analyzeInternal(SharkSemanticAnalyzer.scala:137) at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:279) at shark.SharkDriver.compile(SharkDriver.scala:215) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:337) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:909) at shark.SharkCliDriver.processCmd(SharkCliDriver.scala:338) at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:413) at shark.SharkCliDriver$.main(SharkCliDriver.scala:235) at shark.SharkCliDriver.main(SharkCliDriver.scala)

cd $SHARK_HOME/lib_managed/jars/edu.berkeley.cs.shark/hive-execunzip hive-exec-0.11.0-shark-0.9.1.jarrm -f com/google/protobuf/*rm  hive-exec-0.11.0-shark-0.9.1.jarzip -r hive-exec-0.11.0-shark-0.9.1.jar *rm -rf com hive-exec-log4j.properties javaewah/ javax/ javolution/ META-INF/ org/

shark> select * from t1;16.452: [GC 282770K->32068K(1005568K), 0.0388780 secs]org.apache.spark.SparkException: Job aborted: Spark cluster looks down        at org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1028)        at org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1026)        at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)        at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$abortStage(DAGScheduler.scala:1026)        at org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)        at org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)        at scala.Option.foreach(Option.scala:236)        at org.apache.spark.scheduler.DAGScheduler.processEvent(DAGScheduler.scala:619)        at org.apache.spark.scheduler.DAGScheduler$$anonfun$start$1$$anon$2$$anonfun$receive$1.applyOrElse(DAGScheduler.scala:207)        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)        at akka.actor.ActorCell.invoke(ActorCell.scala:456)        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)        at akka.dispatch.Mailbox.run(Mailbox.scala:219)        at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)FAILED: Execution Error, return code -101 from shark.execution.SparkTask

Release date: April 10, 2014Shark 0.9.1 is a maintenance release that stabilizes 0.9.0, which bumps up Scala compatibility to 2.10.3 and Hive compliance to 0.11. The core dependencies for this version are:Scala 2.10.3Spark 0.9.1AMPLab’s Hive 0.9.0(Optional) Tachyon 0.4.1