首页 > 代码库 > spark项目启动报内存溢出不能创建线程错误
spark项目启动报内存溢出不能创建线程错误
1.错误信息如下:
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method)
at java.lang.Thread.start(Thread.java:714)
at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
at org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:349)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
at org.jboss.netty.channel.socket.nio.NioClientBoss.<init>(NioClientBoss.java:63)
at org.jboss.netty.channel.socket.nio.NioClientBossPool.newBoss(NioClientBossPool.java:61)
at org.jboss.netty.channel.socket.nio.NioClientBossPool.newBoss(NioClientBossPool.java:27)
at org.jboss.netty.channel.socket.nio.AbstractNioBossPool.init(AbstractNioBossPool.java:75)
at org.jboss.netty.channel.socket.nio.NioClientBossPool.<init>(NioClientBossPool.java:45)
at org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory.<init>(NioClientSocketChannelFactory.java:185)
at akka.remote.transport.netty.NettyTransport.<init>(NettyTransport.scala:283)
at akka.remote.transport.netty.NettyTransport.<init>(NettyTransport.scala:240)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78)
at scala.util.Try$.apply(Try.scala:161)
at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73)
at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84)
at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84)
at scala.util.Success.flatMap(Try.scala:200)
at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84)
at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711)
at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703)
at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721)
at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703)
at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491)
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)
at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
17/04/01 19:20:02 ERROR ActorSystemImpl: Uncaught fatal error from thread [sparkDriver-akka.actor.default-dispatcher-3] shutting down ActorSystem [sparkDriver]
2.解决方案:
修改Linux服务器下用户所允许的最大线程数
修改/etc/security/limits.d/90-nproc.conf下的文件(文件名可能不同的系统不一样)
* soft nproc 4096
root soft nproc unlimited
除了root用户外其他用户最大线程是4096
3.常用命令
ulimit -u 此命令可以查看当前用户的最大线程数
命令参数
-a 显示所有限制
-c core文件大小的上限
-d 进程数据段大小的上限
-f shell所能创建的文件大小的上限
-m 驻留内存大小的上限
-s 堆栈大小的上限
-t 每秒可占用的CPU时间上限
-p 管道大小
-n 打开文件数的上限
-u 进程数的上限
-v 虚拟内存的上限
spark项目启动报内存溢出不能创建线程错误