tomcat突然不接收任何请求。机器CPU，内存都正常。jvm的CPU，内存都正常。用arthas查看，发现非常多的线程处于WAITING，没有BLOCKED的线程。怀疑是线程池里的线程都处于WAITING，导致线程消耗光了，所以不响应任何请求。

查看具体线程，主要有3类线程：
第一类业务是发http请求，卡在sun.misc.Unsafe.park这句。
第二类业务是用了个guava cache，当缓存不存在时候，发第一类请求，也卡在sun.misc.Unsafe.park这句。
第三类感觉是dubbo通信的。卡在sun.misc.Unsafe.park这句。

现在问题是http请求这个是有超时的，guava cache也是线程安全的。所以就不明白为什么会有这么多的waiting的了。这个堆栈是出问题后，把业务流量切走，过了几个小时后看的。也不存在实时的问题。
请教各位，这种问题从何处下手

"http-nio-8284-exec-3863" daemon prio=5 tid=155190 WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
       Local Variable: java.util.concurrent.locks.AbstractQueuedSynchronizer$Node#15499
    at org.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:393)
    at org.apache.http.pool.AbstractConnPool.access$300(AbstractConnPool.java:70)
    at org.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:253)
    at org.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:198)
    at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:306)
       Local Variable: org.apache.http.pool.AbstractConnPool$2#45
    at org.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:282)
       Local Variable: org.apache.http.impl.conn.PoolingHttpClientConnectionManager$1#37
    at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:190)
       Local Variable: org.apache.http.auth.AuthState#74
       Local Variable: org.apache.http.auth.AuthState#73
    at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
    at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
       Local Variable: org.apache.http.Header[]#37
    at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
       Local Variable: org.apache.http.conn.routing.HttpRoute#45
       Local Variable: org.apache.http.client.protocol.HttpClientContext#37
       Local Variable: org.apache.http.client.methods.HttpRequestWrapper#37
    at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)

"http-nio-8284-exec-3856" daemon prio=5 tid=155182 WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
    at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:523)
       Local Variable: com.google.common.util.concurrent.AbstractFuture$Waiter#104
    at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:90)
    at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:199)
       Local Variable: com.google.common.util.concurrent.SettableFuture#36
    at com.google.common.cache.LocalCache$LoadingValueReference.waitForValue(LocalCache.java:3581)
    at com.google.common.cache.LocalCache$Segment.waitForLoadingValue(LocalCache.java:2174)
       Local Variable: java.lang.String#492150
    at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2161)
    at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
    at com.google.common.cache.LocalCache.get(LocalCache.java:3952)
    at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4871)
    at com.xxx.jsr203.objstorage.CosFileSystem.doesObjectExist(CosFileSystem.java:504)

"DubboServerHandler-172.17.16.56:20883-thread-22" daemon prio=5 tid=9599 WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
    at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
    at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
       Local Variable: java.util.concurrent.SynchronousQueue$TransferStack$SNode#17
    at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
       Local Variable: java.util.concurrent.ThreadPoolExecutor$Worker#879
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
复制