2017-03-03 00:53:03 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "DestroyJavaVM" prio=10 tid=0x00007f8f64012000 nid=0x7cda waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Attach Listener" daemon prio=10 tid=0x00007f8f65f37800 nid=0x42e6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "IPC Client (554828123) connection to master19876-1000.dev.continuuity.net/10.128.0.10:16020 from cdap/master19876-1000.dev.continuuity.net@CONTINUUITY.NET" daemon prio=10 tid=0x00007f8f65e66800 nid=0x1fb1 in Object.wait() [0x00007f8f530ef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000f58258e8> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:544) - locked <0x00000000f58258e8> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:559) "streams-worker-thread-1-EventThread" daemon prio=10 tid=0x00007f8f54365000 nid=0x1f58 waiting on condition [0x00007f8f590d4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f57b78f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) "streams-worker-thread-1-SendThread(master19876-1000.dev.continuuity.net:2181)" daemon prio=10 tid=0x00007f8f54364000 nid=0x1f56 runnable [0x00007f8f594d8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000000f57b71a8> (a sun.nio.ch.Util$2) - locked <0x00000000f57b7198> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000f57b6cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1125) "stream-cleanup" daemon prio=10 tid=0x00007f8f5c426000 nid=0x238 waiting on condition [0x00007f8f534f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f5665248> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "pool-6-thread-1" prio=10 tid=0x00007f8f65c3b000 nid=0x20b waiting on condition [0x00007f8f5897d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f56c84f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-2" daemon prio=10 tid=0x00000000028fa800 nid=0x7fbc waiting on condition [0x00007f8f58cd0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f53def38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-1" daemon prio=10 tid=0x00000000024aa800 nid=0x7fba waiting on condition [0x00007f8f58dd1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f53def38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "com.google.inject.internal.util.$Finalizer" daemon prio=10 tid=0x00007f8f6593a000 nid=0x7fb3 in Object.wait() [0x00007f8f58ed2000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000f5353f20> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000000f5353f20> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at com.google.inject.internal.util.$Finalizer.run(Finalizer.java:114) "TGT Renewer for cdap/master19876-1000.dev.continuuity.net@CONTINUUITY.NET" daemon prio=10 tid=0x00007f8f6555c000 nid=0x7edb waiting on condition [0x00007f8f5a34e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$1.run(UserGroupInformation.java:896) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00007f8f640a0000 nid=0x7cec runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00007f8f6409e000 nid=0x7cea waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00007f8f6409a800 nid=0x7ce9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00007f8f64098800 nid=0x7ce8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00007f8f64078800 nid=0x7ce6 in Object.wait() [0x00007f8f5bffe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000f4c609d0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000000f4c609d0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x00007f8f64076800 nid=0x7ce5 in Object.wait() [0x00007f8f6014b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000f4c601d0> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x00000000f4c601d0> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00007f8f64072000 nid=0x7ce4 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x00007f8f64027800 nid=0x7cde runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x00007f8f64029800 nid=0x7cdf runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x00007f8f6402b800 nid=0x7ce2 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x00007f8f6402d800 nid=0x7ce3 runnable "VM Periodic Task Thread" prio=10 tid=0x00007f8f640ab000 nid=0x7ced waiting on condition JNI global references: 358