Hey all,
I don’t have access to run that config script or to access the /systemInfo page on the Jenkins instance I’m using.
What I have having is an error where my agents lost connections after various periods of time.
For example:
This job failed after 1h35 with this:
java.nio.channels.ClosedChannelException
at jenkins.agents.WebSocketAgents$Session.closed(WebSocketAgents.java:157)
at jenkins.websocket.WebSockets$1.onWebSocketClose(WebSockets.java:88)
at jenkins.websocket.Jetty10Provider$2.onWebSocketClose(Jetty10Provider.java:164)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.notifyOnClose(JettyWebSocketFrameHandler.java:308)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onClosed(JettyWebSocketFrameHandler.java:292)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$0(WebSocketCoreSession.java:272)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1450)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$1(WebSocketCoreSession.java:272)
at org.eclipse.jetty.util.Callback$4.completed(Callback.java:184)
at org.eclipse.jetty.util.Callback$Completing.succeeded(Callback.java:344)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onError(JettyWebSocketFrameHandler.java:268)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$2(WebSocketCoreSession.java:284)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1468)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.closeConnection(WebSocketCoreSession.java:284)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$sendFrame$7(WebSocketCoreSession.java:519)
at org.eclipse.jetty.util.Callback$3.succeeded(Callback.java:155)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher.notifyCallbackSuccess(TransformingFlusher.java:197)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher$Flusher.process(TransformingFlusher.java:154)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:232)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:214)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher.sendFrame(TransformingFlusher.java:77)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.sendFrame(WebSocketCoreSession.java:522)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.close(WebSocketCoreSession.java:239)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.processHandlerError(WebSocketCoreSession.java:371)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.onIdleExpired(WebSocketConnection.java:233)
at org.eclipse.jetty.io.ssl.SslConnection.onIdleExpired(SslConnection.java:360)
at org.eclipse.jetty.io.AbstractEndPoint.onIdleExpired(AbstractEndPoint.java:407)
at org.eclipse.jetty.io.IdleTimeout.checkIdleTimeout(IdleTimeout.java:170)
at org.eclipse.jetty.io.IdleTimeout.idleCheck(IdleTimeout.java:112)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused: java.io.IOException: Backing channel ‘PO-BUKATMTSTBuildBox-395’ is disconnected.
at hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:215)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:285)
at com.sun.proxy.$Proxy275.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1215)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1207)
at hudson.tasks.CommandInterpreter.join(CommandInterpreter.java:195)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:145)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:92)
at hudson.plugins.powershell.PowerShell.perform(PowerShell.java:48)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:818)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:526)
at hudson.model.Run.execute(Run.java:1900)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
FATAL: Unable to delete script file C:\Windows\TEMP\jenkins8107610045699559202.ps1
java.nio.channels.ClosedChannelException
at jenkins.agents.WebSocketAgents$Session.closed(WebSocketAgents.java:157)
at jenkins.websocket.WebSockets$1.onWebSocketClose(WebSockets.java:88)
at jenkins.websocket.Jetty10Provider$2.onWebSocketClose(Jetty10Provider.java:164)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.notifyOnClose(JettyWebSocketFrameHandler.java:308)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onClosed(JettyWebSocketFrameHandler.java:292)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$0(WebSocketCoreSession.java:272)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1450)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$1(WebSocketCoreSession.java:272)
at org.eclipse.jetty.util.Callback$4.completed(Callback.java:184)
at org.eclipse.jetty.util.Callback$Completing.succeeded(Callback.java:344)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onError(JettyWebSocketFrameHandler.java:268)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$2(WebSocketCoreSession.java:284)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1468)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.closeConnection(WebSocketCoreSession.java:284)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$sendFrame$7(WebSocketCoreSession.java:519)
at org.eclipse.jetty.util.Callback$3.succeeded(Callback.java:155)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher.notifyCallbackSuccess(TransformingFlusher.java:197)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher$Flusher.process(TransformingFlusher.java:154)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:232)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:214)
at org.eclipse.jetty.websocket.core.internal.TransformingFlusher.sendFrame(TransformingFlusher.java:77)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.sendFrame(WebSocketCoreSession.java:522)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.close(WebSocketCoreSession.java:239)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.processHandlerError(WebSocketCoreSession.java:371)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.onIdleExpired(WebSocketConnection.java:233)
at org.eclipse.jetty.io.ssl.SslConnection.onIdleExpired(SslConnection.java:360)
at org.eclipse.jetty.io.AbstractEndPoint.onIdleExpired(AbstractEndPoint.java:407)
at org.eclipse.jetty.io.IdleTimeout.checkIdleTimeout(IdleTimeout.java:170)
at org.eclipse.jetty.io.IdleTimeout.idleCheck(IdleTimeout.java:112)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused: hudson.remoting.ChannelClosedException: Channel “hudson.remoting.Channel@186c8e96:PO-BUKATMTSTBuildBox-395”: Remote call on PO-BUKATMTSTBuildBox-395 failed. The channel is closing down or has closed down
at hudson.remoting.Channel.call(Channel.java:993)
at hudson.FilePath.act(FilePath.java:1192)
at hudson.FilePath.act(FilePath.java:1181)
at hudson.FilePath.delete(FilePath.java:1728)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:163)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:92)
at hudson.plugins.powershell.PowerShell.perform(PowerShell.java:48)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:818)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:526)
at hudson.model.Run.execute(Run.java:1900)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
Build step ‘PowerShell’ marked build as failure
Finished: FAILURE
This other job failed after 2.5 hours:
FATAL: command execution failed
java.nio.channels.ClosedChannelException
at jenkins.agents.WebSocketAgents$Session.closed(WebSocketAgents.java:157)
at jenkins.websocket.WebSockets$1.onWebSocketClose(WebSockets.java:88)
at jenkins.websocket.WebSockets$1.onWebSocketError(WebSockets.java:94)
at jenkins.websocket.Jetty10Provider$2.onWebSocketError(Jetty10Provider.java:174)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onError(JettyWebSocketFrameHandler.java:260)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$2(WebSocketCoreSession.java:284)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1468)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.closeConnection(WebSocketCoreSession.java:284)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.onEof(WebSocketCoreSession.java:254)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.fillAndParse(WebSocketConnection.java:482)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.onFillable(WebSocketConnection.java:340)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:558)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:379)
at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:146)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:934)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1078)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused: java.io.IOException: Backing channel ‘PO-GBRDSM020007906’ is disconnected.
at hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:215)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:285)
at com.sun.proxy.$Proxy205.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1215)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1207)
at hudson.tasks.CommandInterpreter.join(CommandInterpreter.java:195)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:145)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:92)
at hudson.plugins.powershell.PowerShell.perform(PowerShell.java:48)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:818)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:526)
at hudson.model.Run.execute(Run.java:1900)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
FATAL: Unable to delete script file C:\windows\TEMP\jenkins10200756588072006815.ps1
java.nio.channels.ClosedChannelException
at jenkins.agents.WebSocketAgents$Session.closed(WebSocketAgents.java:157)
at jenkins.websocket.WebSockets$1.onWebSocketClose(WebSockets.java:88)
at jenkins.websocket.WebSockets$1.onWebSocketError(WebSockets.java:94)
at jenkins.websocket.Jetty10Provider$2.onWebSocketError(Jetty10Provider.java:174)
at org.eclipse.jetty.websocket.common.JettyWebSocketFrameHandler.onError(JettyWebSocketFrameHandler.java:260)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.lambda$closeConnection$2(WebSocketCoreSession.java:284)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1468)
at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1487)
at org.eclipse.jetty.websocket.core.server.internal.AbstractHandshaker$1.handle(AbstractHandshaker.java:212)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.closeConnection(WebSocketCoreSession.java:284)
at org.eclipse.jetty.websocket.core.internal.WebSocketCoreSession.onEof(WebSocketCoreSession.java:254)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.fillAndParse(WebSocketConnection.java:482)
at org.eclipse.jetty.websocket.core.internal.WebSocketConnection.onFillable(WebSocketConnection.java:340)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:558)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:379)
at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:146)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:934)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1078)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused: hudson.remoting.ChannelClosedException: Channel “hudson.remoting.Channel@71a3ba68:PO-GBRDSM020007906”: Remote call on PO-GBRDSM020007906 failed. The channel is closing down or has closed down
at hudson.remoting.Channel.call(Channel.java:993)
at hudson.FilePath.act(FilePath.java:1192)
at hudson.FilePath.act(FilePath.java:1181)
at hudson.FilePath.delete(FilePath.java:1728)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:163)
at hudson.tasks.CommandInterpreter.perform(CommandInterpreter.java:92)
at hudson.plugins.powershell.PowerShell.perform(PowerShell.java:48)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:818)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:526)
at hudson.model.Run.execute(Run.java:1900)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
Build step ‘PowerShell’ marked build as failure
Finished: FAILURE
Then I have another job running right now that is still running and it’s been going since 11am and it’s now 4:15pm.
In one job, it is waiting for another process to resume and nothing is being sent up to the jenkins console. In the second, the same is happening except regular outputs of “write-host” are being sent to the console, which are coming up into Jenkins and being visible. In the third one that hasn’t failed, there is a large file copy taking place which hasn’t yet completed.
I’m not sure where to look - I’ve tried asking about timeouts but the admins of Jenkins here are saying it’s network related - but I just wanted to check with you all before I have to engage our networks team to investigate.
Many thanks