I currently have two problems at the same time that might be connected.
The first problem is that I have builds that are using up executors but that can’t be killed and the job page itself gives a 404. As far as I can tell they don’t use up any real resources but the executor slot is blocked. The only way I’ve found to get rid of the jobs is to restart the controller.
The second problem is that every second there are multiple similar exceptions logged on the controller. As far as I can tell they all look like this or at least similar to the sample in the bottom. I have written a very extensive pipeline library and I assume that it might be the cause but the exception give me absolutely no indication on where to look.
I was wondering if someone here has any idea how to approach this.
2023-05-30 11:47:15.953+0000 [id=413031] WARNING o.j.p.w.f.FlowExecutionList$StepExecutionIteratorImpl$1#onFailure
org.jenkinsci.remoting.util.ExecutorServiceUtils$FatalRejectedExecutionException: Cannot execute the command java.util.concurrent.FutureTask@6b8d8226[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@f9079cb[Wrapped task = org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1@7b8a7469]]. The executor service is shutting down
at hudson.remoting.SingleLaneExecutorService.execute(SingleLaneExecutorService.java:116)
at java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:118)
at jenkins.util.InterceptingExecutorService.submit(InterceptingExecutorService.java:46)
at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution$5.onSuccess(CpsFlowExecution.java:936)
at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution$5.onSuccess(CpsFlowExecution.java:931)
at org.jenkinsci.plugins.workflow.support.concurrent.Futures$1.run(Futures.java:147)
at org.jenkinsci.plugins.workflow.support.concurrent.DirectExecutor.execute(DirectExecutor.java:33)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1270)
at com.google.common.util.concurrent.AbstractFuture.addListener(AbstractFuture.java:761)
at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.addListener(AbstractFuture.java:136)
at org.jenkinsci.plugins.workflow.support.concurrent.Futures.addCallback(Futures.java:157)
at org.jenkinsci.plugins.workflow.support.concurrent.Futures.addCallback(Futures.java:97)
at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.runInCpsVmThread(CpsFlowExecution.java:931)
at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.getCurrentExecutions(CpsFlowExecution.java:1023)
at org.jenkinsci.plugins.workflow.flow.FlowExecutionList$StepExecutionIteratorImpl.apply(FlowExecutionList.java:215)
at org.jenkinsci.plugins.workflow.steps.StepExecution.applyAll(StepExecution.java:179)
at org.jenkinsci.plugins.workflow.steps.StepExecution.applyAll(StepExecution.java:187)
at org.jenkinsci.plugins.workflow.steps.WaitForConditionStep$Execution.retry(WaitForConditionStep.java:121)
at org.jenkinsci.plugins.workflow.steps.WaitForConditionStep$Callback.onSuccess(WaitForConditionStep.java:181)
at org.jenkinsci.plugins.workflow.cps.CpsBodyExecution$SuccessAdapter.receive(CpsBodyExecution.java:371)
at com.cloudbees.groovy.cps.impl.ContinuationGroup.castToBoolean(ContinuationGroup.java:230)
at com.cloudbees.groovy.cps.impl.LogicalOpBlock$ContinuationImpl.castRhs(LogicalOpBlock.java:49)
at jdk.internal.reflect.GeneratedMethodAccessor207.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:93)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:116)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:85)
at jdk.internal.reflect.GeneratedMethodAccessor158.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.LocalVariableBlock$LocalVariable.get(LocalVariableBlock.java:39)
at com.cloudbees.groovy.cps.LValueBlock$GetAdapter.receive(LValueBlock.java:30)
at com.cloudbees.groovy.cps.impl.LocalVariableBlock.evalLValue(LocalVariableBlock.java:28)
at com.cloudbees.groovy.cps.LValueBlock$BlockImpl.eval(LValueBlock.java:55)
at com.cloudbees.groovy.cps.LValueBlock.eval(LValueBlock.java:16)
at com.cloudbees.groovy.cps.Next.step(Next.java:83)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:152)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:146)
at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:136)
at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:275)
at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:146)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:187)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:420)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:330)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:294)
at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:67)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139)
at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:30)
at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:70)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)