Stale file handle exception when calling safeExit API

I have a Jenkins master running inside a K8s pod. When the pod restarts, I’m calling a script that executes the safeExit API. I’m seeing a Stale file exception when during the API execution; the LTS version is 2.375.2. I’m using a curl command to call the API, and when I execute this command inside the Jenkins master K8s pod, I don’t see an exception - it happens only when the script executes. Any ideas on what could be happening?

2023-05-26 22:31:37.946+0000 [id=20] WARNING o.e.j.s.h.ContextHandler$Context#log: Error while serving http://localhost:8080/ipp-blossom-dev-shutdownscript/safeExit

java.io.IOException: Stale file handle

at java.base/java.io.RandomAccessFile.readBytes(Native Method)

at java.base/java.io.RandomAccessFile.read(RandomAccessFile.java:406)

at java.base/java.io.RandomAccessFile.readFully(RandomAccessFile.java:470)

at java.base/java.util.zip.ZipFile$Source.readFullyAt(ZipFile.java:1339)

at java.base/java.util.zip.ZipFile$ZipFileInputStream.initDataOffset(ZipFile.java:1004)

at java.base/java.util.zip.ZipFile$ZipFileInputStream.read(ZipFile.java:1019)

at java.base/java.util.zip.ZipFile$ZipFileInflaterInputStream.fill(ZipFile.java:469)

at java.base/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:159)

at java.base/jdk.internal.loader.Resource.getBytes(Resource.java:124)

at java.base/jdk.internal.loader.URLClassPath$JarLoader$2.getBytes(URLClassPath.java:882)

at java.base/java.net.URLClassLoader.defineClass(URLClassLoader.java:550)

at java.base/java.net.URLClassLoader$1.run(URLClassLoader.java:458)

Caused: java.lang.ClassNotFoundException: jenkins.model.Jenkins$22

at java.base/java.net.URLClassLoader$1.run(URLClassLoader.java:460)

at java.base/java.net.URLClassLoader$1.run(URLClassLoader.java:452)

at java.base/java.security.AccessController.doPrivileged(Native Method)

at java.base/java.net.URLClassLoader.findClass(URLClassLoader.java:451)

at org.eclipse.jetty.webapp.WebAppClassLoader.foundClass(WebAppClassLoader.java:594)

at org.eclipse.jetty.webapp.WebAppClassLoader.loadAsResource(WebAppClassLoader.java:567)

at org.eclipse.jetty.webapp.WebAppClassLoader.loadClass(WebAppClassLoader.java:502)

at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:522)

Caused: java.lang.NoClassDefFoundError: jenkins/model/Jenkins$22

at jenkins.model.Jenkins.doSafeExit(Jenkins.java:4649)

at java.base/java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:710)

at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:397)

Caused: java.lang.reflect.InvocationTargetException

at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:401)

at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:409)

at org.kohsuke.stapler.interceptor.RequirePOST$Processor.invoke(RequirePOST.java:78)

at org.kohsuke.stapler.PreInvokeInterceptedFunction.invoke(PreInvokeInterceptedFunction.java:26)

at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:207)

at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.java:140)

at org.kohsuke.stapler.MetaClass$11.doDispatch(MetaClass.java:558)

at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:59)

at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:762)

at org.kohsuke.stapler.Stapler.invoke(Stapler.java:894)

at org.kohsuke.stapler.Stapler.invoke(Stapler.java:690)

at org.kohsuke.stapler.Stapler.service(Stapler.java:240)

at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)

at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)

at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:157)

at org.jenkinsci.plugins.ssegateway.Endpoint$SSEListenChannelFilter.doFilter(Endpoint.java:248)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:129)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:81)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at io.jenkins.blueocean.ResourceCacheControl.doFilter(ResourceCacheControl.java:134)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at io.jenkins.blueocean.auth.jwt.impl.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:60)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at jenkins.metrics.impl.MetricsFilter.doFilter(MetricsFilter.java:125)

at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)

at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:160)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:154)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:94)

at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:54)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:122)

at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:116)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:109)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:106)

at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:97)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:223)

at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:217)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at jenkins.security.BasicHeaderProcessor.success(BasicHeaderProcessor.java:139)

at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:86)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:112)

at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:82)

at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:63)

at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99)

at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:111)

at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:172)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:53)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:86)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:38)

at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)

at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)

at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)

at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:549)

at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)

at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)

at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1571)

at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)

at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1383)

at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)

at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)

at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1544)

at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)

at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1305)

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)

at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)

at org.eclipse.jetty.server.Server.handle(Server.java:563)

at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)

at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)

at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)

at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)

at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)

at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)

at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)

at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421)

at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390)

at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277)

at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:139)

at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)

at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:933)

at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1077)

at java.base/java.lang.Thread.run(Thread.java:829)

Rebooting the nodes where the instances were running on helped.