Hi There
We are facing a memory leak somewhere in Jenkins or its plugins.
The memory usage grows up until it eventually reaches the max heap size and then web interface becomes unresponsive, jobs fail to start and we see the following errors in logs:
java.lang.OutOfMemoryError: Java heap space
It consumes all the heap memory (we tried up to 32GB)
We tried to get a sheep dump and saw that DelayBufferedOutputStream object occupied 12 of 20 GB (almost 60%)
Looks like it’s related to something related to ssh commands.
We use jenkins pipelines to run ssh commands on different remote servers and pipelines look like this:
def remote = [:]
remote.name = "some-remote-name"
remote.host = "some-remote.host"
remote.allowAnyHosts = true
def colors = [:]
colors.failure = 'danger'
colors.fixed = 'good'
colors.unstable = 'warning'
colors.aborted = '#808080'
pipeline {
agent { node { label 'master' } }
options {
timeout(time: 600, unit: 'SECONDS') // timeout on whole pipeline job
}
stages {
stage('Run command') {
steps {
script {
withCredentials([sshUserPrivateKey(credentialsId: 'some_user', keyFileVariable: 'identity', passphraseVariable: '', usernameVariable: 'userName')]) {
remote.user = userName
remote.identityFile = identity
sshCommand remote: remote, command: "cd /some/path; php artisan some-command ", sudo: false
}
}
}
}
}
post {
failure {
script {
sh 'echo failure'
mail bcc: '', body: "<b>PROD FAILURE</b><br>Project: ${env.JOB_NAME} <br>Build Number: ${env.BUILD_NUMBER} <br> Build URL: ${env.BUILD_URL}", cc: '', charset: 'UTF-8', from: 'some-sender@domain.com', mimeType: 'text/html', replyTo: '', subject: "PROD FAILURE: Project name -> ${env.JOB_NAME}", to: "some-email@domain.com";
// Send the Slack message
slackSend(channel: "#some-slack-channel", message: "PROD FAILURE:: Job ${env.JOB_NAME} failure. <${env.BUILD_URL}console|Build #${env.BUILD_NUMBER}>", color: colors.failure);
}
}
fixed {
script {
sh 'echo fixed'
mail bcc: '', body: "<b>PROD FIXED</b><br>Project: ${env.JOB_NAME} <br>Build Number: ${env.BUILD_NUMBER} <br> Build URL: ${env.BUILD_URL}", cc: '', charset: 'UTF-8', from: 'some-sender@domain.com', mimeType: 'text/html', replyTo: '', subject: "PROD FIXED: Project name -> ${env.JOB_NAME}", to: "some-email@domain.com";
// Send the Slack message
slackSend(channel: "#some-slack-channel", message: "PROD FIXED:: Job ${env.JOB_NAME} fixed. <${env.BUILD_URL}console|Build #${env.BUILD_NUMBER}>", color: colors.fixed);
}
}
unstable {
script {
sh 'echo unstable'
mail bcc: '', body: "<b>PROD UNSTABLE</b><br>Project: ${env.JOB_NAME} <br>Build Number: ${env.BUILD_NUMBER} <br> Build URL: ${env.BUILD_URL}", cc: '', charset: 'UTF-8', from: 'some-sender@domain.com', mimeType: 'text/html', replyTo: '', subject: "PROD UNSTABLE: Project name -> ${env.JOB_NAME}", to: "some-email@domain.com";
// Send the Slack message
slackSend(channel: "#some-slack-channel", message: "PROD UNSTABLE:: Job ${env.JOB_NAME} unstable. <${env.BUILD_URL}console|Build #${env.BUILD_NUMBER}>", color: colors.unstable);
}
}
aborted {
script {
sh 'echo aborted'
mail bcc: '', body: "<b>PROD ABORTED</b><br>Project: ${env.JOB_NAME} <br>Build Number: ${env.BUILD_NUMBER} <br> Build URL: ${env.BUILD_URL}", cc: '', charset: 'UTF-8', from: 'some-sender@domain.com', mimeType: 'text/html', replyTo: '', subject: "PROD ABORTED: Project name -> ${env.JOB_NAME}", to: "some-email@domain.com";
// Send the Slack message
slackSend(channel: "#some-slack-channel", message: "PROD ABORTED:: Job ${env.JOB_NAME} aborted. <${env.BUILD_URL}console|Build #${env.BUILD_NUMBER}>", color: colors.aborted);
}
}
}
}
I’ll be happy to get any helpful feedback. It might be we are doing something wrong 
Jenkins setup:
Jenkins: 2.452.2
OS: Linux - 5.15.0-1049-aws
Java: 17.0.11 - Ubuntu (OpenJDK 64-Bit Server VM)JiraTestResultReporter:213.vccc8cf83e157
allure-jenkins-plugin:2.31.1
ansicolor:1.0.4
ant:497.v94e7d9fffa_b_9
antisamy-markup-formatter:162.v0e6ec0fcfcf6
apache-httpcomponents-client-4-api:4.5.14-208.v438351942757
asm-api:9.7-33.v4d23ef79fcc8
authentication-tokens:1.113.v81215a_241826
bootstrap5-api:5.3.3-1
bouncycastle-api:2.30.1.78.1-233.vfdcdeb_0a_08a_a_
branch-api:2.1169.va_f810c56e895
build-name-setter:2.4.2
build-timeout:1.33
build-user-vars-plugin:166.v52976843b_435
caffeine-api:3.1.8-133.v17b_1ff2e0599
checks-api:2.2.0
cloudbees-folder:6.928.v7c780211d66e
command-launcher:107.v773860566e2e
commons-httpclient3-api:3.1-3
commons-lang3-api:3.14.0-76.vda_5591261cfe
commons-text-api:1.12.0-119.v73ef73f2345d
credentials:1337.v60b_d7b_c7b_c9f
credentials-binding:677.vdc9d38cb_254d
datadog:7.1.0
display-url-api:2.204.vf6fddd8a_8b_e9
docker-commons:439.va_3cb_0a_6a_fb_29
docker-workflow:580.vc0c340686b_54
durable-task:555.v6802fe0f0b_82
dynamic-search-view:0.4.0
echarts-api:5.5.0-1
eddsa-api:0.3.0-4.v84c6f0f4969e
email-ext:1814.v404722f34263
font-awesome-api:6.5.2-1
git:5.2.2
git-client:5.0.0
github:1.39.0
github-api:1.318-461.v7a_c09c9fa_d63
github-branch-source:1789.v5b_0c0cea_18c3
global-slack-notifier:1.5
gradle:2.12
gson-api:2.11.0-41.v019fcf6125dc
instance-identity:185.v303dc7c645f9
ionicons-api:74.v93d5eb_813d5f
jackson2-api:2.17.0-379.v02de8ec9f64c
jakarta-activation-api:2.1.3-1
jakarta-mail-api:2.1.3-1
javadoc:243.vb_b_503b_b_45537
javax-activation-api:1.2.0-7
javax-mail-api:1.6.2-10
jaxb:2.3.9-1
jdk-tool:73.vddf737284550
jjwt-api:0.11.5-112.ve82dfb_224b_a_d
jnr-posix-api:3.1.19-2
job-dsl:1.87
jobtag:25.va_12b_1dd63367
joda-time-api:2.12.7-29.v5a_b_e3a_82269a_
jquery3-api:3.7.1-2
jsch:0.2.16-86.v42e010d9484b_
json-api:20240303-41.v94e11e6de726
json-path-api:2.9.0-58.v62e3e85b_a_655
junit:1265.v65b_14fa_f12f0
junit-attachments:205.vc0677977deb_0
junit-realtime-test-reporter:149.v05a_d403e2f48
kotlin-v1-stdlib-jdk8:1.3.20-1.4
ldap:725.v3cb_b_711b_1a_ef
mailer:472.vf7c289a_4b_420
matrix-auth:3.2.2
matrix-project:832.va_66e270d2946
maven-plugin:3.23
mina-sshd-api-common:2.12.1-113.v4d3ea_5eb_7f72
mina-sshd-api-core:2.12.1-113.v4d3ea_5eb_7f72
miniorange-saml-sp:2.4.2
nvm-wrapper:0.1.7
okhttp-api:4.11.0-172.vda_da_1feeb_c6e
pam-auth:1.11
pipeline-build-step:540.vb_e8849e1a_b_d8
pipeline-github-lib:61.v629f2cc41d83
pipeline-graph-analysis:216.vfd8b_ece330ca_
pipeline-groovy-lib:727.ve832a_9244dfa_
pipeline-input-step:495.ve9c153f6067b_
pipeline-milestone-step:119.vdfdc43fc3b_9a_
pipeline-model-api:2.2198.v41dd8ef6dd56
pipeline-model-definition:2.2198.v41dd8ef6dd56
pipeline-model-extensions:2.2198.v41dd8ef6dd56
pipeline-rest-api:2.34
pipeline-stage-step:312.v8cd10304c27a_
pipeline-stage-tags-metadata:2.2198.v41dd8ef6dd56
pipeline-stage-view:2.34
plain-credentials:182.v468b_97b_9dcb_8
plugin-util-api:4.1.0
rebuild:332.va_1ee476d8f6d
resource-disposer:0.23
role-strategy:727.vd344b_eec783d
saml:4.464.vea_cb_75d7f5e0
scm-api:690.vfc8b_54395023
script-security:1341.va_2819b_414686
simple-build-for-pipeline:0.2
slack:722.vd07f1ea_7ff40
snakeyaml-api:2.2-111.vc6598e30cc65
ssh:2.6.1
ssh-agent:367.vf9076cd4ee21
ssh-credentials:337.v395d2403ccd4
ssh-slaves:2.973.v0fa_8c0dea_f9f
ssh-steps:2.0.68.va_d21a_12a_6476
sshd:3.330.vc866a_8389b_58
structs:337.v1b_04ea_4df7c8
timestamper:1.27
token-macro:400.v35420b_922dcb_
trilead-api:2.147.vb_73cc728a_32e
uno-choice:2.8.3
variant:60.v7290fc0eb_b_cd
workflow-aggregator:596.v8c21c963d92d
workflow-api:1316.v33eb_726c50b_a_
workflow-basic-steps:1058.vcb_fc1e3a_21a_9
workflow-cps:3903.v48a_8836749e9
workflow-durable-task-step:1353.v1891a_b_01da_18
workflow-job:1400.v7fd111b_ec82f
workflow-multibranch:783.787.v50539468395f
workflow-scm-step:427.v4ca_6512e7df1
workflow-step-api:657.v03b_e8115821b_
workflow-support:907.v6713a_ed8a_573
ws-cleanup:0.46
