Title: Message Title
| |
|
|
|
|
*When the Github Org Folder Scan runs and tries to clean up old branch and PR builds, we see errors like this ".nfsXXXXX : Device or Resource Busy". See stack trace below: *
{{FATAL: Failed to recompute children of Bluton Pull Requests ยป bluton}}
{{jenkins.util.io.CompositeIOException: Unable to delete '/opt/apache/.jenkins/jobs/Bluton Pull Requests/jobs/bluton/branches/Docker123'. Tried 3 times (of a maximum of 3) waiting 0.1 sec between attempts.}}
\ {{ at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:95)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:294)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:283)}}
\ {{ at hudson.model.AbstractItem.performDelete(AbstractItem.java:792)}}
\ {{ at org.jenkinsci.plugins.workflow.job.WorkflowJob.performDelete(WorkflowJob.java:652)}}
\ {{ at hudson.model.AbstractItem.delete(AbstractItem.java:776)}}
\ {{ at hudson.model.Job.delete(Job.java:677)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.ComputedFolder.updateChildren(ComputedFolder.java:290)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.FolderComputation.run(FolderComputation.java:165)}}
\ {{ at jenkins.branch.MultiBranchProject$BranchIndexing.run(MultiBranchProject.java:1025)}}
\ {{ at hudson.model.ResourceController.execute(ResourceController.java:97)}}
\ {{ at hudson.model.Executor.run(Executor.java:429)}}
{{jenkins.util.io.CompositeIOException: Unable to remove file /opt/apache/.jenkins/jobs/Bluton Pull Requests/jobs/bluton/branches/Docker123/builds/41/.nfs00000000080f52850000009d}}
\ {{ at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:248)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:201)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:212)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:92)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:294)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:283)}}
\ {{ at hudson.model.AbstractItem.performDelete(AbstractItem.java:792)}}
\ {{ at org.jenkinsci.plugins.workflow.job.WorkflowJob.performDelete(WorkflowJob.java:652)}}
\ {{ at hudson.model.AbstractItem.delete(AbstractItem.java:776)}}
\ {{ at hudson.model.Job.delete(Job.java:677)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.ComputedFolder.updateChildren(ComputedFolder.java:290)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.FolderComputation.run(FolderComputation.java:165)}}
\ {{ at jenkins.branch.MultiBranchProject$BranchIndexing.run(MultiBranchProject.java:1025)}}
\ {{ at hudson.model.ResourceController.execute(ResourceController.java:97)}}
\ {{ at hudson.model.Executor.run(Executor.java:429)}}
{{java.nio.file.FileSystemException: /opt/apache/.jenkins/jobs/Bluton Pull Requests/jobs/bluton/branches/Docker123/builds/41/.nfs00000000080f52850000009d: Device or resource busy}}
\ {{ at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)}}
\ {{ at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)}}
\ {{ at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)}}
\ {{ at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)}}
\ {{ at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)}}
\ {{ at java.nio.file.Files.deleteIfExists(Files.java:1165)}}
\ {{ at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:233)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:201)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:212)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:222)}}
\ {{ at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:211)}}
\ {{ at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:92)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:294)}}
\ {{ at hudson.Util.deleteRecursive(Util.java:283)}}
\ {{ at hudson.model.AbstractItem.performDelete(AbstractItem.java:792)}}
\ {{ at org.jenkinsci.plugins.workflow.job.WorkflowJob.performDelete(WorkflowJob.java:652)}}
\ {{ at hudson.model.AbstractItem.delete(AbstractItem.java:776)}}
\ {{ at hudson.model.Job.delete(Job.java:677)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.ComputedFolder.updateChildren(ComputedFolder.java:290)}}
\ {{ at com.cloudbees.hudson.plugins.folder.computed.FolderComputation.run(FolderComputation.java:165)}}
\ {{ at jenkins.branch.MultiBranchProject$BranchIndexing.run(MultiBranchProject.java:1025)}}
\ {{ at hudson.model.ResourceController.execute(ResourceController.java:97)}}
\ {{ at hudson.model.Executor.run(Executor.java:429)}}
{color:#172b4d} {color}
{color:#172b4d}*{{When I go look at the open files on the Jenkins java process, I see a bunch of .nfs files which appear to be all related to EC2 instances managed by the EC2 Plugin.}}*
{color}
{{root@foo:/proc/15266/fd# ls -l | grep -i \.nfs}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 101 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-03fd7b1343b0782b8)/.nfs00000000080a6a830000009a}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 108 -> /srv/jenkins/jobs/Bluton Pull Requests/jobs/bluton/branches/Docker123/builds/41/.nfs00000000080f52850000009d}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 109 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-07baa1e7f3d95a94e)/.nfs00000000080ae75e00000095}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 110 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-019098927c3f4c8a9)/.nfs000000000805bd9000000097}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1413 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-03460b045be5a9faa)/.nfs000000000807291b00000088}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1423 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-0cfbcbfdf761f9d07)/.nfs000000000807599500000094}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1450 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-029f40ef47d320089)/.nfs000000000807203f00000087}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1454 -> /srv/jenkins/jobs/Bluton Pull Requests/jobs/tools/branches/PR-799/builds/24/.nfs0000000008056f6500000092}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1463 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-0cfbcbfdf761f9d07)/.nfs000000000807598f00000093}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1487 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-05c354bbe104d7f4b)/.nfs0000000008017b5b00000090}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1497 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-05f3b733d4a484bb1)/.nfs00000000080429ae0000008d}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1536 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-0afc0ecc2e3f5e1f6)/.nfs00000000080752030000009b}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1559 -> /srv/jenkins/jobs/FOO Org (synapse)/jobs/bluton/branches/TCAR12345-Bla.7k9tn5.onfiguration/builds/2/.nfs00000000080ae4fc000000a1}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 1562 -> /srv/jenkins/jobs/FOO Org (synapse)/jobs/bluton/branches/TCAR12345-Bla.7k9tn5.onfiguration/builds/2/.nfs00000000080ae4fd000000a2}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 66 -> /srv/jenkins/jobs/Bluton Pull Requests/jobs/tools/branches/PR-799/builds/24/.nfs0000000008081d1b00000091}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 67 -> /srv/jenkins/jobs/FOO Org (synapse)/jobs/bluton/branches/TCAR12345-Bla.7k9tn5.onfiguration/builds/1/.nfs00000000080b960d0000009f}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 70 -> /srv/jenkins/jobs/FOO Org (synapse)/jobs/bluton/branches/TCAR12345-Bla.7k9tn5.onfiguration/builds/1/.nfs00000000080b9611000000a0}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 79 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-07baa1e7f3d95a94e)/.nfs00000000080ae75800000096}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 83 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-05f3b733d4a484bb1)/.nfs00000000080429aa0000008e}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 84 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-019098927c3f4c8a9)/.nfs000000000805bd9600000098}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 87 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-019098927c3f4c8a9)/.nfs000000000805bd8f00000099}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 93 -> /srv/jenkins/logs/slaves/foo.bluton-large (i-00db25a98b6ffee4c)/.nfs00000000080b9e7800000089}}
{{l-wx------ 1 tomcat tomcat 64 Sep 4 08:58 95 -> /srv/jenkins/jobs/Bluton Pull Requests/jobs/bluton/branches/Docker123/builds/41/.nfs00000000080f52840000009c}} |
|
|
|
| |
|
--
You received this message because you are subscribed to the Google Groups "Jenkins Issues" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
To view this discussion on the web visit https://groups.google.com/d/msgid/jenkinsci-issues/JIRA.201690.1567621668000.6781.1567622520365%40Atlassian.JIRA.
|
