From 503747ba1e59450c06e70edcb263ebffb85517af Mon Sep 17 00:00:00 2001 From: Shubham Verma Date: Mon, 26 Mar 2018 10:25:31 -0400 Subject: [PATCH 1/5] Added storage monitoring Jenkins job Implemented storage monitoring job for linux, aix, and mac Updated the readme to reflect the new Jenkins job --- README.md | 10 ++- WorkspaceInfo.groovy | 157 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 WorkspaceInfo.groovy diff --git a/README.md b/README.md index eafa43c..2623a9e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # openjdk-jenkins-helper -The NodeHelper API contains helper functions to query basic machine stats in real time, add new machines, update/overwrite labels. -## Functions +## Files +### API Functions (NodeHelper.groovy) +The NodeHelper API contains helper functions to query basic machine stats in real time, add new machines, update/overwrite labels. * Get CPU count ```String getCpuCount(String computerName)``` * Get Installed memory ```Tuple getMemory(String computerName)``` * Get OS information ```Tuple getOsInfo(String computerName)``` @@ -17,6 +18,11 @@ The NodeHelper API contains helper functions to query basic machine stats in rea * Add Label ```String addLabel(String computerName, String label)``` * Append Label ```String appendlabel(String computerName, String label)``` +### Space Monitoring (WorkspaceInfo.groovy) +Iterates over online nodes on Jenkins and prints the contents of the workspace directory along with the space they occupy +* The computers it iterates over can be limited by parameter input +* As of now, it only works for linux, aix, and mac + ## How-to ### Setup diff --git a/WorkspaceInfo.groovy b/WorkspaceInfo.groovy new file mode 100644 index 0000000..9baaa96 --- /dev/null +++ b/WorkspaceInfo.groovy @@ -0,0 +1,157 @@ +@Library('NodeHelper') _ +import jenkins.model.Jenkins; +import hudson.model.Computer; + +clones = [:] +List newMachines = new ArrayList(); + +/* Iterates over all the online nodes in Jenkins and + * prints contents of workspace folder along with + * the space they occupy. With exception of tmp directory + */ +node { + stage('Print_Space_Monitoring_Data') { + NodeHelper nodeHelper = new NodeHelper(); + + String projectLabel = params.projectLabel; + + def currentInstance = Jenkins.getInstance(); + Computer[] computers; + + if (currentInstance != null) { + computers = currentInstance.getComputers(); + } + + for (Computer computer : computers) { + String machineName = computer.getName(); + + if (computer.isOnline() && computer.getName() != "") { + String kernelName = nodeHelper.getOsKernelInfo(computer.getName()).get(0).toLowerCase() + + if (nodeHelper.getLabels(computer.getName()).contains(projectLabel)) { + + String workspaceDirectory = nodeHelper.getHomeDirectoryPath(machineName); + + switch (kernelName) { + case 'linux': + clones[machineName] = { + node (machineName) { + workspaceDirectory += "/workspace" + + /* #!/bin/sh -e\n is there to keep the output to console as clean as + * possible + */ + String workspaceStats = sh ( + script: '#!/bin/sh -e\n' + 'du -sh ' + workspaceDirectory, + returnStdout: true).trim() + + String subdirectories = sh ( + script: '#!/bin/sh -e\n du -sh ' + workspaceDirectory + '/* | sort -h', + returnStdout: true).trim() + + println beautify(machineName,workspaceDirectory, workspaceStats,subdirectories) + + cleanWs() + } + } + break; + case 'aix': + clones[machineName] = { + node (machineName) { + workspaceDirectory += "/workspace" + + String workspaceStats = sh ( + script: '#!/bin/sh -e\n' + 'du -sg ' + workspaceDirectory, + returnStdout: true).trim() + + String subdirectories = sh ( + script: '#!/bin/sh -e\n du -sg ' + workspaceDirectory + '/* | sort -n', + returnStdout: true).trim() + + println beautify(machineName,workspaceDirectory, workspaceStats,subdirectories) + + cleanWs() + } + } + break; + case 'mac': + clones[machineName] = { + node (machineName) { + workspaceDirectory += "/workspace" + + String workspaceStats = sh ( + script: '#!/bin/sh -e\n' + 'du -sh ' + workspaceDirectory, + returnStdout: true).trim() + + String subdirectories = sh ( + script: '#!/bin/sh -e\n du -sh ' + workspaceDirectory + '/* | sort -n', + returnStdout: true).trim() + + println beautify(machineName,workspaceDirectory, workspaceStats,subdirectories) + + cleanWs() + } + } + break; + /* This is commented out because it takes way too long to return + * it isn't a top priority + * dir /s /-c + */ + case 'windows': + // clones[machineName] = { + // node(machineName) { + // cleanWs() + // NodeHelper nodeHelper = new NodeHelper(); + // workspaceDirectory += "\\workspace"; + // println workspaceDirectory + // // TODO: get path to directory from the api + // String workspaceStats = sh ( + // script: '#!/bin/sh -e\n' 'du -sh ' workspaceDirectory, + // returnStdout: true).trim() + + // String script = 'set x; du -sh ' workspaceDirectory '\\* | sort -rn'; + // String subdirectories = sh ( + // script: 'script', + // returnStdout: true).trim() + + + // println beautify(machineName,workspaceDirectory, workspaceStats,subdirectories); + + // cleanWs() + // } + // } + // } + // break; + case 'zos': + default: + println ("Support for ${kernelName} is yet to be implemented"); + break; + } + } + } + } + currentInstance = null; + computers = null; + + cleanWs() + } +} + +@NonCPS +def beautify(machineName, workspaceDirectory, workspaceStats, subdirectories) { + workspaceStats = workspaceStats.replaceAll("\\s+", " "); + subdirectories = subdirectories.replace(workspaceDirectory + "/", ""); + String output = "\n\n=======================================================================\n"; + output += "Disk stats for ${machineName}" + output += "\nWorkspace:\n${workspaceStats}"; + def subdirectoriesArray = subdirectories.split("\n"); + for (String line : subdirectoriesArray) { + if (!line.contains("tmp")) { + line = line.replaceAll("\\s+", " ") + output += "\n ${line}"; + } + } + output += "\n=======================================================================\n\n"; +} + +parallel clones \ No newline at end of file From cf4c8e04f936e571e94b25822f3e46c43a7e46f5 Mon Sep 17 00:00:00 2001 From: Shubham Verma Date: Mon, 26 Mar 2018 10:29:12 -0400 Subject: [PATCH 2/5] Moved WorkspaceInfo.groovy to a more appropriate folder --- WorkspaceInfo.groovy => Jenkins_jobs/WorkspaceInfo.groovy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename WorkspaceInfo.groovy => Jenkins_jobs/WorkspaceInfo.groovy (100%) diff --git a/WorkspaceInfo.groovy b/Jenkins_jobs/WorkspaceInfo.groovy similarity index 100% rename from WorkspaceInfo.groovy rename to Jenkins_jobs/WorkspaceInfo.groovy From 397c2a7a583af24a1395a8f72dd23f435dc3e988 Mon Sep 17 00:00:00 2001 From: Shubham Verma Date: Mon, 26 Mar 2018 14:31:49 -0400 Subject: [PATCH 3/5] Added the input parameter name for WorkspaceInfo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2623a9e..d3c09b0 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The NodeHelper API contains helper functions to query basic machine stats in rea ### Space Monitoring (WorkspaceInfo.groovy) Iterates over online nodes on Jenkins and prints the contents of the workspace directory along with the space they occupy -* The computers it iterates over can be limited by parameter input +* The computers it iterates over can be limited by parameter input ```projectLabel``` * As of now, it only works for linux, aix, and mac ## How-to From 4f9a6983a7cc65876a691f3c091f861f53d1c15e Mon Sep 17 00:00:00 2001 From: Shubham Verma Date: Tue, 27 Mar 2018 08:48:42 -0400 Subject: [PATCH 4/5] Requested Changes and Minor Fixes - Added new line at the end of file - Fixed formatting in README --- Jenkins_jobs/WorkspaceInfo.groovy | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkins_jobs/WorkspaceInfo.groovy b/Jenkins_jobs/WorkspaceInfo.groovy index 9baaa96..32797e2 100644 --- a/Jenkins_jobs/WorkspaceInfo.groovy +++ b/Jenkins_jobs/WorkspaceInfo.groovy @@ -68,7 +68,7 @@ node { script: '#!/bin/sh -e\n du -sg ' + workspaceDirectory + '/* | sort -n', returnStdout: true).trim() - println beautify(machineName,workspaceDirectory, workspaceStats,subdirectories) + println beautify(machineName + 'in Gb',workspaceDirectory, workspaceStats,subdirectories) cleanWs() } @@ -154,4 +154,4 @@ def beautify(machineName, workspaceDirectory, workspaceStats, subdirectories) { output += "\n=======================================================================\n\n"; } -parallel clones \ No newline at end of file +parallel clones diff --git a/README.md b/README.md index d3c09b0..a1b5dd2 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The NodeHelper API contains helper functions to query basic machine stats in rea ### Space Monitoring (WorkspaceInfo.groovy) Iterates over online nodes on Jenkins and prints the contents of the workspace directory along with the space they occupy -* The computers it iterates over can be limited by parameter input ```projectLabel``` +* The computers it iterates over can be limited by input parameter, ```projectLabel``` * As of now, it only works for linux, aix, and mac ## How-to From 01459bd9d0ee918e0e975702ca88902db549df2d Mon Sep 17 00:00:00 2001 From: Shubham Verma Date: Thu, 29 Mar 2018 11:26:29 -0400 Subject: [PATCH 5/5] Added timeout parameter and default project label --- Jenkins_jobs/WorkspaceInfo.groovy | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Jenkins_jobs/WorkspaceInfo.groovy b/Jenkins_jobs/WorkspaceInfo.groovy index 32797e2..51172ee 100644 --- a/Jenkins_jobs/WorkspaceInfo.groovy +++ b/Jenkins_jobs/WorkspaceInfo.groovy @@ -13,7 +13,12 @@ node { stage('Print_Space_Monitoring_Data') { NodeHelper nodeHelper = new NodeHelper(); - String projectLabel = params.projectLabel; + String projectLabel; + if (params.projectLabel.length() > 1) { + projectLabel = params.projectLabel; + } else { + projectLabel = "all"; + } def currentInstance = Jenkins.getInstance(); Computer[] computers; @@ -28,7 +33,8 @@ node { if (computer.isOnline() && computer.getName() != "") { String kernelName = nodeHelper.getOsKernelInfo(computer.getName()).get(0).toLowerCase() - if (nodeHelper.getLabels(computer.getName()).contains(projectLabel)) { + if (projectLabel.equals("all") + || nodeHelper.getLabels(computer.getName()).contains(projectLabel)) { String workspaceDirectory = nodeHelper.getHomeDirectoryPath(machineName); @@ -154,4 +160,8 @@ def beautify(machineName, workspaceDirectory, workspaceStats, subdirectories) { output += "\n=======================================================================\n\n"; } -parallel clones + +timeout(time: params.timeout, unit: 'HOURS') { + parallel clones +} +