Enable monitoring to work with multiple SDP instances on a single hos…

scripts/monitor_metrics.py

@@ -110,6 +110,7 @@ def add_parse_args(self, parser, default_log_file=None, default_verbosity=None):
         parser.add_argument('-L', '--log', default=default_log_file, help="Default: " + default_log_file)
         parser.add_argument('-i', '--sdp-instance', help="SDP instance")
         parser.add_argument('-m', '--metrics-root', default=metrics_root, help="Metrics directory to use. Default: " + metrics_root)
+        parser.add_argument('-o', '--metrics-file', default=metrics_file, help="Metrics file to use. Default: " + metrics_file)
         parser.add_argument('-v', '--verbosity',
                             nargs='?',
                             const="INFO",
@@ -312,7 +313,7 @@ def formatMetrics(self, metrics):
         return lines
 
     def writeMetrics(self, lines):
-        fname = os.path.join(self.options.metrics_root, metrics_file)
+        fname = os.path.join(self.options.metrics_root, self.options.metrics_file)
         self.logger.debug("Writing to metrics file: %s", fname)
         self.logger.debug("Metrics: %s\n", "\n".join(lines))
         tmpfname = fname + ".tmp"

scripts/monitor_metrics.sh

@@ -135,6 +135,9 @@ if [[ $UseSDP -eq 1 ]]; then
         exit 1
     fi
 
+    # Since metrics root is shared, adjust tmp_info_data when using SDP:
+    tmp_info_data="$metrics_root/tmp_info-${SDP_INSTANCE}.dat"
+
     # Load SDP controlled shell environment.
     # shellcheck disable=SC1091
     source /p4/common/bin/p4_vars "$SDP_INSTANCE" ||\
@@ -217,7 +220,11 @@ monitor_license () {
     # Note that sometimes you only get supportExpires - we calculate licenseTimeRemaining in that case
     fname="$metrics_root/p4_license${sdpinst_suffix}-${SERVER_ID}.prom"
     tmpfname="$fname.$$"
-    tmp_license_data="$metrics_root/tmp_license"
+    if [[ $UseSDP -eq 1 ]]; then
+        tmp_license_data="$metrics_root/tmp_license-${SDP_INSTANCE}"
+    else
+        tmp_license_data="$metrics_root/tmp_license"
+    fi
     # Don't update if there is no license for this server, e.g. a replica
     no_license=$(grep -c "Server license: none" "$tmp_info_data")
     # Update every 60 mins
@@ -292,7 +299,11 @@ monitor_filesys () {
     #   filesys.P4ROOT.min=250M (default)
     fname="$metrics_root/p4_filesys${sdpinst_suffix}-${SERVER_ID}.prom"
     tmpfname="$fname.$$"
-    tmp_filesys_data="$metrics_root/tmp_filesys"
+    if [[ $UseSDP -eq 1 ]]; then
+        tmp_filesys_data="$metrics_root/tmp_filesys-${SDP_INSTANCE}"
+    else
+        tmp_filesys_data="$metrics_root/tmp_filesys"
+    fi
     # Update every 60 mins
     [[ ! -f "$tmp_filesys_data" || $(find "$tmp_filesys_data" -mmin +60) ]] || return
     configurables="filesys.depot.min filesys.P4ROOT.min filesys.P4JOURNAL.min filesys.P4LOG.min filesys.TEMP.min"
@@ -403,7 +414,11 @@ monitor_processes () {
     # Monitor metrics summarised by cmd or user
     fname="$metrics_root/p4_monitor${sdpinst_suffix}-${SERVER_ID}.prom"
     tmpfname="$fname.$$"
-    monfile="/tmp/mon.out"
+    if [[ $UseSDP -eq 1 ]]; then
+        monfile="/tmp/mon-${SDP_INSTANCE}.out"
+    else
+        monfile="/tmp/mon.out"
+    fi
 
     $p4 monitor show -l > "$monfile" 2> /dev/null
     rm -f "$tmpfname"
@@ -576,7 +591,11 @@ monitor_pull () {
 
     fname="$metrics_root/p4_pull${sdpinst_suffix}-${SERVER_ID}.prom"
     tmpfname="$fname.$$"
-    tmp_pull_queue="$metrics_root/pullq.out"
+    if [[ $UseSDP -eq 1 ]]; then
+        tmp_pull_queue="$metrics_root/pullq-${SDP_INSTANCE}.out"
+    else
+        tmp_pull_queue="$metrics_root/pullq.out"
+    fi
     $p4 pull -l > "$tmp_pull_queue" 2> /dev/null 
     rm -f "$tmpfname"
 
@@ -637,7 +656,11 @@ monitor_pull () {
 # ... currentJournalNumberLEOF 0
 # ... currentJournalSequenceLEOF -1
 
-    tmp_pull_stats="$metrics_root/pull-lj.out"
+    if [[ $UseSDP -eq 1 ]]; then
+        tmp_pull_stats="$metrics_root/pull-lj-${SDP_INSTANCE}.out"
+    else
+        tmp_pull_stats="$metrics_root/pull-lj.out"
+    fi
     $p4 -Ztag pull -lj > "$tmp_pull_stats" 2> /dev/null 
 
     replica_jnl_file=$(grep "replicaJournalCounter " "$tmp_pull_stats" | awk '{print $3}' )

scripts/monitor_wrapper.sh

@@ -22,6 +22,9 @@
 
 # This might also be /hxlogs/metrics or /var/metrics, and can be set via the "-m" parameter to script.
 metrics_root=/p4/metrics
+# This Can be set via the "-m" parameter to script. For SDP installs the default vaule will be changed
+# to locks_<SDP_INSTANCE>.prom
+metrics_file=locks.prom
 
 function msg () { echo -e "$*"; }
 function bail () { msg "\nError: ${1:-Unknown Error}\n"; exit ${2:-1}; }
@@ -37,7 +40,7 @@ function usage
 
    echo "USAGE for monitor_wrapper.sh:
 
-monitor_wrapper.sh [<instance> | -nosdp] [-p <port>] | [-u <user>] | [-m <metrics_dir>]
+monitor_wrapper.sh [<instance> | -nosdp] [-p <port>] | [-u <user>] | [-m <metrics_dir>] [-o <metrics_file>]
 
    or
 
@@ -58,6 +61,7 @@ while [[ $# -gt 0 ]]; do
         (-p) Port=$2; shiftArgs=1;;
         (-u) User=$2; shiftArgs=1;;
         (-m) metrics_root=$2; shiftArgs=1;;
+        (-o) metrics_file=$2; shiftArgs=1;;
         (-nosdp) UseSDP=0;;
         (-*) usage -h "Unknown command line option ($1)." && exit 1;;
         (*) export SDP_INSTANCE=$1;;
@@ -84,6 +88,9 @@ if [[ $UseSDP -eq 1 ]]; then
         echo "You must supply the Perforce SDP instance as a parameter to this script or use flag: -nosdp."
         exit 1
     fi
+    if [ ${metrics_file} = "locks.prom" ]; then
+        metrics_file=locks-${SDP_INSTANCE}.prom
+    fi
     source /p4/common/bin/p4_vars "$SDP_INSTANCE"
 else
     p4port=${Port:-$P4PORT}
@@ -95,7 +102,7 @@ fi
 
 # Adjust to your script location if required
 if [[ $UseSDP -eq 1 ]]; then
-    "$SCRIPT_DIR"/monitor_metrics.py  -i "$SDP_INSTANCE" -m "$metrics_root"
+    "$SCRIPT_DIR"/monitor_metrics.py  -i "$SDP_INSTANCE" -m "$metrics_root" -o "$metrics_file"
 else
     "$SCRIPT_DIR"/monitor_metrics.py -m "$metrics_root" -p "$p4port" -u "$p4user"
 fi