workbench: automation improvements all over the place

Makefile

@@ -57,7 +57,9 @@ cluster-shell-dev-trace: ARGS += --arg 'autoStartCluster' true --arg 'workbenchD
 fixed:                   ARGS += --arg 'autoStartCluster' true
 fixed:                   PROFILE = fixed-alzo
 forge-stress:            PROFILE = forge-stress-alzo
-shell-dev cluster-shell-dev cluster-shell-trace cluster-shell-dev-trace fixed forge-stress: shell
+quick:                   PROFILE = quick-alzo
+quick:                   ARGS += --arg 'workbenchDevMode' true
+shell-dev cluster-shell-dev cluster-shell-trace cluster-shell-dev-trace fixed forge-stress quick: shell
 
 test-smoke: smoke ## Build the 'workbench-smoke-test', same as the Hydra job
 smoke:

bench/locli/src/Cardano/Analysis/Context.hs

@@ -58,6 +58,5 @@ data Metadata
   { tag       :: Text
   , profile   :: Text
   , era       :: Text
-  , timestamp :: UTCTime
   }
   deriving (Generic, Show, FromJSON, ToJSON)

bench/locli/src/Cardano/Analysis/Run.hs

@@ -8,11 +8,9 @@ import Cardano.Prelude
 
 import Control.Monad (fail)
 import Data.Aeson qualified as Aeson
-import Data.Aeson.Types qualified as Aeson
 import Data.Aeson (FromJSON(..), Object, ToJSON(..), withObject, (.:), (.:?))
 import Data.ByteString.Lazy.Char8 qualified as LBS
 import Data.Text qualified as T
-import Data.Time.Clock.POSIX qualified as Time
 
 import Cardano.Analysis.ChainFilter
 import Cardano.Analysis.Context
@@ -55,12 +53,9 @@ instance FromJSON RunPartial where
 
     eraGtor   <- generator       .:? "era"
     eraTop    <- profile_content .:? "era"
-    era <- case eraGtor <> eraTop of
+    era <- case eraGtor <|> eraTop of
       Just x -> pure x
       Nothing -> fail "While parsing run metafile:  missing era specification"
-
-    timestamp <- (meta .: "timestamp" :: Aeson.Parser Integer)
-                  <&> Time.posixSecondsToUTCTime . realToFrac
     --
     let metadata = Metadata{..}
         genesis  = ()

bench/locli/src/Cardano/Util.hs

@@ -1,6 +1,7 @@
 module Cardano.Util
   ( module Cardano.Util
   , module Control.Arrow
+  , module Control.Applicative
   , module Control.Concurrent.Async
   , module Control.Monad.Trans.Except.Extra
   , module Text.Printf
@@ -11,6 +12,7 @@ import Prelude                          (String)
 import Cardano.Prelude
 
 import Control.Arrow                    ((&&&), (***))
+import Control.Applicative              ((<|>))
 import Control.Concurrent.Async         (forConcurrently, forConcurrently_, mapConcurrently, mapConcurrently_)
 import Control.DeepSeq                  qualified as DS
 import Control.Monad.Trans.Except.Extra (firstExceptT, newExceptT)

nix/workbench/genesis.sh

@@ -2,7 +2,7 @@ global_genesis_format_version=March-14-2022
 
 usage_genesis() {
      usage "genesis" "Genesis" <<EOF
-    prepare [--force] CACHEDIR PROFILE-JSON TOPO-DIR OUTDIR
+    prepare-cache-entry [--force] PROFILE-JSON RUN-META-JSON CACHEDIR TOPO-DIR OUTDIR
                      Prepare a genesis cache entry for the specified profile.
                        Cache entry regeneration can be --force 'd
 
@@ -16,7 +16,7 @@ usage_genesis() {
     actually-genesis PROFILE-JSON TOPO-DIR DIR
                      (DEV) Internal procedure to actually generate genesis
 
-    finalise-cache-entry PROFILE-JSON DIR
+    finalise-cache-entry PROFILE-JSON TIMING-JSON-EXPR DIR
                      (DEV) Update a genesis cache entry to the given profile
 EOF
 }
@@ -25,18 +25,18 @@ genesis() {
 local op=${1:-$(usage_genesis)}; shift
 
 case "$op" in
-    prepare )
-        local usage="USAGE:  wb genesis $op [--force] CACHEDIR PROFILE-JSON TOPO-DIR OUTDIR"
+    prepare-cache-entry )
+        local usage="USAGE:  wb genesis $op [--force] PROFILE-JSON CACHEDIR NODE-SPECS OUTDIR"
 
         local regenesis_causes=()
         while test $# -gt 0
         do case "$1" in
                --force ) regenesis_causes+=('has--force');;
                * ) break;; esac; shift; done
 
-        local cachedir=${1:?$usage}
-        local profile_json=${2:?$usage}
-        local topo_dir=${3:?$usage}
+        local profile_json=${1:?$usage}
+        local cachedir=${2:?$usage}
+        local node_specs=${3:?$usage}
         local outdir=${4:?$usage}
 
         local cache_key_input=$(genesis profile-cache-key-input "$profile_json")
@@ -52,7 +52,7 @@ case "$op" in
         if genesis cache-test "$cache_path"
         then cache_hit=t; cache_hit_desc='hit'
         else cache_hit=;  cache_hit_desc='miss'; fi
-        msg "genesis: preparing cache entry $cache_key:  $cache_hit_desc ($cache_path)"
+        msg "genesis | cache:  preparing entry $cache_key:  $cache_hit_desc ($cache_path)"
 
         if   test -z "$cache_hit"
         then regenesis_causes+=('cache-miss')
@@ -68,15 +68,15 @@ case "$op" in
              if profile has-preset "$profile_json"
              then local preset=$(jq .preset "$profile_json" -r)
                   genesis genesis-from-preset "$preset" "$cache_path"
-             else genesis actually-genesis "$profile_json" "$topo_dir" "$cache_path" "$cache_key_input" "$cache_key"
+             else genesis actually-genesis "$profile_json" "$node_specs" "$cache_path" "$cache_key_input" "$cache_key"
              fi
         fi
 
-        genesis finalise-cache-entry "$profile_json" "$cache_path"
-
         ## Install the cache entry:
         rm -f $outdir ## Must be a symlink for this to succeed
-        ln -s "$(realpath "$cache_path")" "$outdir";;
+        ln -s "$(realpath "$cache_path")" "$outdir"
+        echo >&2
+        ;;
 
     cache-test )
         local usage="USAGE:  wb genesis $op GENESIS-CACHE-DIR"
@@ -141,9 +141,9 @@ case "$op" in
         done;;
 
     actually-genesis )
-        local usage="USAGE:  wb genesis $op PROFILE-JSON TOPO-DIR DIR"
+        local usage="USAGE:  wb genesis $op PROFILE-JSON NODE-SPECS DIR"
         local profile_json=${1:?$usage}
-        local topo_dir=${2:?$usage}
+        local node_specs=${2:?$usage}
         local dir=${3:?$usage}
         local cache_key_input=$4
         local cache_key=$5
@@ -189,16 +189,17 @@ case "$op" in
 
         msg "genesis:  moving keys"
         ## TODO: try to get rid of this step:
-        Massage_the_key_file_layout_to_match_AWS "$profile_json" "$topo_dir" "$dir";;
+        Massage_the_key_file_layout_to_match_AWS "$profile_json" "$node_specs" "$dir";;
 
     derive-from-cache )
-        local usage="USAGE:  wb genesis $op PROFILE-OUT CACHE-ENTRY-DIR OUTDIR"
+        local usage="USAGE:  wb genesis $op PROFILE-OUT TIMING-JSON-EXPR CACHE-ENTRY-DIR OUTDIR"
         local profile=${1:?$usage}
-        local cache_entry=${2:?$usage}
-        local outdir=${3:?$usage}
+        local timing=${2:?$usage}
+        local cache_entry=${3:?$usage}
+        local outdir=${4:?$usage}
 
         msg "genesis | derive-from-cache:  $cache_entry -> $outdir"
-        ls -l $cache_entry
+        # ls -l $cache_entry
 
         mkdir -p "$outdir"
         ( cd $outdir
@@ -216,37 +217,29 @@ case "$op" in
           ln -s $cache_entry/stake-delegator-keys
           ln -s $cache_entry/utxo-keys
           ## JSON
-          cp -v $cache_entry/genesis*.json .
+          cp    $cache_entry/genesis*.json .
           chmod u+w          genesis*.json
         )
-        genesis finalise-cache-entry $profile/profile.json $outdir
-
-        ls -l $outdir/node-keys
+        genesis finalise-cache-entry $profile/profile.json "$timing" $outdir
+        # ls -l $outdir/node-keys
         ;;
 
     finalise-cache-entry )
-        local usage="USAGE:  wb genesis $op PROFILE-JSON DIR"
+        local usage="USAGE:  wb genesis $op PROFILE-JSON TIMING-JSON-EXPR DIR"
         local profile_json=${1:?$usage}
-        local dir=${2:?$usage}
+        local timing=${2:?$usage}
+        local dir=${3:?$usage}
 
         if profile has-preset "$profile_json"
         then return; fi
 
         ## Decide start time:
-        local future_offset=$(jq .derived.genesis_future_offset "$profile_json" --raw-output)
-        local system_start_epoch=$(date '+%s' --date="now + $future_offset")
-        local system_start_human=$(date --date=@$system_start_epoch --utc +"%Y-%m-%dT%H:%M:%SZ")
-        local start_time=$(date --date=@$system_start_epoch --utc --iso-8601=s |
-                           cut -c-19)
-
-        msg "genesis:  future offset $future_offset"
-        genesis_byron "$system_start_epoch" "$dir"
+        genesis_byron "$(jq '.start' -r <<<$timing)" "$dir"
 
-        jq ' $prof[0] as $p
-           | . * ($p.genesis.shelley // {})
-           | . * { systemStart: $start_time }
-           ' --slurpfile prof       "$profile_json"  \
-             --arg       start_time "${start_time}Z" \
+        jq ' . * ($prof[0].genesis.shelley // {})
+           | . * { systemStart: $timing.systemStart }
+           ' --slurpfile prof        "$profile_json" \
+             --argjson   timing      "$timing"       \
                "$dir"/genesis-shelley.json |
         sponge "$dir"/genesis-shelley.json
 
@@ -262,13 +255,13 @@ case "$op" in
 __KEY_ROOT=
 Massage_the_key_file_layout_to_match_AWS() {
     local profile_json=${1:?$usage}
-    local topo_dir=${2:?$usage}
+    local node_specs=${2:?$usage}
     local dir=${3:?$usage}
     local ids
 
     set -euo pipefail
 
-    local pool_density_map=$(topology density-map "$profile_json" "$topo_dir")
+    local pool_density_map=$(topology density-map "$profile_json" "$node_specs")
     msg "genesis: pool density map:  $pool_density_map"
 
     __KEY_ROOT=$dir

nix/workbench/lib-cabal.sh

@@ -10,7 +10,7 @@ function workbench-prebuild-executables()
 
     echo "workbench:  prebuilding executables (because of useCabalRun)"
     unset NIX_ENFORCE_PURITY
-    for exe in cardano-node cardano-cli cardano-topology
+    for exe in cardano-node cardano-cli cardano-topology tx-generator
     do echo "workbench:    $(with_color blue prebuilding) $(with_color red $exe)"
        cabal -v0 build -- exe:$exe 2>&1 >/dev/null |
            { grep -v 'exprType TYPE'; true; } || return 1
@@ -45,5 +45,3 @@ function tx-generator() {
 export WORKBENCH_CABAL_MODE=t
 
 export -f cardano-cli cardano-node cardano-topology locli tx-generator
-
-workbench-prebuild-executables

nix/workbench/lib.jq

@@ -0,0 +1,13 @@
+def color:
+  { red:     "\u001b[31m"
+  , green:   "\u001b[32m"
+  , yellow:  "\u001b[33m"
+  , blue:    "\u001b[34m"
+  , magenta: "\u001b[35m"
+  , cyan:    "\u001b[36m"
+  , white:   "\u001b[37m"
+  , off:     "\u001b[39m"
+  };
+
+def colorly($col; $x):
+  "\(color[$col])\($x)\(color["off"])";

nix/workbench/lib.sh

@@ -68,8 +68,13 @@ msg_ne() {
     echo -ne "workbench:  $*" >&2
 }
 
+progress() {
+    local subsys=$1; shift
+    msg "$(with_color green $subsys):  $(with_color blue $*)"
+}
+
 fail() {
-    msg "$*"
+    msg "$(with_color red $*)"
     exit 1
 }
 

nix/workbench/manifest.sh

@@ -55,20 +55,40 @@ case "${op}" in
         local usage="USAGE: wb manifest $0 MANIFEST-JSON-VALUE"
         local json=${1:?$usage}
 
-        msg "component manifest:"
-        jq '
+        jq 'include "lib";
+
+        def repo_color($repo):
+          { "cardano-node":      "yellow"
+          , "ouroboros-network": "white"
+          , "cardano-ledger":    "red"
+          , "plutus":            "cyan"
+          }[$repo] // "off";
+        def repo_colorly($repo; $hash):
+          colorly(repo_color($repo); $hash[:5]) + $hash[5:];
+        def repo_status($status):
+          { modified:   colorly("red";   $status)
+          , clean:      colorly("cyan"; $status)
+          }[$status];
+        def repo_comment($manifest; $repo):
+          { "cardano-node":
+              " (branch \(colorly("yellow"; $manifest."cardano-node-branch")) - \(repo_status($manifest."cardano-node-status")))"
+          }[$repo] // "";
+
         . as $manifest
         | ($manifest
           | del(."cardano-node-status")
           | del(."cardano-node-branch")
           | to_entries
           | (map(.key | length) | max | . + 1) as $maxlen
-          | map( (if .key == "cardano-node" then " (branch \($manifest."cardano-node-branch"), \($manifest."cardano-node-status"))" else "" end)
-                 as $mod
-               | "   \(.key): \(" " * (.key | $maxlen - length))\(.value)\($mod)\n")
+          | map([ "   \(.key): "
+                , " " * (.key | $maxlen - length)
+                , repo_colorly(.key; .value)
+                , repo_comment($manifest; .key)
+                , "\n"
+                ] | add)
           | add
           )
-        ' --raw-output <<<$json
+        ' --raw-output -L$global_basedir <<<$json
         ;;
 
     * ) usage_manifest;; esac