Merge #3812

3812: Benchmarking: forgestress r=deepfire a=deepfire 1. Log analysis has been rewritten as a composition of orthogonal chunks -- micro-ops, with clear inputs and outputs 2. The workbench log output has been clarified 3. Log analysis filters are now specified by profiles 4. Profiles now provide run-time expectations to the workbench -- informed by shutdown slot and tx workload 5. The workbench now enforces the above run-time expectations on the `fixed` scenario profiles (time-bounded) 6. Log analysis can now be requested for runs 7. Runs can be iterated (only makes sense for `fixed` scenario profiles) 8. Genesis handling has been properly moved 9. The profile structure has been cleaned up a lot 10. More intermediate computed values are now specified as part of profile output 11. Log analysis now emits separate components that are easier to use for reports Co-authored-by: Kosyrev Serge <[email protected]> Co-authored-by: MarcFontaine <[email protected]>
IntersectMBO · May 10, 2022 · b699ce0 · b699ce0
2 parents f8550f7 + eac697d
commit b699ce0
Show file tree

Hide file tree

Showing 49 changed files with 2,356 additions and 1,652 deletions.
diff --git a/Makefile b/Makefile
@@ -54,9 +54,13 @@ cluster-shell:           ARGS += --arg 'autoStartCluster' true ## Enter Nix shel
 cluster-shell-dev:       ARGS += --arg 'autoStartCluster' true --arg 'workbenchDevMode' true ## Enter Nix shell, dev mode, and start workbench cluster
 cluster-shell-trace:     ARGS += --arg 'autoStartCluster' true --argstr 'autoStartClusterArgs' '--trace --trace-workbench' ## Enter Nix shell, start workbench cluster, with shell tracing
 cluster-shell-dev-trace: ARGS += --arg 'autoStartCluster' true --arg 'workbenchDevMode' true --argstr 'autoStartClusterArgs' '--trace --trace-workbench' ## Enter Nix shell, dev mode, start workbench cluster, with shell tracing
+fixed:                   PROFILE = fixed-${ERA}
 fixed:                   ARGS += --arg 'autoStartCluster' true
-fixed:                   PROFILE = fixed-alzo
-shell-dev cluster-shell-dev cluster-shell-trace cluster-shell-dev-trace fixed: shell
+forge-stress:            PROFILE = forge-stress-${ERA}
+forge-stress:            ARGS += --arg 'workbenchDevMode' true
+quick:                   PROFILE = quick-${ERA}
+quick:                   ARGS += --arg 'workbenchDevMode' true
+shell-dev cluster-shell-dev cluster-shell-trace cluster-shell-dev-trace fixed forge-stress quick: shell
 
 test-smoke: smoke ## Build the 'workbench-smoke-test', same as the Hydra job
 smoke:
@@ -66,12 +70,14 @@ smoke-analysis:
 	nix build -f 'default.nix' 'workbench-smoke-analysis' --out-link result-smoke-analysis --cores 0 --show-trace
 ci-analysis:
 	nix build -f 'default.nix' 'workbench-ci-analysis'    --out-link result-ci-analysis    --cores 0 --show-trace
+
 list-profiles: ## List workbench profiles
 	nix build .#workbench.profile-names-json --json | jq '.[0].outputs.out' -r | xargs jq .
 show-profile: ## NAME=profile-name
 	@test -n "${NAME}" || { echo 'HELP:  to specify profile to show, add NAME=profle-name' && exit 1; }
 	nix build .#all-profiles-json --json --option substitute false | jq '.[0].outputs.out' -r | xargs jq ".\"${NAME}\" | if . == null then error(\"\n###\n### Error:  unknown profile: ${NAME}  Please consult:  make list-profiles\n###\") else . end"
 ps: list-profiles
+
 bump-cardano-node-workbench: ## Update the cardano-node-workbench flake input
 	nix flake lock --update-input cardano-node-workbench
 bump-node-measured: ## Update the node-measured flake input

diff --git a/bench/locli/app/locli.hs b/bench/locli/app/locli.hs
@@ -5,9 +5,8 @@ import           Cardano.Prelude hiding (option)
 import           Control.Monad.Trans.Except.Exit (orDie)
 import qualified Options.Applicative as Opt
 
-import           Cardano.Analysis.TopHandler
-import           Cardano.Unlog.Parsers (opts, pref)
-import           Cardano.Unlog.Run (renderCommandError, runCommand)
+import           Cardano.TopHandler
+import           Cardano.Command (opts, pref, renderCommandError, runCommand)
 
 
 main :: IO ()

diff --git a/bench/locli/locli.cabal b/bench/locli/locli.cabal
@@ -17,23 +17,24 @@ library
   exposed-modules:     Data.Accum
                        Data.Distribution
 
-                       Cardano.Analysis.TopHandler
+                       Cardano.Command
+                       Cardano.TopHandler
+                       Cardano.Util
+
                        Cardano.Analysis.Run
 
                        Cardano.Analysis.API
                        Cardano.Analysis.BlockProp
                        Cardano.Analysis.Chain
                        Cardano.Analysis.ChainFilter
-                       Cardano.Analysis.Driver
+                       Cardano.Analysis.Context
+                       Cardano.Analysis.Ground
                        Cardano.Analysis.MachTimeline
                        Cardano.Analysis.Version
 
-                       Cardano.Unlog.Commands
                        Cardano.Unlog.LogObject
-                       Cardano.Unlog.Parsers
                        Cardano.Unlog.Render
                        Cardano.Unlog.Resources
-                       Cardano.Unlog.Run
 
   other-modules:       Paths_locli
 

diff --git a/bench/locli/src/Cardano/Analysis/API.hs b/bench/locli/src/Cardano/Analysis/API.hs
@@ -8,19 +8,16 @@ import Prelude                  ((!!), error)
 import Cardano.Prelude          hiding (head)
 
 import Data.Aeson               (ToJSON(..), FromJSON(..))
-import Data.List.Split          (splitOn)
 import Data.Text   qualified as T
 import Data.Text.Short          (toText)
 import Data.Time.Clock          (NominalDiffTime)
 import Data.Time                (UTCTime)
 import Text.Printf              (PrintfArg, printf)
 
-import Cardano.Slotting.Slot    (EpochNo(..), SlotNo(..))
-import Ouroboros.Network.Block  (BlockNo(..))
-
 import Cardano.Analysis.Chain
 import Cardano.Analysis.ChainFilter
-import Cardano.Analysis.Run
+import Cardano.Analysis.Context
+import Cardano.Analysis.Ground
 import Cardano.Analysis.Version
 import Cardano.Logging.Resources.Types
 import Cardano.Unlog.LogObject  hiding (Text)
@@ -36,7 +33,8 @@ import Data.Distribution
 data BlockPropagation
   = BlockPropagation
     { bpVersion             :: !Version
-    , bpSlotRange           :: !(SlotNo, SlotNo) -- ^ Analysis range, inclusive.
+    , bpDomainSlots         :: !(DataDomain SlotNo)
+    , bpDomainBlocks        :: !(DataDomain BlockNo)
     , bpForgerChecks        :: !(Distribution Float NominalDiffTime)
     , bpForgerLeads         :: !(Distribution Float NominalDiffTime)
     , bpForgerForges        :: !(Distribution Float NominalDiffTime)
@@ -51,7 +49,6 @@ data BlockPropagation
     , bpPeerSends           :: !(Distribution Float NominalDiffTime)
     , bpPropagation         :: ![(Float, Distribution Float NominalDiffTime)]
     , bpSizes               :: !(Distribution Float Int)
-    , bpChainBlockEvents    :: [BlockEvents]
     }
   deriving (Generic, FromJSON, ToJSON, Show)
 
@@ -136,21 +133,32 @@ data BPErrorKind
   | BPEFork                  !Hash
   deriving (FromJSON, Generic, NFData, Show, ToJSON)
 
-data DataDomain
+data DataDomain a
   = DataDomain
-    { ddRawSlotFirst      :: !SlotNo
-    , ddRawSlotLast       :: !SlotNo
-    , ddAnalysisSlotFirst :: !SlotNo
-    , ddAnalysisSlotLast  :: !SlotNo
+    { ddRawFirst      :: !a
+    , ddRawLast       :: !a
+    , ddFilteredFirst :: !a
+    , ddFilteredLast  :: !a
+    , ddRawCount      :: Int
+    , ddFilteredCount :: Int
     }
-  deriving (Generic, Show, ToJSON)
+  deriving (Generic, Show, ToJSON, FromJSON)
+  deriving anyclass NFData
 -- Perhaps:  Plutus.V1.Ledger.Slot.SlotRange = Interval Slot
 
+mkDataDomainInj :: a -> a -> (a -> Int) -> DataDomain a
+mkDataDomainInj f l measure = DataDomain f l f l delta delta
+  where delta = measure l - measure f
+
+mkDataDomain :: a -> a -> a -> a -> (a -> Int) -> DataDomain a
+mkDataDomain f l f' l' measure =
+  DataDomain f l f' l' (measure l - measure f) (measure l' - measure f')
+
 -- | The top-level representation of the machine timeline analysis results.
 data MachTimeline
   = MachTimeline
     { sVersion           :: !Version
-    , sSlotRange         :: (SlotNo, SlotNo) -- ^ Analysis range, inclusive.
+    , sDomain            :: !(DataDomain SlotNo)
     , sMaxChecks         :: !Word64
     , sSlotMisses        :: ![Word64]
     , sSpanLensCPU85     :: ![Int]
@@ -171,7 +179,7 @@ data MachTimeline
     , sSpanLensCPU85RwdDistrib  :: !(Distribution Float Int)
     , sResourceDistribs  :: !(Resources (Distribution Float Word64))
     }
-  deriving (Generic, Show, ToJSON)
+  deriving (Generic, NFData, Show, ToJSON)
 
 data SlotStats
   = SlotStats
@@ -201,6 +209,7 @@ data SlotStats
     , slResources    :: !(Resources (Maybe Word64))
     }
   deriving (Generic, Show, ToJSON)
+  deriving anyclass NFData
 
 --
 -- * Key properties
@@ -253,6 +262,18 @@ testSlotStats g SlotStats{..} = \case
 --
 -- * Timeline rendering instances
 --
+bpFieldsForger :: DField a -> Bool
+bpFieldsForger Field{fId} = elem fId
+  [ "fChecked", "fLeading", "fForged", "fAdopted", "fAnnounced", "fSendStart" ]
+
+bpFieldsPeers :: DField a -> Bool
+bpFieldsPeers Field{fId} = elem fId
+  [ "noticedVal", "requestedVal", "fetchedVal", "pAdoptedVal", "pAnnouncedVal", "pSendStartVal" ]
+
+bpFieldsPropagation :: DField a -> Bool
+bpFieldsPropagation Field{fHead2} = elem fHead2
+  [ "0.50", "0.80", "0.90", "0.92", "0.94", "0.96", "0.98", "1.00" ]
+
 instance RenderDistributions BlockPropagation where
   rdFields _ =
     --  Width LeftPad
@@ -355,6 +376,36 @@ instance RenderTimeline BlockEvents where
 
   rtCommentary BlockEvents{..} = ("    " <>) . show <$> beErrors
 
+mtFieldsReport :: DField a -> Bool
+mtFieldsReport Field{fId} = elem fId
+  [ "CPU", "GC", "MUT", "RSS", "Heap", "Live", "Alloc" ]
+
+instance RenderDistributions MachTimeline where
+  rdFields _ =
+    --  Width LeftPad
+    [ Field 4 0 "missR"    "Miss"  "ratio"  $ DFloat   sMissDistrib
+    , Field 5 0 "CheckΔ"   (d!!0)  "Check"  $ DDeltaT  sSpanCheckDistrib
+    , Field 5 0 "LeadΔ"    (d!!1)  "Lead"   $ DDeltaT  sSpanLeadDistrib
+    , Field 5 0 "ForgeΔ"   (d!!2)  "Forge"  $ DDeltaT  sSpanForgeDistrib
+    , Field 4 0 "BlkGap"   "Block" "gap"    $ DWord64  sBlocklessDistrib
+    , Field 5 0 "chDensity" "Dens" "ity"    $ DFloat   sDensityDistrib
+    , Field 3 0 "CPU"      "CPU"   "%"      $ DWord64 (rCentiCpu . sResourceDistribs)
+    , Field 3 0 "GC"       "GC"    "%"      $ DWord64 (rCentiGC . sResourceDistribs)
+    , Field 3 0 "MUT"      "MUT"    "%"     $ DWord64 (fmap (min 999) . rCentiMut . sResourceDistribs)
+    , Field 3 0 "GcMaj"    "GC "   "Maj"    $ DWord64 (rGcsMajor . sResourceDistribs)
+    , Field 3 0 "GcMin"    "flt "  "Min"    $ DWord64 (rGcsMinor . sResourceDistribs)
+    , Field 5 0 "RSS"      (m!!0)  "RSS"    $ DWord64 (rRSS . sResourceDistribs)
+    , Field 5 0 "Heap"     (m!!1)  "Heap"   $ DWord64 (rHeap . sResourceDistribs)
+    , Field 5 0 "Live"     (m!!2)  "Live"   $ DWord64 (rLive . sResourceDistribs)
+    , Field 5 0 "Allocd"   "Alloc" "MB"     $ DWord64 (rAlloc . sResourceDistribs)
+    , Field 5 0 "CPU85%LensAll"  (c!!0) "All"   $ DInt     sSpanLensCPU85Distrib
+    , Field 5 0 "CPU85%LensEBnd" (c!!1) "EBnd"  $ DInt     sSpanLensCPU85EBndDistrib
+    ]
+   where
+     d = nChunksEachOf  3 6 "---- Δt ----"
+     m = nChunksEachOf  3 6 "Memory usage, MB"
+     c = nChunksEachOf  2 6 "CPU85% spans"
+
 instance RenderTimeline SlotStats where
   rtFields _ =
     --  Width LeftPad
@@ -399,9 +450,9 @@ instance RenderTimeline SlotStats where
                 <*> (fromIntegral . max 1 . (1024 *) <$> rCentiMut slResources)))
     , Field 7 0 "mempoolTxs"   "Mempool" "txs"   $ IWord64 slMempoolTxs
     , Field 9 0 "utxoEntries"  "UTxO"  "entries" $ IWord64 slUtxoSize
-    , Field 10 0 "absSlotTime" "Absolute" "slot time" $ IText
-      (\SlotStats{..}->
-         T.pack $ " " `splitOn` show slStart !! 1)
+    -- , Field 10 0 "absSlotTime" "Absolute" "slot time" $ IText
+    --   (\SlotStats{..}->
+    --      T.pack $ " " `splitOn` show slStart !! 1)
     ]
    where
      t w = nChunksEachOf 4 (w + 1) "mempool tx"