Merge remote-tracking branch 'upstream/master'

Author: NanuIjaz

.github/ISSUE_TEMPLATE/user-facing-feature.md

@@ -56,4 +56,4 @@ assignees: ''
 
 Acceptance Criteria & User Stories define here (or in a separate file (linked here) for a big feature)
 
-Example - https://github.com/input-output-hk/cardano-node/issues/4453
+Example - https://github.com/intersectmbo/cardano-node/issues/4453

.github/workflows/haskell.yml

@@ -31,7 +31,7 @@ jobs:
 
     env:
       # Modify this value to "invalidate" the cabal cache.
-      CABAL_CACHE_VERSION: "2023-11-20-3"
+      CABAL_CACHE_VERSION: "2023-12-15"
 
     concurrency:
       group: >

.github/workflows/release-ghcr.yaml

@@ -41,7 +41,7 @@ jobs:
               sleep 30;;
           esac
         done
-    
+
   build:
     needs: [wait-for-hydra]
     name: "Upload to ghcr.io"
@@ -64,7 +64,7 @@ jobs:
     # pushed the tagged commit somewhere, and Hydra will have had the
     # change to build the image.
 
-    - name: Uploading input-output-hk/cardano-node
+    - name: Uploading intersectmbo/cardano-node
       run: |
         echo "::group::Downloading from cache"
         nix build --accept-flake-config --print-out-paths --builders "" --max-jobs 0 ${{ steps.flake-metadata.outputs.LOCKED_URL }}#dockerImage/node

CONTRIBUTING.rst

@@ -14,7 +14,7 @@ We maintain a [CODEOWNERS file][CODEOWNERS] which provides information who
 should review a contributing PR.  Note that you might need to get approvals
 from all code owners (even though GitHub doesn't give a way to enforce it).
 
-[CODEOWNERS]: https://github.com/input-output-hk/cardano-node/blob/master//CODEOWNERS
+[CODEOWNERS]: https://github.com/intersectmbo/cardano-node/blob/master//CODEOWNERS
 
 
 Supplementary tooling
@@ -105,12 +105,12 @@ You can quickly spin up a local cluster (on Linux and Darwin), based on any of a
     - ``devops`` is an unloaded profile (no transaction generation) with short slots -- ``0.2`` sec.
     - ..and many more -- which can be either:
         - listed, by ``make ps``
-        - observed at their point of definition: `nix/workbench/profiles/prof1-variants.jq <https://github.com/input-output-hk/cardano-node/tree/master/nix/workbench/profiles/prof1-variants.jq#L333-L526>`_
+        - observed at their point of definition: `nix/workbench/profiles/prof1-variants.jq <https://github.com/intersectmbo/cardano-node/tree/master/nix/workbench/profiles/prof1-variants.jq#L333-L526>`_
 2. Optional: select mode of operation, by optionally providing a suffix:
     - default -- no suffix -- just enter the workbench shell, allowing you to run ``start-cluster`` at any time.  Binaries will be built locally, by ``cabal``.
     - ``autostay`` suffix -- enter the workbench shell, start the cluster, and stay in the shell afterwards.  Binaries will be built locally, by ``cabal``.
     - ``autonix`` suffix -- enter the workbench shell, start the cluster.  All binaries will be provided by the Nix CI.
-    - ..there are other modes, as per `lib.mk <https://github.com/input-output-hk/cardano-node/tree/master/lib.mk>`_
+    - ..there are other modes, as per `lib.mk <https://github.com/intersectmbo/cardano-node/tree/master/lib.mk>`_
 3. Enter the workbench shell for the chosen profile & mode:
     ``make <PROFILE-NAME>`` or ``make <PROFILE-NAME>-<SUFFIX>`` (when there is a suffix).
 4. Optional: start cluster:

Makefile

@@ -14,6 +14,7 @@ PROFILE ?= default-${ERA}
 BACKEND ?= supervisor
 REV     ?= master
 ITER    ?=
+BATCH   ?=
 ARGS    ?=
 CMD     ?=
 RUN     ?=
@@ -73,11 +74,12 @@ ps:                                              ## Plain-text list of profiles
 ##
 ## Profile-based cluster shells (autogenerated targets)
 ##
-PROFILES_BASE             := default plutus plutus-secp-ecdsa plutus-secp-schnorr oldtracing idle tracer-only
+PROFILES_BASE             := default default-p2p plutus plutus-secp-ecdsa plutus-secp-schnorr oldtracing idle tracer-only
 PROFILES_FAST             := fast fast-p2p fast-plutus fast-notracer fast-oldtracing
 PROFILES_CI_TEST          := ci-test ci-test-p2p ci-test-plutus ci-test-notracer ci-test-rtview ci-test-dense10
 PROFILES_CI_BENCH         := ci-bench ci-bench-p2p ci-bench-plutus ci-bench-plutus-secp-ecdsa ci-bench-plutus-secp-schnorr ci-bench-notracer ci-bench-rtview
 PROFILES_TRACE_BENCH      := trace-bench trace-bench-notracer trace-bench-oldtracing trace-bench-rtview
+PROFILES_TRACE_FULL       := trace-full trace-full-rtview
 PROFILES_EPOCHTRANS       := epoch-transition
 PROFILES_PLUTUSCALL       := plutuscall-loop-plain plutuscall-secp-ecdsa-plain plutuscall-secp-schnorr-plain
 PROFILES_PLUTUSCALL       += plutuscall-loop-half plutuscall-secp-ecdsa-half plutuscall-secp-schnorr-half
@@ -99,12 +101,16 @@ PROFILES_NOMAD_CW_QA      := default-nomadcwqa ci-test-nomadcwqa ci-bench-nomadc
 # Cloud version of "default", "ci-test" and "ci-bench" plus value (52+explorer)
 # Not all local profiles are compatible or tested (yet) with a cloud runs
 PROFILES_NOMAD_PERF       := default-nomadperf ci-test-nomadperf ci-bench-nomadperf value-nomadperf oldtracing-nomadperf ci-test-oldtracing-nomadperf ci-bench-oldtracing-nomadperf value-oldtracing-nomadperf
+PROFILES_NOMAD_PERF       += plutus-nomadperf
+PROFILES_NOMAD_PERF_NOP2P := default-nomadperf-nop2p oldtracing-nomadperf-nop2p ci-test-nomadperf-nop2p ci-bench-nomadperf-nop2p
+PROFILES_NOMAD_PERF_NOP2P += value-nomadperf-nop2p value-oldtracing-nomadperf-nop2p plutus-nomadperf-nop2p
 
 LOCAL_PROFILES += $(PROFILES_BASE)
 LOCAL_PROFILES += $(PROFILES_FAST)
 LOCAL_PROFILES += $(PROFILES_CI_TEST)
 LOCAL_PROFILES += $(PROFILES_CI_BENCH)
 LOCAL_PROFILES += $(PROFILES_TRACE_BENCH)
+LOCAL_PROFILES += $(PROFILES_TRACE_FULL)
 LOCAL_PROFILES += $(PROFILES_EPOCHTRANS)
 LOCAL_PROFILES += $(PROFILES_PLUTUSCALL)
 LOCAL_PROFILES += $(PROFILES_MODEL)
@@ -115,6 +121,8 @@ LOCAL_PROFILES += $(PROFILES_FORGE_STRESS_RTS)
 LOCAL_PROFILES += $(PROFILES_CHAINSYNC)
 LOCAL_PROFILES += $(PROFILES_VENDOR)
 CLOUD_PROFILES += $(PROFILES_NOMAD_CW_QA) $(PROFILES_NOMAD_PERF)
+CLOUD_PROFILES += $(PROFILES_NOMAD_PERF_NOP2P)
+
 
 ## Note:  to enable a shell for a profile, just add its name (one of names from 'make ps') to SHELL_PROFILES
 

README.rst

@@ -1,15 +1,15 @@
 .. raw:: html
 
   <p align="center">
-    <a href="https://github.com/input-output-hk/cardano-node/releases"><img src="https://img.shields.io/github/release-pre/input-output-hk/cardano-node.svg?style=for-the-badge" /></a>
+    <a href="https://github.com/intersectmbo/cardano-node/releases"><img src="https://img.shields.io/github/release-pre/intersectmbo/cardano-node.svg?style=for-the-badge" /></a>
   </p>
 
   <table align="center">
     <tr><td colspan="2" align="center">GitHub Actions</td></tr>
     <tr>
       <td>
-        <a href="https://github.com/input-output-hk/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (master)" src="https://img.shields.io/github/workflow/status/input-output-hk/cardano-node/Haskell%20CI/master" /></a>
-        <a href="https://github.com/input-output-hk/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (branch)" src="https://img.shields.io/github/workflow/status/input-output-hk/cardano-node/Haskell%20CI/nightly?label=nightly" /></a>
+        <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (master)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=master" /></a>
+        <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (branch)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=nightly" /></a>
       </td>
     </tr>
   </table>
@@ -20,7 +20,7 @@
 Overview of the ``cardano-node`` repository
 *************************
 
-Integration of the `ledger <https://github.com/input-output-hk/cardano-ledger-specs>`_, `consensus <https://github.com/input-output-hk/ouroboros-network/tree/master/ouroboros-consensus>`_,
+Integration of the `ledger <https://github.com/input-output-hk/cardano-ledger-specs>`_, `consensus <https://github.com/input-output-hk/ouroboros-consensus>`_,
 `networking <https://github.com/input-output-hk/ouroboros-network/tree/master/ouroboros-network>`_ and
 `node shell <https://github.com/input-output-hk/cardano-shell>`_ repositories.
 
@@ -52,7 +52,7 @@ Documentation for building the node can be found `here <https://docs.cardano.org
 Executables
 ===========
 
-You can download the hydra binaries of ``cardano-node`` and ``cardano-cli`` from the `release notes <https://github.com/input-output-hk/cardano-node/releases>`_
+You can download the hydra binaries of ``cardano-node`` and ``cardano-cli`` from the `release notes <https://github.com/intersectmbo/cardano-node/releases>`_
 
 
 Running the node on Windows
@@ -135,7 +135,7 @@ Configuration
 ====
 
 The ``--config`` flag points to a ``.yaml`` (or a structurally equivalent ``.json``) file that is responsible to configuring the logging & other important settings for the node. E.g. see the Byron mainnet configuration in this
-`configuration.yaml <https://github.com/input-output-hk/cardano-node/blob/master/configuration/defaults/byron-mainnet/configuration.yaml>`_.
+`configuration.yaml <https://github.com/intersectmbo/cardano-node/blob/master/configuration/defaults/byron-mainnet/configuration.yaml>`_.
 
 Some of the more important settings are as follows:
 
@@ -437,13 +437,13 @@ To help you get started, see:
 
 - `Native tokens exercises <https://github.com/input-output-hk/cardano-node-wiki/wiki/03-exercises>`_
 
-To start, please ensure that you are familiar with setting up and operating the `Cardano node <https://github.com/input-output-hk/cardano-node>`_. Alternatively, see instructions on how to `start your node <https://github.com/input-output-hk/cardano-node-wiki/wiki/2_start_your_nodes>`_ to submit the commands. You will not need to set up and start a full block producing node ('stake pool'), just a much simpler relay node. This node will need to connect to a Cardano network that is capable of processing native tokens (e.g., the native token pre-production environment (PPE), or the Cardano mainnet).
+To start, please ensure that you are familiar with setting up and operating the `Cardano node <https://github.com/intersectmbo/cardano-node>`_. Alternatively, see instructions on how to `start your node <https://github.com/input-output-hk/cardano-node-wiki/wiki/2_start_your_nodes>`_ to submit the commands. You will not need to set up and start a full block producing node ('stake pool'), just a much simpler relay node. This node will need to connect to a Cardano network that is capable of processing native tokens (e.g., the native token pre-production environment (PPE), or the Cardano mainnet).
 
 ****
 API Documentation
 ****
 
-The API documentation is published `here <https://input-output-hk.github.io/cardano-node/>`_.
+The API documentation is published `here <https://cardano-node.cardano.intersectmbo.org/>`_.
 
 The documentation is built with each push, but is only published from ``master`` branch.  In order to
 test if the documentation is working, build the documentation locally with ``cabal haddock-project --local --output=./haddocks`` and

bench/locli/src/Cardano/Unlog/LogObject.hs

@@ -31,6 +31,7 @@ module Cardano.Unlog.LogObject
 where
 
 import           Cardano.Prelude hiding (Text, show)
+import           GHC.Conc (numCapabilities)
 import           Prelude (id, show, unzip3)
 
 import qualified Data.Aeson as AE
@@ -95,7 +96,7 @@ data HostLogs a
     , hlRawFirstAt     :: Maybe UTCTime
     , hlRawLastAt      :: Maybe UTCTime
     }
-  deriving (Generic)
+  deriving (Generic, NFData)
 
 deriving instance FromJSON a => FromJSON (HostLogs a)
 deriving instance   ToJSON a =>   ToJSON (HostLogs a)
@@ -117,11 +118,24 @@ rlLogs = fmap hlLogs . Map.elems . rlHostLogs
 
 runLiftLogObjects :: RunLogs () -> Bool -> Maybe [LOAnyType]
                   -> ExceptT LText.Text IO (RunLogs [LogObject])
-runLiftLogObjects rl@RunLogs{..} okDErr loAnyLimit = liftIO $ do
-  forConcurrently (Map.toList rlHostLogs)
-    (uncurry readHostLogs)
-    <&> \kvs -> rl { rlHostLogs = Map.fromList kvs }
+runLiftLogObjects rl@RunLogs{..} okDErr loAnyLimit = liftIO $
+ go Map.empty 0 simultaneousReads
  where
+   go (force -> !acc) batchBase = \case
+     []    -> pure $ rl{ rlHostLogs = acc }
+     c:cs  -> do
+       let batchBase' = batchBase + length c
+       when (length c > 1) $
+         progress "logs" (Q $ printf "processing batch %d - %d" batchBase (batchBase' - 1))
+       hlsMap <- readHostLogChunk c
+       go (acc `Map.union` hlsMap) batchBase' cs
+
+   simultaneousReads = chunksOf numCapabilities (Map.toList rlHostLogs)
+
+   readHostLogChunk :: [(Host, HostLogs ())] -> IO (Map Host (HostLogs [LogObject]))
+   readHostLogChunk hls =
+     Map.fromList <$> forConcurrently hls (uncurry readHostLogs)
+
    readHostLogs :: Host -> HostLogs () -> IO (Host, HostLogs [LogObject])
    readHostLogs h hl@HostLogs{..} =
      readLogObjectStream (unJsonLogfile $ fst hlLogs) okDErr loAnyLimit
@@ -523,14 +537,14 @@ parsePartialResourceStates =
       rGcsMajor   <- o .:  "GcsMajor"
       rGcsMinor   <- o .:  "GcsMinor"
       rAlloc      <- o .:  "Alloc"
-      rLive       <- o .:? "Heap"       <&> fromMaybe 0
-      rHeap       <- o .:  "Live"
+      rLive       <- o .:  "Live"
+      rHeap       <- o .:? "Heap"       .!= 0
       rRSS        <- o .:  "RSS"
       rCentiBlkIO <- o .:  "CentiBlkIO"
-      rNetRd      <- o .:? "NetRd"      <&> fromMaybe 0
-      rNetWr      <- o .:? "NetWr"      <&> fromMaybe 0
-      rFsRd       <- o .:? "FsRd"       <&> fromMaybe 0
-      rFsWr       <- o .:? "FsWr"       <&> fromMaybe 0
+      rNetRd      <- o .:? "NetRd"      .!= 0
+      rNetWr      <- o .:? "NetWr"      .!= 0
+      rFsRd       <- o .:? "FsRd"       .!= 0
+      rFsWr       <- o .:? "FsWr"       .!= 0
       rThreads    <- o .:  "Threads"
       pure Resources{..}
 

bench/locli/src/Cardano/Util.hs

@@ -44,7 +44,7 @@ import GHC.Utils.Misc                   as Util
                                  hiding (fst3, snd3, third3, uncurry3)
 #endif
 
-import Data.Aeson                       (FromJSON (..), ToJSON (..), Object, Value (..), (.:), (.:?), withObject, object)
+import Data.Aeson                       (FromJSON (..), ToJSON (..), Object, Value (..), (.:), (.:?), (.!=), withObject, object)
 import Data.Aeson                       qualified as AE
 import Control.Arrow                    ((&&&), (***))
 import Control.Applicative              ((<|>))

bench/plutus-scripts-bench/plutus-scripts-bench.cabal

@@ -73,7 +73,7 @@ library
   -- IOG dependencies
   --------------------------
   build-depends:
-    , cardano-api ^>= 8.35
+    , cardano-api ^>= 8.36.0.1
     , plutus-ledger-api       >=1.0.0
     , plutus-tx               >=1.0.0
     , plutus-tx-plugin        >=1.0.0

bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs

@@ -108,13 +108,12 @@ measureBSCosts era = map (metadataSize era . Just . bsMetadata) [0..maxBSSize]
 metadataSize :: forall era . IsShelleyBasedEra era => AsType era -> Maybe TxMetadata -> Int
 metadataSize p m = dummyTxSize p m - dummyTxSize p Nothing
 
-dummyTxSizeInEra :: forall era . IsShelleyBasedEra era => TxMetadataInEra era -> Int
-dummyTxSizeInEra metadata = case createAndValidateTransactionBody (shelleyBasedEra @era) dummyTx of
+dummyTxSizeInEra :: IsShelleyBasedEra era => TxMetadataInEra era -> Int
+dummyTxSizeInEra metadata = case createAndValidateTransactionBody shelleyBasedEra dummyTx of
   Right b -> BS.length $ serialiseToCBOR b
   Left err -> error $ "metaDataSize " ++ show err
  where
-  dummyTx :: TxBodyContent BuildTx era
-  dummyTx = defaultTxBodyContent (cardanoEra @era)
+  dummyTx = defaultTxBodyContent shelleyBasedEra
     & setTxIns
       [ ( TxIn "dbaff4e270cfb55612d9e2ac4658a27c79da4a5271c6f90853042d1403733810" (TxIx 0)
         , BuildTxWith $ KeyWitness KeyWitnessForSpending