[main] Update 2024-06-25.23 (#188)

Reference commit: c6ea53392f

Co-authored-by: Canton <[email protected]>

community/admin-api/src/main/protobuf/com/digitalasset/canton/admin/participant/v30/inspection_service.proto

@@ -18,8 +18,6 @@ service InspectionService {
   // Lookup the domain where a contract is currently active.
   // Supports querying many contracts at once.
   rpc LookupContractDomain(LookupContractDomain.Request) returns (LookupContractDomain.Response);
-  // Lookup the domain that the transaction was committed over. Can fail with NOT_FOUND if no domain was found.
-  rpc LookupTransactionDomain(LookupTransactionDomain.Request) returns (LookupTransactionDomain.Response);
   // Look up the ledger offset corresponding to the timestamp, specifically the largest offset such that no later
   // offset corresponds to a later timestamp than the specified one.
   rpc LookupOffsetByTime(LookupOffsetByTime.Request) returns (LookupOffsetByTime.Response);

community/admin-api/src/main/protobuf/com/digitalasset/canton/admin/participant/v30/traffic_control_service.proto

@@ -31,8 +31,10 @@ message TrafficState {
   int64 extra_traffic_consumed = 2;
   // Amount of base traffic remaining
   int64 base_traffic_remainder = 3;
+  // Cost deducted at `timestamp`, only present when traffic was consumed at `timestamp`, otherwise is set to 0
+  uint64 last_consumed_cost = 4;
   // Timestamp at which the state is valid
-  int64 timestamp = 4;
+  int64 timestamp = 5;
   // Optional serial of the balance update that updated the extra traffic limit
-  google.protobuf.UInt32Value serial = 5;
+  google.protobuf.UInt32Value serial = 6;
 }

community/admin-api/src/main/protobuf/com/digitalasset/canton/version/ProtocolVersionAnnotation.scala

@@ -10,6 +10,9 @@ object ProtocolVersionAnnotation {
 
   /** Marker for stable protocol versions */
   sealed trait Stable extends Status
+
+  /** Marker for beta protocol versions */
+  sealed trait Beta extends Status
 }
 
 /** Marker trait for Protobuf messages generated by scalapb

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala

@@ -4,6 +4,14 @@
 package com.digitalasset.canton.admin.api.client.commands
 
 import cats.syntax.either.*
+import cats.syntax.traverse.*
+import com.daml.ledger.api.v2.admin.command_inspection_service.CommandInspectionServiceGrpc.CommandInspectionServiceStub
+import com.daml.ledger.api.v2.admin.command_inspection_service.{
+  CommandInspectionServiceGrpc,
+  CommandState,
+  GetCommandStatusRequest,
+  GetCommandStatusResponse,
+}
 import com.daml.ledger.api.v2.admin.identity_provider_config_service.IdentityProviderConfigServiceGrpc.IdentityProviderConfigServiceStub
 import com.daml.ledger.api.v2.admin.identity_provider_config_service.*
 import com.daml.ledger.api.v2.admin.metering_report_service.MeteringReportServiceGrpc.MeteringReportServiceStub
@@ -135,6 +143,7 @@ import com.digitalasset.canton.ledger.client.services.admin.IdentityProviderConf
 import com.digitalasset.canton.logging.ErrorLoggingContext
 import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting}
 import com.digitalasset.canton.networking.grpc.ForwardingStreamObserver
+import com.digitalasset.canton.platform.apiserver.execution.CommandStatus
 import com.digitalasset.canton.protocol.LfContractId
 import com.digitalasset.canton.serialization.ProtoConverter
 import com.digitalasset.canton.topology.{DomainId, PartyId}
@@ -352,6 +361,34 @@ object LedgerApiCommands {
     }
 
   }
+
+  object CommandInspectionService {
+    abstract class BaseCommand[Req, Resp, Res] extends GrpcAdminCommand[Req, Resp, Res] {
+      override type Svc = CommandInspectionServiceStub
+
+      override def createService(channel: ManagedChannel): CommandInspectionServiceStub =
+        CommandInspectionServiceGrpc.stub(channel)
+    }
+
+    final case class GetCommandStatus(commandIdPrefix: String, state: CommandState, limit: Int)
+        extends BaseCommand[GetCommandStatusRequest, GetCommandStatusResponse, Seq[CommandStatus]] {
+      override def createRequest(): Either[String, GetCommandStatusRequest] = Right(
+        GetCommandStatusRequest(commandIdPrefix = commandIdPrefix, state = state, limit = limit)
+      )
+
+      override def submitRequest(
+          service: CommandInspectionServiceStub,
+          request: GetCommandStatusRequest,
+      ): Future[GetCommandStatusResponse] = service.getCommandStatus(request)
+
+      override def handleResponse(
+          response: GetCommandStatusResponse
+      ): Either[String, Seq[CommandStatus]] = {
+        response.commandStatus.traverse(CommandStatus.fromProto).leftMap(_.message)
+      }
+    }
+  }
+
   object ParticipantPruningService {
     abstract class BaseCommand[Req, Resp, Res] extends GrpcAdminCommand[Req, Resp, Res] {
       override type Svc = ParticipantPruningServiceStub

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/commands/ParticipantAdminCommands.scala

@@ -57,7 +57,7 @@ import com.digitalasset.canton.topology.{DomainId, ParticipantId, PartyId}
 import com.digitalasset.canton.tracing.TraceContext
 import com.digitalasset.canton.util.BinaryFileUtil
 import com.digitalasset.canton.version.ProtocolVersion
-import com.digitalasset.canton.{DomainAlias, LedgerTransactionId, SequencerCounter, config}
+import com.digitalasset.canton.{DomainAlias, SequencerCounter, config}
 import com.google.protobuf.ByteString
 import com.google.protobuf.empty.Empty
 import com.google.protobuf.timestamp.Timestamp
@@ -1057,29 +1057,6 @@ object ParticipantAdminCommands {
 
     }
 
-    final case class LookupTransactionDomain(transactionId: LedgerTransactionId)
-        extends Base[
-          v30.LookupTransactionDomain.Request,
-          v30.LookupTransactionDomain.Response,
-          DomainId,
-        ] {
-      override def createRequest() = Right(v30.LookupTransactionDomain.Request(transactionId))
-
-      override def submitRequest(
-          service: InspectionServiceStub,
-          request: v30.LookupTransactionDomain.Request,
-      ): Future[v30.LookupTransactionDomain.Response] =
-        service.lookupTransactionDomain(request)
-
-      override def handleResponse(
-          response: v30.LookupTransactionDomain.Response
-      ): Either[String, DomainId] =
-        DomainId.fromString(response.domainId)
-
-      override def timeoutType: TimeoutType = DefaultUnboundedTimeout
-
-    }
-
     final case class LookupOffsetByTime(ts: Timestamp)
         extends Base[v30.LookupOffsetByTime.Request, v30.LookupOffsetByTime.Response, String] {
       override def createRequest() = Right(v30.LookupOffsetByTime.Request(Some(ts)))

community/app-base/src/main/scala/com/digitalasset/canton/config/CantonConfig.scala

@@ -70,6 +70,7 @@ import com.digitalasset.canton.participant.ParticipantNodeParameters
 import com.digitalasset.canton.participant.admin.AdminWorkflowConfig
 import com.digitalasset.canton.participant.config.ParticipantInitConfig.ParticipantLedgerApiInitConfig
 import com.digitalasset.canton.participant.config.*
+import com.digitalasset.canton.participant.sync.CommandProgressTrackerConfig
 import com.digitalasset.canton.platform.apiserver.SeedService.Seeding
 import com.digitalasset.canton.platform.apiserver.configuration.{
   EngineLoggingConfig,
@@ -238,6 +239,7 @@ final case class RetentionPeriodDefaults(
   * @param startupParallelism Start up to N nodes in parallel (default is num-threads)
   * @param nonStandardConfig don't fail config validation on non-standard configuration settings
   * @param devVersionSupport If true, allow domain nodes to use unstable protocol versions and participant nodes to connect to such domains
+  * @param betaVersionSupport If true, allow domain nodes to use beta protocol versions and participant nodes to connect to such domains
   * @param timeouts Sets the timeouts used for processing and console
   * @param portsFile A ports file name, where the ports of all participants will be written to after startup
   * @param exitOnFatalFailures If true the node will exit/stop the process in case of fatal failures
@@ -251,6 +253,7 @@ final case class CantonParameters(
     nonStandardConfig: Boolean = true,
     // TODO(i15561): Revert back to `false` once there is a stable Daml 3 protocol version
     devVersionSupport: Boolean = true,
+    betaVersionSupport: Boolean = false,
     portsFile: Option[String] = None,
     timeouts: TimeoutSettings = TimeoutSettings(),
     retentionPeriodDefaults: RetentionPeriodDefaults = RetentionPeriodDefaults(),
@@ -378,6 +381,7 @@ trait CantonConfig {
         protocolConfig = ParticipantProtocolConfig(
           minimumProtocolVersion = participantParameters.minimumProtocolVersion.map(_.unwrap),
           devVersionSupport = participantParameters.devVersionSupport,
+          betaVersionSupport = participantParameters.BetaVersionSupport,
           dontWarnOnDeprecatedPV = participantParameters.dontWarnOnDeprecatedPV,
         ),
         ledgerApiServerParameters = participantParameters.ledgerApiServer,
@@ -388,6 +392,7 @@ trait CantonConfig {
         disableUpgradeValidation = participantParameters.disableUpgradeValidation,
         allowForUnauthenticatedContractIds =
           participantParameters.allowForUnauthenticatedContractIds,
+        commandProgressTracking = participantParameters.commandProgressTracker,
       )
     }
 
@@ -519,6 +524,7 @@ private[canton] object CantonNodeParameterConverter {
   def protocol(parent: CantonConfig, config: ProtocolConfig): CantonNodeParameters.Protocol =
     CantonNodeParameters.Protocol.Impl(
       devVersionSupport = parent.parameters.devVersionSupport || config.devVersionSupport,
+      betaVersionSupport = parent.parameters.betaVersionSupport || config.betaVersionSupport,
       dontWarnOnDeprecatedPV = config.dontWarnOnDeprecatedPV,
     )
 
@@ -971,9 +977,12 @@ object CantonConfig {
       deriveReader[EngineLoggingConfig]
     lazy implicit val cantonEngineConfigReader: ConfigReader[CantonEngineConfig] =
       deriveReader[CantonEngineConfig]
-    lazy implicit val participantNodeParameterConfigReader
-        : ConfigReader[ParticipantNodeParameterConfig] =
+    @nowarn("cat=unused") lazy implicit val participantNodeParameterConfigReader
+        : ConfigReader[ParticipantNodeParameterConfig] = {
+      implicit val commandProgressTrackerConfigReader: ConfigReader[CommandProgressTrackerConfig] =
+        deriveReader[CommandProgressTrackerConfig]
       deriveReader[ParticipantNodeParameterConfig]
+    }
     lazy implicit val timeTrackerConfigReader: ConfigReader[DomainTimeTrackerConfig] =
       deriveReader[DomainTimeTrackerConfig]
     lazy implicit val timeRequestConfigReader: ConfigReader[TimeProofRequestConfig] =
@@ -1385,9 +1394,12 @@ object CantonConfig {
       deriveWriter[EngineLoggingConfig]
     lazy implicit val cantonEngineConfigWriter: ConfigWriter[CantonEngineConfig] =
       deriveWriter[CantonEngineConfig]
-    lazy implicit val participantNodeParameterConfigWriter
-        : ConfigWriter[ParticipantNodeParameterConfig] =
+    @nowarn("cat=unused") lazy implicit val participantNodeParameterConfigWriter
+        : ConfigWriter[ParticipantNodeParameterConfig] = {
+      implicit val commandProgressTrackerConfigWriter: ConfigWriter[CommandProgressTrackerConfig] =
+        deriveWriter[CommandProgressTrackerConfig]
       deriveWriter[ParticipantNodeParameterConfig]
+    }
     lazy implicit val timeTrackerConfigWriter: ConfigWriter[DomainTimeTrackerConfig] =
       deriveWriter[DomainTimeTrackerConfig]
     lazy implicit val timeRequestConfigWriter: ConfigWriter[TimeProofRequestConfig] =

community/app-base/src/main/scala/com/digitalasset/canton/config/CommunityConfigValidations.scala

@@ -9,7 +9,6 @@ import cats.syntax.functor.*
 import cats.syntax.functorFilter.*
 import com.daml.nonempty.NonEmpty
 import com.daml.nonempty.catsinstances.*
-import com.digitalasset.canton.discard.Implicits.DiscardOps
 import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging}
 import com.digitalasset.canton.tracing.TraceContext
 import com.digitalasset.canton.version.HandshakeErrors.DeprecatedProtocolVersion
@@ -53,19 +52,17 @@ object CommunityConfigValidations
   type Validation = CantonCommunityConfig => Validated[NonEmpty[Seq[String]], Unit]
 
   override protected val validations: List[Validation] =
-    List[Validation](noDuplicateStorage, atLeastOneNode) ++ genericValidations[
-      CantonCommunityConfig
-    ]
+    List[Validation](noDuplicateStorage, atLeastOneNode) ++
+      genericValidations[CantonCommunityConfig]
 
   /** Validations applied to all community and enterprise Canton configurations. */
   private[config] def genericValidations[C <: CantonConfig]
-      : List[C => Validated[NonEmpty[Seq[String]], Unit]] = {
+      : List[C => Validated[NonEmpty[Seq[String]], Unit]] =
     List(
       developmentProtocolSafetyCheck,
       warnIfUnsafeMinProtocolVersion,
       adminTokenSafetyCheckParticipants,
     )
-  }
 
   /** Group node configs by db access to find matching db storage configs.
     * Overcomplicated types used are to work around that at this point nodes could have conflicting names so we can't just
@@ -207,19 +204,22 @@ object CommunityConfigValidations
         devVersionSupport = nodeConfig.parameters.devVersionSupport,
       )
     }
+
   }
 
   private def warnIfUnsafeMinProtocolVersion(
       config: CantonConfig
   ): Validated[NonEmpty[Seq[String]], Unit] = {
-    config.participants.toSeq.foreach { case (name, config) =>
+    val errors = config.participants.toSeq.mapFilter { case (name, config) =>
       val minimum = config.parameters.minimumProtocolVersion.map(_.unwrap)
       val isMinimumDeprecatedVersion = minimum.getOrElse(ProtocolVersion.minimum).isDeprecated
 
-      if (isMinimumDeprecatedVersion && !config.parameters.dontWarnOnDeprecatedPV)
-        DeprecatedProtocolVersion.WarnParticipant(name, minimum).discard
+      Option.when(isMinimumDeprecatedVersion && !config.parameters.dontWarnOnDeprecatedPV)(
+        DeprecatedProtocolVersion.WarnParticipant(name, minimum).cause
+      )
     }
-    Validated.valid(())
+
+    NonEmpty.from(errors).map(Validated.invalid).getOrElse(Validated.valid(()))
   }
 
   private def adminTokenSafetyCheckParticipants(

community/app-base/src/main/scala/com/digitalasset/canton/console/InstanceReference.scala

@@ -401,9 +401,6 @@ class ExternalLedgerApiClient(
   override val loggerFactory: NamedLoggerFactory =
     consoleEnvironment.environment.loggerFactory.append("client", name)
 
-  override protected def domainOfTransaction(transactionId: String): DomainId =
-    throw new NotImplementedError("domain_of is not implemented for external ledger api clients")
-
   override protected[console] def ledgerApiCommand[Result](
       command: GrpcAdminCommand[?, ?, Result]
   ): ConsoleCommandResult[Result] =
@@ -413,6 +410,7 @@ class ExternalLedgerApiClient(
   override protected def optionallyAwait[Tx](
       tx: Tx,
       txId: String,
+      txDomainId: String,
       optTimeout: Option[NonNegativeDuration],
   ): Tx = tx