fix: allow unlimited reconnect attempts on compute nats client (#4116)

- closes #4114

When this value is set to `-1` the client will never stop attempting to
reconnect to the server and thus never remove the server from its
connection pool:
https://docs.nats.io/using-nats/developer/connecting/reconnect/max +
https://github.com/nats-io/nats.go/blob/main/nats.go#L1674: setting
maxReconnect to -1 when only a single server exists in the pool means
that the server will never be removed from the pool. Since we only have
one server (at the moment) this seems like reasonable behavior. But in
the future I expect more advanced re-connection logic will be required.

Co-authored-by: frrist <[email protected]>

pkg/nats/transport/nats.go

@@ -208,6 +208,7 @@ func CreateClient(ctx context.Context, config *NATSTransportConfig) (*nats_helpe
 	log.Debug().Msgf("Creating NATS client with servers: %s", strings.Join(config.Orchestrators, ","))
 	clientOptions := []nats.Option{
 		nats.Name(config.NodeID),
+		nats.MaxReconnects(-1),
 	}
 	if config.AuthSecret != "" {
 		clientOptions = append(clientOptions, nats.Token(config.AuthSecret))