drop store service in favor of a micro store implementation (#8419)

Signed-off-by: Jörn Friedrich Dreyer <[email protected]>

changelog/unreleased/change-presigned-key-store.md

@@ -0,0 +1,5 @@
+Change: change the default store for presigned keys to nats-js-kv
+
+We wrapped the store service in a micro store implementation and changed the default to the built-in NATS instance.
+
+https://github.com/owncloud/ocis/pull/8419

ocis/pkg/runtime/service/service.go

@@ -57,7 +57,6 @@ import (
 	storageshares "github.com/owncloud/ocis/v2/services/storage-shares/pkg/command"
 	storageSystem "github.com/owncloud/ocis/v2/services/storage-system/pkg/command"
 	storageusers "github.com/owncloud/ocis/v2/services/storage-users/pkg/command"
-	store "github.com/owncloud/ocis/v2/services/store/pkg/command"
 	thumbnails "github.com/owncloud/ocis/v2/services/thumbnails/pkg/command"
 	userlog "github.com/owncloud/ocis/v2/services/userlog/pkg/command"
 	users "github.com/owncloud/ocis/v2/services/users/pkg/command"
@@ -245,11 +244,6 @@ func NewService(options ...Option) (*Service, error) {
 		cfg.StorageUsers.Commons = cfg.Commons
 		return storageusers.Execute(cfg.StorageUsers)
 	})
-	reg(3, opts.Config.Store.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
-		cfg.Store.Context = ctx
-		cfg.Store.Commons = cfg.Commons
-		return store.Execute(cfg.Store)
-	})
 	reg(3, opts.Config.Thumbnails.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
 		cfg.Thumbnails.Context = ctx
 		cfg.Thumbnails.Commons = cfg.Commons

services/ocs/README.md

@@ -5,3 +5,20 @@ The `ocs` service (open collaboration services) serves one purpose: it has an en
 ## Signing-Keys Endpoint
 
 The `ocs` service contains an endpoint `/cloud/user/signing-key` on which a user can GET a signing key. Note, this functionality might be deprecated or moved in the future.
+
+## Signing-Keys Store
+
+To authenticate presigned URLs the proxy service needs to read the signing keys from a store that is populated by the ocs service.
+Possible stores that can be configured via `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE` are:
+  -   `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
+  -   `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
+  -   `ocisstoreservice`:  Stores data in the legacy ocis store service. Requires setting `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to `com.owncloud.api.store`.
+
+The `memory` or `ocmem` stores cannot be used as they do not share the memory from the ocs service signing key memory store, even in a single process.
+
+Make sure to configure the same store in the proxy service.
+
+Store specific notes:
+  -   When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
+  -   When using `nats-js-kv` it is recommended to set `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to the same value as `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES`. That way the proxy uses the same nats instance as the ocs service.
+  -   When using `ocisstoreservice` the `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` must be set to the service name `com.owncloud.api.store`. It does not support TTL and stores the presigning keys indefinitely. Also, the store service needs to be started.

services/ocs/pkg/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"context"
+	"time"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/shared"
 	"go-micro.dev/v4/client"
@@ -22,7 +23,18 @@ type Config struct {
 	GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
 	GrpcClient    client.Client         `yaml:"-"`
 
+	SigningKeys *SigningKeys `yaml:"signing_keys"`
+
 	TokenManager *TokenManager `yaml:"token_manager"`
 
 	Context context.Context `yaml:"-"`
 }
+
+// SigningKeys is a store configuration.
+type SigningKeys struct {
+	Store        string        `yaml:"store" env:"OCIS_CACHE_STORE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel' and 'nats-js-kv'. See the text description for details."`
+	Nodes        []string      `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
+	TTL          time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details."`
+	AuthUsername string        `yaml:"username" env:"OCIS_CACHE_AUTH_USERNAME;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."`
+	AuthPassword string        `yaml:"password" env:"OCIS_CACHE_AUTH_PASSWORD;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."`
+}

services/ocs/pkg/config/defaults/defaultconfig.go

@@ -2,6 +2,7 @@ package defaults
 
 import (
 	"strings"
+	"time"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/structs"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/config"
@@ -38,6 +39,11 @@ func DefaultConfig() *config.Config {
 		Service: config.Service{
 			Name: "ocs",
 		},
+		SigningKeys: &config.SigningKeys{
+			Store: "nats-js-kv", // signing keys are read by proxy, so we cannot use memory. It is not shared.
+			Nodes: []string{"127.0.0.1:9233"},
+			TTL:   time.Hour * 12,
+		},
 	}
 }
 

services/ocs/pkg/server/http/server.go

@@ -3,14 +3,17 @@ package http
 import (
 	"fmt"
 
+	"github.com/cs3org/reva/v2/pkg/store"
 	chimiddleware "github.com/go-chi/chi/v5/middleware"
 	"github.com/owncloud/ocis/v2/ocis-pkg/cors"
 	"github.com/owncloud/ocis/v2/ocis-pkg/middleware"
 	"github.com/owncloud/ocis/v2/ocis-pkg/service/http"
 	"github.com/owncloud/ocis/v2/ocis-pkg/version"
 	ocsmw "github.com/owncloud/ocis/v2/services/ocs/pkg/middleware"
 	svc "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0"
+	ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store"
 	"go-micro.dev/v4"
+	microstore "go-micro.dev/v4/store"
 )
 
 // Server initializes the http service and server.
@@ -34,6 +37,24 @@ func Server(opts ...Option) (http.Service, error) {
 		return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
 	}
 
+	var signingKeyStore microstore.Store
+	if options.Config.SigningKeys.Store == "ocisstoreservice" {
+		signingKeyStore = ocisstore.NewStore(
+			microstore.Nodes(options.Config.SigningKeys.Nodes...),
+			microstore.Database("proxy"),
+			microstore.Table("signing-keys"),
+		)
+	} else {
+		signingKeyStore = store.Create(
+			store.Store(options.Config.SigningKeys.Store),
+			store.TTL(options.Config.SigningKeys.TTL),
+			microstore.Nodes(options.Config.SigningKeys.Nodes...),
+			microstore.Database("proxy"),
+			microstore.Table("signing-keys"),
+			store.Authentication(options.Config.SigningKeys.AuthUsername, options.Config.SigningKeys.AuthPassword),
+		)
+	}
+
 	handle := svc.NewService(
 		svc.Logger(options.Logger),
 		svc.Config(options.Config),
@@ -56,6 +77,7 @@ func Server(opts ...Option) (http.Service, error) {
 			middleware.Logger(options.Logger),
 			ocsmw.LogTrace,
 		),
+		svc.Store(signingKeyStore),
 	)
 
 	{

services/ocs/pkg/service/v0/option.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/config"
+	"go-micro.dev/v4/store"
 )
 
 // Option defines a single option function.
@@ -15,6 +16,7 @@ type Options struct {
 	Logger     log.Logger
 	Config     *config.Config
 	Middleware []func(http.Handler) http.Handler
+	Store      store.Store
 }
 
 // newOptions initializes the available default options.
@@ -48,3 +50,10 @@ func Middleware(val ...func(http.Handler) http.Handler) Option {
 		o.Middleware = val
 	}
 }
+
+// Store provides a function to set the store option.
+func Store(s store.Store) Option {
+	return func(o *Options) {
+		o.Store = s
+	}
+}

services/ocs/pkg/service/v0/service.go

@@ -14,6 +14,7 @@ import (
 	ocsm "github.com/owncloud/ocis/v2/services/ocs/pkg/middleware"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/data"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/response"
+	microstore "go-micro.dev/v4/store"
 )
 
 // Service defines the service handlers.
@@ -32,6 +33,7 @@ func NewService(opts ...Option) Service {
 		config: options.Config,
 		mux:    m,
 		logger: options.Logger,
+		store:  options.Store,
 	}
 
 	m.Route(options.Config.HTTP.Root, func(r chi.Router) {
@@ -61,6 +63,7 @@ type Ocs struct {
 	config *config.Config
 	logger log.Logger
 	mux    *chi.Mux
+	store  microstore.Store
 }
 
 // ServeHTTP implements the Service interface.

services/ocs/pkg/service/v0/users.go

@@ -3,15 +3,13 @@ package svc
 import (
 	"crypto/rand"
 	"encoding/hex"
+	"errors"
 	"net/http"
 
-	storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0"
-	storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
-
 	revactx "github.com/cs3org/reva/v2/pkg/ctx"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/data"
 	"github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/response"
-	merrors "go-micro.dev/v4/errors"
+	"go-micro.dev/v4/store"
 )
 
 // GetSigningKey returns the signing key for the current user. It will create it on the fly if it does not exist
@@ -28,24 +26,16 @@ func (o Ocs) GetSigningKey(w http.ResponseWriter, r *http.Request) {
 	// use the user's UUID
 	userID := u.Id.OpaqueId
 
-	c := storesvc.NewStoreService("com.owncloud.api.store", o.config.GrpcClient)
-	res, err := c.Read(r.Context(), &storesvc.ReadRequest{
-		Options: &storemsg.ReadOptions{
-			Database: "proxy",
-			Table:    "signing-keys",
-		},
-		Key: userID,
-	})
-	if err == nil && len(res.Records) > 0 {
+	res, err := o.store.Read(userID)
+	if err == nil && len(res) > 0 {
 		o.mustRender(w, r, response.DataRender(&data.SigningKey{
 			User:       userID,
-			SigningKey: string(res.Records[0].Value),
+			SigningKey: string(res[0].Value),
 		}))
 		return
 	}
 	if err != nil {
-		e := merrors.Parse(err.Error())
-		if e.Code == http.StatusNotFound {
+		if errors.Is(err, store.ErrNotFound) {
 			// not found is ok, so we can continue and generate the key on the fly
 		} else {
 			o.logger.Error().Err(err).Msg("error reading from server")
@@ -63,20 +53,8 @@ func (o Ocs) GetSigningKey(w http.ResponseWriter, r *http.Request) {
 	}
 	signingKey := hex.EncodeToString(key)
 
-	_, err = c.Write(r.Context(), &storesvc.WriteRequest{
-		Options: &storemsg.WriteOptions{
-			Database: "proxy",
-			Table:    "signing-keys",
-		},
-		Record: &storemsg.Record{
-			Key:   userID,
-			Value: []byte(signingKey),
-			// TODO Expiry?
-		},
-	})
-
+	err = o.store.Write(&store.Record{Key: userID, Value: []byte(signingKey)})
 	if err != nil {
-		//o.logger.Error().Err(err).Msg("error writing key")
 		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, "could not persist signing key"))
 		return
 	}

services/proxy/README.md

@@ -153,6 +153,25 @@ Store specific notes:
   -   When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
   -   When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
 
+
+## Presigned Urls
+
+To authenticate presigned URLs the proxy service needs to read signing keys from a store that is populated by the ocs service. Possible stores are:
+  -   `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
+  -   `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
+  -   `ocisstoreservice`:  Stores data in the legacy ocis store service. Requires setting `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to `com.owncloud.api.store`.
+
+The `memory` or `ocmem` stores cannot be used as they do not share the memory from the ocs service signing key memory store, even in a single process.
+
+Make sure to configure the same store in the ocs service.
+
+Store specific notes:
+  -   When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
+  -   When using `nats-js-kv` it is recommended to set `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to the same value as `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES`. That way the ocs uses the same nats instance as the proxy service.
+  -   When using the `nats-js-kv` store, it is possible to set `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE` to instruct nats to not persist signing key data on disc.
+  -   When using `ocisstoreservice` the `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` must be set to the service name `com.owncloud.api.store`. It does not support TTL and stores the presigning keys indefinitely. Also, the store service needs to be started.
+
+
 ## Special Settings
 
 When using the ocis IDP service instead of an external IDP:

services/proxy/pkg/command/server.go

@@ -31,7 +31,6 @@ import (
 	"github.com/owncloud/ocis/v2/ocis-pkg/version"
 	policiessvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
-	storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
 	"github.com/owncloud/ocis/v2/services/proxy/pkg/config"
 	"github.com/owncloud/ocis/v2/services/proxy/pkg/config/parser"
 	"github.com/owncloud/ocis/v2/services/proxy/pkg/logging"
@@ -43,6 +42,7 @@ import (
 	proxyHTTP "github.com/owncloud/ocis/v2/services/proxy/pkg/server/http"
 	"github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend"
 	"github.com/owncloud/ocis/v2/services/proxy/pkg/userroles"
+	ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store"
 )
 
 // Server is the entrypoint for the server command.
@@ -66,6 +66,24 @@ func Server(cfg *config.Config) *cli.Command {
 				store.Authentication(cfg.OIDC.UserinfoCache.AuthUsername, cfg.OIDC.UserinfoCache.AuthPassword),
 			)
 
+			var signingKeyStore microstore.Store
+			if cfg.PreSignedURL.SigningKeys.Store == "ocisstoreservice" {
+				signingKeyStore = ocisstore.NewStore(
+					microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...),
+					microstore.Database("proxy"),
+					microstore.Table("signing-keys"),
+				)
+			} else {
+				signingKeyStore = store.Create(
+					store.Store(cfg.PreSignedURL.SigningKeys.Store),
+					store.TTL(cfg.PreSignedURL.SigningKeys.TTL),
+					microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...),
+					microstore.Database("proxy"),
+					microstore.Table("signing-keys"),
+					store.Authentication(cfg.PreSignedURL.SigningKeys.AuthUsername, cfg.PreSignedURL.SigningKeys.AuthPassword),
+				)
+			}
+
 			logger := logging.Configure(cfg.Service.Name, cfg.Log)
 			traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name)
 			if err != nil {
@@ -130,7 +148,7 @@ func Server(cfg *config.Config) *cli.Command {
 			}
 
 			{
-				middlewares := loadMiddlewares(ctx, logger, cfg, userInfoCache, traceProvider, *m)
+				middlewares := loadMiddlewares(ctx, logger, cfg, userInfoCache, signingKeyStore, traceProvider, *m)
 				server, err := proxyHTTP.Server(
 					proxyHTTP.Handler(lh.handler()),
 					proxyHTTP.Logger(logger),
@@ -271,7 +289,7 @@ func (h *StaticRouteHandler) backchannelLogout(w http.ResponseWriter, r *http.Re
 	render.JSON(w, r, nil)
 }
 
-func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, userInfoCache microstore.Store, traceProvider trace.TracerProvider, metrics metrics.Metrics) alice.Chain {
+func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, userInfoCache, signingKeyStore microstore.Store, traceProvider trace.TracerProvider, metrics metrics.Metrics) alice.Chain {
 	rolesClient := settingssvc.NewRoleService("com.owncloud.api.settings", cfg.GrpcClient)
 	policiesProviderClient := policiessvc.NewPoliciesProviderService("com.owncloud.api.policies", cfg.GrpcClient)
 	gatewaySelector, err := pool.GatewaySelector(
@@ -322,13 +340,6 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config,
 		logger.Fatal().Msgf("Invalid role assignment driver '%s'", cfg.RoleAssignment.Driver)
 	}
 
-	storeClient := storesvc.NewStoreService("com.owncloud.api.store", cfg.GrpcClient)
-	if err != nil {
-		logger.Error().Err(err).
-			Str("gateway", cfg.Reva.Address).
-			Msg("Failed to create reva gateway service client")
-	}
-
 	oidcHTTPClient := &http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
@@ -373,7 +384,7 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config,
 		PreSignedURLConfig: cfg.PreSignedURL,
 		UserProvider:       userProvider,
 		UserRoleAssigner:   roleAssigner,
-		Store:              storeClient,
+		Store:              signingKeyStore,
 		Now:                time.Now,
 	})
 

services/proxy/pkg/config/config.go

@@ -166,8 +166,19 @@ type StaticSelectorConf struct {
 
 // PreSignedURL is the config for the presigned url middleware
 type PreSignedURL struct {
-	AllowedHTTPMethods []string `yaml:"allowed_http_methods"`
-	Enabled            bool     `yaml:"enabled" env:"PROXY_ENABLE_PRESIGNEDURLS" desc:"Allow OCS to get a signing key to sign requests."`
+	AllowedHTTPMethods []string     `yaml:"allowed_http_methods"`
+	Enabled            bool         `yaml:"enabled" env:"PROXY_ENABLE_PRESIGNEDURLS" desc:"Allow OCS to get a signing key to sign requests."`
+	SigningKeys        *SigningKeys `yaml:"signing_keys"`
+}
+
+// SigningKeys is a store configuration.
+type SigningKeys struct {
+	Store              string        `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel', 'nats-js-kv' and 'ocisstoreservice' (deprecated). See the text description for details."`
+	Nodes              []string      `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
+	TTL                time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details."`
+	DisablePersistence bool          `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE" desc:"Disables persistence of the store. Only applies when store type 'nats-js-kv' is configured. Defaults to true."`
+	AuthUsername       string        `yaml:"username" env:"OCIS_CACHE_AUTH_USERNAME;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."`
+	AuthPassword       string        `yaml:"password" env:"OCIS_CACHE_AUTH_PASSWORD;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."`
 }
 
 // ClaimsSelectorConf is the config for the claims-selector