diff --git a/go.mod b/go.mod index 1dcb46d53dd..df25fb12c72 100644 --- a/go.mod +++ b/go.mod @@ -346,3 +346,5 @@ require ( ) replace github.com/go-micro/plugins/v4/store/nats-js => github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a + +replace github.com/cs3org/reva/v2 => github.com/butonic/reva/v2 v2.0.0-20231205112716-b643432e6672 diff --git a/go.sum b/go.sum index 5be68902c96..0aa76f575bb 100644 --- a/go.sum +++ b/go.sum @@ -941,6 +941,8 @@ github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHA github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/butonic/reva/v2 v2.0.0-20231205112716-b643432e6672 h1:86ZSMr1/7kc0aLi5rcPo0pEsbB5VDI9MTKiKIRpKuOs= +github.com/butonic/reva/v2 v2.0.0-20231205112716-b643432e6672/go.mod h1:zcrrYVsBv/DwhpyO2/W5hoSZ/k6az6Z2EYQok65uqZY= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= @@ -1017,8 +1019,6 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.16.1-0.20231201122033-a389ddc645c4 h1:61AwMfov2OxrUElWXXKHZfBsuxgNIVwZVQW4PlJoqnM= -github.com/cs3org/reva/v2 v2.16.1-0.20231201122033-a389ddc645c4/go.mod h1:zcrrYVsBv/DwhpyO2/W5hoSZ/k6az6Z2EYQok65uqZY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/services/graph/pkg/service/v0/users.go b/services/graph/pkg/service/v0/users.go index 04e4a2c23e5..1c721e65a4e 100644 --- a/services/graph/pkg/service/v0/users.go +++ b/services/graph/pkg/service/v0/users.go @@ -333,7 +333,7 @@ func (g Graph) PostUser(w http.ResponseWriter, r *http.Request) { if g.roleService != nil && g.config.API.AssignDefaultUserRole { // All users get the user role by default currently. // to all new users for now, as create Account request does not have any role field - if _, err = g.roleService.AssignRoleToUser(r.Context(), &settings.AssignRoleToUserRequest{ + if _, err := g.roleService.AssignRoleToUser(r.Context(), &settings.AssignRoleToUserRequest{ AccountUuid: *u.Id, RoleId: ocissettingssvc.BundleUUIDRoleUser, }); err != nil { diff --git a/services/proxy/pkg/middleware/accesslog.go b/services/proxy/pkg/middleware/accesslog.go index 00afc71c436..072bd07c399 100644 --- a/services/proxy/pkg/middleware/accesslog.go +++ b/services/proxy/pkg/middleware/accesslog.go @@ -6,6 +6,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/owncloud/ocis/v2/ocis-pkg/log" + "go.opentelemetry.io/otel/trace" ) // AccessLog is a middleware to log http requests at info level logging. @@ -19,15 +20,19 @@ func AccessLog(logger log.Logger) func(http.Handler) http.Handler { wrap := middleware.NewWrapResponseWriter(w, r.ProtoMajor) next.ServeHTTP(wrap, r) + spanContext := trace.SpanContextFromContext(r.Context()) + logger.Info(). Str("proto", r.Proto). Str(log.RequestIDString, requestID). + Str("traceid", spanContext.TraceID().String()). Str("remote-addr", r.RemoteAddr). Str("method", r.Method). Int("status", wrap.Status()). Str("path", r.URL.Path). Dur("duration", time.Since(start)). Int("bytes", wrap.BytesWritten()). + Int("content-length", int(r.ContentLength)). Msg("access-log") }) } diff --git a/services/settings/pkg/store/metadata/assignments.go b/services/settings/pkg/store/metadata/assignments.go index 003b248cf33..b3315357c59 100644 --- a/services/settings/pkg/store/metadata/assignments.go +++ b/services/settings/pkg/store/metadata/assignments.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" + "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/gofrs/uuid" settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" @@ -16,6 +17,7 @@ import ( func (s *Store) ListRoleAssignments(accountUUID string) ([]*settingsmsg.UserRoleAssignment, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) assIDs, err := s.mdc.ReadDir(ctx, accountPath(accountUUID)) switch err.(type) { case nil: @@ -41,6 +43,7 @@ func (s *Store) ListRoleAssignments(accountUUID string) ([]*settingsmsg.UserRole a := &settingsmsg.UserRoleAssignment{} err = json.Unmarshal(b, a) if err != nil { + s.Logger.Error().Err(err).Str("data", string(b)).Msg("could not parse json") return nil, err } @@ -53,6 +56,7 @@ func (s *Store) ListRoleAssignments(accountUUID string) ([]*settingsmsg.UserRole func (s *Store) WriteRoleAssignment(accountUUID, roleID string) (*settingsmsg.UserRoleAssignment, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) // as per https://github.com/owncloud/product/issues/103 "Each user can have exactly one role" err := s.mdc.Delete(ctx, accountPath(accountUUID)) switch err.(type) { @@ -78,13 +82,15 @@ func (s *Store) WriteRoleAssignment(accountUUID, roleID string) (*settingsmsg.Us if err != nil { return nil, err } - return ass, s.mdc.SimpleUpload(ctx, assignmentPath(accountUUID, ass.Id), b) + err = s.mdc.SimpleUpload(ctx, assignmentPath(accountUUID, ass.Id), b) + return ass, err } // RemoveRoleAssignment deletes the given role assignment from the existing assignments of the respective account. func (s *Store) RemoveRoleAssignment(assignmentID string) error { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) accounts, err := s.mdc.ReadDir(ctx, accountsFolderLocation) switch err.(type) { case nil: diff --git a/services/settings/pkg/store/metadata/bundles.go b/services/settings/pkg/store/metadata/bundles.go index 9025c97893c..97f07cc6d24 100644 --- a/services/settings/pkg/store/metadata/bundles.go +++ b/services/settings/pkg/store/metadata/bundles.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" + "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/gofrs/uuid" settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" @@ -17,6 +18,7 @@ import ( func (s *Store) ListBundles(bundleType settingsmsg.Bundle_Type, bundleIDs []string) ([]*settingsmsg.Bundle, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) if len(bundleIDs) == 0 { bIDs, err := s.mdc.ReadDir(ctx, bundleFolderLocation) @@ -61,6 +63,8 @@ func (s *Store) ListBundles(bundleType settingsmsg.Bundle_Type, bundleIDs []stri func (s *Store) ReadBundle(bundleID string) (*settingsmsg.Bundle, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) + b, err := s.mdc.SimpleDownload(ctx, bundlePath(bundleID)) switch err.(type) { case nil: @@ -79,6 +83,7 @@ func (s *Store) ReadBundle(bundleID string) (*settingsmsg.Bundle, error) { func (s *Store) ReadSetting(settingID string) (*settingsmsg.Setting, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) ids, err := s.mdc.ReadDir(ctx, bundleFolderLocation) switch err.(type) { @@ -114,6 +119,7 @@ func (s *Store) ReadSetting(settingID string) (*settingsmsg.Setting, error) { func (s *Store) WriteBundle(record *settingsmsg.Bundle) (*settingsmsg.Bundle, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) b, err := json.Marshal(record) if err != nil { diff --git a/services/settings/pkg/store/metadata/store.go b/services/settings/pkg/store/metadata/store.go index 68946a3a80d..8358ea702ce 100644 --- a/services/settings/pkg/store/metadata/store.go +++ b/services/settings/pkg/store/metadata/store.go @@ -7,6 +7,7 @@ import ( "log" "sync" + "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/gofrs/uuid" olog "github.com/owncloud/ocis/v2/ocis-pkg/log" @@ -99,6 +100,8 @@ func NewMetadataClient(cfg config.Metadata) MetadataClient { // we need to lazy initialize the MetadataClient because metadata service might not be ready func (s *Store) initMetadataClient(mdc MetadataClient) error { ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) + err := mdc.Init(ctx, settingsSpaceID) if err != nil { return err diff --git a/services/settings/pkg/store/metadata/values.go b/services/settings/pkg/store/metadata/values.go index c40f859fa73..285731c4009 100644 --- a/services/settings/pkg/store/metadata/values.go +++ b/services/settings/pkg/store/metadata/values.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" + "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/gofrs/uuid" settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0" @@ -19,6 +20,7 @@ import ( func (s *Store) ListValues(bundleID, accountUUID string) ([]*settingsmsg.Value, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) vIDs, err := s.mdc.ReadDir(ctx, valuesFolderLocation) switch err.(type) { @@ -70,6 +72,7 @@ func (s *Store) ListValues(bundleID, accountUUID string) ([]*settingsmsg.Value, func (s *Store) ReadValue(valueID string) (*settingsmsg.Value, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) b, err := s.mdc.SimpleDownload(ctx, valuePath(valueID)) switch err.(type) { @@ -91,6 +94,7 @@ func (s *Store) ReadValueByUniqueIdentifiers(accountUUID, settingID string) (*se } s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) vIDs, err := s.mdc.ReadDir(ctx, valuesFolderLocation) if err != nil { @@ -125,6 +129,7 @@ func (s *Store) ReadValueByUniqueIdentifiers(accountUUID, settingID string) (*se func (s *Store) WriteValue(value *settingsmsg.Value) (*settingsmsg.Value, error) { s.Init() ctx := context.TODO() + ctx = appctx.WithLogger(ctx, &s.Logger.Logger) if value.Id == "" { value.Id = uuid.Must(uuid.NewV4()).String() diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 4070394e98c..c295476ab41 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -127,14 +127,19 @@ type S3NGDriver struct { Propagator string `yaml:"propagator" env:"OCIS_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_S3NG_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option."` AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"` // Root is the absolute path to the location of the data - Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."` - UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."` - PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."` - Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."` - AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."` - SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."` - Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."` - Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."` + Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."` + UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."` + PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."` + Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."` + AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."` + SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."` + Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."` + Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."` + UploadObjectPrefix string `yaml:"upload_object_prefix" env:"STORAGE_USERS_S3NG_UPLOAD_OBJECT_PREFIX" desc:"This object prefix is prepended to the name of each S3 object that is created to store uploaded files. It can be used to create a pseudo-directory structure in the bucket, like 'path/to/my/uploads'."` + UploadMetadataPrefix string `yaml:"upload_metadata_prefix" env:"STORAGE_USERS_S3NG_UPLOAD_METADATA_PREFIX" desc:"The metadata object prefix is prepended to the name of each .info and .part S3 object that is created. If it is not set, then object prefix is used."` + UploadTemporaryDirectory string `yaml:"upload_temporary_directory" env:"STORAGE_USERS_S3NG_UPLOAD_TEMPORARY_DIRECTORY" desc:"Path where temporary files will be stored on disk during the upload."` + DisableSSL bool `yaml:"disable_ssl" env:"STORAGE_USERS_S3NG_DISABLE_SSL" desc:"Disable SSL when accessing the S3 bucket."` + ForcePathStyle bool `yaml:"force_path_style" env:"STORAGE_USERS_S3NG_FORCE_PATH_STYLE" desc:"Force path style S3 requests."` // PersonalSpaceAliasTemplate contains the template used to construct // the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"` PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_S3NG_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases."` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 1bb34eab579..7dc1fd367e8 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -73,6 +73,7 @@ func DefaultConfig() *config.Config { PermissionsEndpoint: "com.owncloud.api.settings", MaxAcquireLockCycles: 20, LockCycleDurationFactor: 30, + UploadObjectPrefix: "uploads", }, OCIS: config.OCISDriver{ MetadataBackend: "messagepack", diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 097f16250f9..1bc14f97ba4 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -237,24 +237,29 @@ func S3NG(cfg *config.Config) map[string]interface{} { "async_propagator_options": map[string]interface{}{ "propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay, }, - "root": cfg.Drivers.S3NG.Root, - "user_layout": cfg.Drivers.S3NG.UserLayout, - "share_folder": cfg.Drivers.S3NG.ShareFolder, - "personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate, - "generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate, - "treetime_accounting": true, - "treesize_accounting": true, - "permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, - "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, - "s3.region": cfg.Drivers.S3NG.Region, - "s3.access_key": cfg.Drivers.S3NG.AccessKey, - "s3.secret_key": cfg.Drivers.S3NG.SecretKey, - "s3.endpoint": cfg.Drivers.S3NG.Endpoint, - "s3.bucket": cfg.Drivers.S3NG.Bucket, - "max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, - "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, - "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, - "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, + "root": cfg.Drivers.S3NG.Root, + "user_layout": cfg.Drivers.S3NG.UserLayout, + "share_folder": cfg.Drivers.S3NG.ShareFolder, + "personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate, + "generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate, + "treetime_accounting": true, + "treesize_accounting": true, + "permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, + "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, + "s3.region": cfg.Drivers.S3NG.Region, + "s3.access_key": cfg.Drivers.S3NG.AccessKey, + "s3.secret_key": cfg.Drivers.S3NG.SecretKey, + "s3.endpoint": cfg.Drivers.S3NG.Endpoint, + "s3.bucket": cfg.Drivers.S3NG.Bucket, + "s3.upload_object_prefix": cfg.Drivers.S3NG.UploadObjectPrefix, + "s3.upload_metadata_prefix": cfg.Drivers.S3NG.UploadMetadataPrefix, + "s3.upload_temporary_directory": cfg.Drivers.S3NG.UploadTemporaryDirectory, + "s3.disable_ssl": cfg.Drivers.S3NG.DisableSSL, + "s3.force_path_style": cfg.Drivers.S3NG.ForcePathStyle, + "max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, + "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, + "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, + "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, "statcache": map[string]interface{}{ "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, @@ -300,23 +305,28 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} { "async_propagator_options": map[string]interface{}{ "propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay, }, - "root": cfg.Drivers.S3NG.Root, - "user_layout": cfg.Drivers.S3NG.UserLayout, - "share_folder": cfg.Drivers.S3NG.ShareFolder, - "personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate, - "generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate, - "treetime_accounting": true, - "treesize_accounting": true, - "permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, - "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, - "s3.region": cfg.Drivers.S3NG.Region, - "s3.access_key": cfg.Drivers.S3NG.AccessKey, - "s3.secret_key": cfg.Drivers.S3NG.SecretKey, - "s3.endpoint": cfg.Drivers.S3NG.Endpoint, - "s3.bucket": cfg.Drivers.S3NG.Bucket, - "max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, - "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, - "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, + "root": cfg.Drivers.S3NG.Root, + "user_layout": cfg.Drivers.S3NG.UserLayout, + "share_folder": cfg.Drivers.S3NG.ShareFolder, + "personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate, + "generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate, + "treetime_accounting": true, + "treesize_accounting": true, + "permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, + "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, + "s3.region": cfg.Drivers.S3NG.Region, + "s3.access_key": cfg.Drivers.S3NG.AccessKey, + "s3.secret_key": cfg.Drivers.S3NG.SecretKey, + "s3.endpoint": cfg.Drivers.S3NG.Endpoint, + "s3.bucket": cfg.Drivers.S3NG.Bucket, + "s3.upload_object_prefix": cfg.Drivers.S3NG.UploadObjectPrefix, + "s3.upload_metadata_prefix": cfg.Drivers.S3NG.UploadMetadataPrefix, + "s3.upload_temporary_directory": cfg.Drivers.S3NG.UploadTemporaryDirectory, + "s3.disable_ssl": cfg.Drivers.S3NG.DisableSSL, + "s3.force_path_style": cfg.Drivers.S3NG.ForcePathStyle, + "max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, + "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, + "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, "statcache": map[string]interface{}{ "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index f7812783cf6..0132e92d8b0 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -338,6 +338,15 @@ func (s *service) InitiateFileUpload(ctx context.Context, req *provider.Initiate }, nil } + // FIXME: This is a hack to transport more metadata to the storage.FS InitiateUpload implementation + // we should use a request object that can carry + // * if-match + // * if-unmodified-since + // * uploadLength from the tus Upload-Length header + // * checksum from the tus Upload-Checksum header + // * mtime from the X-OC-Mtime header + // * expires from the s.conf.UploadExpiration ... should that not be part of the driver? + // * providerID metadata := map[string]string{} ifMatch := req.GetIfMatch() if ifMatch != "" { diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/dataprovider/dataprovider.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/dataprovider/dataprovider.go index 8c3a31195ff..2d7fca75e6a 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/dataprovider/dataprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/dataprovider/dataprovider.go @@ -143,7 +143,13 @@ func getDataTXs(c *config, fs storage.FS, publisher events.Publisher) (map[strin if tx, err := f(c.DataTXs[t], publisher); err == nil { if handler, err := tx.Handler(fs); err == nil { txs[t] = handler + // FIXME we at least need to log this. the ocm received storage e.g. does not support tus + // } else { + // return nil, err } + // FIXME we at least need to log this. the ocm received storage e.g. does not support tus + // } else { + // return nil, err } } } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go index e648582c0d1..475580ce0be 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go @@ -302,11 +302,13 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, r.Body) if err != nil { + log.Error().Err(err).Msg("error creating new request to data service") w.WriteHeader(http.StatusInternalServerError) return } Propagator.Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) httpReq.Header.Set(datagateway.TokenTransportHeader, token) + httpReq.ContentLength = r.ContentLength httpRes, err := s.client.Do(httpReq) if err != nil { diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/tus.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/tus.go index 6fbbbb6e43d..96c06c83f78 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/tus.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/tus.go @@ -88,6 +88,10 @@ func (s *svc) handleSpacesTusPost(w http.ResponseWriter, r *http.Request, spaceI sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger() + // use filename to build a storage space reference + // but what if upload happens directly to toh resourceid .. and filename is empty? + // currently there is always a validator thet requires the filename is not empty ... + // hm -> bug: clients currently cannot POST to an existing source with a resource id only ref, err := spacelookup.MakeStorageSpaceReference(spaceID, path.Join(r.URL.Path, meta["filename"])) if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/versions.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/versions.go index 1f6099597da..21e0721086a 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/versions.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/versions.go @@ -180,6 +180,7 @@ func (h *VersionsHandler) doListVersions(w http.ResponseWriter, r *http.Request, Type: provider.ResourceType_RESOURCE_TYPE_FILE, Id: &provider.ResourceId{ StorageId: "versions", + SpaceId: info.Id.SpaceId, OpaqueId: info.Id.OpaqueId + "@" + versions[i].GetKey(), }, // Checksum diff --git a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go index a46bd177392..05993668243 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go @@ -16,6 +16,9 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package tus implements a data tx manager that handles uploads using the TUS protocol. +// reva storage drivers should implement the hasTusDatastore interface by using composition +// of an upstream tusd.DataStore. If necessary they can also implement a tusd.DataStore directly. package tus import ( @@ -33,12 +36,12 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/logger" "github.com/cs3org/reva/v2/pkg/rhttp/datatx" "github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/registry" "github.com/cs3org/reva/v2/pkg/rhttp/datatx/metrics" "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/cache" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/mitchellh/mapstructure" ) @@ -76,24 +79,49 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e } func (m *manager) Handler(fs storage.FS) (http.Handler, error) { - composable, ok := fs.(composable) - if !ok { - return nil, errtypes.NotSupported("file system does not support the tus protocol") + zlog, err := logger.FromConfig(&logger.LogConf{ + Output: "stderr", + Mode: "json", + Level: "error", // FIXME introduce shared config for logging + }) + if err != nil { + return nil, errtypes.NotSupported("could not initialize log") } - // A storage backend for tusd may consist of multiple different parts which - // handle upload creation, locking, termination and so on. The composer is a - // place where all those separated pieces are joined together. In this example - // we only use the file store but you may plug in multiple. composer := tusd.NewStoreComposer() - // let the composable storage tell tus which extensions it supports - composable.UseIn(composer) - config := tusd.Config{ - StoreComposer: composer, + StoreComposer: composer, + PreUploadCreateCallback: func(hook tusd.HookEvent) error { + return errors.New("uploads must be created with a cs3 InitiateUpload call") + }, NotifyCompleteUploads: true, - Logger: log.New(appctx.GetLogger(context.Background()), "", 0), + Logger: log.New(zlog, "", 0), + } + + var dataStore tusd.DataStore + + cb, ok := fs.(hasTusDatastore) + if ok { + dataStore = cb.GetDataStore() + composable, ok := dataStore.(composable) + if !ok { + return nil, errtypes.NotSupported("tus datastore is not composable") + } + composable.UseIn(composer) + config.PreFinishResponseCallback = cb.PreFinishResponseCallback + } else { + composable, ok := fs.(composable) + if !ok { + return nil, errtypes.NotSupported("storage driver does not support the tus protocol") + } + + // let the composable storage tell tus which extensions it supports + composable.UseIn(composer) + dataStore, ok = fs.(tusd.DataStore) + if !ok { + return nil, errtypes.NotSupported("storage driver does not support the tus datastore") + } } handler, err := tusd.NewUnroutedHandler(config) @@ -101,21 +129,30 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return nil, err } - if _, ok := fs.(storage.UploadSessionLister); ok { + usl, ok := fs.(storage.UploadSessionLister) + if ok { // We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info go func() { for { ev := <-handler.CompleteUploads // We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files // so we create a Progress instance here that is used to read the correct properties - up := upload.Progress{ - Info: ev.Upload, + sessions, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) + if err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Str("id", ev.Upload.ID).Msg("failed to list upload session for upload") + continue + } + if len(sessions) != 1 { + appctx.GetLogger(context.Background()).Error().Err(err).Str("id", ev.Upload.ID).Msg("no upload session found") + continue } - executant := up.Executant() - ref := up.Reference() + us := sessions[0] + + executant := us.Executant() + ref := us.Reference() datatx.InvalidateCache(&executant, &ref, m.statCache) if m.publisher != nil { - if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { + if err := datatx.EmitFileUploadedEvent(us.SpaceOwner(), &executant, &ref, m.publisher); err != nil { appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") } } @@ -137,7 +174,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(dataStore, usl, w, r) handler.PostFile(w, r) case "HEAD": handler.HeadFile(w, r) @@ -147,7 +184,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(dataStore, usl, w, r) handler.PatchFile(w, r) case "DELETE": handler.DelFile(w, r) @@ -174,14 +211,14 @@ type composable interface { UseIn(composer *tusd.StoreComposer) } -func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { +type hasTusDatastore interface { + PreFinishResponseCallback(hook tusd.HookEvent) error + GetDataStore() tusd.DataStore +} + +func setHeaders(datastore tusd.DataStore, usl storage.UploadSessionLister, w http.ResponseWriter, r *http.Request) { ctx := r.Context() id := path.Base(r.URL.Path) - datastore, ok := fs.(tusd.DataStore) - if !ok { - appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore") - return - } upload, err := datastore.GetUpload(ctx, id) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage") @@ -192,14 +229,51 @@ func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload info for upload") return } - expires := info.MetaData["expires"] + var expires string + + var resourceid provider.ResourceId + var uploadSession storage.UploadSession + if usl != nil { + sessions, err := usl.ListUploadSessions(ctx, storage.UploadSessionFilter{ID: &id}) + if err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Str("id", id).Msg("failed to list upload session for upload") + return + } + if len(sessions) != 1 { + appctx.GetLogger(context.Background()).Error().Err(err).Str("id", id).Msg("no upload session found") + return + } + uploadSession = sessions[0] + + t := time.Time{} + if uploadSession.Expires() != t { + expires = uploadSession.Expires().Format(net.RFC1123) + } + + reference := uploadSession.Reference() + resourceid = *reference.GetResourceId() + } + + // FIXME expires should be part of the tus handler + // fallback for outdated storageproviders that implement a tus datastore + if expires == "" { + expires = info.MetaData["expires"] + } + if expires != "" { w.Header().Set(net.HeaderTusUploadExpires, expires) } - resourceid := provider.ResourceId{ - StorageId: info.MetaData["providerID"], - SpaceId: info.Storage["SpaceRoot"], - OpaqueId: info.Storage["NodeId"], + + // fallback for outdated storageproviders that implement a tus datastore + if resourceid.GetStorageId() == "" { + resourceid.StorageId = info.MetaData["providerID"] } + if resourceid.GetSpaceId() == "" { + resourceid.SpaceId = info.MetaData["SpaceRoot"] + } + if resourceid.GetOpaqueId() == "" { + resourceid.OpaqueId = info.MetaData["NodeId"] + } + w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(resourceid)) } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go index 3bb95e02a9e..a23b5533f2e 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go @@ -113,7 +113,8 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"path":"/file"},"uploadLength":0,"metadata":{"providerID":""}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":0,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":1,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":2,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/yes"},"mdKeys":[]}`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/yes"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/yes","permission_set":{},"size":1,"canonical_metadata":{},"arbitrary_metadata":{}}`, serverStateEmpty}, diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/ocis.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/ocis.go index 812b44e4f71..a9583f7c91c 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/ocis.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/ocis.go @@ -20,6 +20,7 @@ package ocis import ( "path" + "path/filepath" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storage" @@ -27,6 +28,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/tus/tusd/pkg/filestore" ) func init() { @@ -46,5 +48,7 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { return nil, err } - return decomposedfs.NewDefault(m, bs, stream) + tusDataStore := filestore.New(filepath.Join(o.Root, "uploads")) + + return decomposedfs.NewDefault(m, bs, tusDataStore, stream) } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go index 9c744e75401..d4b6926ae72 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go @@ -63,6 +63,22 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, err }, nil } +func (bs *Blobstore) MoveBlob(node *node.Node, source, bucket, key string) error { + + _, err := bs.client.CopyObject(context.Background(), minio.CopyDestOptions{ + Bucket: bs.bucket, + Object: bs.path(node), + }, minio.CopySrcOptions{ + Bucket: bucket, + Object: key, + }) + + if err != nil { + return errors.Wrapf(err, "could not copy object bucket:'%s' key:'%s' to bucket:'%s' key'%s'", bs.bucket, bs.path(node), bucket, key) + } + return nil +} + // Upload stores some data in the blobstore under the given key func (bs *Blobstore) Upload(node *node.Node, source string) error { reader, err := os.Open(source) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go index 877a7d71891..cf8fc08ecc8 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go @@ -43,6 +43,21 @@ type Options struct { // Secret key for the s3 blobstore S3SecretKey string `mapstructure:"s3.secret_key"` + + // UploadObjectPrefix for the s3 blobstore + S3UploadObjectPrefix string `mapstructure:"s3.upload_object_prefix"` + + // UploadMetadataPrefix for the s3 blobstore + S3UploadMetadataPrefix string `mapstructure:"s3.upload_metadata_prefix"` + + // UploadTemporaryDirectory for the s3 blobstore + S3UploadTemporaryDirectory string `mapstructure:"s3.upload_temporary_directory"` + + // DisableSSL for the s3 blobstore + S3DisableSSL bool `mapstructure:"s3.disable_ssl"` + + // ForcePathStyle for the s3 blobstore + S3ForcePathStyle bool `mapstructure:"s3.force_path_style"` } // S3ConfigComplete return true if all required s3 fields are set diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go index 5cc7f8873b6..ef5eca87a88 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go @@ -21,11 +21,16 @@ package s3ng import ( "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/fs/registry" "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" + "github.com/tus/tusd/pkg/s3store" ) func init() { @@ -49,5 +54,17 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { return nil, err } - return decomposedfs.NewDefault(m, bs, stream) + s3Config := aws.NewConfig() + s3Config.WithCredentials(credentials.NewStaticCredentials(o.S3AccessKey, o.S3SecretKey, "")). + WithEndpoint(o.S3Endpoint). + WithRegion(o.S3Region). + WithS3ForcePathStyle(o.S3ForcePathStyle). + WithDisableSSL(o.S3DisableSSL) + + tusDataStore := s3store.New(o.S3Bucket, s3.New(session.Must(session.NewSession()), s3Config)) + tusDataStore.ObjectPrefix = o.S3UploadObjectPrefix + tusDataStore.MetadataObjectPrefix = o.S3UploadMetadataPrefix + tusDataStore.TemporaryDirectory = o.S3UploadTemporaryDirectory + + return decomposedfs.NewDefault(m, bs, tusDataStore, stream) } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 5c341c73570..e42d707c145 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -34,7 +34,6 @@ import ( "strings" "time" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" @@ -61,6 +60,7 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" "github.com/jellydator/ttlcache/v2" "github.com/pkg/errors" + tusHandler "github.com/tus/tusd/pkg/handler" microstore "go-micro.dev/v4/store" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -96,10 +96,6 @@ type Tree interface { RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, target *node.Node) (*node.Node, *node.Node, func() error, error) PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*node.Node, func() error, error) - WriteBlob(node *node.Node, source string) error - ReadBlob(node *node.Node) (io.ReadCloser, error) - DeleteBlob(node *node.Node) error - Propagate(ctx context.Context, node *node.Node, sizeDiff int64) (err error) } @@ -112,6 +108,8 @@ type Decomposedfs struct { chunkHandler *chunking.ChunkHandler stream events.Stream cache cache.StatCache + tusDataStore tusHandler.DataStore + blobstore tree.Blobstore UserCache *ttlcache.Cache userSpaceIndex *spaceidindex.Index @@ -120,7 +118,7 @@ type Decomposedfs struct { } // NewDefault returns an instance with default components -func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) (storage.FS, error) { +func NewDefault(m map[string]interface{}, bs tree.Blobstore, tusDataStore tusHandler.DataStore, es events.Stream) (storage.FS, error) { o, err := options.New(m) if err != nil { return nil, err @@ -152,12 +150,12 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( permissions := NewPermissions(node.NewPermissions(lu), permissionsSelector) - return New(o, lu, permissions, tp, es) + return New(o, lu, permissions, tp, es, tusDataStore, bs) } // New returns an implementation of the storage.FS interface that talks to // a local filesystem. -func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es events.Stream) (storage.FS, error) { +func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es events.Stream, tusDataStore tusHandler.DataStore, blobstore tree.Blobstore) (storage.FS, error) { log := logger.New() err := tp.Setup() if err != nil { @@ -208,6 +206,8 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event userSpaceIndex: userSpaceIndex, groupSpaceIndex: groupSpaceIndex, spaceTypeIndex: spaceTypeIndex, + tusDataStore: tusDataStore, + blobstore: blobstore, } if o.AsyncFileUploads { @@ -226,258 +226,13 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event } for i := 0; i < o.Events.NumConsumers; i++ { - go fs.Postprocessing(ch) + go upload.Postprocessing(lu, tp, fs.cache, es, tusDataStore, blobstore, fs.downloadURL, ch) } } return fs, nil } -// Postprocessing starts the postprocessing result collector -func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { - ctx := context.TODO() // we should pass the trace id in the event and initialize the trace provider here - ctx, span := tracer.Start(ctx, "Postprocessing") - defer span.End() - log := logger.New() - for event := range ch { - switch ev := event.Event.(type) { - case events.PostprocessingFinished: - up, err := upload.Get(ctx, ev.UploadID, fs.lu, fs.tp, fs.o.Root, fs.stream, fs.o.AsyncFileUploads, fs.o.Tokens) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") - continue // NOTE: since we can't get the upload, we can't delete the blob - } - - var ( - failed bool - keepUpload bool - ) - - n, err := node.ReadNode(ctx, fs.lu, up.Info.Storage["SpaceRoot"], up.Info.Storage["NodeId"], false, nil, true) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not read node") - continue - } - up.Node = n - - switch ev.Outcome { - default: - log.Error().Str("outcome", string(ev.Outcome)).Str("uploadID", ev.UploadID).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - keepUpload = true - case events.PPOutcomeContinue: - if err := up.Finalize(); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not finalize upload") - keepUpload = true // should we keep the upload when assembling failed? - failed = true - } - case events.PPOutcomeDelete: - failed = true - } - - getParent := func() *node.Node { - p, err := up.Node.Parent(ctx) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not read parent") - return nil - } - return p - } - - now := time.Now() - if failed { - // propagate sizeDiff after failed postprocessing - if err := fs.tp.Propagate(ctx, up.Node, -up.SizeDiff); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not propagate tree size change") - } - } else if p := getParent(); p != nil { - // update parent tmtime to propagate etag change after successful postprocessing - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not propagate etag change") - } - } - - upload.Cleanup(up, failed, keepUpload) - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - - if err := events.Publish( - ctx, - fs.stream, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: up.Info.MetaData["providerID"], - SpaceId: up.Info.Storage["SpaceRoot"], - OpaqueId: up.Info.Storage["SpaceRoot"], - }, - Path: utils.MakeRelativePath(filepath.Join(up.Info.MetaData["dir"], up.Info.MetaData["filename"])), - }, - Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), - }, - ); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to publish UploadReady event") - } - case events.RestartPostprocessing: - up, err := upload.Get(ctx, ev.UploadID, fs.lu, fs.tp, fs.o.Root, fs.stream, fs.o.AsyncFileUploads, fs.o.Tokens) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") - continue - } - n, err := node.ReadNode(ctx, fs.lu, up.Info.Storage["SpaceRoot"], up.Info.Storage["NodeId"], false, nil, true) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not read node") - continue - } - s, err := up.URL(up.Ctx) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not create url") - continue - } - // restart postprocessing - if err := events.Publish(ctx, fs.stream, events.BytesReceived{ - UploadID: up.Info.ID, - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(up.Ctx), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: up.Info.Storage["NodeName"], - Filesize: uint64(up.Info.Size), - }); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to publish BytesReceived event") - } - case events.PostprocessingStepFinished: - if ev.FinishedStep != events.PPStepAntivirus { - // atm we are only interested in antivirus results - continue - } - - res := ev.Result.(events.VirusscanResult) - if res.ErrorMsg != "" { - // scan failed somehow - // Should we handle this here? - continue - } - - var n *node.Node - switch ev.UploadID { - case "": - // uploadid is empty -> this was an on-demand scan - /* ON DEMAND SCANNING NOT SUPPORTED ATM - ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) - ref := &provider.Reference{ResourceId: ev.ResourceID} - - no, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") - continue - - } - n = no - if ev.Outcome == events.PPOutcomeDelete { - // antivir wants us to delete the file. We must obey and need to - - // check if there a previous versions existing - revs, err := fs.ListRevisions(ctx, ref) - if len(revs) == 0 { - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") - } - - // no versions -> trash file - err := fs.Delete(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") - continue - } - - // now purge it from the recycle bin - if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - - // we have versions - find the newest - versions := make(map[uint64]string) // remember all versions - we need them later - var nv uint64 - for _, v := range revs { - versions[v.Mtime] = v.Key - if v.Mtime > nv { - nv = v.Mtime - } - } - - // restore newest version - if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") - continue - } - - // now find infected version - revs, err = fs.ListRevisions(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") - } - - for _, v := range revs { - // we looking for a version that was previously not there - if _, ok := versions[v.Mtime]; ok { - continue - } - - if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") - } - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - */ - default: - // uploadid is not empty -> this is an async upload - up, err := upload.Get(ctx, ev.UploadID, fs.lu, fs.tp, fs.o.Root, fs.stream, fs.o.AsyncFileUploads, fs.o.Tokens) - if err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") - continue - } - - no, err := node.ReadNode(up.Ctx, fs.lu, up.Info.Storage["SpaceRoot"], up.Info.Storage["NodeId"], false, nil, false) - if err != nil { - log.Error().Err(err).Interface("uploadID", ev.UploadID).Msg("Failed to get node after scan") - continue - } - - n = no - } - - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - log.Error().Err(err).Str("uploadID", ev.UploadID).Interface("resourceID", res.ResourceID).Msg("Failed to set scan results") - continue - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - default: - log.Error().Interface("event", ev).Msg("Unknown event") - } - } -} - // Shutdown shuts down the storage func (fs *Decomposedfs) Shutdown(ctx context.Context) error { return nil @@ -1027,9 +782,12 @@ func (fs *Decomposedfs) Download(ctx context.Context, ref *provider.Reference) ( if currentEtag != expectedEtag { return nil, errtypes.Aborted(fmt.Sprintf("file changed from etag %s to %s", expectedEtag, currentEtag)) } - reader, err := fs.tp.ReadBlob(n) + if n.Blobsize == 0 { + return io.NopCloser(strings.NewReader("")), nil + } + reader, err := fs.blobstore.Download(n) if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error download blob '"+n.ID+"'") + return nil, errors.Wrap(err, "Decomposedfs: error downloading blob '"+n.BlobID+"' for node '"+n.ID+"'") } return reader, nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go index 9e0ffd55b14..79efb1f3fc8 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -273,6 +273,11 @@ func (lu *Lookup) InternalPath(spaceID, nodeID string) string { return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2)) } +// UploadPath returns the upload path for a given upload ID +func (lu *Lookup) UploadPath(uploadID string) string { + return filepath.Join(lu.Options.Root, "uploads", uploadID+".mpk") +} + // // ReferenceFromAttr returns a CS3 reference from xattr of a node. // // Supported formats are: "cs3:storageid/nodeid" // func ReferenceFromAttr(b []byte) (*provider.Reference, error) { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index 7c60043590c..ef36c9c2006 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -159,7 +159,7 @@ func (XattrsBackend) Remove(ctx context.Context, filePath string, key string, ac } // IsMetaFile returns whether the given path represents a meta file -func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".meta.lock") } +func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".mlock") } // Purge purges the data of a given path func (XattrsBackend) Purge(path string) error { return nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go index 3eb29ea4d54..add903eb48b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go @@ -72,8 +72,7 @@ const ( QuotaUnknown = "-2" // TrashIDDelimiter represents the characters used to separate the nodeid and the deletion time. - TrashIDDelimiter = ".T." - RevisionIDDelimiter = ".REV." + TrashIDDelimiter = ".T." // RootID defines the root node's ID RootID = "root" @@ -114,6 +113,7 @@ func New(spaceID, id, parentID, name string, blobsize int64, blobID string, t pr if blobID == "" { blobID = uuid.New().String() } + // hm but dirs have no blob id return &Node{ SpaceID: spaceID, ID: id, @@ -127,6 +127,40 @@ func New(spaceID, id, parentID, name string, blobsize int64, blobID string, t pr } } +// RevisionNode will return a node for the revision without reading the metadata +func (n *Node) RevisionNode(ctx context.Context, revision string) *Node { + return &Node{ + SpaceID: n.SpaceID, + ID: JoinRevisionKey(n.ID, revision), + ParentID: n.ParentID, + Name: n.Name, + owner: n.owner, + lu: n.lu, + nodeType: n.nodeType, + } +} + +// ReadRevision will return a node for the revision and read the metadata +func (n *Node) ReadRevision(ctx context.Context, revision string) (*Node, error) { + rn := n.RevisionNode(ctx, revision) + attrs, err := rn.Xattrs(ctx) + switch { + case metadata.IsNotExist(err): + return rn, nil // swallow not found, the node defaults to exists = false + case err != nil: + return nil, err + } + rn.Exists = true + + rn.BlobID = attrs.String(prefixes.BlobIDAttr) + rn.Blobsize, err = attrs.Int64(prefixes.BlobsizeAttr) + if err != nil { + return nil, err + } + + return rn, nil +} + // Type returns the node's resource type func (n *Node) Type(ctx context.Context) provider.ResourceType { if n.nodeType != nil { @@ -257,17 +291,8 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis } // are we reading a revision? - revisionSuffix := "" - if strings.Contains(nodeID, RevisionIDDelimiter) { - // verify revision key format - kp := strings.SplitN(nodeID, RevisionIDDelimiter, 2) - if len(kp) == 2 { - // use the actual node for the metadata lookup - nodeID = kp[0] - // remember revision for blob metadata - revisionSuffix = RevisionIDDelimiter + kp[1] - } - } + var revision string + nodeID, revision = SplitRevisionKey(nodeID) // read node n := &Node{ @@ -282,7 +307,9 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis defer func() { // when returning errors n is nil if n != nil { - n.ID += revisionSuffix + if revision != "" { + n.ID = JoinRevisionKey(n.ID, revision) + } } }() @@ -305,7 +332,7 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis return nil, errtypes.InternalError("Missing parent ID on node") } - if revisionSuffix == "" { + if revision == "" { n.BlobID = attrs.String(prefixes.BlobIDAttr) if n.BlobID != "" { blobSize, err := attrs.Int64(prefixes.BlobsizeAttr) @@ -315,13 +342,13 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis n.Blobsize = blobSize } } else { - n.BlobID, err = lu.ReadBlobIDAttr(ctx, nodePath+revisionSuffix) + n.BlobID, err = lu.ReadBlobIDAttr(ctx, JoinRevisionKey(nodePath, revision)) if err != nil { return nil, err } // Lookup blobsize - n.Blobsize, err = lu.ReadBlobSizeAttr(ctx, nodePath+revisionSuffix) + n.Blobsize, err = lu.ReadBlobSizeAttr(ctx, JoinRevisionKey(nodePath, revision)) if err != nil { return nil, err } @@ -895,7 +922,7 @@ func (n *Node) GetTMTime(ctx context.Context) (time.Time, error) { // GetMTime reads the mtime from the extended attributes, falling back to disk func (n *Node) GetMTime(ctx context.Context) (time.Time, error) { b, err := n.XattrString(ctx, prefixes.MTimeAttr) - if err != nil { + if err != nil || len(b) == 0 { fi, err := os.Lstat(n.InternalPath()) if err != nil { return time.Time{}, err diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go index 84814a1641f..6ebfc07fd79 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go @@ -20,7 +20,6 @@ package node import ( "context" - "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -132,17 +131,7 @@ func (p *Permissions) assemblePermissions(ctx context.Context, n *Node, failOnTr if u.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE { return ServiceAccountPermissions(), nil } - - // are we reading a revision? - if strings.Contains(n.ID, RevisionIDDelimiter) { - // verify revision key format - kp := strings.SplitN(n.ID, RevisionIDDelimiter, 2) - if len(kp) != 2 { - return NoPermissions(), errtypes.NotFound(n.ID) - } - // use the actual node for the permission assembly - n.ID = kp[0] - } + n.ID, _ = SplitRevisionKey(n.ID) // determine root rn := n.SpaceRoot diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/revisions.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/revisions.go new file mode 100644 index 00000000000..084080971ae --- /dev/null +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/revisions.go @@ -0,0 +1,40 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package node + +import "strings" + +// Define keys and values used in the node metadata +const ( + RevisionIDDelimiter = ".REV." +) + +// SplitRevisionKey splits revision key into nodeid and revisionTime +func SplitRevisionKey(revisionKey string) (string, string) { + parts := strings.SplitN(revisionKey, RevisionIDDelimiter, 2) + if len(parts) != 2 { + return revisionKey, "" + } + return parts[0], parts[1] +} + +// JoinRevisionKey joins nodeid and revision into revision key +func JoinRevisionKey(nodeID, revision string) string { + return nodeID + RevisionIDDelimiter + revision +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/revisions.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/revisions.go index e078d78bacc..0203b16d058 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/revisions.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/revisions.go @@ -50,6 +50,8 @@ func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Referen if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { return } + sublog := appctx.GetLogger(ctx).With().Str("node", n.ID).Logger() + if !n.Exists { err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) return @@ -69,35 +71,44 @@ func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Referen revisions = []*provider.FileVersion{} np := n.InternalPath() - if items, err := filepath.Glob(np + node.RevisionIDDelimiter + "*"); err == nil { + if items, err := filepath.Glob(node.JoinRevisionKey(np, "*")); err == nil { for i := range items { - if fs.lu.MetadataBackend().IsMetaFile(items[i]) || strings.HasSuffix(items[i], ".mlock") { + if fs.lu.MetadataBackend().IsMetaFile(items[i]) { + continue + } + _, revision := node.SplitRevisionKey(items[i]) + if revision == "" { + sublog.Err(err).Str("revisionKey", items[i]).Msg("invalid revision key, skipping") + continue + } + sublog = sublog.With().Str("revision", revision).Logger() + rn, err := n.ReadRevision(ctx, revision) + if err != nil { + sublog.Error().Err(err).Msg("could not read revision, skipping") + continue + } + if !rn.Exists { + sublog.Error().Msg("revision does not exist, skipping") continue } - if fi, err := os.Stat(items[i]); err == nil { - parts := strings.SplitN(fi.Name(), node.RevisionIDDelimiter, 2) - if len(parts) != 2 { - appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("invalid revision name, skipping") - continue - } - mtime := fi.ModTime() - rev := &provider.FileVersion{ - Key: n.ID + node.RevisionIDDelimiter + parts[1], - Mtime: uint64(mtime.Unix()), - } - blobSize, err := fs.lu.ReadBlobSizeAttr(ctx, items[i]) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0") - } - rev.Size = uint64(blobSize) - etag, err := node.CalculateEtag(n, mtime) - if err != nil { - return nil, errors.Wrapf(err, "error calculating etag") - } - rev.Etag = etag - revisions = append(revisions, rev) + rmtime, err := rn.GetMTime(ctx) + if err != nil { + sublog.Error().Err(err).Msg("could not get revision mtime, skipping") + continue + } + etag, err := node.CalculateEtag(rn, rmtime) + if err != nil { + sublog.Error().Err(err).Msg("could not calculate etag, skipping") + continue } + rev := &provider.FileVersion{ + Key: rn.ID, + Mtime: uint64(rmtime.Unix()), + Size: uint64(rn.Blobsize), + Etag: etag, + } + revisions = append(revisions, rev) } } // maybe we need to sort the list by key @@ -113,19 +124,16 @@ func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Referen // DownloadRevision returns a reader for the specified revision // FIXME the CS3 api should explicitly allow initiating revision and trash download, a related issue is https://github.com/cs3org/reva/issues/1813 func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") + nodeID, revision := node.SplitRevisionKey(revisionKey) + if revision == "" { return nil, errtypes.NotFound(revisionKey) } - log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") + sublog := appctx.GetLogger(ctx).With().Str("node", nodeID).Str("revision", revision).Logger() + sublog.Debug().Msg("DownloadRevision") spaceID := ref.ResourceId.SpaceId // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, false, nil, false) if err != nil { return nil, err } @@ -146,40 +154,29 @@ func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Refe return nil, errtypes.NotFound(f) } - contentPath := fs.lu.InternalPath(spaceID, revisionKey) - - blobid, err := fs.lu.ReadBlobIDAttr(ctx, contentPath) + revisionNode, err := n.ReadRevision(ctx, revision) if err != nil { - return nil, errors.Wrapf(err, "Decomposedfs: could not read blob id of revision '%s' for node '%s'", n.ID, revisionKey) + return nil, errors.Wrapf(err, "Decomposedfs: could not read revision '%s' for node '%s'", revision, n.ID) } - blobsize, err := fs.lu.ReadBlobSizeAttr(ctx, contentPath) + reader, err := fs.blobstore.Download(revisionNode) if err != nil { - return nil, errors.Wrapf(err, "Decomposedfs: could not read blob size of revision '%s' for node '%s'", n.ID, revisionKey) - } - - revisionNode := node.Node{SpaceID: spaceID, BlobID: blobid, Blobsize: blobsize} // blobsize is needed for the s3ng blobstore - - reader, err := fs.tp.ReadBlob(&revisionNode) - if err != nil { - return nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", n.ID, revisionKey) + return nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", revision, n.ID) } return reader, nil } // RestoreRevision restores the specified revision of the resource func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (returnErr error) { - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") + nodeID, revision := node.SplitRevisionKey(revisionKey) + if revision == "" { return errtypes.NotFound(revisionKey) } + sublog := appctx.GetLogger(ctx).With().Str("node", nodeID).Str("revision", revision).Logger() + sublog.Debug().Msg("RestoreRevision") spaceID := ref.ResourceId.SpaceId // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, false, nil, false) if err != nil { return err } @@ -215,16 +212,15 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer } defer f.Close() - // move current version to new revision - nodePath := fs.lu.InternalPath(spaceID, kp[0]) + nodePath := fs.lu.InternalPath(spaceID, nodeID) mtime, err := n.GetMTime(ctx) if err != nil { - log.Error().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("cannot read mtime") + sublog.Error().Err(err).Interface("ref", ref).Msg("cannot read mtime") return err } // revisions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - newRevisionPath := fs.lu.InternalPath(spaceID, kp[0]+node.RevisionIDDelimiter+mtime.UTC().Format(time.RFC3339Nano)) + newRevisionPath := fs.lu.InternalPath(spaceID, node.JoinRevisionKey(nodeID, mtime.Format(time.RFC3339Nano))) // touch new revision if _, err := os.Create(newRevisionPath); err != nil { @@ -233,10 +229,10 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer defer func() { if returnErr != nil { if err := os.Remove(newRevisionPath); err != nil { - log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") + sublog.Error().Err(err).Str("newRevision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") } if err := fs.lu.MetadataBackend().Purge(newRevisionPath); err != nil { - log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") + sublog.Error().Err(err).Str("newRevision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") } } }() @@ -283,16 +279,16 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer // drop old revision if err := os.Remove(restoredRevisionPath); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision, continuing") + sublog.Warn().Err(err).Interface("ref", ref).Msg("could not delete old revision, continuing") } if err := os.Remove(fs.lu.MetadataBackend().MetadataPath(restoredRevisionPath)); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision metadata, continuing") + sublog.Warn().Err(err).Interface("ref", ref).Msg("could not delete old revision metadata, continuing") } if err := os.Remove(fs.lu.MetadataBackend().LockfilePath(restoredRevisionPath)); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision metadata lockfile, continuing") + sublog.Warn().Err(err).Interface("ref", ref).Msg("could not delete old revision metadata lockfile, continuing") } if err := fs.lu.MetadataBackend().Purge(restoredRevisionPath); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not purge old revision from cache, continuing") + sublog.Warn().Err(err).Interface("ref", ref).Msg("could not purge old revision from cache, continuing") } // revision 5, current 10 (restore a smaller blob) -> 5-10 = -5 @@ -315,23 +311,20 @@ func (fs *Decomposedfs) DeleteRevision(ctx context.Context, ref *provider.Refere return err } - return fs.tp.DeleteBlob(n) + return fs.blobstore.Delete(n) } func (fs *Decomposedfs) getRevisionNode(ctx context.Context, ref *provider.Reference, revisionKey string, hasPermission func(*provider.ResourcePermissions) bool) (*node.Node, error) { - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") + nodeID, revision := node.SplitRevisionKey(revisionKey) + if revision == "" { return nil, errtypes.NotFound(revisionKey) } - log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") + sublog := appctx.GetLogger(ctx).With().Str("node", nodeID).Str("revision", revision).Logger() + sublog.Debug().Msg("getRevisionNode") spaceID := ref.ResourceId.SpaceId // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, false, nil, false) if err != nil { return nil, err } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go index 4d9add38951..46607533e76 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go @@ -2,6 +2,7 @@ package spaceidindex import ( "io" + "io/fs" "os" "path/filepath" "time" @@ -44,6 +45,9 @@ func (i *Index) Load(index string) (map[string]string, error) { indexPath := filepath.Join(i.root, i.name, index+".mpk") fi, err := os.Stat(indexPath) if err != nil { + if _, ok := err.(*fs.PathError); ok { + return map[string]string{}, nil + } return nil, err } return i.readSpaceIndex(indexPath, i.name+":"+index, fi.ModTime()) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/blobstore.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/blobstore.go new file mode 100644 index 00000000000..589ea8c39ce --- /dev/null +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/blobstore.go @@ -0,0 +1,42 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package tree + +import ( + "errors" + "io" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" +) + +//go:generate make --no-print-directory -C ../../../../.. mockery NAME=Blobstore + +// Blobstore defines an interface for storing blobs in a blobstore +type Blobstore interface { + Upload(node *node.Node, source string) error + Download(node *node.Node) (io.ReadCloser, error) + Delete(node *node.Node) error +} + +// BlobstoreMover is used to move a file from the upload to the final destination +type BlobstoreMover interface { + MoveBlob(n *node.Node, source, bucket, key string) error +} + +var ErrBlobstoreCannotMove = errors.New("cannot move") diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go index e0610ed5d50..e6832f4271b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go @@ -19,10 +19,7 @@ package tree import ( - "bytes" "context" - "fmt" - "io" "io/fs" iofs "io/fs" "os" @@ -42,7 +39,6 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" "github.com/google/uuid" "github.com/pkg/errors" - "github.com/rs/zerolog/log" "go-micro.dev/v4/store" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -55,15 +51,6 @@ func init() { tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree") } -//go:generate make --no-print-directory -C ../../../../.. mockery NAME=Blobstore - -// Blobstore defines an interface for storing blobs in a blobstore -type Blobstore interface { - Upload(node *node.Node, source string) error - Download(node *node.Node) (io.ReadCloser, error) - Delete(node *node.Node) error -} - // Tree manages a hierarchical tree type Tree struct { lookup lookup.PathLookup @@ -391,7 +378,8 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, nodeID, false, n.SpaceRoot, true) if err != nil { - return err + appctx.GetLogger(ctx).Error().Err(err).Str("space", n.SpaceID).Str("node", nodeID).Msg("cannot read node") + continue } // prevent listing denied resources @@ -598,7 +586,7 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa deletePath = filepath.Join(resolvedTrashRoot, trashPath) } if err = os.Remove(deletePath); err != nil { - log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trash item") + appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trash item") } var sizeDiff int64 @@ -638,7 +626,7 @@ func (t *Tree) PurgeRecycleItemFunc(ctx context.Context, spaceid, key string, pa deletePath = filepath.Join(resolvedTrashRoot, path) } if err = os.Remove(deletePath); err != nil { - log.Error().Err(err).Str("deletePath", deletePath).Msg("error deleting trash item") + appctx.GetLogger(ctx).Error().Err(err).Str("deletePath", deletePath).Msg("error deleting trash item") return err } @@ -649,6 +637,7 @@ func (t *Tree) PurgeRecycleItemFunc(ctx context.Context, spaceid, key string, pa } func (t *Tree) removeNode(ctx context.Context, path string, n *node.Node) error { + log := appctx.GetLogger(ctx) // delete the actual node if err := utils.RemoveItem(path); err != nil { log.Error().Err(err).Str("path", path).Msg("error purging node") @@ -662,16 +651,16 @@ func (t *Tree) removeNode(ctx context.Context, path string, n *node.Node) error // delete blob from blobstore if n.BlobID != "" { - if err := t.DeleteBlob(n); err != nil { + if err := t.blobstore.Delete(n); err != nil { log.Error().Err(err).Str("blobID", n.BlobID).Msg("error purging nodes blob") return err } } // delete revisions - revs, err := filepath.Glob(n.InternalPath() + node.RevisionIDDelimiter + "*") + revs, err := filepath.Glob(node.JoinRevisionKey(n.InternalPath(), "*")) if err != nil { - log.Error().Err(err).Str("path", n.InternalPath()+node.RevisionIDDelimiter+"*").Msg("glob failed badly") + log.Error().Err(err).Str("node", n.ID).Msg("glob failed badly") return err } for _, rev := range revs { @@ -691,7 +680,7 @@ func (t *Tree) removeNode(ctx context.Context, path string, n *node.Node) error } if bID != "" { - if err := t.DeleteBlob(&node.Node{SpaceID: n.SpaceID, BlobID: bID}); err != nil { + if err := t.blobstore.Delete(&node.Node{SpaceID: n.SpaceID, BlobID: bID}); err != nil { log.Error().Err(err).Str("revision", rev).Str("blobID", bID).Msg("error removing revision node blob") return err } @@ -707,32 +696,6 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err return t.propagator.Propagate(ctx, n, sizeDiff) } -// WriteBlob writes a blob to the blobstore -func (t *Tree) WriteBlob(node *node.Node, source string) error { - return t.blobstore.Upload(node, source) -} - -// ReadBlob reads a blob from the blobstore -func (t *Tree) ReadBlob(node *node.Node) (io.ReadCloser, error) { - if node.BlobID == "" { - // there is no blob yet - we are dealing with a 0 byte file - return io.NopCloser(bytes.NewReader([]byte{})), nil - } - return t.blobstore.Download(node) -} - -// DeleteBlob deletes a blob from the blobstore -func (t *Tree) DeleteBlob(node *node.Node) error { - if node == nil { - return fmt.Errorf("could not delete blob, nil node was given") - } - if node.BlobID == "" { - return fmt.Errorf("could not delete blob, node with empty blob id was given") - } - - return t.blobstore.Delete(node) -} - // TODO check if node exists? func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { ctx, span := tracer.Start(ctx, "createDirNode") @@ -817,7 +780,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( if attrBytes, err = backend.Get(ctx, resolvedTrashItem, prefixes.TrashOriginAttr); err == nil { origin = filepath.Join(string(attrBytes), path) } else { - log.Error().Err(err).Str("trashItem", trashItem).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") + appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") } return diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload.go index 59ef12a4e17..67000008ff1 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload.go @@ -20,234 +20,662 @@ package decomposedfs import ( "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" "fmt" + "hash" + "hash/adler32" + "io" + "net/url" "os" "path/filepath" "regexp" "strings" "time" + "github.com/golang-jwt/jwt" tusd "github.com/tus/tusd/pkg/handler" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/internal/grpc/services/storageprovider" "github.com/cs3org/reva/v2/pkg/appctx" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/logger" "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/utils/chunking" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload" + "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) -var _idRegexp = regexp.MustCompile(".*/([^/]+).info") +var _idRegexp = regexp.MustCompile(".*/([^/]+).mpk") -// Upload uploads data to the given resource -// TODO Upload (and InitiateUpload) needs a way to receive the expected checksum. -// Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (provider.ResourceInfo, error) { - up, err := fs.GetUpload(ctx, req.Ref.GetPath()) +// InitiateUpload returns upload ids corresponding to different protocols it supports +// It creates a node for new files to persist the fileid for the new child. +// TODO read optional content for small files in this request +// TODO InitiateUpload (and Upload) needs a way to receive the expected checksum. Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? +// TODO needs a way to handle unknown filesize, currently uses the context +// FIXME headers is actually used to carry all kinds of headers +func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, headers map[string]string) (map[string]string, error) { + + n, err := fs.lu.NodeFromResource(ctx, ref) + switch err.(type) { + case nil: + // ok + case errtypes.IsNotFound: + return nil, errtypes.PreconditionFailed(err.Error()) + default: + return nil, err + } + + sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Int64("uploadLength", uploadLength).Interface("headers", headers).Logger() + + // permissions are checked in NewUpload below + + relative, err := fs.lu.Path(ctx, n, node.NoCheck) if err != nil { - return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload") + return nil, err } - uploadInfo := up.(*upload.Upload) + usr := ctxpkg.ContextMustGetUser(ctx) + uploadMetadata := upload.Metadata{ + Filename: n.Name, + SpaceRoot: n.SpaceRoot.ID, + SpaceOwnerOrManager: n.SpaceOwnerOrManager(ctx).GetOpaqueId(), + ProviderID: headers["providerID"], + MTime: time.Now().UTC().Format(time.RFC3339Nano), + NodeID: n.ID, + NodeParentID: n.ParentID, + ExecutantIdp: usr.Id.Idp, + ExecutantID: usr.Id.OpaqueId, + ExecutantType: utils.UserTypeToString(usr.Id.Type), + ExecutantUserName: usr.Username, + LogLevel: sublog.GetLevel().String(), + } - p := uploadInfo.Info.Storage["NodeName"] - if chunking.IsChunked(p) { // check chunking v1 - var assembledFile string - p, assembledFile, err = fs.chunkHandler.WriteChunk(p, req.Body) - if err != nil { - return provider.ResourceInfo{}, err + tusMetadata := tusd.MetaData{} + + // checksum is sent as tus Upload-Checksum header and should not magically become a metadata property + if checksum, ok := headers["checksum"]; ok { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") } - if p == "" { - if err = uploadInfo.Terminate(ctx); err != nil { - return provider.ResourceInfo{}, errors.Wrap(err, "ocfs: error removing auxiliary files") - } - return provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) + switch parts[0] { + case "sha1", "md5", "adler32": + uploadMetadata.Checksum = checksum + default: + return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) } - uploadInfo.Info.Storage["NodeName"] = p - fd, err := os.Open(assembledFile) + } + + // if mtime has been set via the headers, expose it as tus metadata + if ocmtime, ok := headers["mtime"]; ok && ocmtime != "null" { + tusMetadata["mtime"] = ocmtime + // overwrite mtime if requested + mtime, err := utils.MTimeToTime(ocmtime) if err != nil { - return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error opening assembled file") + return nil, err } - defer fd.Close() - defer os.RemoveAll(assembledFile) - req.Body = fd + uploadMetadata.MTime = mtime.UTC().Format(time.RFC3339Nano) } - if _, err := uploadInfo.WriteChunk(ctx, 0, req.Body); err != nil { - return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") + _, err = node.CheckQuota(ctx, n.SpaceRoot, n.Exists, uint64(n.Blobsize), uint64(uploadLength)) + if err != nil { + return nil, err } - if err := uploadInfo.FinishUpload(ctx); err != nil { - return provider.ResourceInfo{}, err + // check permissions + var ( + checkNode *node.Node + path string + ) + if n.Exists { + // check permissions of file to be overwritten + checkNode = n + path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ + SpaceId: checkNode.SpaceID, + OpaqueId: checkNode.ID, + }}) + } else { + // check permissions of parent + parent, perr := n.Parent(ctx) + if perr != nil { + return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID) + } + checkNode = parent + path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ + SpaceId: checkNode.SpaceID, + OpaqueId: checkNode.ID, + }, Path: n.Name}) + } + rp, err := fs.p.AssemblePermissions(ctx, checkNode) // context does not have a user? + switch { + case err != nil: + return nil, err + case !rp.InitiateFileUpload: + return nil, errtypes.PermissionDenied(path) } - if uff != nil { - info := uploadInfo.Info - uploadRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: info.MetaData["providerID"], - SpaceId: info.Storage["SpaceRoot"], - OpaqueId: info.Storage["SpaceRoot"], - }, - Path: utils.MakeRelativePath(filepath.Join(info.MetaData["dir"], info.MetaData["filename"])), - } - executant, ok := ctxpkg.ContextGetUser(uploadInfo.Ctx) - if !ok { - return provider.ResourceInfo{}, errtypes.PreconditionFailed("error getting user from uploadinfo context") + // are we trying to overwrite a folder with a file? + if n.Exists && n.IsDir(ctx) { + return nil, errtypes.PreconditionFailed("resource is not a file") + } + + // check lock + // FIXME we cannot check the lock of a new file, because it would have to use the name ... + if err := n.CheckLock(ctx); err != nil { + return nil, err + } + + info := tusd.FileInfo{ + MetaData: tusMetadata, + Size: uploadLength, + SizeIsDeferred: uploadLength == 0, // treat 0 length uploads as deferred + } + if lockID, ok := ctxpkg.ContextGetLockID(ctx); ok { + uploadMetadata.LockID = lockID + } + uploadMetadata.Dir = filepath.Dir(relative) + + // rewrite filename for old chunking v1 + if chunking.IsChunked(n.Name) { + uploadMetadata.Chunk = n.Name + bi, err := chunking.GetChunkBLOBInfo(n.Name) + if err != nil { + return nil, err } - spaceOwner := &userpb.UserId{ - OpaqueId: info.Storage["SpaceOwnerOrManager"], + n.Name = bi.Path + } + + // TODO at this point we have no way to figure out the output or mode of the logger. we need that to reinitialize a logger in PreFinishResponseCallback + // or better create a config option for the log level during PreFinishResponseCallback? might be easier for now + + // expires has been set by the storageprovider, do not expose as metadata. It is sent as a tus Upload-Expires header + if expiration, ok := headers["expires"]; ok && expiration != "null" { // TODO this is set by the storageprovider ... it cannot be set by cliensts, so it can never be the string 'null' ... or can it??? + uploadMetadata.Expires, err = utils.MTimeToTime(expiration) + if err != nil { + return nil, err } - uff(spaceOwner, executant.Id, uploadRef) + } + // only check preconditions if they are not empty + // do not expose as metadata + if headers["if-match"] != "" { + uploadMetadata.HeaderIfMatch = headers["if-match"] // TODO drop? + } + if headers["if-none-match"] != "" { + uploadMetadata.HeaderIfNoneMatch = headers["if-none-match"] + } + if headers["if-unmodified-since"] != "" { + uploadMetadata.HeaderIfUnmodifiedSince = headers["if-unmodified-since"] } - ri := provider.ResourceInfo{ - // fill with at least fileid, mtime and etag - Id: &provider.ResourceId{ - StorageId: uploadInfo.Info.MetaData["providerID"], - SpaceId: uploadInfo.Info.Storage["SpaceRoot"], - OpaqueId: uploadInfo.Info.Storage["NodeId"], - }, - Etag: uploadInfo.Info.MetaData["etag"], + if uploadMetadata.HeaderIfNoneMatch == "*" && n.Exists { + return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s", n.ID, n.Name)) } - if mtime, err := utils.MTimeToTS(uploadInfo.Info.MetaData["mtime"]); err == nil { - ri.Mtime = &mtime + // create the upload + u, err := fs.tusDataStore.NewUpload(ctx, info) + if err != nil { + return nil, err } - return ri, nil -} + info, err = u.GetInfo(ctx) + if err != nil { + return nil, err + } -// InitiateUpload returns upload ids corresponding to different protocols it supports -// TODO read optional content for small files in this request -// TODO InitiateUpload (and Upload) needs a way to receive the expected checksum. Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - log := appctx.GetLogger(ctx) + uploadMetadata.ID = info.ID - n, err := fs.lu.NodeFromResource(ctx, ref) - switch err.(type) { - case nil: - // ok - case errtypes.IsNotFound: - return nil, errtypes.PreconditionFailed(err.Error()) - default: + // keep track of upload + err = upload.WriteMetadata(ctx, fs.lu, info.ID, uploadMetadata) + if err != nil { return nil, err } - // permissions are checked in NewUpload below + sublog.Debug().Interface("info", info).Msg("Decomposedfs: initiated upload") - relative, err := fs.lu.Path(ctx, n, node.NoCheck) + return map[string]string{ + "simple": info.ID, + "tus": info.ID, + }, nil +} + +// GetDataStore returns the initialized Datastore +func (fs *Decomposedfs) GetDataStore() tusd.DataStore { + return fs.tusDataStore +} + +// PreFinishResponseCallback is called by the tus datatx, after all bytes have been transferred +func (fs *Decomposedfs) PreFinishResponseCallback(hook tusd.HookEvent) error { + ctx := context.TODO() + appctx.GetLogger(ctx).Debug().Interface("hook", hook).Msg("got PreFinishResponseCallback") + ctx, span := tracer.Start(ctx, "PreFinishResponseCallback") + defer span.End() + + info := hook.Upload + up, err := fs.tusDataStore.GetUpload(ctx, info.ID) if err != nil { - return nil, err + return err + } + + uploadMetadata, err := upload.ReadMetadata(ctx, fs.lu, info.ID) + if err != nil { + return err } - lockID, _ := ctxpkg.ContextGetLockID(ctx) + // put lockID from upload back into context + if uploadMetadata.LockID != "" { + ctx = ctxpkg.ContextSetLockID(ctx, uploadMetadata.LockID) + } - info := tusd.FileInfo{ - MetaData: tusd.MetaData{ - "filename": filepath.Base(relative), - "dir": filepath.Dir(relative), - "lockid": lockID, - }, - Size: uploadLength, - Storage: map[string]string{ - "SpaceRoot": n.SpaceRoot.ID, - "SpaceOwnerOrManager": n.SpaceOwnerOrManager(ctx).GetOpaqueId(), - }, + // restore logger from file info + log, err := logger.FromConfig(&logger.LogConf{ + Output: "stderr", + Mode: "json", + Level: uploadMetadata.LogLevel, + }) + if err != nil { + return err } - if metadata != nil { - info.MetaData["providerID"] = metadata["providerID"] - if mtime, ok := metadata["mtime"]; ok { - if mtime != "null" { - info.MetaData["mtime"] = mtime - } + ctx = appctx.WithLogger(ctx, log) + + // calculate the checksum of the written bytes + // they will all be written to the metadata later, so we cannot omit any of them + // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present + // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... + + log.Debug().Msg("calculating checksums") + sha1h := sha1.New() + md5h := md5.New() + adler32h := adler32.New() + { + _, subspan := tracer.Start(ctx, "GetReader") + reader, err := up.GetReader(ctx) + subspan.End() + if err != nil { + // we can continue if no oc checksum header is set + log.Info().Err(err).Interface("info", info).Msg("error getting Reader from upload") } - if expiration, ok := metadata["expires"]; ok { - if expiration != "null" { - info.MetaData["expires"] = expiration + if readCloser, ok := reader.(io.ReadCloser); ok { + defer readCloser.Close() + } + + r1 := io.TeeReader(reader, sha1h) + r2 := io.TeeReader(r1, md5h) + + _, subspan = tracer.Start(ctx, "io.Copy") + /*bytesCopied*/ _, err = io.Copy(adler32h, r2) + subspan.End() + if err != nil { + log.Info().Err(err).Msg("error copying checksums") + } + /* + if bytesCopied != info.Size { + msg := fmt.Sprintf("mismatching upload length. expected %d, could only copy %d", info.Size, bytesCopied) + log.Error().Interface("info", info).Msg(msg) + return errtypes.InternalError(msg) } + */ + } + + // compare if they match the sent checksum + // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads + if uploadMetadata.Checksum != "" { + var err error + parts := strings.SplitN(uploadMetadata.Checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") } - if _, ok := metadata["sizedeferred"]; ok { - info.SizeIsDeferred = true + switch parts[0] { + case "sha1": + err = checkHash(parts[1], sha1h) + case "md5": + err = checkHash(parts[1], md5h) + case "adler32": + err = checkHash(parts[1], adler32h) + default: + err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) } - if checksum, ok := metadata["checksum"]; ok { - parts := strings.SplitN(checksum, " ", 2) - if len(parts) != 2 { - return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + if err != nil { + if tup, ok := up.(tusd.TerminatableUpload); ok { + terr := tup.Terminate(ctx) + if terr != nil { + log.Error().Err(terr).Interface("info", info).Msg("failed to terminate upload") + } } - switch parts[0] { - case "sha1", "md5", "adler32": - info.MetaData["checksum"] = checksum - default: - return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + return err + } + } + + // update checksums + uploadMetadata.ChecksumSHA1 = sha1h.Sum(nil) + uploadMetadata.ChecksumMD5 = md5h.Sum(nil) + uploadMetadata.ChecksumADLER32 = adler32h.Sum(nil) + + log.Debug().Str("id", info.ID).Msg("upload.UpdateMetadata") + uploadMetadata, n, err := upload.UpdateMetadata(ctx, fs.lu, info.ID, info.Size, uploadMetadata) + if err != nil { + upload.Cleanup(ctx, fs.lu, n, info.ID, uploadMetadata.MTime, true) + if tup, ok := up.(tusd.TerminatableUpload); ok { + terr := tup.Terminate(ctx) + if terr != nil { + log.Error().Err(terr).Interface("info", info).Msg("failed to terminate upload") } } + return err + } - // only check preconditions if they are not empty // TODO or is this a bad request? - if metadata["if-match"] != "" { - info.MetaData["if-match"] = metadata["if-match"] + if fs.stream != nil { + user := &userpb.User{ + Id: &userpb.UserId{ + Type: userpb.UserType(userpb.UserType_value[uploadMetadata.ExecutantType]), + Idp: uploadMetadata.ExecutantIdp, + OpaqueId: uploadMetadata.ExecutantID, + }, + Username: uploadMetadata.ExecutantUserName, } - if metadata["if-none-match"] != "" { - info.MetaData["if-none-match"] = metadata["if-none-match"] + s, err := fs.downloadURL(ctx, info.ID) + if err != nil { + return err } - if metadata["if-unmodified-since"] != "" { - info.MetaData["if-unmodified-since"] = metadata["if-unmodified-since"] + + log.Debug().Str("id", info.ID).Msg("events.Publish BytesReceived") + if err := events.Publish(ctx, fs.stream, events.BytesReceived{ + UploadID: info.ID, + URL: s, + SpaceOwner: n.SpaceOwnerOrManager(ctx), + ExecutingUser: user, + ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, + Filename: uploadMetadata.Filename, // TODO what and when do we publish chunking v2 names? Currently, this uses the chunk name. + Filesize: uint64(info.Size), + }); err != nil { + return err } } - log.Debug().Interface("info", info).Interface("node", n).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename") + if n.Exists { + // // copy metadata to a revision node + log.Debug().Str("id", info.ID).Msg("copy metadata to a revision node") + currentAttrs, err := n.Xattrs(ctx) + if err != nil { + return err + } + previousRevisionTime, err := n.GetMTime(ctx) + if err != nil { + return err + } + rm := upload.RevisionMetadata{ + MTime: previousRevisionTime.UTC().Format(time.RFC3339Nano), + BlobID: n.BlobID, + BlobSize: n.Blobsize, + ChecksumSHA1: currentAttrs[prefixes.ChecksumPrefix+storageprovider.XSSHA1], + ChecksumMD5: currentAttrs[prefixes.ChecksumPrefix+storageprovider.XSMD5], + ChecksumADLER32: currentAttrs[prefixes.ChecksumPrefix+storageprovider.XSAdler32], + } + revisionNode := n.RevisionNode(ctx, rm.MTime) - _, err = node.CheckQuota(ctx, n.SpaceRoot, n.Exists, uint64(n.Blobsize), uint64(info.Size)) - if err != nil { - return nil, err + rh, err := upload.CreateRevisionNode(ctx, fs.lu, revisionNode) + if err != nil { + return err + } + defer rh.Close() + err = upload.WriteRevisionMetadataToNode(ctx, revisionNode, rm) + if err != nil { + return err + } + } + + sizeDiff := info.Size - n.Blobsize + if !fs.o.AsyncFileUploads { + // handle postprocessing synchronously + log.Debug().Str("id", info.ID).Msg("upload.Finalize") + err = upload.Finalize(ctx, fs.blobstore, uploadMetadata.MTime, info, n, uploadMetadata.BlobID) // moving or copying the blob only reads the blobid, no need to change the revision nodes nodeid + + log.Debug().Str("id", info.ID).Msg("upload.Cleanup") + upload.Cleanup(ctx, fs.lu, n, info.ID, uploadMetadata.MTime, err != nil) + if tup, ok := up.(tusd.TerminatableUpload); ok { + log.Debug().Str("id", info.ID).Msg("tup.Terminate") + terr := tup.Terminate(ctx) + if terr != nil { + log.Error().Err(terr).Interface("info", info).Msg("failed to terminate upload") + } + } + if err != nil { + log.Error().Err(err).Msg("failed to upload") + return err + } + log.Debug().Str("id", info.ID).Msg("upload.SetNodeToUpload") + sizeDiff, err = upload.SetNodeToUpload(ctx, fs.lu, n, uploadMetadata) + if err != nil { + log.Error().Err(err).Msg("failed update Node to revision") + return err + } } + log.Debug().Str("id", info.ID).Msg("fs.tp.Propagate") + return fs.tp.Propagate(ctx, n, sizeDiff) +} - upload, err := fs.NewUpload(ctx, info) +// URL returns a url to download an upload +func (fs *Decomposedfs) downloadURL(_ context.Context, id string) (string, error) { + type transferClaims struct { + jwt.StandardClaims + Target string `json:"target"` + } + + u, err := url.JoinPath(fs.o.Tokens.DownloadEndpoint, "tus/", id) if err != nil { - return nil, err + return "", errors.Wrapf(err, "error joinging URL path") + } + ttl := time.Duration(fs.o.Tokens.TransferExpires) * time.Second + claims := transferClaims{ + StandardClaims: jwt.StandardClaims{ + ExpiresAt: time.Now().Add(ttl).Unix(), + Audience: "reva", + IssuedAt: time.Now().Unix(), + }, + Target: u, } - info, _ = upload.GetInfo(ctx) + t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) - return map[string]string{ - "simple": info.ID, - "tus": info.ID, - }, nil + tkn, err := t.SignedString([]byte(fs.o.Tokens.TransferSharedSecret)) + if err != nil { + return "", errors.Wrapf(err, "error signing token with claims %+v", claims) + } + + return url.JoinPath(fs.o.Tokens.DataGatewayEndpoint, tkn) } -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *Decomposedfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - composer.UseConcater(fs) - composer.UseLengthDeferrer(fs) +func checkHash(expected string, h hash.Hash) error { + if expected != hex.EncodeToString(h.Sum(nil)) { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, h.Sum(nil))) + } + return nil } -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload +// Upload uploads data to the given resource +// is used by the simple datatx, after an InitiateUpload call +// TODO Upload (and InitiateUpload) needs a way to receive the expected checksum. +// Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? +func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (provider.ResourceInfo, error) { + sublog := appctx.GetLogger(ctx).With().Str("path", req.Ref.Path).Int64("uploadLength", req.Length).Logger() + up, err := fs.tusDataStore.GetUpload(ctx, req.Ref.GetPath()) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error retrieving upload") + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload") + } -// NewUpload returns a new tus Upload instance -func (fs *Decomposedfs) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) { - return upload.New(ctx, info, fs.lu, fs.tp, fs.p, fs.o.Root, fs.stream, fs.o.AsyncFileUploads, fs.o.Tokens) -} + uploadInfo, err := up.GetInfo(ctx) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error retrieving upload info") + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload info") + } -// GetUpload returns the Upload for the given upload id -func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - return upload.Get(ctx, id, fs.lu, fs.tp, fs.o.Root, fs.stream, fs.o.AsyncFileUploads, fs.o.Tokens) + uploadMetadata, err := upload.ReadMetadata(ctx, fs.lu, uploadInfo.ID) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error retrieving upload metadata") + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload metadata") + } + + if chunking.IsChunked(uploadMetadata.Chunk) { // check chunking v1, TODO, actually there is a 'OC-Chunked: 1' header, at least when the testsuite uses chunking v1 + var assembledFile, p string + p, assembledFile, err = fs.chunkHandler.WriteChunk(uploadMetadata.Chunk, req.Body) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: could not write chunk") + return provider.ResourceInfo{}, err + } + if p == "" { + sublog.Debug().Err(err).Str("chunk", uploadMetadata.Chunk).Msg("Decomposedfs: wrote chunk") + return provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) + } + uploadMetadata.Filename = p + fd, err := os.Open(assembledFile) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error opening assembled file") + } + defer fd.Close() + defer os.RemoveAll(assembledFile) + + chunkStat, err := fd.Stat() + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: could not stat assembledFile for legacy chunking") + } + + // fake a new upload with the correct size + newInfo := tusd.FileInfo{ + Size: chunkStat.Size(), + MetaData: uploadInfo.MetaData, + } + nup, err := fs.tusDataStore.NewUpload(ctx, newInfo) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: could not create new tus upload for legacy chunking") + } + newInfo, err = nup.GetInfo(ctx) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: could not get info from upload") + } + uploadMetadata.ID = newInfo.ID + uploadMetadata.BlobSize = newInfo.Size + err = upload.WriteMetadata(ctx, fs.lu, newInfo.ID, uploadMetadata) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing upload metadata for legacy chunking") + } + + _, err = nup.WriteChunk(ctx, 0, fd) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file for legacy chunking") + } + // use new upload and info + up = nup + uploadInfo, err = up.GetInfo(ctx) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: could not get info for legacy chunking") + } + } else { + // we need to call up.DeclareLength() before writing the chunk, but only if we actually got a length + if req.Length > 0 { + if ldx, ok := up.(tusd.LengthDeclarableUpload); ok { + if err := ldx.DeclareLength(ctx, req.Length); err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error declaring length") + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error declaring length") + } + } + } + bytesWritten, err := up.WriteChunk(ctx, 0, req.Body) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error writing to binary file") + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") + } + uploadInfo.Offset += bytesWritten + if uploadInfo.SizeIsDeferred { + // update the size and offset + uploadInfo.SizeIsDeferred = false + uploadInfo.Size = bytesWritten + } + } + + // This finishes the tus upload + sublog.Debug().Msg("finishing upload") + if err := up.FinishUpload(ctx); err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error finishing upload") + return provider.ResourceInfo{}, err + } + + // we now need to handle to move/copy&delete to the target blobstore + sublog.Debug().Msg("executing tusd prefinish callback") + err = fs.PreFinishResponseCallback(tusd.HookEvent{Upload: uploadInfo}) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: tusd callback failed") + return provider.ResourceInfo{}, err + } + + n, err := upload.ReadNode(ctx, fs.lu, uploadMetadata) + if err != nil { + sublog.Debug().Err(err).Msg("Decomposedfs: error reading node") + return provider.ResourceInfo{}, err + } + + if uff != nil { + // TODO search needs to index the full path, so we return a reference relative to the space root. + // but then the search has to walk the path. it might be more efficient if search called GetPath itself ... or we send the path as additional metadata in the event + uploadRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: uploadMetadata.ProviderID, + SpaceId: n.SpaceID, + OpaqueId: n.SpaceID, + }, + Path: utils.MakeRelativePath(filepath.Join(uploadMetadata.Dir, uploadMetadata.Filename)), + } + executant, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + return provider.ResourceInfo{}, errtypes.PreconditionFailed("error getting user from context") + } + + sublog.Debug().Msg("calling upload finished func") + uff(n.SpaceOwnerOrManager(ctx), executant.Id, uploadRef) + } + + mtime, err := n.GetMTime(ctx) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error getting mtime for '"+n.ID+"'") + } + etag, err := node.CalculateEtag(n, mtime) + if err != nil { + return provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error calculating etag '"+n.ID+"'") + } + ri := provider.ResourceInfo{ + // fill with at least fileid, mtime and etag + Id: &provider.ResourceId{ + StorageId: uploadMetadata.ProviderID, + SpaceId: n.SpaceID, + OpaqueId: n.ID, + }, + Etag: etag, + } + + if mtime, err := utils.MTimeToTS(uploadInfo.MetaData["mtime"]); err == nil { + ri.Mtime = &mtime + } + sublog.Debug().Msg("Decomposedfs: finished upload") + + return ri, nil } // ListUploadSessions returns the upload sessions for the given filter func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { var sessions []storage.UploadSession if filter.ID != nil && *filter.ID != "" { - session, err := fs.getUploadSession(ctx, filepath.Join(fs.o.Root, "uploads", *filter.ID+".info")) + session, err := fs.getUploadSession(ctx, fs.lu.UploadPath("*")) if err != nil { return nil, err } @@ -281,42 +709,65 @@ func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.U return filteredSessions, nil } -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (fs *Decomposedfs) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.Upload) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (fs *Decomposedfs) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.Upload) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (fs *Decomposedfs) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.Upload) -} - func (fs *Decomposedfs) uploadSessions(ctx context.Context) ([]storage.UploadSession, error) { uploads := []storage.UploadSession{} - infoFiles, err := filepath.Glob(filepath.Join(fs.o.Root, "uploads", "*.info")) + sessionFiles, err := filepath.Glob(fs.lu.UploadPath("*")) if err != nil { return nil, err } - for _, info := range infoFiles { - progress, err := fs.getUploadSession(ctx, info) - if err != nil { - appctx.GetLogger(ctx).Error().Interface("path", info).Msg("Decomposedfs: could not getUploadSession") - continue + numWorkers := fs.o.MaxConcurrency + if len(sessionFiles) < numWorkers { + numWorkers = len(sessionFiles) + } + + work := make(chan string, len(sessionFiles)) + results := make(chan storage.UploadSession, len(sessionFiles)) + + g, ctx := errgroup.WithContext(ctx) + + // Distribute work + g.Go(func() error { + defer close(work) + for _, itemPath := range sessionFiles { + select { + case work <- itemPath: + case <-ctx.Done(): + return ctx.Err() + } } + return nil + }) + + // Spawn workers that'll concurrently work the queue + for i := 0; i < numWorkers; i++ { + g.Go(func() error { + for path := range work { + session, err := fs.getUploadSession(ctx, path) + if err != nil { + appctx.GetLogger(ctx).Error().Interface("path", path).Msg("Decomposedfs: could not getUploadSession") + continue + } - uploads = append(uploads, progress) + select { + case results <- session: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + } + + // Wait for things to settle down, then close results chan + go func() { + _ = g.Wait() // error is checked later + close(results) + }() + + // Collect results + for ri := range results { + uploads = append(uploads, ri) } return uploads, nil } @@ -326,23 +777,37 @@ func (fs *Decomposedfs) getUploadSession(ctx context.Context, path string) (stor if match == nil || len(match) < 2 { return nil, fmt.Errorf("invalid upload path") } - up, err := fs.GetUpload(ctx, match[1]) + + metadata, err := upload.ReadMetadata(ctx, fs.lu, match[1]) if err != nil { return nil, err } - info, err := up.GetInfo(context.Background()) + // upload processing state is stored in the node, for decomposedfs the NodeId is always set by InitiateUpload + var n *node.Node + if metadata.NodeID == "" { + // read parent first + n, err = node.ReadNode(ctx, fs.lu, metadata.SpaceRoot, metadata.NodeParentID, true, nil, true) + if err != nil { + return nil, err + } + n, err = n.Child(ctx, metadata.Filename) + } else { + n, err = node.ReadNode(ctx, fs.lu, metadata.SpaceRoot, metadata.NodeID, true, nil, true) + } if err != nil { return nil, err } - // upload processing state is stored in the node, for decomposedfs the NodeId is always set by InitiateUpload - n, err := node.ReadNode(ctx, fs.lu, info.Storage["SpaceRoot"], info.Storage["NodeId"], true, nil, true) + tusUpload, err := fs.tusDataStore.GetUpload(ctx, metadata.ID) if err != nil { return nil, err } + progress := upload.Progress{ + Upload: tusUpload, Path: path, - Info: info, + Metadata: metadata, Processing: n.IsProcessing(ctx), } + return progress, nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/metadata.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/metadata.go new file mode 100644 index 00000000000..86ff40b7706 --- /dev/null +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/metadata.go @@ -0,0 +1,264 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "os" + "path/filepath" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rogpeppe/go-internal/lockedfile" + "github.com/shamaton/msgpack/v2" +) + +type Metadata struct { + ID string + Filename string + SpaceRoot string + SpaceOwnerOrManager string + ProviderID string + MTime string + NodeID string + NodeParentID string + ExecutantIdp string + ExecutantID string + ExecutantType string + ExecutantUserName string + LogLevel string + Checksum string + ChecksumSHA1 []byte + ChecksumADLER32 []byte + ChecksumMD5 []byte + + BlobID string + BlobSize int64 + + Chunk string + Dir string + LockID string + HeaderIfMatch string + HeaderIfNoneMatch string + HeaderIfUnmodifiedSince string + Expires time.Time +} + +// WriteMetadata will create a metadata file to keep track of an upload +func WriteMetadata(ctx context.Context, lu *lookup.Lookup, uploadID string, metadata Metadata) error { + _, span := tracer.Start(ctx, "WriteMetadata") + defer span.End() + + uploadPath := lu.UploadPath(uploadID) + + // create folder structure (if needed) + if err := os.MkdirAll(filepath.Dir(uploadPath), 0700); err != nil { + return err + } + + var d []byte + d, err := msgpack.Marshal(metadata) + if err != nil { + return err + } + + _, subspan := tracer.Start(ctx, "os.Writefile") + err = os.WriteFile(uploadPath, d, 0600) + subspan.End() + if err != nil { + return err + } + + return nil +} +func ReadMetadata(ctx context.Context, lu *lookup.Lookup, uploadID string) (Metadata, error) { + _, span := tracer.Start(ctx, "ReadMetadata") + defer span.End() + + uploadPath := lu.UploadPath(uploadID) + + _, subspan := tracer.Start(ctx, "os.ReadFile") + msgBytes, err := os.ReadFile(uploadPath) + subspan.End() + if err != nil { + return Metadata{}, err + } + + metadata := Metadata{} + if len(msgBytes) > 0 { + err = msgpack.Unmarshal(msgBytes, &metadata) + if err != nil { + return Metadata{}, err + } + } + return metadata, nil +} + +// UpdateMetadata will create the target node for the Upload +// - if the node does not exist it is created and assigned an id, no blob id? +// - then always write out a revision node +// - when postprocessing finishes copy metadata to node and replace latest revision node with previous blob info. if blobid is empty delete previous revision completely? +func UpdateMetadata(ctx context.Context, lu *lookup.Lookup, uploadID string, size int64, uploadMetadata Metadata) (Metadata, *node.Node, error) { + ctx, span := tracer.Start(ctx, "UpdateMetadata") + defer span.End() + log := appctx.GetLogger(ctx).With().Str("uploadID", uploadID).Logger() + + // check lock + if uploadMetadata.LockID != "" { + ctx = ctxpkg.ContextSetLockID(ctx, uploadMetadata.LockID) + } + + var err error + var n *node.Node + var nodeHandle *lockedfile.File + if uploadMetadata.NodeID == "" { + // we need to check if the node exists via parentid & child name + p, err := node.ReadNode(ctx, lu, uploadMetadata.SpaceRoot, uploadMetadata.NodeParentID, false, nil, true) + if err != nil { + log.Error().Err(err).Msg("could not read parent node") + return Metadata{}, nil, err + } + if !p.Exists { + return Metadata{}, nil, errtypes.PreconditionFailed("parent does not exist") + } + n, err = p.Child(ctx, uploadMetadata.Filename) + if err != nil { + log.Error().Err(err).Msg("could not read parent node") + return Metadata{}, nil, err + } + if !n.Exists { + n.ID = uuid.New().String() + nodeHandle, err = initNewNode(ctx, lu, uploadID, uploadMetadata.MTime, n) + if err != nil { + log.Error().Err(err).Msg("could not init new node") + return Metadata{}, nil, err + } + } else { + nodeHandle, err = openExistingNode(ctx, lu, n) + if err != nil { + log.Error().Err(err).Msg("could not open existing node") + return Metadata{}, nil, err + } + } + } + + if nodeHandle == nil { + n, err = node.ReadNode(ctx, lu, uploadMetadata.SpaceRoot, uploadMetadata.NodeID, false, nil, true) + if err != nil { + log.Error().Err(err).Msg("could not read parent node") + return Metadata{}, nil, err + } + nodeHandle, err = openExistingNode(ctx, lu, n) + if err != nil { + log.Error().Err(err).Msg("could not open existing node") + return Metadata{}, nil, err + } + } + defer func() { + if nodeHandle == nil { + return + } + if err := nodeHandle.Close(); err != nil { + log.Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock") + } + }() + + err = validateRequest(ctx, size, uploadMetadata, n) + if err != nil { + return Metadata{}, nil, err + } + + newBlobID := uuid.New().String() + + // set processing status of node + nodeAttrs := node.Attributes{} + // store Blobsize in node so we can propagate a sizediff + // do not yet update the blobid ... urgh this is fishy + nodeAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+uploadID) + err = n.SetXattrsWithContext(ctx, nodeAttrs, false) + if err != nil { + return Metadata{}, nil, errors.Wrap(err, "Decomposedfs: could not write metadata") + } + + uploadMetadata.BlobID = newBlobID + uploadMetadata.BlobSize = size + // TODO we should persist all versions as writes with ranges and the blobid in the node metadata + + err = WriteMetadata(ctx, lu, uploadID, uploadMetadata) + if err != nil { + return Metadata{}, nil, errors.Wrap(err, "Decomposedfs: could not write upload metadata") + } + + return uploadMetadata, n, nil +} + +func (m Metadata) GetID() string { + return m.ID +} +func (m Metadata) GetFilename() string { + return m.Filename +} + +// TODO use uint64? use SizeDeferred flag is in tus? cleaner then int64 and a negative value +func (m Metadata) GetSize() int64 { + return m.BlobSize +} +func (m Metadata) GetResourceID() provider.ResourceId { + return provider.ResourceId{ + StorageId: m.ProviderID, + SpaceId: m.SpaceRoot, + OpaqueId: m.NodeID, + } +} +func (m Metadata) GetReference() provider.Reference { + return provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: m.ProviderID, + SpaceId: m.SpaceRoot, + OpaqueId: m.NodeID, + }, + // Path is not used + } +} +func (m Metadata) GetExecutantID() userpb.UserId { + return userpb.UserId{ + Type: userpb.UserType(userpb.UserType_value[m.ExecutantType]), + Idp: m.ExecutantIdp, + OpaqueId: m.ExecutantID, + } +} +func (m Metadata) GetSpaceOwner() userpb.UserId { + return userpb.UserId{ + // idp and type do not seem to be consumed and the node currently only stores the user id anyway + OpaqueId: m.SpaceOwnerOrManager, + } + +} +func (m Metadata) GetExpires() time.Time { + return m.Expires +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/processing.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/processing.go index a025c54ed75..7bf5bb7e54b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/processing.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/processing.go @@ -20,544 +20,397 @@ package upload import ( "context" - "encoding/json" - stderrors "errors" - "fmt" - iofs "io/fs" + "errors" "os" "path/filepath" - "strconv" - "strings" "time" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" - ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/logger" - "github.com/cs3org/reva/v2/pkg/storage/utils/chunking" + "github.com/cs3org/reva/v2/pkg/storage/cache" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/cs3org/reva/v2/pkg/utils" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" tusd "github.com/tus/tusd/pkg/handler" ) -var defaultFilePerm = os.FileMode(0664) - // PermissionsChecker defines an interface for checking permissions on a Node type PermissionsChecker interface { AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error) } -// New returns a new processing instance -func New(ctx context.Context, info tusd.FileInfo, lu *lookup.Lookup, tp Tree, p PermissionsChecker, fsRoot string, pub events.Publisher, async bool, tknopts options.TokenOptions) (upload *Upload, err error) { - - log := appctx.GetLogger(ctx) - log.Debug().Interface("info", info).Msg("Decomposedfs: NewUpload") - - if info.MetaData["filename"] == "" { - return nil, errors.New("Decomposedfs: missing filename in metadata") - } - if info.MetaData["dir"] == "" { - return nil, errors.New("Decomposedfs: missing dir in metadata") - } - - n, err := lu.NodeFromSpaceID(ctx, info.Storage["SpaceRoot"]) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error getting space root node") - } - - n, err = lookupNode(ctx, n, filepath.Join(info.MetaData["dir"], info.MetaData["filename"]), lu) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error walking path") - } - - log.Debug().Interface("info", info).Interface("node", n).Msg("Decomposedfs: resolved filename") - - // the parent owner will become the new owner - parent, perr := n.Parent(ctx) - if perr != nil { - return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID) - } - - // check permissions - var ( - checkNode *node.Node - path string - ) - if n.Exists { - // check permissions of file to be overwritten - checkNode = n - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }}) - } else { - // check permissions of parent - checkNode = parent - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }, Path: n.Name}) - } - rp, err := p.AssemblePermissions(ctx, checkNode) - switch { - case err != nil: - return nil, err - case !rp.InitiateFileUpload: - return nil, errtypes.PermissionDenied(path) - } - - // are we trying to overwriting a folder with a file? - if n.Exists && n.IsDir(ctx) { - return nil, errtypes.PreconditionFailed("resource is not a file") - } - - // check lock - if info.MetaData["lockid"] != "" { - ctx = ctxpkg.ContextSetLockID(ctx, info.MetaData["lockid"]) - } - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - info.ID = uuid.New().String() - - binPath := filepath.Join(fsRoot, "uploads", info.ID) - usr := ctxpkg.ContextMustGetUser(ctx) - - var ( - spaceRoot string - ok bool - ) - if info.Storage != nil { - if spaceRoot, ok = info.Storage["SpaceRoot"]; !ok { - spaceRoot = n.SpaceRoot.ID - } - } else { - spaceRoot = n.SpaceRoot.ID - } - - info.Storage = map[string]string{ - "Type": "OCISStore", - "BinPath": binPath, - - "NodeId": n.ID, - "NodeExists": "true", - "NodeParentId": n.ParentID, - "NodeName": n.Name, - "SpaceRoot": spaceRoot, - "SpaceOwnerOrManager": info.Storage["SpaceOwnerOrManager"], - - "Idp": usr.Id.Idp, - "UserId": usr.Id.OpaqueId, - "UserType": utils.UserTypeToString(usr.Id.Type), - "UserName": usr.Username, - - "LogLevel": log.GetLevel().String(), - } - if !n.Exists { - // fill future node info - info.Storage["NodeId"] = uuid.New().String() - info.Storage["NodeExists"] = "false" - } - if info.MetaData["if-none-match"] == "*" && info.Storage["NodeExists"] == "true" { - return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s", n.ID, n.Name)) - } - // Create binary file in the upload folder with no content - log.Debug().Interface("info", info).Msg("Decomposedfs: built storage info") - file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return nil, err - } - defer file.Close() - - u := buildUpload(ctx, info, binPath, filepath.Join(fsRoot, "uploads", info.ID+".info"), lu, tp, pub, async, tknopts) - - // writeInfo creates the file by itself if necessary - err = u.writeInfo() - if err != nil { - return nil, err - } - - return u, nil -} - -// Get returns the Upload for the given upload id -func Get(ctx context.Context, id string, lu *lookup.Lookup, tp Tree, fsRoot string, pub events.Publisher, async bool, tknopts options.TokenOptions) (*Upload, error) { - infoPath := filepath.Join(fsRoot, "uploads", id+".info") - - info := tusd.FileInfo{} - data, err := os.ReadFile(infoPath) - if err != nil { - if errors.Is(err, iofs.ErrNotExist) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - if err := json.Unmarshal(data, &info); err != nil { - return nil, err - } - - stat, err := os.Stat(info.Storage["BinPath"]) - if err != nil { - return nil, err - } - - info.Offset = stat.Size() - - u := &userpb.User{ - Id: &userpb.UserId{ - Idp: info.Storage["Idp"], - OpaqueId: info.Storage["UserId"], - Type: utils.UserTypeMap(info.Storage["UserType"]), - }, - Username: info.Storage["UserName"], - } - - ctx = ctxpkg.ContextSetUser(ctx, u) - - // restore logger from file info - log, err := logger.FromConfig(&logger.LogConf{ - Output: "stderr", // TODO use config from decomposedfs - Mode: "json", // TODO use config from decomposedfs - Level: info.Storage["LogLevel"], - }) - if err != nil { - return nil, err - } - sub := log.With().Int("pid", os.Getpid()).Logger() - ctx = appctx.WithLogger(ctx, &sub) - - // TODO store and add traceid in file info - - up := buildUpload(ctx, info, info.Storage["BinPath"], infoPath, lu, tp, pub, async, tknopts) - up.versionsPath = info.MetaData["versionsPath"] - up.SizeDiff, _ = strconv.ParseInt(info.MetaData["sizeDiff"], 10, 64) - return up, nil +type Propagator interface { + Propagate(ctx context.Context, node *node.Node, sizeDiff int64) (err error) } -// CreateNodeForUpload will create the target node for the Upload -func CreateNodeForUpload(upload *Upload, initAttrs node.Attributes) (*node.Node, error) { - ctx, span := tracer.Start(upload.Ctx, "CreateNodeForUpload") +// Postprocessing starts the postprocessing result collector +func Postprocessing(lu *lookup.Lookup, propagator Propagator, cache cache.StatCache, es events.Stream, tusDataStore tusd.DataStore, blobstore tree.Blobstore, downloadURLfunc func(ctx context.Context, id string) (string, error), ch <-chan events.Event) { + ctx := context.TODO() // we should pass the trace id in the event and initialize the trace provider here + ctx, span := tracer.Start(ctx, "Postprocessing") defer span.End() - _, subspan := tracer.Start(ctx, "os.Stat") - fi, err := os.Stat(upload.binPath) - subspan.End() - if err != nil { - return nil, err - } - - fsize := fi.Size() - spaceID := upload.Info.Storage["SpaceRoot"] - n := node.New( - spaceID, - upload.Info.Storage["NodeId"], - upload.Info.Storage["NodeParentId"], - upload.Info.Storage["NodeName"], - fsize, - upload.Info.ID, - provider.ResourceType_RESOURCE_TYPE_FILE, - nil, - upload.lu, - ) - n.SpaceRoot, err = node.ReadNode(ctx, upload.lu, spaceID, spaceID, false, nil, false) - if err != nil { - return nil, err - } - - // check lock - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - var f *lockedfile.File - switch upload.Info.Storage["NodeExists"] { - case "false": - f, err = initNewNode(upload, n, uint64(fsize)) - if f != nil { - appctx.GetLogger(upload.Ctx).Info().Str("lockfile", f.Name()).Interface("err", err).Msg("got lock file from initNewNode") - } - default: - f, err = updateExistingNode(upload, n, spaceID, uint64(fsize)) - if f != nil { - appctx.GetLogger(upload.Ctx).Info().Str("lockfile", f.Name()).Interface("err", err).Msg("got lock file from updateExistingNode") - } - } - defer func() { - if f == nil { - return - } - if err := f.Close(); err != nil { - appctx.GetLogger(upload.Ctx).Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock") - } - }() - if err != nil { - return nil, err - } - - mtime := time.Now() - if upload.Info.MetaData["mtime"] != "" { - // overwrite mtime if requested - mtime, err = utils.MTimeToTime(upload.Info.MetaData["mtime"]) - if err != nil { - return nil, err - } - } - - // overwrite technical information - initAttrs.SetString(prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) - initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) - initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) - initAttrs.SetString(prefixes.NameAttr, n.Name) - initAttrs.SetString(prefixes.BlobIDAttr, n.BlobID) - initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) - initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+upload.Info.ID) - - // update node metadata with new blobid etc - err = n.SetXattrsWithContext(ctx, initAttrs, false) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: could not write metadata") - } - - // add etag to metadata - upload.Info.MetaData["etag"], _ = node.CalculateEtag(n, mtime) - - // update nodeid for later - upload.Info.Storage["NodeId"] = n.ID - if err := upload.writeInfo(); err != nil { - return nil, err - } - - return n, nil -} - -func initNewNode(upload *Upload, n *node.Node, fsize uint64) (*lockedfile.File, error) { - // create folder structure (if needed) - if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { - return nil, err - } - - // create and write lock new node metadata - f, err := lockedfile.OpenFile(upload.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return nil, err - } - - // we also need to touch the actual node file here it stores the mtime of the resource - h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return f, err - } - h.Close() - - if _, err := node.CheckQuota(upload.Ctx, n.SpaceRoot, false, 0, fsize); err != nil { - return f, err - } - - // link child name to parent if it is new - childNameLink := filepath.Join(n.ParentPath(), n.Name) - relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - log := appctx.GetLogger(upload.Ctx).With().Str("childNameLink", childNameLink).Str("relativeNodePath", relativeNodePath).Logger() - log.Info().Msg("initNewNode: creating symlink") - - if err = os.Symlink(relativeNodePath, childNameLink); err != nil { - log.Info().Err(err).Msg("initNewNode: symlink failed") - if errors.Is(err, iofs.ErrExist) { - log.Info().Err(err).Msg("initNewNode: symlink already exists") - return f, errtypes.AlreadyExists(n.Name) - } - return f, errors.Wrap(err, "Decomposedfs: could not symlink child entry") - } - log.Info().Msg("initNewNode: symlink created") - - // on a new file the sizeDiff is the fileSize - upload.SizeDiff = int64(fsize) - upload.Info.MetaData["sizeDiff"] = strconv.Itoa(int(upload.SizeDiff)) - return f, nil -} + log := logger.New() + for event := range ch { + switch ev := event.Event.(type) { + case events.PostprocessingFinished: + up, err := tusDataStore.GetUpload(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") + continue // NOTE: since we can't get the upload, we can't delete the blob + } + info, err := up.GetInfo(ctx) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload info") + continue // NOTE: since we can't get the upload, we can't delete the blob + } + uploadMetadata, err := ReadMetadata(ctx, lu, info.ID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload metadata") + continue // NOTE: since we can't get the upload, we can't delete the blob + } -func updateExistingNode(upload *Upload, n *node.Node, spaceID string, fsize uint64) (*lockedfile.File, error) { - targetPath := n.InternalPath() + var ( + failed bool + keepUpload bool + ) - // write lock existing node before reading any metadata - f, err := lockedfile.OpenFile(upload.lu.MetadataBackend().LockfilePath(targetPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return nil, err - } + var sizeDiff int64 + // propagate sizeDiff after failed postprocessing - old, _ := node.ReadNode(upload.Ctx, upload.lu, spaceID, n.ID, false, nil, false) - if _, err := node.CheckQuota(upload.Ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil { - return f, err - } + n, err := ReadNode(ctx, lu, uploadMetadata) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Interface("metadata", uploadMetadata).Msg("could not read revision node on postprocessing finished") + continue + } - oldNodeMtime, err := old.GetMTime(upload.Ctx) - if err != nil { - return f, err - } - oldNodeEtag, err := node.CalculateEtag(old, oldNodeMtime) - if err != nil { - return f, err - } + switch ev.Outcome { + default: + log.Error().Str("outcome", string(ev.Outcome)).Str("uploadID", ev.UploadID).Msg("unknown postprocessing outcome - aborting") + fallthrough + case events.PPOutcomeAbort: + failed = true + keepUpload = true + case events.PPOutcomeContinue: + if err := Finalize(ctx, blobstore, uploadMetadata.MTime, info, n, uploadMetadata.BlobID); err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not finalize upload") + keepUpload = true // should we keep the upload when assembling failed? + failed = true + } + sizeDiff, err = SetNodeToUpload(ctx, lu, n, uploadMetadata) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could set node to revision upload") + keepUpload = true // should we keep the upload when assembling failed? + failed = true + } + case events.PPOutcomeDelete: + failed = true + } - // When the if-match header was set we need to check if the - // etag still matches before finishing the upload. - if ifMatch, ok := upload.Info.MetaData["if-match"]; ok { - if ifMatch != oldNodeEtag { - return f, errtypes.Aborted("etag mismatch") - } - } + getParent := func() *node.Node { + p, err := n.Parent(ctx) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not read parent") + return nil + } + return p + } - // When the if-none-match header was set we need to check if any of the - // etags matches before finishing the upload. - if ifNoneMatch, ok := upload.Info.MetaData["if-none-match"]; ok { - if ifNoneMatch == "*" { - return f, errtypes.Aborted("etag mismatch, resource exists") - } - for _, ifNoneMatchTag := range strings.Split(ifNoneMatch, ",") { - if ifNoneMatchTag == oldNodeEtag { - return f, errtypes.Aborted("etag mismatch") + now := time.Now() + if failed { + // propagate sizeDiff after failed postprocessing + if err := propagator.Propagate(ctx, n, -sizeDiff); err != nil { // FIXME revert sizediff .,.. and write an issue that condemns this + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not propagate tree size change") + } + + } else if p := getParent(); p != nil { + // update parent tmtime to propagate etag change after successful postprocessing + _ = p.SetTMTime(ctx, &now) + if err := propagator.Propagate(ctx, p, 0); err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not propagate etag change") + } } - } - } - // When the if-unmodified-since header was set we need to check if the - // etag still matches before finishing the upload. - if ifUnmodifiedSince, ok := upload.Info.MetaData["if-unmodified-since"]; ok { - if err != nil { - return f, errtypes.InternalError(fmt.Sprintf("failed to read mtime of node: %s", err)) - } - ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, ifUnmodifiedSince) - if err != nil { - return f, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) - } + previousRevisionTime, err := n.GetMTime(ctx) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not get mtime") + } + revision := previousRevisionTime.UTC().Format(time.RFC3339Nano) + Cleanup(ctx, lu, n, info.ID, revision, failed) + if !keepUpload { + if tup, ok := up.(tusd.TerminatableUpload); ok { + terr := tup.Terminate(ctx) + if terr != nil { + log.Error().Err(terr).Interface("info", info).Msg("failed to terminate upload") + } + } + } - if oldNodeMtime.After(ifUnmodifiedSince) { - return f, errtypes.Aborted("if-unmodified-since mismatch") - } - } + // remove cache entry in gateway + cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) + + if err := events.Publish( + ctx, + es, + events.UploadReady{ + UploadID: ev.UploadID, + Failed: failed, + ExecutingUser: &user.User{ + Id: &user.UserId{ + Type: user.UserType(user.UserType_value[uploadMetadata.ExecutantType]), + Idp: uploadMetadata.ExecutantIdp, + OpaqueId: uploadMetadata.ExecutantID, + }, + Username: uploadMetadata.ExecutantUserName, + }, + Filename: ev.Filename, + FileRef: &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: uploadMetadata.ProviderID, + SpaceId: uploadMetadata.SpaceRoot, + OpaqueId: uploadMetadata.SpaceRoot, + }, + // FIXME this seems wrong, path is not really relative to space root + // actually it is: InitiateUpload calls fs.lu.Path to get the path relative to the root so soarch can index the path + // hm is that robust? what if the file is moved? shouldn't we store the parent id, then? + Path: utils.MakeRelativePath(filepath.Join(uploadMetadata.Dir, uploadMetadata.Filename)), + }, + Timestamp: utils.TimeToTS(now), + SpaceOwner: n.SpaceOwnerOrManager(ctx), + }, + ); err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to publish UploadReady event") + } + case events.RestartPostprocessing: + up, err := tusDataStore.GetUpload(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") + continue // NOTE: since we can't get the upload, we can't restart postprocessing + } + info, err := up.GetInfo(ctx) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload info") + continue // NOTE: since we can't get the upload, we can't restart postprocessing + } + uploadMetadata, err := ReadMetadata(ctx, lu, info.ID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload metadata") + continue // NOTE: since we can't get the upload, we can't delete the blob + } - upload.versionsPath = upload.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - upload.SizeDiff = int64(fsize) - old.Blobsize - upload.Info.MetaData["versionsPath"] = upload.versionsPath - upload.Info.MetaData["sizeDiff"] = strconv.Itoa(int(upload.SizeDiff)) + n, err := ReadNode(ctx, lu, uploadMetadata) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Interface("metadata", uploadMetadata).Msg("could not read revision node on restart postprocessing") + continue + } - // create version node - if _, err := os.Create(upload.versionsPath); err != nil { - return f, err - } + s, err := downloadURLfunc(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not create url") + continue + } + // restart postprocessing + if err := events.Publish(ctx, es, events.BytesReceived{ + UploadID: info.ID, + URL: s, + SpaceOwner: n.SpaceOwnerOrManager(ctx), + ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? + ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, + Filename: uploadMetadata.Filename, + Filesize: uint64(info.Size), + }); err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to publish BytesReceived event") + } + case events.PostprocessingStepFinished: + if ev.FinishedStep != events.PPStepAntivirus { + // atm we are only interested in antivirus results + continue + } - // copy blob metadata to version node - if err := upload.lu.CopyMetadataWithSourceLock(upload.Ctx, targetPath, upload.versionsPath, func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr - }, f, true); err != nil { - return f, err - } + res := ev.Result.(events.VirusscanResult) + if res.ErrorMsg != "" { + // scan failed somehow + // Should we handle this here? + continue + } - // keep mtime from previous version - if err := os.Chtimes(upload.versionsPath, oldNodeMtime, oldNodeMtime); err != nil { - return f, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err)) - } + var n *node.Node + switch ev.UploadID { + case "": + // uploadid is empty -> this was an on-demand scan + /* ON DEMAND SCANNING NOT SUPPORTED ATM + ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) + ref := &provider.Reference{ResourceId: ev.ResourceID} + + no, err := fs.lu.NodeFromResource(ctx, ref) + if err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") + continue + + } + n = no + if ev.Outcome == events.PPOutcomeDelete { + // antivir wants us to delete the file. We must obey and need to + + // check if there a previous versions existing + revs, err := fs.ListRevisions(ctx, ref) + if len(revs) == 0 { + if err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") + } + + // no versions -> trash file + err := fs.Delete(ctx, ref) + if err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") + continue + } + + // now purge it from the recycle bin + if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") + } + + // remove cache entry in gateway + fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) + continue + } + + // we have versions - find the newest + versions := make(map[uint64]string) // remember all versions - we need them later + var nv uint64 + for _, v := range revs { + versions[v.Mtime] = v.Key + if v.Mtime > nv { + nv = v.Mtime + } + } + + // restore newest version + if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") + continue + } + + // now find infected version + revs, err = fs.ListRevisions(ctx, ref) + if err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") + } + + for _, v := range revs { + // we looking for a version that was previously not there + if _, ok := versions[v.Mtime]; ok { + continue + } + + if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { + log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") + } + } + + // remove cache entry in gateway + fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) + continue + } + */ + default: + // uploadid is not empty -> this is an async upload + up, err := tusDataStore.GetUpload(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload") + continue + } + info, err := up.GetInfo(ctx) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload info") + continue + } + uploadMetadata, err := ReadMetadata(ctx, lu, info.ID) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("Failed to get upload metadata") + continue // NOTE: since we can't get the upload, we can't delete the blob + } + + // scan data should be set on the node revision not the node ... then when postprocessing finishes we need to copy the state to the node + + n, err = ReadNode(ctx, lu, uploadMetadata) + if err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Interface("metadata", uploadMetadata).Msg("could not read revision node on default event") + continue + } + } - return f, nil -} + if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { + log.Error().Err(err).Str("uploadID", ev.UploadID).Interface("resourceID", res.ResourceID).Msg("Failed to set scan results") + continue + } -// lookupNode looks up nodes by path. -// This method can also handle lookups for paths which contain chunking information. -func lookupNode(ctx context.Context, spaceRoot *node.Node, path string, lu *lookup.Lookup) (*node.Node, error) { - p := path - isChunked := chunking.IsChunked(path) - if isChunked { - chunkInfo, err := chunking.GetChunkBLOBInfo(path) - if err != nil { - return nil, err + // remove cache entry in gateway + cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) + default: + log.Error().Interface("event", ev).Msg("Unknown event") } - p = chunkInfo.Path - } - - n, err := lu.WalkPath(ctx, spaceRoot, p, true, func(ctx context.Context, n *node.Node) error { return nil }) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error walking path") - } - - if isChunked { - n.Name = filepath.Base(path) } - return n, nil } // Progress adapts the persisted upload metadata for the UploadSessionLister interface type Progress struct { + Upload tusd.Upload Path string - Info tusd.FileInfo + Metadata Metadata Processing bool } // ID implements the storage.UploadSession interface func (p Progress) ID() string { - return p.Info.ID + return p.Metadata.ID } // Filename implements the storage.UploadSession interface func (p Progress) Filename() string { - return p.Info.MetaData["filename"] + return p.Metadata.Filename } // Size implements the storage.UploadSession interface func (p Progress) Size() int64 { - return p.Info.Size + info, err := p.Upload.GetInfo(context.Background()) + if err != nil { + return p.Metadata.GetSize() + } + return info.Offset } // Offset implements the storage.UploadSession interface func (p Progress) Offset() int64 { - return p.Info.Offset + info, err := p.Upload.GetInfo(context.Background()) + if err != nil { + return 0 + } + return info.Offset } // Reference implements the storage.UploadSession interface func (p Progress) Reference() provider.Reference { - return provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: p.Info.MetaData["providerID"], - SpaceId: p.Info.Storage["SpaceRoot"], - OpaqueId: p.Info.Storage["NodeId"], // Node id is always set in InitiateUpload - }, - } + return p.Metadata.GetReference() } // Executant implements the storage.UploadSession interface -func (p Progress) Executant() userpb.UserId { - return userpb.UserId{ - Idp: p.Info.Storage["Idp"], - OpaqueId: p.Info.Storage["UserId"], - Type: utils.UserTypeMap(p.Info.Storage["UserType"]), - } +func (p Progress) Executant() user.UserId { + return p.Metadata.GetExecutantID() } // SpaceOwner implements the storage.UploadSession interface -func (p Progress) SpaceOwner() *userpb.UserId { - return &userpb.UserId{ - // idp and type do not seem to be consumed and the node currently only stores the user id anyway - OpaqueId: p.Info.Storage["SpaceOwnerOrManager"], - } +func (p Progress) SpaceOwner() *user.UserId { + u := p.Metadata.GetSpaceOwner() + return &u } // Expires implements the storage.UploadSession interface func (p Progress) Expires() time.Time { - mt, _ := utils.MTimeToTime(p.Info.MetaData["expires"]) - return mt + return p.Metadata.Expires } // IsProcessing implements the storage.UploadSession interface @@ -567,16 +420,22 @@ func (p Progress) IsProcessing() bool { // Purge implements the storage.UploadSession interface func (p Progress) Purge(ctx context.Context) error { - berr := os.Remove(p.Info.Storage["BinPath"]) - if berr != nil { - appctx.GetLogger(ctx).Error().Str("id", p.Info.ID).Interface("path", p.Info.Storage["BinPath"]).Msg("Decomposedfs: could not purge bin path for upload session") + // terminate tus upload + var terr error + if terminatableUpload, ok := p.Upload.(tusd.TerminatableUpload); ok { + terr = terminatableUpload.Terminate(ctx) + if terr != nil { + appctx.GetLogger(ctx).Error().Str("id", p.Metadata.ID).Msg("Decomposedfs: could not terminate tus upload for session") + } + } else { + terr = errors.New("tus upload does not implement TerminatableUpload interface") } - // remove upload metadata + // remove upload session metadata merr := os.Remove(p.Path) if merr != nil { - appctx.GetLogger(ctx).Error().Str("id", p.Info.ID).Interface("path", p.Path).Msg("Decomposedfs: could not purge metadata path for upload session") + appctx.GetLogger(ctx).Error().Str("id", p.Metadata.ID).Interface("path", p.Path).Msg("Decomposedfs: could not purge metadata path for upload session") } - return stderrors.Join(berr, merr) + return errors.Join(terr, merr) } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go index c3e04460542..9f4e34492cc 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go @@ -16,37 +16,74 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. +// Package upload handles the processing of uploads. +// In general this is the lifecycle of an upload from the perspective of a storageprovider: +// 1. To start an upload a client makes a call to InitializeUpload which will return protocols and urls that he can use to append bytes to the upload. +// 2. When the client has sent all bytes the tusd handler will call a PreFinishResponseCallback which marks the end of the transfer and the start of postprocessing. +// 3. When async uploads are enabled the storageprovider emits an BytesReceived event, otherwise a FileUploaded event and the upload lifcycle ends. +// 4. During async postprocessing the uploaded bytes might be read at the upload URL to determine the outcome of the postprocessing steps +// 5. To handle async postprocessing the storageprovider has to listen to multiple events: +// - PostprocessingFinished determines what should happen with the upload: +// - abort - the upload is cancelled but the bytes are kept in the upload folder, eg. when antivirus scanning encounters an error +// then what? can the admin retrigger the upload? +// - continue - the upload is moved to its final destination (eventually being marked with pp results) +// - delete - the file and the upload should be deleted +// - RestartPostprocessing +// - PostprocessingStepFinished is used to set scan data on an upload +// +// 6. The storageprovider emits an UploadReady event that can be used by eg. the search or thumbnails services to do update their metadata. +// +// There are two interesting scenarios: +// 1. Two concurrent requests try to create the same file +// 2. Two concurrent requests try to overwrite the same file +// The first step to upload a file is making an InitiateUpload call to the storageprovider via CS3. It will return an upload id that can be used to append bytes to the upload. +// With an upload id clients can append bytes to the upload. +// When all bytes have been received tusd will call PreFinishResponseCallback on the storageprovider. +// The storageprovider cannot use the tus upload metadata to persist a postprocessing status, we have to store the processing status on a revision node instead. +// On disk the layout for a node consists of the actual node metadata and revision nodes. +// The revision nodes are used to capture the different revsions ... +// * so every upload always creates a revision node first? +// * and in PreFinishResponseCallback we update or create? the actual node? or do we create the node in the InitiateUpload call? +// * We need to skip unfinished revisions when listing versions? +// The size diff is always calculated when updating the node +// +// ## Client considerations +// When do we propagate the etag? Currently, already when an upload is in postprocessing ... why? because we update the node when all bytes are transferred? +// Does the client expect an etag change when it uploads a file? it should not ... sync and uploads are independent last someone explained it to me +// postprocessing might change the content, leading to an etag change as a result +// +// When the client finishes transferring all bytes it gets the 'future' etag of the resource which it currently stores as the etag for the file in its local db. +// When the next propfind happens before postprocessing finishes the client would see the old etag and download the old version. Then, when postprocessing causes +// the next etag change, the client will download the file it previously uploaded. +// +// For the new file scenario, the desktop client would delete the uploaded file locally, when it is not listed in the next propfind. +// +// The graph api exposes pending uploads explicitly using the pendingOperations property, which carries a pendingContentUpdate resource with a +// queuedDateTime property: Date and time the pending binary operation was queued in UTC time. Read-only. +// +// So, until clients learn to keep track of their uploads we need to return 425 when an upload is in progress ಠ_ಠ package upload import ( "context" - "crypto/md5" - "crypto/sha1" - "encoding/hex" - "encoding/json" "fmt" - "hash" - "hash/adler32" - "io" - "io/fs" + iofs "io/fs" "os" "path/filepath" "strings" "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/internal/grpc/services/storageprovider" "github.com/cs3org/reva/v2/pkg/appctx" - ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" - "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/cs3org/reva/v2/pkg/utils" - "github.com/golang-jwt/jwt" "github.com/pkg/errors" - "github.com/rs/zerolog" + "github.com/rogpeppe/go-internal/lockedfile" tusd "github.com/tus/tusd/pkg/handler" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -58,417 +95,298 @@ func init() { tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/upload") } -// Tree is used to manage a tree hierarchy -type Tree interface { - Setup() error - - GetMD(ctx context.Context, node *node.Node) (os.FileInfo, error) - ListFolder(ctx context.Context, node *node.Node) ([]*node.Node, error) - // CreateHome(owner *userpb.UserId) (n *node.Node, err error) - CreateDir(ctx context.Context, node *node.Node) (err error) - // CreateReference(ctx context.Context, node *node.Node, targetURI *url.URL) error - Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error) - Delete(ctx context.Context, node *node.Node) (err error) - RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, target *node.Node) (*node.Node, *node.Node, func() error, error) - PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*node.Node, func() error, error) - - WriteBlob(node *node.Node, binPath string) error - ReadBlob(node *node.Node) (io.ReadCloser, error) - DeleteBlob(node *node.Node) error - - Propagate(ctx context.Context, node *node.Node, sizeDiff int64) (err error) -} - -// Upload processes the upload -// it implements tus tusd.Upload interface https://tus.io/protocols/resumable-upload.html#core-protocol -// it also implements its termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// it also implements its creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// it also implements its concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -type Upload struct { - // we use a struct field on the upload as tus pkg will give us an empty context.Background - Ctx context.Context - // info stores the current information about the upload - Info tusd.FileInfo - // node for easy access - Node *node.Node - // SizeDiff size difference between new and old file version - SizeDiff int64 - // infoPath is the path to the .info file - infoPath string - // binPath is the path to the binary file (which has no extension) - binPath string - // lu and tp needed for file operations - lu *lookup.Lookup - tp Tree - // versionsPath will be empty if there was no file before - versionsPath string - // and a logger as well - log zerolog.Logger - // publisher used to publish events - pub events.Publisher - // async determines if uploads shoud be done asynchronously - async bool - // tknopts hold token signing information - tknopts options.TokenOptions -} - -func buildUpload(ctx context.Context, info tusd.FileInfo, binPath string, infoPath string, lu *lookup.Lookup, tp Tree, pub events.Publisher, async bool, tknopts options.TokenOptions) *Upload { - return &Upload{ - Info: info, - binPath: binPath, - infoPath: infoPath, - lu: lu, - tp: tp, - Ctx: ctx, - pub: pub, - async: async, - tknopts: tknopts, - log: appctx.GetLogger(ctx). - With(). - Interface("info", info). - Str("binPath", binPath). - Logger(), +func validateRequest(ctx context.Context, size int64, uploadMetadata Metadata, n *node.Node) error { + if err := n.CheckLock(ctx); err != nil { + return err } -} -// Cleanup cleans the upload -func Cleanup(upload *Upload, failure bool, keepUpload bool) { - ctx, span := tracer.Start(upload.Ctx, "Cleanup") - defer span.End() - upload.cleanup(failure, !keepUpload, !keepUpload) - - // unset processing status - if upload.Node != nil { // node can be nil when there was an error before it was created (eg. checksum-mismatch) - if err := upload.Node.UnmarkProcessing(ctx, upload.Info.ID); err != nil { - upload.log.Info().Str("path", upload.Node.InternalPath()).Err(err).Msg("unmarking processing failed") - } + if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(n.Blobsize), uint64(size)); err != nil { + return err } -} -// WriteChunk writes the stream from the reader to the given offset of the upload -func (upload *Upload) WriteChunk(_ context.Context, offset int64, src io.Reader) (int64, error) { - ctx, span := tracer.Start(upload.Ctx, "WriteChunk") - defer span.End() - _, subspan := tracer.Start(ctx, "os.OpenFile") - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - subspan.End() + mtime, err := n.GetMTime(ctx) if err != nil { - return 0, err + return err } - defer file.Close() - - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... - _, subspan = tracer.Start(ctx, "io.Copy") - n, err := io.Copy(file, src) - subspan.End() - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the ocis driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil && err != io.ErrUnexpectedEOF { - return n, err + currentEtag, err := node.CalculateEtag(n, mtime) + if err != nil { + return err } - upload.Info.Offset += n - return n, upload.writeInfo() -} - -// GetInfo returns the FileInfo -func (upload *Upload) GetInfo(_ context.Context) (tusd.FileInfo, error) { - return upload.Info, nil -} - -// GetReader returns an io.Reader for the upload -func (upload *Upload) GetReader(_ context.Context) (io.Reader, error) { - _, span := tracer.Start(upload.Ctx, "GetReader") - defer span.End() - return os.Open(upload.binPath) -} - -// FinishUpload finishes an upload and moves the file to the internal destination -func (upload *Upload) FinishUpload(_ context.Context) error { - ctx, span := tracer.Start(upload.Ctx, "FinishUpload") - defer span.End() - // set lockID to context - if upload.Info.MetaData["lockid"] != "" { - upload.Ctx = ctxpkg.ContextSetLockID(upload.Ctx, upload.Info.MetaData["lockid"]) + // When the if-match header was set we need to check if the + // etag still matches before finishing the upload. + if uploadMetadata.HeaderIfMatch != "" { + if uploadMetadata.HeaderIfMatch != currentEtag { + return errtypes.Aborted("etag mismatch") + } } - log := appctx.GetLogger(upload.Ctx) - - // calculate the checksum of the written bytes - // they will all be written to the metadata later, so we cannot omit any of them - // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present - // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - { - _, subspan := tracer.Start(ctx, "os.Open") - f, err := os.Open(upload.binPath) - subspan.End() - if err != nil { - // we can continue if no oc checksum header is set - log.Info().Err(err).Str("binPath", upload.binPath).Msg("error opening binPath") + // When the if-none-match header was set we need to check if any of the + // etags matches before finishing the upload. + if uploadMetadata.HeaderIfNoneMatch != "" { + if uploadMetadata.HeaderIfNoneMatch == "*" { + return errtypes.Aborted("etag mismatch, resource exists") } - defer f.Close() - - r1 := io.TeeReader(f, sha1h) - r2 := io.TeeReader(r1, md5h) - - _, subspan = tracer.Start(ctx, "io.Copy") - _, err = io.Copy(adler32h, r2) - subspan.End() - if err != nil { - log.Info().Err(err).Msg("error copying checksums") + for _, ifNoneMatchTag := range strings.Split(uploadMetadata.HeaderIfNoneMatch, ",") { + if ifNoneMatchTag == currentEtag { + return errtypes.Aborted("etag mismatch") + } } } - // compare if they match the sent checksum - // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - if upload.Info.MetaData["checksum"] != "" { - var err error - parts := strings.SplitN(upload.Info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = upload.checkHash(parts[1], sha1h) - case "md5": - err = upload.checkHash(parts[1], md5h) - case "adler32": - err = upload.checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + // When the if-unmodified-since header was set we need to check if the + // etag still matches before finishing the upload. + if uploadMetadata.HeaderIfUnmodifiedSince != "" { + if err != nil { + return errtypes.InternalError(fmt.Sprintf("failed to read mtime of node: %s", err)) } + ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, uploadMetadata.HeaderIfUnmodifiedSince) if err != nil { - Cleanup(upload, true, false) - return err + return errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) + } + + if mtime.After(ifUnmodifiedSince) { + return errtypes.Aborted("if-unmodified-since mismatch") } } + return nil +} - // update checksums - attrs := node.Attributes{ - prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), - prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), - prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), +func openExistingNode(ctx context.Context, lu *lookup.Lookup, n *node.Node) (*lockedfile.File, error) { + // create and read lock existing node metadata + return lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_RDONLY, 0600) +} +func initNewNode(ctx context.Context, lu *lookup.Lookup, uploadID, mtime string, n *node.Node) (*lockedfile.File, error) { + nodePath := n.InternalPath() + // create folder structure (if needed) + if err := os.MkdirAll(filepath.Dir(nodePath), 0700); err != nil { + return nil, err } - n, err := CreateNodeForUpload(upload, attrs) + // create and write lock new node metadata + f, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(nodePath), os.O_RDWR|os.O_CREATE, 0600) if err != nil { - Cleanup(upload, true, false) - return err + return nil, err } - upload.Node = n + // FIXME if this is removed links to files will be dangling, causing subsequest stats to files to fail + // we also need to touch the actual node file here it stores the mtime of the resource + h, err := os.OpenFile(nodePath, os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return f, err + } + h.Close() - if upload.pub != nil { - u, _ := ctxpkg.ContextGetUser(upload.Ctx) - s, err := upload.URL(upload.Ctx) - if err != nil { - return err - } + // link child name to parent if it is new + childNameLink := filepath.Join(n.ParentPath(), n.Name) + relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - if err := events.Publish(ctx, upload.pub, events.BytesReceived{ - UploadID: upload.Info.ID, - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(upload.Ctx), - ExecutingUser: u, - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: upload.Info.Storage["NodeName"], - Filesize: uint64(upload.Info.Size), - }); err != nil { - return err + if err = os.Symlink(relativeNodePath, childNameLink); err != nil { + if errors.Is(err, iofs.ErrExist) { + return f, errtypes.AlreadyExists(n.Name) } + return f, errors.Wrap(err, "Decomposedfs: could not symlink child entry") } - if !upload.async { - // handle postprocessing synchronously - err = upload.Finalize() - Cleanup(upload, err != nil, false) - if err != nil { - log.Error().Err(err).Msg("failed to upload") - return err - } - } + attrs := node.Attributes{} + attrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) + attrs.SetString(prefixes.ParentidAttr, n.ParentID) + attrs.SetString(prefixes.NameAttr, n.Name) + attrs.SetString(prefixes.MTimeAttr, mtime) - return upload.tp.Propagate(upload.Ctx, n, upload.SizeDiff) -} + // here we set the status the first time. + attrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+uploadID) -// Terminate terminates the upload -func (upload *Upload) Terminate(_ context.Context) error { - upload.cleanup(true, true, true) - return nil + // update node metadata with basic metadata + err = n.SetXattrsWithContext(ctx, attrs, false) + if err != nil { + return nil, errors.Wrap(err, "Decomposedfs: could not write metadata") + } + return f, nil } -// DeclareLength updates the upload length information -func (upload *Upload) DeclareLength(_ context.Context, length int64) error { - upload.Info.Size = length - upload.Info.SizeIsDeferred = false - return upload.writeInfo() -} +func CreateRevisionNode(ctx context.Context, lu *lookup.Lookup, revisionNode *node.Node) (*lockedfile.File, error) { + revisionPath := revisionNode.InternalPath() -// ConcatUploads concatenates multiple uploads -func (upload *Upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + // write lock existing node before reading any metadata + f, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(revisionPath), os.O_RDWR|os.O_CREATE, 0600) if err != nil { - return err + return nil, err } - defer file.Close() - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*Upload) + // FIXME if this is removed listing revisions breaks because it globs the dir but then filters all metadata files + // we also need to touch the versions node here to list revisions + h, err := os.OpenFile(revisionPath, os.O_CREATE /*|os.O_EXCL*/, 0600) // we have to allow overwriting revisions to be oc10 compatible + if err != nil { + return f, err + } + h.Close() + return f, nil +} - src, err := os.Open(fileUpload.binPath) - if err != nil { - return err - } - defer src.Close() +func SetNodeToUpload(ctx context.Context, lu *lookup.Lookup, n *node.Node, uploadMetadata Metadata) (int64, error) { - if _, err := io.Copy(file, src); err != nil { - return err - } + nodePath := n.InternalPath() + // lock existing node metadata + nh, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(nodePath), os.O_RDWR, 0600) + if err != nil { + return 0, err } + defer nh.Close() + // read nodes - return -} - -// writeInfo updates the entire information. Everything will be overwritten. -func (upload *Upload) writeInfo() error { - _, span := tracer.Start(upload.Ctx, "writeInfo") - defer span.End() - data, err := json.Marshal(upload.Info) + n, err = node.ReadNode(ctx, lu, n.SpaceID, n.ID, false, n.SpaceRoot, true) if err != nil { - return err + return 0, err } - return os.WriteFile(upload.infoPath, data, defaultFilePerm) -} -// Finalize finalizes the upload (eg moves the file to the internal destination) -func (upload *Upload) Finalize() (err error) { - ctx, span := tracer.Start(upload.Ctx, "Finalize") - defer span.End() - n := upload.Node - if n == nil { - var err error - n, err = node.ReadNode(ctx, upload.lu, upload.Info.Storage["SpaceRoot"], upload.Info.Storage["NodeId"], false, nil, false) - if err != nil { - return err - } - upload.Node = n + sizeDiff := uploadMetadata.BlobSize - n.Blobsize + + n.BlobID = uploadMetadata.BlobID + n.Blobsize = uploadMetadata.BlobSize + + rm := RevisionMetadata{ + MTime: uploadMetadata.MTime, + BlobID: uploadMetadata.BlobID, + BlobSize: uploadMetadata.BlobSize, + ChecksumSHA1: uploadMetadata.ChecksumSHA1, + ChecksumMD5: uploadMetadata.ChecksumMD5, + ChecksumADLER32: uploadMetadata.ChecksumADLER32, } - // upload the data to the blobstore - _, subspan := tracer.Start(ctx, "WriteBlob") - err = upload.tp.WriteBlob(n, upload.binPath) - subspan.End() + if rm.MTime == "" { + rm.MTime = time.Now().UTC().Format(time.RFC3339Nano) + } + + // update node + err = WriteRevisionMetadataToNode(ctx, n, rm) if err != nil { - return errors.Wrap(err, "failed to upload file to blobstore") + return 0, errors.Wrap(err, "Decomposedfs: could not write metadata") } - return nil + return sizeDiff, nil } -func (upload *Upload) checkHash(expected string, h hash.Hash) error { - if expected != hex.EncodeToString(h.Sum(nil)) { - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", upload.Info.MetaData["checksum"], h.Sum(nil))) - } - return nil +type RevisionMetadata struct { + MTime string + BlobID string + BlobSize int64 + ChecksumSHA1 []byte + ChecksumMD5 []byte + ChecksumADLER32 []byte } -// cleanup cleans up after the upload is finished -func (upload *Upload) cleanup(cleanNode, cleanBin, cleanInfo bool) { - if cleanNode && upload.Node != nil { - switch p := upload.versionsPath; p { - case "": - // remove node - if err := utils.RemoveItem(upload.Node.InternalPath()); err != nil { - upload.log.Info().Str("path", upload.Node.InternalPath()).Err(err).Msg("removing node failed") - } - - // no old version was present - remove child entry - src := filepath.Join(upload.Node.ParentPath(), upload.Node.Name) - if err := os.Remove(src); err != nil { - upload.log.Info().Str("path", upload.Node.ParentPath()).Err(err).Msg("removing node from parent failed") - } - - // remove node from upload as it no longer exists - upload.Node = nil - default: +func WriteRevisionMetadataToNode(ctx context.Context, n *node.Node, revisionMetadata RevisionMetadata) error { + attrs := node.Attributes{} + attrs.SetString(prefixes.BlobIDAttr, revisionMetadata.BlobID) + attrs.SetInt64(prefixes.BlobsizeAttr, revisionMetadata.BlobSize) + attrs.SetString(prefixes.MTimeAttr, revisionMetadata.MTime) + attrs[prefixes.ChecksumPrefix+storageprovider.XSSHA1] = revisionMetadata.ChecksumSHA1 + attrs[prefixes.ChecksumPrefix+storageprovider.XSMD5] = revisionMetadata.ChecksumMD5 + attrs[prefixes.ChecksumPrefix+storageprovider.XSAdler32] = revisionMetadata.ChecksumADLER32 - if err := upload.lu.CopyMetadata(upload.Ctx, p, upload.Node.InternalPath(), func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr - }, true); err != nil { - upload.log.Info().Str("versionpath", p).Str("nodepath", upload.Node.InternalPath()).Err(err).Msg("renaming version node failed") - } - - if err := os.RemoveAll(p); err != nil { - upload.log.Info().Str("versionpath", p).Str("nodepath", upload.Node.InternalPath()).Err(err).Msg("error removing version") - } + return n.SetXattrsWithContext(ctx, attrs, false) +} +func ReadNode(ctx context.Context, lu *lookup.Lookup, uploadMetadata Metadata) (*node.Node, error) { + var n *node.Node + var err error + if uploadMetadata.NodeID == "" { + p, err := node.ReadNode(ctx, lu, uploadMetadata.SpaceRoot, uploadMetadata.NodeParentID, false, nil, true) + if err != nil { + return nil, err } - } - - if cleanBin { - if err := os.Remove(upload.binPath); err != nil && !errors.Is(err, fs.ErrNotExist) { - upload.log.Error().Str("path", upload.binPath).Err(err).Msg("removing upload failed") + n, err = p.Child(ctx, uploadMetadata.Filename) + if err != nil { + return nil, err + } + } else { + n, err = node.ReadNode(ctx, lu, uploadMetadata.SpaceRoot, uploadMetadata.NodeID, false, nil, true) + if err != nil { + return nil, err } } + return n, nil +} + +// Cleanup cleans the upload +func Cleanup(ctx context.Context, lu *lookup.Lookup, n *node.Node, uploadID, revision string, failure bool) { + ctx, span := tracer.Start(ctx, "Cleanup") + defer span.End() - if cleanInfo { - if err := os.Remove(upload.infoPath); err != nil && !errors.Is(err, fs.ErrNotExist) { - upload.log.Error().Str("path", upload.infoPath).Err(err).Msg("removing upload info failed") + if n != nil { // node can be nil when there was an error before it was created (eg. checksum-mismatch) + if failure { + removeRevision(ctx, lu, n, revision) + } + // unset processing status + if err := n.UnmarkProcessing(ctx, uploadID); err != nil { + log := appctx.GetLogger(ctx) + log.Info().Str("path", n.InternalPath()).Err(err).Msg("unmarking processing failed") } } } -// URL returns a url to download an upload -func (upload *Upload) URL(_ context.Context) (string, error) { - type transferClaims struct { - jwt.StandardClaims - Target string `json:"target"` +// removeRevision cleans up after the upload is finished +func removeRevision(ctx context.Context, lu *lookup.Lookup, n *node.Node, revision string) { + log := appctx.GetLogger(ctx) + nodePath := n.InternalPath() + revisionPath := node.JoinRevisionKey(nodePath, revision) + // remove revision + if err := utils.RemoveItem(revisionPath); err != nil { + log.Info().Str("path", revisionPath).Err(err).Msg("removing revision failed") } - - u := joinurl(upload.tknopts.DownloadEndpoint, "tus/", upload.Info.ID) - ttl := time.Duration(upload.tknopts.TransferExpires) * time.Second - claims := transferClaims{ - StandardClaims: jwt.StandardClaims{ - ExpiresAt: time.Now().Add(ttl).Unix(), - Audience: "reva", - IssuedAt: time.Now().Unix(), - }, - Target: u, + // purge revision metadata to clean up cache + if err := lu.MetadataBackend().Purge(revisionPath); err != nil { + log.Info().Str("path", revisionPath).Err(err).Msg("purging revision metadata failed") } - t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) + if n.BlobID == "" { // FIXME ... this is brittle + // no old version was present - remove child entry symlink from directory + src := filepath.Join(n.ParentPath(), n.Name) + if err := os.Remove(src); err != nil { + log.Info().Str("path", n.ParentPath()).Err(err).Msg("removing node from parent failed") + } - tkn, err := t.SignedString([]byte(upload.tknopts.TransferSharedSecret)) - if err != nil { - return "", errors.Wrapf(err, "error signing token with claims %+v", claims) - } + // delete node + if err := utils.RemoveItem(nodePath); err != nil { + log.Info().Str("path", nodePath).Err(err).Msg("removing node failed") + } - return joinurl(upload.tknopts.DataGatewayEndpoint, tkn), nil + // purge node metadata to clean up cache + if err := lu.MetadataBackend().Purge(nodePath); err != nil { + log.Info().Str("path", nodePath).Err(err).Msg("purging node metadata failed") + } + } } -// replace with url.JoinPath after switching to go1.19 -func joinurl(paths ...string) string { - var s strings.Builder - l := len(paths) - for i, p := range paths { - s.WriteString(p) - if !strings.HasSuffix(p, "/") && i != l-1 { - s.WriteString("/") +// Finalize finalizes the upload (eg moves the file to the internal destination) +func Finalize(ctx context.Context, blobstore tree.Blobstore, revision string, info tusd.FileInfo, n *node.Node, blobID string) error { + _, span := tracer.Start(ctx, "Finalize") + defer span.End() + + rn := n.RevisionNode(ctx, revision) + rn.BlobID = blobID + var err error + if mover, ok := blobstore.(tree.BlobstoreMover); ok { + err = mover.MoveBlob(rn, "", info.Storage["Bucket"], info.Storage["Key"]) + switch err { + case nil: + return nil + case tree.ErrBlobstoreCannotMove: + // fallback below + default: + return err } } - return s.String() + // upload the data to the blobstore + _, subspan := tracer.Start(ctx, "WriteBlob") + err = blobstore.Upload(rn, info.Storage["Path"]) // FIXME where do we read from + subspan.End() + if err != nil { + return errors.Wrap(err, "failed to upload file to blobstore") + } + + // FIXME use a reader + return nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go index 2f0fed087ed..335de51d5b4 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go @@ -181,6 +181,9 @@ func (cs3 *CS3) Upload(ctx context.Context, req UploadRequest) (*UploadResponse, ifuReq.Opaque = utils.AppendPlainToOpaque(ifuReq.Opaque, "X-OC-Mtime", strconv.Itoa(int(req.MTime.Unix()))+"."+strconv.Itoa(req.MTime.Nanosecond())) } + // FIXME ... we need a better way to transport filesize + ifuReq.Opaque = utils.AppendPlainToOpaque(ifuReq.Opaque, net.HeaderUploadLength, strconv.FormatInt(int64(len(req.Content)), 10)) + res, err := client.InitiateFileUpload(ctx, ifuReq) if err != nil { return nil, err @@ -215,7 +218,9 @@ func (cs3 *CS3) Upload(ctx context.Context, req UploadRequest) (*UploadResponse, if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { + go func() { resp.Body.Close() }() + }() if err := errtypes.NewErrtypeFromHTTPStatusCode(resp.StatusCode, httpReq.URL.Path); err != nil { return nil, err } diff --git a/vendor/github.com/tus/tusd/internal/uid/uid.go b/vendor/github.com/tus/tusd/internal/uid/uid.go new file mode 100644 index 00000000000..71f0cb397dd --- /dev/null +++ b/vendor/github.com/tus/tusd/internal/uid/uid.go @@ -0,0 +1,23 @@ +package uid + +import ( + "crypto/rand" + "encoding/hex" + "io" +) + +// uid returns a unique id. These ids consist of 128 bits from a +// cryptographically strong pseudo-random generator and are like uuids, but +// without the dashes and significant bits. +// +// See: http://en.wikipedia.org/wiki/UUID#Random_UUID_probability_of_duplicates +func Uid() string { + id := make([]byte, 16) + _, err := io.ReadFull(rand.Reader, id) + if err != nil { + // This is probably an appropriate way to handle errors from our source + // for random bits. + panic(err) + } + return hex.EncodeToString(id) +} diff --git a/vendor/github.com/tus/tusd/pkg/filestore/filestore.go b/vendor/github.com/tus/tusd/pkg/filestore/filestore.go new file mode 100644 index 00000000000..882ae2c85fb --- /dev/null +++ b/vendor/github.com/tus/tusd/pkg/filestore/filestore.go @@ -0,0 +1,225 @@ +// Package filestore provide a storage backend based on the local file system. +// +// FileStore is a storage backend used as a handler.DataStore in handler.NewHandler. +// It stores the uploads in a directory specified in two different files: The +// `[id].info` files are used to store the fileinfo in JSON format. The +// `[id]` files without an extension contain the raw binary data uploaded. +// No cleanup is performed so you may want to run a cronjob to ensure your disk +// is not filled up with old and finished uploads. +package filestore + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/tus/tusd/internal/uid" + "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// See the handler.DataStore interface for documentation about the different +// methods. +type FileStore struct { + // Relative or absolute path to store files in. FileStore does not check + // whether the path exists, use os.MkdirAll in this case on your own. + Path string +} + +// New creates a new file based storage backend. The directory specified will +// be used as the only storage entry. This method does not check +// whether the path exists, use os.MkdirAll to ensure. +// In addition, a locking mechanism is provided. +func New(path string) FileStore { + return FileStore{path} +} + +// UseIn sets this store as the core data store in the passed composer and adds +// all possible extension to it. +func (store FileStore) UseIn(composer *handler.StoreComposer) { + composer.UseCore(store) + composer.UseTerminater(store) + composer.UseConcater(store) + composer.UseLengthDeferrer(store) +} + +func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { + if info.ID == "" { + info.ID = uid.Uid() + } + binPath := store.binPath(info.ID) + info.Storage = map[string]string{ + "Type": "filestore", + "Path": binPath, + } + + // Create binary file with no content + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + if os.IsNotExist(err) { + err = fmt.Errorf("upload directory does not exist: %s", store.Path) + } + return nil, err + } + err = file.Close() + if err != nil { + return nil, err + } + + upload := &fileUpload{ + info: info, + infoPath: store.infoPath(info.ID), + binPath: binPath, + } + + // writeInfo creates the file by itself if necessary + err = upload.writeInfo() + if err != nil { + return nil, err + } + + return upload, nil +} + +func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) { + info := handler.FileInfo{} + data, err := ioutil.ReadFile(store.infoPath(id)) + if err != nil { + if os.IsNotExist(err) { + // Interpret os.ErrNotExist as 404 Not Found + err = handler.ErrNotFound + } + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + binPath := store.binPath(id) + infoPath := store.infoPath(id) + stat, err := os.Stat(binPath) + if err != nil { + if os.IsNotExist(err) { + // Interpret os.ErrNotExist as 404 Not Found + err = handler.ErrNotFound + } + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + }, nil +} + +func (store FileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + return upload.(*fileUpload) +} + +func (store FileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { + return upload.(*fileUpload) +} + +// binPath returns the path to the file storing the binary data. +func (store FileStore) binPath(id string) string { + return filepath.Join(store.Path, id) +} + +// infoPath returns the path to the .info file storing the file's info. +func (store FileStore) infoPath(id string) string { + return filepath.Join(store.Path, id+".info") +} + +type fileUpload struct { + // info stores the current information about the upload + info handler.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string +} + +func (upload *fileUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) { + return upload.info, nil +} + +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + upload.info.Offset += n + return n, err +} + +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} + +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []handler.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +} + +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + return nil +} diff --git a/vendor/github.com/tus/tusd/pkg/s3store/multi_error.go b/vendor/github.com/tus/tusd/pkg/s3store/multi_error.go new file mode 100644 index 00000000000..a2d9e3db985 --- /dev/null +++ b/vendor/github.com/tus/tusd/pkg/s3store/multi_error.go @@ -0,0 +1,13 @@ +package s3store + +import ( + "errors" +) + +func newMultiError(errs []error) error { + message := "Multiple errors occurred:\n" + for _, err := range errs { + message += "\t" + err.Error() + "\n" + } + return errors.New(message) +} diff --git a/vendor/github.com/tus/tusd/pkg/s3store/s3store.go b/vendor/github.com/tus/tusd/pkg/s3store/s3store.go new file mode 100644 index 00000000000..e7fabc589ec --- /dev/null +++ b/vendor/github.com/tus/tusd/pkg/s3store/s3store.go @@ -0,0 +1,1006 @@ +// Package s3store provides a storage backend using AWS S3 or compatible servers. +// +// Configuration +// +// In order to allow this backend to function properly, the user accessing the +// bucket must have at least following AWS IAM policy permissions for the +// bucket and all of its subresources: +// s3:AbortMultipartUpload +// s3:DeleteObject +// s3:GetObject +// s3:ListMultipartUploadParts +// s3:PutObject +// +// While this package uses the official AWS SDK for Go, S3Store is able +// to work with any S3-compatible service such as Riak CS. In order to change +// the HTTP endpoint used for sending requests to, consult the AWS Go SDK +// (http://docs.aws.amazon.com/sdk-for-go/api/aws/Config.html#WithEndpoint-instance_method). +// +// Implementation +// +// Once a new tus upload is initiated, multiple objects in S3 are created: +// +// First of all, a new info object is stored which contains a JSON-encoded blob +// of general information about the upload including its size and meta data. +// This kind of objects have the suffix ".info" in their key. +// +// In addition a new multipart upload +// (http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) is +// created. Whenever a new chunk is uploaded to tusd using a PATCH request, a +// new part is pushed to the multipart upload on S3. +// +// If meta data is associated with the upload during creation, it will be added +// to the multipart upload and after finishing it, the meta data will be passed +// to the final object. However, the metadata which will be attached to the +// final object can only contain ASCII characters and every non-ASCII character +// will be replaced by a question mark (for example, "MenĂ¼" will be "Men?"). +// However, this does not apply for the metadata returned by the GetInfo +// function since it relies on the info object for reading the metadata. +// Therefore, HEAD responses will always contain the unchanged metadata, Base64- +// encoded, even if it contains non-ASCII characters. +// +// Once the upload is finished, the multipart upload is completed, resulting in +// the entire file being stored in the bucket. The info object, containing +// meta data is not deleted. It is recommended to copy the finished upload to +// another bucket to avoid it being deleted by the Termination extension. +// +// If an upload is about to being terminated, the multipart upload is aborted +// which removes all of the uploaded parts from the bucket. In addition, the +// info object is also deleted. If the upload has been finished already, the +// finished object containing the entire upload is also removed. +// +// Considerations +// +// In order to support tus' principle of resumable upload, S3's Multipart-Uploads +// are internally used. +// +// When receiving a PATCH request, its body will be temporarily stored on disk. +// This requirement has been made to ensure the minimum size of a single part +// and to allow the AWS SDK to calculate a checksum. Once the part has been uploaded +// to S3, the temporary file will be removed immediately. Therefore, please +// ensure that the server running this storage backend has enough disk space +// available to hold these caches. +// +// In addition, it must be mentioned that AWS S3 only offers eventual +// consistency (https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel). +// Therefore, it is required to build additional measurements in order to +// prevent concurrent access to the same upload resources which may result in +// data corruption. See handler.LockerDataStore for more information. +package s3store + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/tus/tusd/internal/uid" + "github.com/tus/tusd/pkg/handler" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" +) + +// This regular expression matches every character which is not +// considered valid into a header value according to RFC2616. +var nonPrintableRegexp = regexp.MustCompile(`[^\x09\x20-\x7E]`) + +// See the handler.DataStore interface for documentation about the different +// methods. +type S3Store struct { + // Bucket used to store the data in, e.g. "tusdstore.example.com" + Bucket string + // ObjectPrefix is prepended to the name of each S3 object that is created + // to store uploaded files. It can be used to create a pseudo-directory + // structure in the bucket, e.g. "path/to/my/uploads". + ObjectPrefix string + // MetadataObjectPrefix is prepended to the name of each .info and .part S3 + // object that is created. If it is not set, then ObjectPrefix is used. + MetadataObjectPrefix string + // Service specifies an interface used to communicate with the S3 backend. + // Usually, this is an instance of github.com/aws/aws-sdk-go/service/s3.S3 + // (http://docs.aws.amazon.com/sdk-for-go/api/service/s3/S3.html). + Service S3API + // MaxPartSize specifies the maximum size of a single part uploaded to S3 + // in bytes. This value must be bigger than MinPartSize! In order to + // choose the correct number, two things have to be kept in mind: + // + // If this value is too big and uploading the part to S3 is interrupted + // expectedly, the entire part is discarded and the end user is required + // to resume the upload and re-upload the entire big part. In addition, the + // entire part must be written to disk before submitting to S3. + // + // If this value is too low, a lot of requests to S3 may be made, depending + // on how fast data is coming in. This may result in an eventual overhead. + MaxPartSize int64 + // MinPartSize specifies the minimum size of a single part uploaded to S3 + // in bytes. This number needs to match with the underlying S3 backend or else + // uploaded parts will be reject. AWS S3, for example, uses 5MB for this value. + MinPartSize int64 + // PreferredPartSize specifies the preferred size of a single part uploaded to + // S3. S3Store will attempt to slice the incoming data into parts with this + // size whenever possible. In some cases, smaller parts are necessary, so + // not every part may reach this value. The PreferredPartSize must be inside the + // range of MinPartSize to MaxPartSize. + PreferredPartSize int64 + // MaxMultipartParts is the maximum number of parts an S3 multipart upload is + // allowed to have according to AWS S3 API specifications. + // See: http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html + MaxMultipartParts int64 + // MaxObjectSize is the maximum size an S3 Object can have according to S3 + // API specifications. See link above. + MaxObjectSize int64 + // MaxBufferedParts is the number of additional parts that can be received from + // the client and stored on disk while a part is being uploaded to S3. This + // can help improve throughput by not blocking the client while tusd is + // communicating with the S3 API, which can have unpredictable latency. + MaxBufferedParts int64 + // TemporaryDirectory is the path where S3Store will create temporary files + // on disk during the upload. An empty string ("", the default value) will + // cause S3Store to use the operating system's default temporary directory. + TemporaryDirectory string + // DisableContentHashes instructs the S3Store to not calculate the MD5 and SHA256 + // hashes when uploading data to S3. These hashes are used for file integrity checks + // and for authentication. However, these hashes also consume a significant amount of + // CPU, so it might be desirable to disable them. + // Note that this property is experimental and might be removed in the future! + DisableContentHashes bool +} + +type S3API interface { + PutObjectWithContext(ctx context.Context, input *s3.PutObjectInput, opt ...request.Option) (*s3.PutObjectOutput, error) + ListPartsWithContext(ctx context.Context, input *s3.ListPartsInput, opt ...request.Option) (*s3.ListPartsOutput, error) + UploadPartWithContext(ctx context.Context, input *s3.UploadPartInput, opt ...request.Option) (*s3.UploadPartOutput, error) + GetObjectWithContext(ctx context.Context, input *s3.GetObjectInput, opt ...request.Option) (*s3.GetObjectOutput, error) + CreateMultipartUploadWithContext(ctx context.Context, input *s3.CreateMultipartUploadInput, opt ...request.Option) (*s3.CreateMultipartUploadOutput, error) + AbortMultipartUploadWithContext(ctx context.Context, input *s3.AbortMultipartUploadInput, opt ...request.Option) (*s3.AbortMultipartUploadOutput, error) + DeleteObjectWithContext(ctx context.Context, input *s3.DeleteObjectInput, opt ...request.Option) (*s3.DeleteObjectOutput, error) + DeleteObjectsWithContext(ctx context.Context, input *s3.DeleteObjectsInput, opt ...request.Option) (*s3.DeleteObjectsOutput, error) + CompleteMultipartUploadWithContext(ctx context.Context, input *s3.CompleteMultipartUploadInput, opt ...request.Option) (*s3.CompleteMultipartUploadOutput, error) + UploadPartCopyWithContext(ctx context.Context, input *s3.UploadPartCopyInput, opt ...request.Option) (*s3.UploadPartCopyOutput, error) +} + +type s3APIForPresigning interface { + UploadPartRequest(input *s3.UploadPartInput) (req *request.Request, output *s3.UploadPartOutput) +} + +// New constructs a new storage using the supplied bucket and service object. +func New(bucket string, service S3API) S3Store { + return S3Store{ + Bucket: bucket, + Service: service, + MaxPartSize: 5 * 1024 * 1024 * 1024, + MinPartSize: 5 * 1024 * 1024, + PreferredPartSize: 50 * 1024 * 1024, + MaxMultipartParts: 10000, + MaxObjectSize: 5 * 1024 * 1024 * 1024 * 1024, + MaxBufferedParts: 20, + TemporaryDirectory: "", + } +} + +// UseIn sets this store as the core data store in the passed composer and adds +// all possible extension to it. +func (store S3Store) UseIn(composer *handler.StoreComposer) { + composer.UseCore(store) + composer.UseTerminater(store) + composer.UseConcater(store) + composer.UseLengthDeferrer(store) +} + +type s3Upload struct { + id string + store *S3Store + + // info stores the upload's current FileInfo struct. It may be nil if it hasn't + // been fetched yet from S3. Never read or write to it directly but instead use + // the GetInfo and writeInfo functions. + info *handler.FileInfo +} + +func (store S3Store) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { + // an upload larger than MaxObjectSize must throw an error + if info.Size > store.MaxObjectSize { + return nil, fmt.Errorf("s3store: upload size of %v bytes exceeds MaxObjectSize of %v bytes", info.Size, store.MaxObjectSize) + } + + var uploadId string + if info.ID == "" { + uploadId = uid.Uid() + } else { + // certain tests set info.ID in advance + uploadId = info.ID + } + + // Convert meta data into a map of pointers for AWS Go SDK, sigh. + metadata := make(map[string]*string, len(info.MetaData)) + for key, value := range info.MetaData { + // Copying the value is required in order to prevent it from being + // overwritten by the next iteration. + v := nonPrintableRegexp.ReplaceAllString(value, "?") + metadata[key] = &v + } + + // Create the actual multipart upload + res, err := store.Service.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + Metadata: metadata, + }) + if err != nil { + return nil, fmt.Errorf("s3store: unable to create multipart upload:\n%s", err) + } + + id := uploadId + "+" + *res.UploadId + info.ID = id + + info.Storage = map[string]string{ + "Type": "s3store", + "Bucket": store.Bucket, + "Key": *store.keyWithPrefix(uploadId), + } + + upload := &s3Upload{id, &store, nil} + err = upload.writeInfo(ctx, info) + if err != nil { + return nil, fmt.Errorf("s3store: unable to create info file:\n%s", err) + } + + return upload, nil +} + +func (store S3Store) GetUpload(ctx context.Context, id string) (handler.Upload, error) { + return &s3Upload{id, &store, nil}, nil +} + +func (store S3Store) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + return upload.(*s3Upload) +} + +func (store S3Store) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + return upload.(*s3Upload) +} + +func (store S3Store) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { + return upload.(*s3Upload) +} + +func (upload *s3Upload) writeInfo(ctx context.Context, info handler.FileInfo) error { + id := upload.id + store := upload.store + + uploadId, _ := splitIds(id) + + upload.info = &info + + infoJson, err := json.Marshal(info) + if err != nil { + return err + } + + // Create object on S3 containing information about the file + _, err = store.Service.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.metadataKeyWithPrefix(uploadId + ".info"), + Body: bytes.NewReader(infoJson), + ContentLength: aws.Int64(int64(len(infoJson))), + }) + + return err +} + +func (upload s3Upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + id := upload.id + store := upload.store + + uploadId, multipartId := splitIds(id) + + // Get the total size of the current upload + info, err := upload.GetInfo(ctx) + if err != nil { + return 0, err + } + + size := info.Size + bytesUploaded := int64(0) + optimalPartSize, err := store.calcOptimalPartSize(size) + if err != nil { + return 0, err + } + + // Get number of parts to generate next number + parts, err := store.listAllParts(ctx, id) + if err != nil { + return 0, err + } + + numParts := len(parts) + nextPartNum := int64(numParts + 1) + + incompletePartFile, incompletePartSize, err := store.downloadIncompletePartForUpload(ctx, uploadId) + if err != nil { + return 0, err + } + if incompletePartFile != nil { + defer cleanUpTempFile(incompletePartFile) + + if err := store.deleteIncompletePartForUpload(ctx, uploadId); err != nil { + return 0, err + } + + src = io.MultiReader(incompletePartFile, src) + } + + fileChan := make(chan *os.File, store.MaxBufferedParts) + doneChan := make(chan struct{}) + defer close(doneChan) + + // If we panic or return while there are still files in the channel, then + // we may leak file descriptors. Let's ensure that those are cleaned up. + defer func() { + for file := range fileChan { + cleanUpTempFile(file) + } + }() + + partProducer := s3PartProducer{ + store: store, + done: doneChan, + files: fileChan, + r: src, + } + go partProducer.produce(optimalPartSize) + + for file := range fileChan { + stat, err := file.Stat() + if err != nil { + return 0, err + } + n := stat.Size() + + isFinalChunk := !info.SizeIsDeferred && (size == (offset-incompletePartSize)+n) + if n >= store.MinPartSize || isFinalChunk { + uploadPartInput := &s3.UploadPartInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + PartNumber: aws.Int64(nextPartNum), + } + if err := upload.putPartForUpload(ctx, uploadPartInput, file, n); err != nil { + return bytesUploaded, err + } + } else { + if err := store.putIncompletePartForUpload(ctx, uploadId, file); err != nil { + return bytesUploaded, err + } + + bytesUploaded += n + + return (bytesUploaded - incompletePartSize), nil + } + + offset += n + bytesUploaded += n + nextPartNum += 1 + } + + return bytesUploaded - incompletePartSize, partProducer.err +} + +func cleanUpTempFile(file *os.File) { + file.Close() + os.Remove(file.Name()) +} + +func (upload *s3Upload) putPartForUpload(ctx context.Context, uploadPartInput *s3.UploadPartInput, file *os.File, size int64) error { + defer cleanUpTempFile(file) + + if !upload.store.DisableContentHashes { + // By default, use the traditional approach to upload data + uploadPartInput.Body = file + _, err := upload.store.Service.UploadPartWithContext(ctx, uploadPartInput) + return err + } else { + // Experimental feature to prevent the AWS SDK from calculating the SHA256 hash + // for the parts we upload to S3. + // We compute the presigned URL without the body attached and then send the request + // on our own. This way, the body is not included in the SHA256 calculation. + s3api, ok := upload.store.Service.(s3APIForPresigning) + if !ok { + return fmt.Errorf("s3store: failed to cast S3 service for presigning") + } + + s3Req, _ := s3api.UploadPartRequest(uploadPartInput) + + url, err := s3Req.Presign(15 * time.Minute) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", url, file) + if err != nil { + return err + } + + // Set the Content-Length manually to prevent the usage of Transfer-Encoding: chunked, + // which is not supported by AWS S3. + req.ContentLength = size + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + buf := new(strings.Builder) + io.Copy(buf, res.Body) + return fmt.Errorf("s3store: unexpected response code %d for presigned upload: %s", res.StatusCode, buf.String()) + } + + return nil + } +} + +func (upload *s3Upload) GetInfo(ctx context.Context) (info handler.FileInfo, err error) { + if upload.info != nil { + return *upload.info, nil + } + + info, err = upload.fetchInfo(ctx) + if err != nil { + return info, err + } + + upload.info = &info + return info, nil +} + +func (upload s3Upload) fetchInfo(ctx context.Context) (info handler.FileInfo, err error) { + id := upload.id + store := upload.store + uploadId, _ := splitIds(id) + + // Get file info stored in separate object + res, err := store.Service.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.metadataKeyWithPrefix(uploadId + ".info"), + }) + if err != nil { + if isAwsError(err, "NoSuchKey") { + return info, handler.ErrNotFound + } + + return info, err + } + + if err := json.NewDecoder(res.Body).Decode(&info); err != nil { + return info, err + } + + // Get uploaded parts and their offset + parts, err := store.listAllParts(ctx, id) + if err != nil { + // Check if the error is caused by the upload not being found. This happens + // when the multipart upload has already been completed or aborted. Since + // we already found the info object, we know that the upload has been + // completed and therefore can ensure the the offset is the size. + // AWS S3 returns NoSuchUpload, but other implementations, such as DigitalOcean + // Spaces, can also return NoSuchKey. + if isAwsError(err, "NoSuchUpload") || isAwsError(err, "NoSuchKey") { + info.Offset = info.Size + return info, nil + } else { + return info, err + } + } + + offset := int64(0) + + for _, part := range parts { + offset += *part.Size + } + + incompletePartObject, err := store.getIncompletePartForUpload(ctx, uploadId) + if err != nil { + return info, err + } + if incompletePartObject != nil { + defer incompletePartObject.Body.Close() + offset += *incompletePartObject.ContentLength + } + + info.Offset = offset + + return +} + +func (upload s3Upload) GetReader(ctx context.Context) (io.Reader, error) { + id := upload.id + store := upload.store + uploadId, multipartId := splitIds(id) + + // Attempt to get upload content + res, err := store.Service.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + }) + if err == nil { + // No error occurred, and we are able to stream the object + return res.Body, nil + } + + // If the file cannot be found, we ignore this error and continue since the + // upload may not have been finished yet. In this case we do not want to + // return a ErrNotFound but a more meaning-full message. + if !isAwsError(err, "NoSuchKey") { + return nil, err + } + + // Test whether the multipart upload exists to find out if the upload + // never existsted or just has not been finished yet + _, err = store.Service.ListPartsWithContext(ctx, &s3.ListPartsInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + MaxParts: aws.Int64(0), + }) + if err == nil { + // The multipart upload still exists, which means we cannot download it yet + return nil, handler.NewHTTPError(errors.New("cannot stream non-finished upload"), http.StatusBadRequest) + } + + if isAwsError(err, "NoSuchUpload") { + // Neither the object nor the multipart upload exists, so we return a 404 + return nil, handler.ErrNotFound + } + + return nil, err +} + +func (upload s3Upload) Terminate(ctx context.Context) error { + id := upload.id + store := upload.store + uploadId, multipartId := splitIds(id) + var wg sync.WaitGroup + wg.Add(2) + errs := make([]error, 0, 3) + + go func() { + defer wg.Done() + + // Abort the multipart upload + _, err := store.Service.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + }) + if err != nil && !isAwsError(err, "NoSuchUpload") { + errs = append(errs, err) + } + }() + + go func() { + defer wg.Done() + + // Delete the info and content files + res, err := store.Service.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(store.Bucket), + Delete: &s3.Delete{ + Objects: []*s3.ObjectIdentifier{ + { + Key: store.keyWithPrefix(uploadId), + }, + { + Key: store.metadataKeyWithPrefix(uploadId + ".part"), + }, + { + Key: store.metadataKeyWithPrefix(uploadId + ".info"), + }, + }, + Quiet: aws.Bool(true), + }, + }) + + if err != nil { + errs = append(errs, err) + return + } + + for _, s3Err := range res.Errors { + if *s3Err.Code != "NoSuchKey" { + errs = append(errs, fmt.Errorf("AWS S3 Error (%s) for object %s: %s", *s3Err.Code, *s3Err.Key, *s3Err.Message)) + } + } + }() + + wg.Wait() + + if len(errs) > 0 { + return newMultiError(errs) + } + + return nil +} + +func (upload s3Upload) FinishUpload(ctx context.Context) error { + id := upload.id + store := upload.store + uploadId, multipartId := splitIds(id) + + // Get uploaded parts + parts, err := store.listAllParts(ctx, id) + if err != nil { + return err + } + + if len(parts) == 0 { + // AWS expects at least one part to be present when completing the multipart + // upload. So if the tus upload has a size of 0, we create an empty part + // and use that for completing the multipart upload. + res, err := store.Service.UploadPartWithContext(ctx, &s3.UploadPartInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + PartNumber: aws.Int64(1), + Body: bytes.NewReader([]byte{}), + }) + if err != nil { + return err + } + + parts = []*s3.Part{ + &s3.Part{ + ETag: res.ETag, + PartNumber: aws.Int64(1), + }, + } + + } + + // Transform the []*s3.Part slice to a []*s3.CompletedPart slice for the next + // request. + completedParts := make([]*s3.CompletedPart, len(parts)) + + for index, part := range parts { + completedParts[index] = &s3.CompletedPart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + } + } + + _, err = store.Service.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + MultipartUpload: &s3.CompletedMultipartUpload{ + Parts: completedParts, + }, + }) + + return err +} + +func (upload *s3Upload) ConcatUploads(ctx context.Context, partialUploads []handler.Upload) error { + hasSmallPart := false + for _, partialUpload := range partialUploads { + info, err := partialUpload.GetInfo(ctx) + if err != nil { + return err + } + + if info.Size < upload.store.MinPartSize { + hasSmallPart = true + } + } + + // If one partial upload is smaller than the the minimum part size for an S3 + // Multipart Upload, we cannot use S3 Multipart Uploads for concatenating all + // the files. + // So instead we have to download them and concat them on disk. + if hasSmallPart { + return upload.concatUsingDownload(ctx, partialUploads) + } else { + return upload.concatUsingMultipart(ctx, partialUploads) + } +} + +func (upload *s3Upload) concatUsingDownload(ctx context.Context, partialUploads []handler.Upload) error { + id := upload.id + store := upload.store + uploadId, multipartId := splitIds(id) + + // Create a temporary file for holding the concatenated data + file, err := ioutil.TempFile(store.TemporaryDirectory, "tusd-s3-concat-tmp-") + if err != nil { + return err + } + defer cleanUpTempFile(file) + + // Download each part and append it to the temporary file + for _, partialUpload := range partialUploads { + partialS3Upload := partialUpload.(*s3Upload) + partialId, _ := splitIds(partialS3Upload.id) + + res, err := store.Service.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(partialId), + }) + if err != nil { + return err + } + defer res.Body.Close() + + if _, err := io.Copy(file, res.Body); err != nil { + return err + } + } + + // Seek to the beginning of the file, so the entire file is being uploaded + file.Seek(0, 0) + + // Upload the entire file to S3 + _, err = store.Service.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + Body: file, + }) + if err != nil { + return err + } + + // Finally, abort the multipart upload since it will no longer be used. + // This happens asynchronously since we do not need to wait for the result. + // Also, the error is ignored on purpose as it does not change the outcome of + // the request. + go func() { + store.Service.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + }) + }() + + return nil +} + +func (upload *s3Upload) concatUsingMultipart(ctx context.Context, partialUploads []handler.Upload) error { + id := upload.id + store := upload.store + uploadId, multipartId := splitIds(id) + + numPartialUploads := len(partialUploads) + errs := make([]error, 0, numPartialUploads) + + // Copy partial uploads concurrently + var wg sync.WaitGroup + wg.Add(numPartialUploads) + for i, partialUpload := range partialUploads { + partialS3Upload := partialUpload.(*s3Upload) + partialId, _ := splitIds(partialS3Upload.id) + + go func(i int, partialId string) { + defer wg.Done() + + _, err := store.Service.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + // Part numbers must be in the range of 1 to 10000, inclusive. Since + // slice indexes start at 0, we add 1 to ensure that i >= 1. + PartNumber: aws.Int64(int64(i + 1)), + CopySource: aws.String(store.Bucket + "/" + *store.keyWithPrefix(partialId)), + }) + if err != nil { + errs = append(errs, err) + return + } + }(i, partialId) + } + + wg.Wait() + + if len(errs) > 0 { + return newMultiError(errs) + } + + return upload.FinishUpload(ctx) +} + +func (upload *s3Upload) DeclareLength(ctx context.Context, length int64) error { + info, err := upload.GetInfo(ctx) + if err != nil { + return err + } + info.Size = length + info.SizeIsDeferred = false + + return upload.writeInfo(ctx, info) +} + +func (store S3Store) listAllParts(ctx context.Context, id string) (parts []*s3.Part, err error) { + uploadId, multipartId := splitIds(id) + + partMarker := int64(0) + for { + // Get uploaded parts + listPtr, err := store.Service.ListPartsWithContext(ctx, &s3.ListPartsInput{ + Bucket: aws.String(store.Bucket), + Key: store.keyWithPrefix(uploadId), + UploadId: aws.String(multipartId), + PartNumberMarker: aws.Int64(partMarker), + }) + if err != nil { + return nil, err + } + + parts = append(parts, (*listPtr).Parts...) + + if listPtr.IsTruncated != nil && *listPtr.IsTruncated { + partMarker = *listPtr.NextPartNumberMarker + } else { + break + } + } + return parts, nil +} + +func (store S3Store) downloadIncompletePartForUpload(ctx context.Context, uploadId string) (*os.File, int64, error) { + incompleteUploadObject, err := store.getIncompletePartForUpload(ctx, uploadId) + if err != nil { + return nil, 0, err + } + if incompleteUploadObject == nil { + // We did not find an incomplete upload + return nil, 0, nil + } + defer incompleteUploadObject.Body.Close() + + partFile, err := ioutil.TempFile(store.TemporaryDirectory, "tusd-s3-tmp-") + if err != nil { + return nil, 0, err + } + + n, err := io.Copy(partFile, incompleteUploadObject.Body) + if err != nil { + return nil, 0, err + } + if n < *incompleteUploadObject.ContentLength { + return nil, 0, errors.New("short read of incomplete upload") + } + + _, err = partFile.Seek(0, 0) + if err != nil { + return nil, 0, err + } + + return partFile, n, nil +} + +func (store S3Store) getIncompletePartForUpload(ctx context.Context, uploadId string) (*s3.GetObjectOutput, error) { + obj, err := store.Service.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.metadataKeyWithPrefix(uploadId + ".part"), + }) + + if err != nil && (isAwsError(err, s3.ErrCodeNoSuchKey) || isAwsError(err, "NotFound") || isAwsError(err, "AccessDenied")) { + return nil, nil + } + + return obj, err +} + +func (store S3Store) putIncompletePartForUpload(ctx context.Context, uploadId string, file *os.File) error { + defer cleanUpTempFile(file) + + _, err := store.Service.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.metadataKeyWithPrefix(uploadId + ".part"), + Body: file, + }) + return err +} + +func (store S3Store) deleteIncompletePartForUpload(ctx context.Context, uploadId string) error { + _, err := store.Service.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(store.Bucket), + Key: store.metadataKeyWithPrefix(uploadId + ".part"), + }) + return err +} + +func splitIds(id string) (uploadId, multipartId string) { + index := strings.Index(id, "+") + if index == -1 { + return + } + + uploadId = id[:index] + multipartId = id[index+1:] + return +} + +// isAwsError tests whether an error object is an instance of the AWS error +// specified by its code. +func isAwsError(err error, code string) bool { + if err, ok := err.(awserr.Error); ok && err.Code() == code { + return true + } + return false +} + +func (store S3Store) calcOptimalPartSize(size int64) (optimalPartSize int64, err error) { + switch { + // When upload is smaller or equal to PreferredPartSize, we upload in just one part. + case size <= store.PreferredPartSize: + optimalPartSize = store.PreferredPartSize + // Does the upload fit in MaxMultipartParts parts or less with PreferredPartSize. + case size <= store.PreferredPartSize*store.MaxMultipartParts: + optimalPartSize = store.PreferredPartSize + // Prerequisite: Be aware, that the result of an integer division (x/y) is + // ALWAYS rounded DOWN, as there are no digits behind the comma. + // In order to find out, whether we have an exact result or a rounded down + // one, we can check, whether the remainder of that division is 0 (x%y == 0). + // + // So if the result of (size/MaxMultipartParts) is not a rounded down value, + // then we can use it as our optimalPartSize. But if this division produces a + // remainder, we have to round up the result by adding +1. Otherwise our + // upload would not fit into MaxMultipartParts number of parts with that + // size. We would need an additional part in order to upload everything. + // While in almost all cases, we could skip the check for the remainder and + // just add +1 to every result, but there is one case, where doing that would + // doom our upload. When (MaxObjectSize == MaxPartSize * MaxMultipartParts), + // by adding +1, we would end up with an optimalPartSize > MaxPartSize. + // With the current S3 API specifications, we will not run into this problem, + // but these specs are subject to change, and there are other stores as well, + // which are implementing the S3 API (e.g. RIAK, Ceph RadosGW), but might + // have different settings. + case size%store.MaxMultipartParts == 0: + optimalPartSize = size / store.MaxMultipartParts + // Having a remainder larger than 0 means, the float result would have + // digits after the comma (e.g. be something like 10.9). As a result, we can + // only squeeze our upload into MaxMultipartParts parts, if we rounded UP + // this division's result. That is what is happending here. We round up by + // adding +1, if the prior test for (remainder == 0) did not succeed. + default: + optimalPartSize = size/store.MaxMultipartParts + 1 + } + + // optimalPartSize must never exceed MaxPartSize + if optimalPartSize > store.MaxPartSize { + return optimalPartSize, fmt.Errorf("calcOptimalPartSize: to upload %v bytes optimalPartSize %v must exceed MaxPartSize %v", size, optimalPartSize, store.MaxPartSize) + } + return optimalPartSize, nil +} + +func (store S3Store) keyWithPrefix(key string) *string { + prefix := store.ObjectPrefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + return aws.String(prefix + key) +} + +func (store S3Store) metadataKeyWithPrefix(key string) *string { + prefix := store.MetadataObjectPrefix + if prefix == "" { + prefix = store.ObjectPrefix + } + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + return aws.String(prefix + key) +} diff --git a/vendor/github.com/tus/tusd/pkg/s3store/s3store_part_producer.go b/vendor/github.com/tus/tusd/pkg/s3store/s3store_part_producer.go new file mode 100644 index 00000000000..80b3f857ef9 --- /dev/null +++ b/vendor/github.com/tus/tusd/pkg/s3store/s3store_part_producer.go @@ -0,0 +1,64 @@ +package s3store + +import ( + "io" + "io/ioutil" + "os" +) + +// s3PartProducer converts a stream of bytes from the reader into a stream of files on disk +type s3PartProducer struct { + store *S3Store + files chan<- *os.File + done chan struct{} + err error + r io.Reader +} + +func (spp *s3PartProducer) produce(partSize int64) { + for { + file, err := spp.nextPart(partSize) + if err != nil { + spp.err = err + close(spp.files) + return + } + if file == nil { + close(spp.files) + return + } + select { + case spp.files <- file: + case <-spp.done: + close(spp.files) + return + } + } +} + +func (spp *s3PartProducer) nextPart(size int64) (*os.File, error) { + // Create a temporary file to store the part + file, err := ioutil.TempFile(spp.store.TemporaryDirectory, "tusd-s3-tmp-") + if err != nil { + return nil, err + } + + limitedReader := io.LimitReader(spp.r, size) + n, err := io.Copy(file, limitedReader) + if err != nil { + return nil, err + } + + // If the entire request body is read and no more data is available, + // io.Copy returns 0 since it is unable to read any bytes. In that + // case, we can close the s3PartProducer. + if n == 0 { + cleanUpTempFile(file) + return nil, nil + } + + // Seek to the beginning of the file + file.Seek(0, 0) + + return file, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ee520d3afe1..85c924cadf1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -357,7 +357,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.16.1-0.20231201122033-a389ddc645c4 +# github.com/cs3org/reva/v2 v2.16.1-0.20231201122033-a389ddc645c4 => github.com/butonic/reva/v2 v2.0.0-20231205112716-b643432e6672 ## explicit; go 1.20 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime @@ -1732,7 +1732,10 @@ github.com/trustelem/zxcvbn/matching github.com/trustelem/zxcvbn/scoring # github.com/tus/tusd v1.13.0 ## explicit; go 1.16 +github.com/tus/tusd/internal/uid +github.com/tus/tusd/pkg/filestore github.com/tus/tusd/pkg/handler +github.com/tus/tusd/pkg/s3store # github.com/urfave/cli/v2 v2.25.7 ## explicit; go 1.18 github.com/urfave/cli/v2 @@ -2276,3 +2279,4 @@ stash.kopano.io/kgol/oidc-go ## explicit; go 1.13 stash.kopano.io/kgol/rndm # github.com/go-micro/plugins/v4/store/nats-js => github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a +# github.com/cs3org/reva/v2 => github.com/butonic/reva/v2 v2.0.0-20231205112716-b643432e6672