Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@butonic
butonic committed Sep 22, 2023
1 parent c5ff99e commit 7bec02a
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 45 deletions.
5 changes: 3 additions & 2 deletions services/storage-users/pkg/command/uploads.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
tusd "github.com/tus/tusd/pkg/handler"
"github.com/urfave/cli/v2"

"github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
Expand Down Expand Up @@ -64,7 +65,7 @@ func ListUploads(cfg *config.Config) *cli.Command {

fmt.Println("Incomplete uploads:")
for _, u := range uploads {
fmt.Printf(" - %s (%s, Size: %d, Expires: %s)\n", u.ID, u.MetaData["filename"], u.Size, expiredString(u.MetaData["expires"]))
fmt.Printf(" - %s (%s, Size: %d, Expires: %s)\n", u.ID, u.MetaData[tus.CS3Prefix+"filename"], u.Size, expiredString(u.MetaData[tus.CS3Prefix+"expires"]))
}
return nil
},
Expand Down Expand Up @@ -105,7 +106,7 @@ func PurgeExpiredUploads(cfg *config.Config) *cli.Command {
fmt.Println("Cleaned uploads:")
go func() {
for purged := range purgedChannel {
fmt.Printf(" - %s (%s, Size: %d, Expires: %s)\n", purged.ID, purged.MetaData["filename"], purged.Size, expiredString(purged.MetaData["expires"]))
fmt.Printf(" - %s (%s, Size: %d, Expires: %s)\n", purged.ID, purged.MetaData[tus.CS3Prefix+"filename"], purged.Size, expiredString(purged.MetaData[tus.CS3Prefix+"expires"]))
}
wg.Done()
}()
Expand Down
21 changes: 13 additions & 8 deletions services/storage-users/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,19 @@ type S3NGDriver struct {
Propagator string `yaml:"propagator" env:"OCIS_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_S3NG_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option."`
AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"`
// Root is the absolute path to the location of the data
Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."`
UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."`
PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."`
Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."`
AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."`
SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."`
Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."`
Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."`
Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."`
UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."`
PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."`
Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."`
AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."`
SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."`
Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."`
Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."`
UploadObjectPrefix string `yaml:"upload_object_prefix" env:"STORAGE_USERS_S3NG_UPLOAD_OBJECT_PREFIX" desc:"Object prefix is prepended to the name of each S3 object that is created to store uploaded files. It can be used to create a pseudo-directory structure in the bucket, e.g. 'path/to/my/uploads'."`
UploadMetadataPrefix string `yaml:"upload_metadata_prefix" env:"STORAGE_USERS_S3NG_UPLOAD_METADATA_PREFIX" desc:"Metadata object prefix is prepended to the name of each .info and .part S3 object that is created. If it is not set, then object prefix is used."`
UploadTemporaryDirectory string `yaml:"upload_temporary_directory" env:"STORAGE_USERS_S3NG_UPLOAD_TEMPORARY_DIRECTORY" desc:"Path where temporary files will be stored on disk during the upload."`
DisableSSL bool `yaml:"disable_ssl" env:"STORAGE_USERS_S3NG_DISABLE_SSL" desc:"Disable SSL when accessing the S3 bucket."`
ForcePathStyle bool `yaml:"force_path_style" env:"STORAGE_USERS_S3NG_FORCE_PATH_STYLE" desc:"Force path style S3 requests."`
// PersonalSpaceAliasTemplate contains the template used to construct
// the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"`
PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_S3NG_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases."`
Expand Down
1 change: 1 addition & 0 deletions services/storage-users/pkg/config/defaults/defaultconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func DefaultConfig() *config.Config {
PermissionsEndpoint: "com.owncloud.api.settings",
MaxAcquireLockCycles: 20,
LockCycleDurationFactor: 30,
UploadObjectPrefix: "uploads",
},
OCIS: config.OCISDriver{
MetadataBackend: "messagepack",
Expand Down
80 changes: 45 additions & 35 deletions services/storage-users/pkg/revaconfig/drivers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,24 +237,29 @@ func S3NG(cfg *config.Config) map[string]interface{} {
"async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay,
},
"root": cfg.Drivers.S3NG.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder,
"personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"root": cfg.Drivers.S3NG.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder,
"personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"s3.upload_object_prefix": cfg.Drivers.S3NG.UploadObjectPrefix,
"s3.upload_metadata_prefix": cfg.Drivers.S3NG.UploadMetadataPrefix,
"s3.upload_temporary_directory": cfg.Drivers.S3NG.UploadTemporaryDirectory,
"s3.disable_ssl": cfg.Drivers.S3NG.DisableSSL,
"s3.force_path_style": cfg.Drivers.S3NG.ForcePathStyle,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
Expand Down Expand Up @@ -300,23 +305,28 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} {
"async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay,
},
"root": cfg.Drivers.S3NG.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder,
"personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"root": cfg.Drivers.S3NG.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder,
"personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"s3.upload_object_prefix": cfg.Drivers.S3NG.UploadObjectPrefix,
"s3.upload_metadata_prefix": cfg.Drivers.S3NG.UploadMetadataPrefix,
"s3.upload_temporary_directory": cfg.Drivers.S3NG.UploadTemporaryDirectory,
"s3.disable_ssl": cfg.Drivers.S3NG.DisableSSL,
"s3.force_path_style": cfg.Drivers.S3NG.ForcePathStyle,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
Expand Down

0 comments on commit 7bec02a

Please sign in to comment.