swap out for parquet-go-source S3 writer

.golangci.yml

@@ -51,6 +51,10 @@ linters-settings:
     disabled-checks:
       - appendAssign
 
+  gosec:
+    excludes:
+      - G601 # memory aliasing -- not a problem in 1.22+
+
   importas:
     no-unaliased: true
     alias:

cmd/args.go

@@ -6,21 +6,21 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/acrlabs/prom2parquet/pkg/remotes"
+	"github.com/acrlabs/prom2parquet/pkg/backends"
 )
 
 const (
-	prefixFlag            = "prefix"
-	serverPortFlag        = "server-port"
-	cleanLocalStorageFlag = "clean-local-storage"
-	remoteFlag            = "remote"
-	verbosityFlag         = "verbosity"
+	prefixFlag      = "prefix"
+	serverPortFlag  = "server-port"
+	backendFlag     = "backend"
+	backendRootFlag = "backend-root"
+	verbosityFlag   = "verbosity"
 )
 
 //nolint:gochecknoglobals
-var supportedRemoteIDs = map[remotes.Endpoint][]string{
-	remotes.None: {"none"},
-	remotes.S3:   {"s3", "aws"},
+var supportedBackendIDs = map[backends.StorageBackend][]string{
+	backends.Local: {"none"},
+	backends.S3:    {"s3", "aws"},
 }
 
 //nolint:gochecknoglobals
@@ -38,8 +38,8 @@ type options struct {
 	prefix string
 	port   int
 
-	cleanLocalStorage bool
-	remote            remotes.Endpoint
+	backend     backends.StorageBackend
+	backendRoot string
 
 	verbosity log.Level
 }

cmd/root.go

@@ -40,21 +40,22 @@ func rootCmd() *cobra.Command {
 		"port for the remote write endpoint to listen on",
 	)
 
-	root.PersistentFlags().BoolVar(
-		&opts.cleanLocalStorage,
-		cleanLocalStorageFlag,
-		false,
-		"delete pod-local parquet files upon flush",
-	)
 	root.PersistentFlags().Var(
-		enumflag.New(&opts.remote, remoteFlag, supportedRemoteIDs, enumflag.EnumCaseInsensitive),
-		remoteFlag,
+		enumflag.New(&opts.backend, backendFlag, supportedBackendIDs, enumflag.EnumCaseInsensitive),
+		backendFlag,
 		fmt.Sprintf(
-			"supported remote endpoints for saving parquet files\n(valid options: %s)",
-			validArgs(supportedRemoteIDs),
+			"supported remote backends for saving parquet files\n(valid options: %s)",
+			validArgs(supportedBackendIDs),
 		),
 	)
 
+	root.PersistentFlags().StringVar(
+		&opts.backendRoot,
+		backendRootFlag,
+		"/data",
+		"root path/location for the specified backend (e.g. bucket name for AWS S3)",
+	)
+
 	root.PersistentFlags().VarP(
 		enumflag.New(&opts.verbosity, verbosityFlag, logLevelIDs, enumflag.EnumCaseInsensitive),
 		verbosityFlag,
@@ -66,6 +67,7 @@ func rootCmd() *cobra.Command {
 
 func start(opts *options) {
 	util.SetupLogging(opts.verbosity)
+	log.Infof("running with options: %v", opts)
 
 	server := newServer(opts)
 	server.run()

cmd/server.go

@@ -148,11 +148,10 @@ func (self *promserver) spawnWriter(ctx context.Context, metricName string) (cha
 	log.Infof("new metric name seen, creating writer %s", metricName)
 	writer, err := parquet.NewProm2ParquetWriter(
 		ctx,
-		"/data",
+		self.opts.backendRoot,
 		self.opts.prefix,
 		metricName,
-		self.opts.cleanLocalStorage,
-		self.opts.remote,
+		self.opts.backend,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("could not create writer for %s: %w", metricName, err)

go.mod

@@ -2,6 +2,8 @@ module github.com/acrlabs/prom2parquet
 
 go 1.22.0
 
+replace github.com/xitongsys/parquet-go => github.com/drmorr0/parquet-go v1.7.0
+
 require (
 	github.com/aws/aws-sdk-go-v2/config v1.25.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0
@@ -10,7 +12,10 @@ require (
 	github.com/prometheus/prometheus v0.49.1
 	github.com/samber/lo v1.39.0
 	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.8.4
+	github.com/thediveo/enumflag/v2 v2.0.5
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18
 )
@@ -21,13 +26,15 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
-	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
-	github.com/apache/thrift v0.14.2 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/apache/arrow/go/v12 v12.0.1 // indirect
+	github.com/apache/thrift v0.16.0 // indirect
 	github.com/aws/aws-sdk-go v1.48.14 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.25.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.2 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
@@ -42,36 +49,45 @@ require (
 	github.com/aws/smithy-go v1.20.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.9.11 // indirect
+	github.com/goccy/go-reflect v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/flatbuffers v2.0.8+incompatible // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/pierrec/lz4/v4 v4.1.8 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/thediveo/enumflag/v2 v2.0.5 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opentelemetry.io/collector/featuregate v1.0.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.90.1 // indirect
@@ -83,15 +99,18 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )