Add script to delete indexes from Redis (#2120)

Add a script to check what indexes are in the EntryIndex table in a MySQL database and deletes those indexes from the Redis database. This is meant to be done after Rekor is migrated to using --search_index.storage_provider=mysql and the MySQL table has been backfilled with all the indexes. Deleting the data allows us to downsize the Redis instance. Signed-off-by: Colleen Murphy <[email protected]>
sigstore · May 15, 2024 · bb41aff · bb41aff
1 parent b047093
commit bb41aff
Show file tree

Hide file tree

Showing 4 changed files with 364 additions and 44 deletions.
diff --git a/cmd/cleanup-index/main.go b/cmd/cleanup-index/main.go
@@ -0,0 +1,161 @@
+// Copyright 2024 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+	cleanup-index checks what index entries are in the MySQL table and deletes those entries from the Redis databse.
+	It does not go the other way
+
+	To run:
+	go run cmd/cleanup-index/main.go --mysql-dsn <mysql connection> --redis-hostname <redis-hostname> --redis-port <redis-port> [--dry-run]
+*/
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jmoiron/sqlx"
+	"github.com/redis/go-redis/v9"
+	"sigs.k8s.io/release-utils/version"
+
+	// these imports are to call the packages' init methods
+	_ "github.com/sigstore/rekor/pkg/types/alpine/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/cose/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/dsse/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/helm/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.2"
+	_ "github.com/sigstore/rekor/pkg/types/jar/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rekord/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rfc3161/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/rpm/v0.0.1"
+	_ "github.com/sigstore/rekor/pkg/types/tuf/v0.0.1"
+)
+
+const (
+	mysqlSelectStmt = "SELECT DISTINCT EntryKey FROM EntryIndex"
+)
+
+var (
+	redisHostname           = flag.String("redis-hostname", "", "Hostname for Redis application")
+	redisPort               = flag.String("redis-port", "", "Port to Redis application")
+	redisPassword           = flag.String("redis-password", "", "Password for Redis authentication")
+	redisEnableTLS          = flag.Bool("redis-enable-tls", false, "Enable TLS for Redis client")
+	redisInsecureSkipVerify = flag.Bool("redis-insecure-skip-verify", false, "Whether to skip TLS verification for Redis client or not")
+	mysqlDSN                = flag.String("mysql-dsn", "", "MySQL Data Source Name")
+	versionFlag             = flag.Bool("version", false, "Print the current version of Backfill MySQL")
+	dryRun                  = flag.Bool("dry-run", false, "Dry run - don't actually insert into MySQL")
+)
+
+func main() {
+	flag.Parse()
+
+	versionInfo := version.GetVersionInfo()
+	if *versionFlag {
+		fmt.Println(versionInfo.String())
+		os.Exit(0)
+	}
+
+	if *mysqlDSN == "" {
+		log.Fatal("mysql-dsn must be set")
+	}
+	if *redisHostname == "" {
+		log.Fatal("redis-hostname must be set")
+	}
+	if *redisPort == "" {
+		log.Fatal("redis-port must be set")
+	}
+
+	log.Printf("running cleanup index Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate)
+
+	redisClient := getRedisClient()
+
+	mysqlClient, err := getMySQLClient()
+	if err != nil {
+		log.Fatalf("creating mysql client: %v", err)
+	}
+
+	ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+
+	keys, err := getKeysToDelete(ctx, mysqlClient)
+	if err != nil {
+		log.Fatalf("getting keys from mysql: %v", err)
+	}
+	err = removeFromRedis(ctx, redisClient, keys)
+	if err != nil {
+		log.Fatalf("deleting keys from redis: %v", err)
+	}
+}
+
+// getRedisClient creates a Redis client.
+func getRedisClient() *redis.Client {
+	opts := &redis.Options{
+		Addr:     fmt.Sprintf("%s:%s", *redisHostname, *redisPort),
+		Password: *redisPassword,
+		Network:  "tcp",
+		DB:       0, // default DB
+	}
+	// #nosec G402
+	if *redisEnableTLS {
+		opts.TLSConfig = &tls.Config{
+			InsecureSkipVerify: *redisInsecureSkipVerify, //nolint: gosec
+		}
+	}
+	return redis.NewClient(opts)
+}
+
+// getMySQLClient creates a MySQL client.
+func getMySQLClient() (*sqlx.DB, error) {
+	dbClient, err := sqlx.Open("mysql", *mysqlDSN)
+	if err != nil {
+		return nil, err
+	}
+	if err = dbClient.Ping(); err != nil {
+		return nil, err
+	}
+	return dbClient, nil
+}
+
+// getKeysToDelete looks up entries in the EntryIndex table in MySQL.
+func getKeysToDelete(ctx context.Context, dbClient *sqlx.DB) ([]string, error) {
+	keys := []string{}
+	err := dbClient.SelectContext(ctx, &keys, mysqlSelectStmt)
+	return keys, err
+}
+
+// removeFromRedis delete the given keys from Redis.
+func removeFromRedis(ctx context.Context, redisClient *redis.Client, keys []string) error {
+	fmt.Printf("attempting to remove %d keys from redis\n", len(keys))
+	if *dryRun {
+		return nil
+	}
+	result, err := redisClient.Del(ctx, keys...).Result()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("removed %d keys from redis\n", result)
+	if result != int64(len(keys)) {
+		fmt.Println("some keys present in mysql may already have been removed from redis")
+	}
+	return nil
+}
diff --git a/tests/backfill-test.sh b/tests/backfill-test.sh
@@ -29,6 +29,10 @@ testdir=$(mktemp -d)
 declare -A expected_artifacts
 declare -A expected_keys
 
+source $(dirname "$0")/index-test-utils.sh
+
+trap cleanup EXIT
+
 make_entries() {
     set -e
     # make 10 unique artifacts and sign each once
@@ -67,50 +71,6 @@ make_entries() {
     set +e
 }
 
-cleanup() {
-    rv=$?
-    if [ $rv -ne 0 ] ; then
-        docker-compose -f docker-compose.yml -f docker-compose.backfill-test.yml logs --no-color > /tmp/docker-compose.log
-    fi
-    docker-compose down --remove-orphans
-    rm -rf $testdir
-    exit $rv
-}
-trap cleanup EXIT
-
-docker_up () {
-    set -e
-    docker-compose -f docker-compose.yml -f docker-compose.backfill-test.yml up -d --build
-    local count=0
-    echo "waiting up to 2 min for system to start"
-    until [ $(docker-compose ps | \
-       grep -E "(rekor[-_]mysql|rekor[-_]redis|rekor[-_]rekor-server)" | \
-       grep -c "(healthy)" ) == 3 ];
-    do
-        if [ $count -eq 24 ]; then
-           echo "! timeout reached"
-           exit 1
-        else
-           echo -n "."
-           sleep 5
-           let 'count+=1'
-        fi
-    done
-    set +e
-}
-
-redis_cli() {
-    set -e
-    redis-cli -h $REDIS_HOST -a $REDIS_PASSWORD $@ 2>/dev/null
-    set +e
-}
-
-mysql_cli() {
-    set -e
-    mysql -h $MYSQL_HOST -P $MYSQL_PORT -u $MYSQL_USER -p${MYSQL_PASSWORD} -D $MYSQL_DB "$@"
-    set +e
-}
-
 remove_keys() {
     set -e
     for i in $@ ; do

diff --git a/tests/cleanup-index-test.sh b/tests/cleanup-index-test.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REKOR_ADDRESS=http://localhost:3000
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_PASSWORD=test
+MYSQL_HOST=127.0.0.1
+MYSQL_PORT=3306
+MYSQL_USER=test
+MYSQL_PASSWORD=zaphod
+MYSQL_DB=test
+
+testdir=$(mktemp -d)
+
+source $(dirname "$0")/index-test-utils.sh
+
+trap cleanup EXIT
+
+make_entries() {
+    set -e
+    # make 10 unique artifacts and sign each once
+    for i in $(seq 0 9) ; do
+        minisign -GW -p $testdir/mini${i}.pub -s $testdir/mini${i}.key
+        echo test${i} > $testdir/blob${i}
+        minisign -S -s $testdir/mini${i}.key -m $testdir/blob${i}
+        rekor-cli --rekor_server $REKOR_ADDRESS upload \
+            --artifact $testdir/blob${i} \
+            --pki-format=minisign \
+            --public-key $testdir/mini${i}.pub \
+            --signature $testdir/blob${i}.minisig \
+            --format json
+    done
+    # double-sign a few artifacts
+    for i in $(seq 7 9) ; do
+        set +e
+        let key_index=$i-5
+        set -e
+        minisign -S -s $testdir/mini${key_index}.key -m $testdir/blob${i}
+        rekor-cli --rekor_server $REKOR_ADDRESS upload \
+            --artifact $testdir/blob${i} \
+            --pki-format=minisign \
+            --public-key $testdir/mini${key_index}.pub \
+            --signature $testdir/blob${i}.minisig \
+            --format json
+    done
+    set +e
+}
+
+docker_up
+
+checkpoints=$(redis_cli --scan)
+
+make_entries
+
+set -e
+loginfo=$(rekor-cli --rekor_server $REKOR_ADDRESS loginfo --format=json)
+let end_index=$(echo $loginfo | jq .ActiveTreeSize)-1
+set +e
+
+# check that the entries are in redis
+if [ $(redis_cli --scan | grep -v '/' | wc -l) -ne 20 ] ; then
+    echo "Setup failed: redis had an unexpected number of index keys."
+    exit 1
+fi
+
+# backfill to mysql - this isn't useful in a real scenario because
+# search_index.storage_provider still points to redis, but it's useful
+# to test that the key cleanup is working
+go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \
+    --mysql-dsn "${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DB}" \
+    --concurrency 5 --start 0 --end $end_index
+
+# run the cleanup script
+go run cmd/cleanup-index/main.go --mysql-dsn "${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DB}" --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD
+
+# there should be no more index entries in redis
+if [ $(redis_cli --scan | grep -v '/' | wc -l) -ne 0 ] ; then
+    echo "Found index keys remaining in redis."
+    exit 1
+fi
+
+# the checkpoints should have been left alone
+for cp in $checkpoints ; do
+    if [ $(redis_cli EXISTS $cp) -ne 1 ] ; then
+        echo "Missing checkpoint $cp"
+        exit 1
+    fi
+done