Adding scrub-deposition-properties cmd to remove non-critical video p…

…roperties accumulated during normal use
boggydigital · Aug 17, 2024 · c447970 · c447970
1 parent e6e8cf6
commit c447970
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 30 deletions.
diff --git a/cli-commands.txt b/cli-commands.txt
@@ -39,8 +39,6 @@ backup
 cleanup-ended-videos
     now
 
-scrub-ended-properties
-
 download-video
     video-id^*
     prefer-single-format
@@ -114,6 +112,10 @@ remove-videos
     source
     force
 
+scrub-deposition-properties
+
+scrub-ended-properties
+
 serve
     port$*
     stderr$

diff --git a/cli/scrub_deposition_properties.go b/cli/scrub_deposition_properties.go
@@ -0,0 +1,82 @@
+package cli
+
+import (
+	"github.com/boggydigital/kevlar"
+	"github.com/boggydigital/nod"
+	"github.com/boggydigital/yet/data"
+	"net/url"
+	"slices"
+)
+
+func ScrubDepositionPropertiesHandler(u *url.URL) error {
+	return ScrubDepositionProperties(nil)
+}
+
+// ScrubDepositionProperties will remove all accumulated property depositions:
+// - search results
+// - older channel and playlist videos properties
+// To do that we start by identifying all critical videos:
+// - part of current channel, playlist data
+// - downloaded, not-ended videos
+// Then we iterate over all non-preserved properties and remove data for all non-critical videos
+func ScrubDepositionProperties(rdx kevlar.WriteableRedux) error {
+	sdpa := nod.NewProgress("scrubbing deposition properties...")
+	defer sdpa.End()
+
+	var err error
+	rdx, err = validateWritableRedux(rdx, data.AllProperties()...)
+	if err != nil {
+		return sdpa.EndWithError(err)
+	}
+
+	currentVideoIds := make(map[string]any)
+
+	for _, videoId := range rdx.Keys(data.VideoDownloadCompletedProperty) {
+		if rdx.HasKey(data.VideoEndedDateProperty, videoId) {
+			continue
+		}
+		currentVideoIds[videoId] = nil
+	}
+
+	for _, channelId := range rdx.Keys(data.ChannelAutoRefreshProperty) {
+		if videos, ok := rdx.GetAllValues(data.ChannelVideosProperty, channelId); ok {
+			for _, videoId := range videos {
+				currentVideoIds[videoId] = nil
+			}
+		}
+	}
+
+	for _, playlistId := range rdx.Keys(data.PlaylistAutoRefreshProperty) {
+		if videos, ok := rdx.GetAllValues(data.PlaylistVideosProperty, playlistId); ok {
+			for _, videoId := range videos {
+				currentVideoIds[videoId] = nil
+			}
+		}
+	}
+
+	properties := data.VideoProperties()
+
+	sdpa.TotalInt(len(properties))
+
+	for _, vp := range properties {
+		if slices.Contains(preserveVideoProperties, vp) {
+			sdpa.Increment()
+			continue
+		}
+		videos := rdx.Keys(vp)
+		for _, videoId := range videos {
+			if _, ok := currentVideoIds[videoId]; !ok {
+				if rdx.HasKey(vp, videoId) {
+					if err := rdx.CutKeys(vp, videoId); err != nil {
+						return sdpa.EndWithError(err)
+					}
+				}
+			}
+		}
+		sdpa.Increment()
+	}
+
+	sdpa.EndWithResult("done")
+
+	return nil
+}
diff --git a/cli/scrub_ended_properties.go b/cli/scrub_ended_properties.go
@@ -18,12 +18,13 @@ var preserveVideoProperties = []string{
 	data.VideoDownloadCleanedUpProperty, // required for cleanup
 }
 
-func ScrubEndedPropertiesHandler(u *url.URL) error {
+func ScrubEndedPropertiesHandler(_ *url.URL) error {
 	return ScrubEndedProperties(nil)
 }
 
+// ScrubEndedProperties will remove all non-preserved properties for ended videos.
+// Preserved properties are required for core functionality - history, cleanup, etc.
 func ScrubEndedProperties(rdx kevlar.WriteableRedux) error {
-
 	sevpa := nod.NewProgress("scrubbing ended videos properties...")
 	defer sevpa.End()
 

diff --git a/cli/sync.go b/cli/sync.go
@@ -56,6 +56,9 @@ func Sync(rdx kevlar.WriteableRedux, opt *VideoOptions) error {
 	if err := ScrubEndedProperties(rdx); err != nil {
 		return sa.EndWithError(err)
 	}
+	if err := ScrubDepositionProperties(rdx); err != nil {
+		return sa.EndWithError(err)
+	}
 	if err := CleanupEndedVideos(false, rdx); err != nil {
 		return sa.EndWithError(err)
 	}

diff --git a/main.go b/main.go
@@ -47,32 +47,33 @@ func main() {
 	}
 
 	clo.HandleFuncs(map[string]clo.Handler{
-		"add-channel":                cli.AddChannelHandler,
-		"add-playlist":               cli.AddPlaylistHandler,
-		"add-video":                  cli.AddVideoHandler,
-		"backup":                     cli.BackupHandler,
-		"cleanup-ended-videos":       cli.CleanupEndedVideosHandler,
-		"download-video":             cli.DownloadVideoHandler,
-		"get-captions":               cli.GetCaptionsHandler,
-		"get-channels-metadata":      cli.GetChannelsMetadataHandler,
-		"get-playlists-metadata":     cli.GetPlaylistsMetadataHandler,
-		"get-poster":                 cli.GetPosterHandler,
-		"get-rutube-video":           cli.GetRuTubeVideoHandler,
-		"get-video-metadata":         cli.GetVideoMetadataHandler,
-		"process-queue":              cli.ProcessQueueHandler,
-		"migrate":                    cli.MigrateHandler,
-		"queue-channels-downloads":   cli.QueueChannelsDownloadsHandler,
-		"queue-playlists-downloads":  cli.QueuePlaylistsDownloadsHandler,
-		"refresh-channels-metadata":  cli.RefreshChannelsMetadataHandler,
-		"refresh-playlists-metadata": cli.RefreshPlaylistsMetadataHandler,
-		"remove-channel":             cli.RemoveChannelHandler,
-		"remove-playlist":            cli.RemovePlaylistHandler,
-		"remove-videos":              cli.RemoveVideosHandler,
-		"scrub-ended-properties":     cli.ScrubEndedPropertiesHandler,
-		"serve":                      cli.ServeHandler,
-		"sync":                       cli.SyncHandler,
-		"test-dependencies":          cli.TestDependenciesHandler,
-		"version":                    cli.VersionHandler,
+		"add-channel":                 cli.AddChannelHandler,
+		"add-playlist":                cli.AddPlaylistHandler,
+		"add-video":                   cli.AddVideoHandler,
+		"backup":                      cli.BackupHandler,
+		"cleanup-ended-videos":        cli.CleanupEndedVideosHandler,
+		"download-video":              cli.DownloadVideoHandler,
+		"get-captions":                cli.GetCaptionsHandler,
+		"get-channels-metadata":       cli.GetChannelsMetadataHandler,
+		"get-playlists-metadata":      cli.GetPlaylistsMetadataHandler,
+		"get-poster":                  cli.GetPosterHandler,
+		"get-rutube-video":            cli.GetRuTubeVideoHandler,
+		"get-video-metadata":          cli.GetVideoMetadataHandler,
+		"process-queue":               cli.ProcessQueueHandler,
+		"migrate":                     cli.MigrateHandler,
+		"queue-channels-downloads":    cli.QueueChannelsDownloadsHandler,
+		"queue-playlists-downloads":   cli.QueuePlaylistsDownloadsHandler,
+		"refresh-channels-metadata":   cli.RefreshChannelsMetadataHandler,
+		"refresh-playlists-metadata":  cli.RefreshPlaylistsMetadataHandler,
+		"remove-channel":              cli.RemoveChannelHandler,
+		"remove-playlist":             cli.RemovePlaylistHandler,
+		"remove-videos":               cli.RemoveVideosHandler,
+		"scrub-deposition-properties": cli.ScrubDepositionPropertiesHandler,
+		"scrub-ended-properties":      cli.ScrubEndedPropertiesHandler,
+		"serve":                       cli.ServeHandler,
+		"sync":                        cli.SyncHandler,
+		"test-dependencies":           cli.TestDependenciesHandler,
+		"version":                     cli.VersionHandler,
 	})
 
 	if err := defs.AssertCommandsHaveHandlers(); err != nil {