diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8abd509483..7512519e84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,25 +134,6 @@ jobs: with: test-results: test.json - test-no-xattrs: - runs-on: ubuntu-latest - env: - GOPRIVATE: github.com/couchbaselabs - SG_TEST_USE_XATTRS: false - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: 1.23.3 - - name: Run Tests - run: go test -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )' - shell: bash - - name: Annotate Failures - if: always() - uses: guyarb/golang-test-annotations@v0.8.0 - with: - test-results: test.json - python-format: runs-on: ubuntu-latest steps: diff --git a/.golangci.yml b/.golangci.yml index 87dd5ce737..352043eb79 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -73,6 +73,7 @@ issues: - path: (_test\.go|utilities_testing\.go) linters: - goconst + - prealloc - path: (_test\.go|utilities_testing\.go) linters: - govet diff --git a/base/bucket_gocb_test.go b/base/bucket_gocb_test.go index 9a73dbf265..9dc9afbb5e 100644 --- a/base/bucket_gocb_test.go +++ b/base/bucket_gocb_test.go @@ -414,11 +414,11 @@ func TestXattrWriteCasSimple(t *testing.T) { assert.Equal(t, Crc32cHashString(valBytes), macroBodyHashString) // Validate against $document.value_crc32c - _, xattrs, _, err = dataStore.GetWithXattrs(ctx, key, []string{"$document"}) + _, xattrs, _, err = dataStore.GetWithXattrs(ctx, key, []string{VirtualDocumentXattr}) require.NoError(t, err) var retrievedVxattr map[string]interface{} - require.NoError(t, json.Unmarshal(xattrs["$document"], &retrievedVxattr)) + require.NoError(t, json.Unmarshal(xattrs[VirtualDocumentXattr], &retrievedVxattr)) vxattrCrc32c, ok := retrievedVxattr["value_crc32c"].(string) assert.True(t, ok, "Unable to retrieve virtual xattr crc32c as string") diff --git a/base/constants.go b/base/constants.go index 0b5044c05d..49530a20ac 100644 --- a/base/constants.go +++ b/base/constants.go @@ -135,7 +135,9 @@ const ( // SyncPropertyName is used when storing sync data inline in a document. SyncPropertyName = "_sync" // SyncXattrName is used when storing sync data in a document's xattrs. - SyncXattrName = "_sync" + SyncXattrName = "_sync" + VvXattrName = "_vv" + GlobalXattrName = "_globalSync" // MouXattrName is used when storing metadata-only update information in a document's xattrs. MouXattrName = "_mou" @@ -143,6 +145,11 @@ const ( // Intended to be used in Meta Map and related tests MetaMapXattrsKey = "xattrs" + // VirtualXattrRevSeqNo is used to fetch rev seq no from documents virtual xattr + VirtualXattrRevSeqNo = "$document.revid" + // VirtualDocumentXattr is used to fetch the documents virtual xattr + VirtualDocumentXattr = "$document" + // Prefix for transaction metadata documents TxnPrefix = "_txn:" diff --git a/base/constants_syncdocs.go b/base/constants_syncdocs.go index a57b7fb9f5..7e3a257906 100644 --- a/base/constants_syncdocs.go +++ b/base/constants_syncdocs.go @@ -11,6 +11,7 @@ licenses/APL2.txt. package base import ( + "context" "errors" "fmt" "strconv" @@ -377,20 +378,22 @@ func CollectionSyncFunctionKeyWithGroupID(groupID string, scopeName, collectionN // SyncInfo documents are stored in collections to identify the metadataID associated with sync metadata in that collection type SyncInfo struct { - MetadataID string `json:"metadataID"` + MetadataID string `json:"metadataID,omitempty"` + MetaDataVersion string `json:"metadata_version,omitempty"` } // initSyncInfo attempts to initialize syncInfo for a datastore // 1. If syncInfo doesn't exist, it is created for the specified metadataID // 2. If syncInfo exists with a matching metadataID, returns requiresResync=false // 3. If syncInfo exists with a non-matching metadataID, returns requiresResync=true -func InitSyncInfo(ds DataStore, metadataID string) (requiresResync bool, err error) { +// If syncInfo exists and has metaDataVersion greater than or equal to 4.0, return requiresAttachmentMigration=false, else requiresAttachmentMigration=true to bring migrate metadata attachments. +func InitSyncInfo(ctx context.Context, ds DataStore, metadataID string) (requiresResync bool, requiresAttachmentMigration bool, err error) { var syncInfo SyncInfo _, fetchErr := ds.Get(SGSyncInfo, &syncInfo) if IsDocNotFoundError(fetchErr) { if metadataID == "" { - return false, nil + return false, true, nil } newSyncInfo := &SyncInfo{MetadataID: metadataID} _, addErr := ds.Add(SGSyncInfo, 0, newSyncInfo) @@ -398,37 +401,110 @@ func InitSyncInfo(ds DataStore, metadataID string) (requiresResync bool, err err // attempt new fetch _, fetchErr = ds.Get(SGSyncInfo, &syncInfo) if fetchErr != nil { - return true, fmt.Errorf("Error retrieving syncInfo (after failed add): %v", fetchErr) + return true, true, fmt.Errorf("Error retrieving syncInfo (after failed add): %v", fetchErr) } } else if addErr != nil { - return true, fmt.Errorf("Error adding syncInfo: %v", addErr) + return true, true, fmt.Errorf("Error adding syncInfo: %v", addErr) } // successfully added - return false, nil + requiresAttachmentMigration, err = CompareMetadataVersion(ctx, syncInfo.MetaDataVersion) + if err != nil { + return syncInfo.MetadataID != metadataID, true, err + } + return false, requiresAttachmentMigration, nil } else if fetchErr != nil { - return true, fmt.Errorf("Error retrieving syncInfo: %v", fetchErr) + return true, true, fmt.Errorf("Error retrieving syncInfo: %v", fetchErr) + } + // check for meta version, if we don't have meta version of 4.0 we need to run migration job + requiresAttachmentMigration, err = CompareMetadataVersion(ctx, syncInfo.MetaDataVersion) + if err != nil { + return syncInfo.MetadataID != metadataID, true, err } - return syncInfo.MetadataID != metadataID, nil + return syncInfo.MetadataID != metadataID, requiresAttachmentMigration, nil } -// SetSyncInfo sets syncInfo in a DataStore to the specified metadataID -func SetSyncInfo(ds DataStore, metadataID string) error { +// SetSyncInfoMetadataID sets syncInfo in a DataStore to the specified metadataID, preserving metadata version if present +func SetSyncInfoMetadataID(ds DataStore, metadataID string) error { // If the metadataID isn't defined, don't persist SyncInfo. Defensive handling for legacy use cases. if metadataID == "" { return nil } - syncInfo := &SyncInfo{ - MetadataID: metadataID, + _, err := ds.Update(SGSyncInfo, 0, func(current []byte) (updated []byte, expiry *uint32, delete bool, err error) { + var syncInfo SyncInfo + if current != nil { + parseErr := JSONUnmarshal(current, &syncInfo) + if parseErr != nil { + return nil, nil, false, parseErr + } + } + // if we have a metadataID to set, set it preserving the metadata version if present + syncInfo.MetadataID = metadataID + bytes, err := JSONMarshal(&syncInfo) + return bytes, nil, false, err + }) + return err +} + +// SetSyncInfoMetaVersion sets sync info in DataStore to specified metadata version, preserving metadataID if present +func SetSyncInfoMetaVersion(ds DataStore, metaVersion string) error { + if metaVersion == "" { + return nil } - return ds.Set(SGSyncInfo, 0, nil, syncInfo) + _, err := ds.Update(SGSyncInfo, 0, func(current []byte) (updated []byte, expiry *uint32, delete bool, err error) { + var syncInfo SyncInfo + if current != nil { + parseErr := JSONUnmarshal(current, &syncInfo) + if parseErr != nil { + return nil, nil, false, parseErr + } + } + // if we have a meta version to set, set it preserving the metadata ID if present + syncInfo.MetaDataVersion = metaVersion + bytes, err := JSONMarshal(&syncInfo) + return bytes, nil, false, err + }) + return err } -// SerializeIfLonger returns name as a sha1 string if the length of the name is greater or equal to the length specificed. Otherwise, returns the original string. +// SerializeIfLonger returns name as a sha1 string if the length of the name is greater or equal to the length specified. Otherwise, returns the original string. func SerializeIfLonger(name string, length int) string { if len(name) < length { return name } return Sha1HashString(name, "") } + +// CompareMetadataVersion Will build comparable build version for comparison with meta version defined in syncInfo, then +// will return true if we require attachment migration, false if not. +func CompareMetadataVersion(ctx context.Context, metaVersion string) (bool, error) { + if metaVersion == "" { + // no meta version passed in, thus attachment migration should take place + return true, nil + } + syncInfoVersion, err := NewComparableBuildVersionFromString(metaVersion) + if err != nil { + return true, err + } + return CheckRequireAttachmentMigration(ctx, syncInfoVersion) +} + +// CheckRequireAttachmentMigration will return true if current metaVersion < 4.0.0, else false +func CheckRequireAttachmentMigration(ctx context.Context, version *ComparableBuildVersion) (bool, error) { + if version == nil { + AssertfCtx(ctx, "failed to build comparable build version for syncInfo metaVersion") + return true, fmt.Errorf("corrupt syncInfo metaVersion value") + } + minVerStr := "4.0.0" // minimum meta version that needs to be defined for metadata migration. Any version less than this will require attachment migration + minVersion, err := NewComparableBuildVersionFromString(minVerStr) + if err != nil { + AssertfCtx(ctx, "failed to build comparable build version for minimum version for attachment migration") + return true, err + } + + if minVersion.AtLeastMinorDowngrade(version) { + return true, nil + } + return false, nil +} diff --git a/base/dcp_common.go b/base/dcp_common.go index 3c54c041e0..50f72f0000 100644 --- a/base/dcp_common.go +++ b/base/dcp_common.go @@ -522,12 +522,13 @@ func dcpKeyFilter(key []byte, metaKeys *MetadataKeys) bool { } // Makes a feedEvent that can be passed to a FeedEventCallbackFunc implementation -func makeFeedEvent(key []byte, value []byte, dataType uint8, cas uint64, expiry uint32, vbNo uint16, collectionID uint32, opcode sgbucket.FeedOpcode) sgbucket.FeedEvent { +func makeFeedEvent(key []byte, value []byte, dataType uint8, cas uint64, expiry uint32, vbNo uint16, collectionID uint32, revNo uint64, opcode sgbucket.FeedOpcode) sgbucket.FeedEvent { // not currently doing rq.Extras handling (as in gocouchbase/upr_feed, makeUprEvent) as SG doesn't use // expiry/flags information, and snapshot handling is done by cbdatasource and sent as // SnapshotStart, SnapshotEnd event := sgbucket.FeedEvent{ + RevNo: revNo, Opcode: opcode, Key: key, Value: value, diff --git a/base/dcp_dest.go b/base/dcp_dest.go index e5d92fb106..df04a4a639 100644 --- a/base/dcp_dest.go +++ b/base/dcp_dest.go @@ -92,7 +92,7 @@ func (d *DCPDest) DataUpdate(partition string, key []byte, seq uint64, if !dcpKeyFilter(key, d.metaKeys) { return nil } - event := makeFeedEventForDest(key, val, cas, partitionToVbNo(d.loggingCtx, partition), collectionIDFromExtras(extras), 0, 0, sgbucket.FeedOpMutation) + event := makeFeedEventForDest(key, val, cas, partitionToVbNo(d.loggingCtx, partition), collectionIDFromExtras(extras), 0, 0, 0, sgbucket.FeedOpMutation) d.dataUpdate(seq, event) return nil } @@ -116,7 +116,7 @@ func (d *DCPDest) DataUpdateEx(partition string, key []byte, seq uint64, val []b if !ok { return errors.New("Unable to cast extras of type DEST_EXTRAS_TYPE_GOCB_DCP to cbgt.GocbExtras") } - event = makeFeedEventForDest(key, val, cas, partitionToVbNo(d.loggingCtx, partition), dcpExtras.CollectionId, dcpExtras.Expiry, dcpExtras.Datatype, sgbucket.FeedOpMutation) + event = makeFeedEventForDest(key, val, cas, partitionToVbNo(d.loggingCtx, partition), dcpExtras.CollectionId, dcpExtras.Expiry, dcpExtras.Datatype, dcpExtras.RevNo, sgbucket.FeedOpMutation) } @@ -131,7 +131,7 @@ func (d *DCPDest) DataDelete(partition string, key []byte, seq uint64, return nil } - event := makeFeedEventForDest(key, nil, cas, partitionToVbNo(d.loggingCtx, partition), collectionIDFromExtras(extras), 0, 0, sgbucket.FeedOpDeletion) + event := makeFeedEventForDest(key, nil, cas, partitionToVbNo(d.loggingCtx, partition), collectionIDFromExtras(extras), 0, 0, 0, sgbucket.FeedOpDeletion) d.dataUpdate(seq, event) return nil } @@ -154,7 +154,7 @@ func (d *DCPDest) DataDeleteEx(partition string, key []byte, seq uint64, if !ok { return errors.New("Unable to cast extras of type DEST_EXTRAS_TYPE_GOCB_DCP to cbgt.GocbExtras") } - event = makeFeedEventForDest(key, dcpExtras.Value, cas, partitionToVbNo(d.loggingCtx, partition), dcpExtras.CollectionId, dcpExtras.Expiry, dcpExtras.Datatype, sgbucket.FeedOpDeletion) + event = makeFeedEventForDest(key, dcpExtras.Value, cas, partitionToVbNo(d.loggingCtx, partition), dcpExtras.CollectionId, dcpExtras.Expiry, dcpExtras.Datatype, dcpExtras.RevNo, sgbucket.FeedOpDeletion) } d.dataUpdate(seq, event) @@ -247,8 +247,8 @@ func collectionIDFromExtras(extras []byte) uint32 { return binary.LittleEndian.Uint32(extras[4:]) } -func makeFeedEventForDest(key []byte, val []byte, cas uint64, vbNo uint16, collectionID uint32, expiry uint32, dataType uint8, opcode sgbucket.FeedOpcode) sgbucket.FeedEvent { - return makeFeedEvent(key, val, dataType, cas, expiry, vbNo, collectionID, opcode) +func makeFeedEventForDest(key []byte, val []byte, cas uint64, vbNo uint16, collectionID uint32, expiry uint32, dataType uint8, revNo uint64, opcode sgbucket.FeedOpcode) sgbucket.FeedEvent { + return makeFeedEvent(key, val, dataType, cas, expiry, vbNo, collectionID, revNo, opcode) } // DCPLoggingDest wraps DCPDest to provide per-callback logging diff --git a/base/dcp_feed_type.go b/base/dcp_feed_type.go index fd915c5ada..e6bef0b62b 100644 --- a/base/dcp_feed_type.go +++ b/base/dcp_feed_type.go @@ -296,7 +296,7 @@ func getCbgtCredentials(dbName string) (cbgtCreds, bool) { return creds, found } -// See the comment of cbgtRootCAsProvider for usage details. +// setCbgtRootCertsForBucket creates root certificates for a given bucket. If TLS should be used, this function must be called. If tls certificate verification is skipped, then this function should be called with pool as nil. See the comment of cbgtRootCAsProvider for usage details. func setCbgtRootCertsForBucket(bucketUUID string, pool *x509.CertPool) { cbgtGlobalsLock.Lock() defer cbgtGlobalsLock.Unlock() diff --git a/base/dcp_receiver.go b/base/dcp_receiver.go index 853a3a59d8..8872b7b9e1 100644 --- a/base/dcp_receiver.go +++ b/base/dcp_receiver.go @@ -26,7 +26,7 @@ const MemcachedDataTypeRaw = 0 // Make a feed event for a gomemcached request. Extracts expiry from extras func makeFeedEventForMCRequest(rq *gomemcached.MCRequest, opcode sgbucket.FeedOpcode) sgbucket.FeedEvent { - return makeFeedEvent(rq.Key, rq.Body, rq.DataType, rq.Cas, ExtractExpiryFromDCPMutation(rq), rq.VBucket, 0, opcode) + return makeFeedEvent(rq.Key, rq.Body, rq.DataType, rq.Cas, ExtractExpiryFromDCPMutation(rq), rq.VBucket, 0, 0, opcode) } // ShardedImportDCPMetadata is an internal struct that is exposed to enable json marshaling, used by sharded import feed. It differs from DCPMetadata because it must match the private struct used by cbgt.metadata. diff --git a/base/dcp_sharded.go b/base/dcp_sharded.go index a489718104..8efc924336 100644 --- a/base/dcp_sharded.go +++ b/base/dcp_sharded.go @@ -451,6 +451,7 @@ func (c *CbgtContext) Stop() { func (c *CbgtContext) RemoveFeedCredentials(dbName string) { removeCbgtCredentials(dbName) + // CBG-4394: removing root certs for the bucket should be done, but it is keyed based on the bucket UUID, and multiple dbs can use the same bucket } // Format of dest key for retrieval of import dest from cbgtDestFactories diff --git a/base/log_keys.go b/base/log_keys.go index 73cb0c60dc..f575495e99 100644 --- a/base/log_keys.go +++ b/base/log_keys.go @@ -53,6 +53,7 @@ const ( KeyReplicate KeySync KeySyncMsg + KeyVV KeyWebSocket KeyWebSocketFrame KeySGTest @@ -87,6 +88,7 @@ var ( KeyReplicate: "Replicate", KeySync: "Sync", KeySyncMsg: "SyncMsg", + KeyVV: "VV", KeyWebSocket: "WS", KeyWebSocketFrame: "WSFrame", KeySGTest: "TEST", diff --git a/base/main_test_bucket_pool.go b/base/main_test_bucket_pool.go index c79d2231d4..31fd3a75fc 100644 --- a/base/main_test_bucket_pool.go +++ b/base/main_test_bucket_pool.go @@ -137,6 +137,15 @@ func NewTestBucketPoolWithOptions(ctx context.Context, bucketReadierFunc TBPBuck } tbp.skipMobileXDCR = !useMobileXDCR + // at least anemone release + if os.Getenv(tbpEnvAllowIncompatibleServerVersion) == "" && !ProductVersion.Less(&ComparableBuildVersion{major: 4}) { + overrideMsg := "Set " + tbpEnvAllowIncompatibleServerVersion + "=true to override this check." + // this check also covers BucketStoreFeatureMultiXattrSubdocOperations, which is Couchbase Server 7.6 + if tbp.skipMobileXDCR { + tbp.Fatalf(ctx, "Sync Gateway %v requires mobile XDCR support, but Couchbase Server %v does not support it. Couchbase Server %s is required. %s", ProductVersion, tbp.cluster.version, firstServerVersionToSupportMobileXDCR, overrideMsg) + } + } + tbp.verbose.Set(tbpVerbose()) // Start up an async readier worker to process dirty buckets @@ -450,6 +459,7 @@ func (tbp *TestBucketPool) setXDCRBucketSetting(ctx context.Context, bucket Buck tbp.Logf(ctx, "Setting crossClusterVersioningEnabled=true") + // retry for 1 minute to get this bucket setting, MB-63675 store, ok := AsCouchbaseBucketStore(bucket) if !ok { tbp.Fatalf(ctx, "unable to get server management endpoints. Underlying bucket type was not GoCBBucket") @@ -459,12 +469,20 @@ func (tbp *TestBucketPool) setXDCRBucketSetting(ctx context.Context, bucket Buck posts.Add("enableCrossClusterVersioning", "true") url := fmt.Sprintf("/pools/default/buckets/%s", store.GetName()) - output, statusCode, err := store.MgmtRequest(ctx, http.MethodPost, url, "application/x-www-form-urlencoded", strings.NewReader(posts.Encode())) + // retry for 1 minute to get this bucket setting, MB-63675 + _, err := RetryLoop(ctx, "setXDCRBucketSetting", func() (bool, error, interface{}) { + output, statusCode, err := store.MgmtRequest(ctx, http.MethodPost, url, "application/x-www-form-urlencoded", strings.NewReader(posts.Encode())) + if err != nil { + tbp.Fatalf(ctx, "request to mobile XDCR bucket setting failed, status code: %d error: %w output: %s", statusCode, err, string(output)) + } + if statusCode != http.StatusOK { + err := fmt.Errorf("request to mobile XDCR bucket setting failed with status code, %d, output: %s", statusCode, string(output)) + return true, err, nil + } + return false, nil, nil + }, CreateMaxDoublingSleeperFunc(200, 500, 500)) if err != nil { - tbp.Fatalf(ctx, "request to mobile XDCR bucket setting failed, status code: %d error: %v output: %s", statusCode, err, string(output)) - } - if statusCode != http.StatusOK { - tbp.Fatalf(ctx, "request to mobile XDCR bucket setting failed with status code, %d, output: %s", statusCode, string(output)) + tbp.Fatalf(ctx, "Couldn't set crossClusterVersioningEnabled: %v", err) } } diff --git a/base/main_test_bucket_pool_config.go b/base/main_test_bucket_pool_config.go index 734e90eb1c..afd2928867 100644 --- a/base/main_test_bucket_pool_config.go +++ b/base/main_test_bucket_pool_config.go @@ -53,11 +53,17 @@ const ( tbpEnvUseDefaultCollection = "SG_TEST_USE_DEFAULT_COLLECTION" + // tbpEnvAllowIncompatibleServerVersion allows tests to run against a server version that is not presumed compatible with version of Couchbase Server running. + tbpEnvAllowIncompatibleServerVersion = "SG_TEST_SKIP_SERVER_VERSION_CHECK" + // wait this long when requesting a test bucket from the pool before giving up and failing the test. waitForReadyBucketTimeout = time.Minute // Creates buckets with a specific number of number of replicas tbpEnvBucketNumReplicas = "SG_TEST_BUCKET_NUM_REPLICAS" + + // Environment variable to specify the topology tests to run + TbpEnvTopologyTests = "SG_TEST_TOPOLOGY_TESTS" ) // TestsUseNamedCollections returns true if the tests use named collections. diff --git a/base/main_test_cluster.go b/base/main_test_cluster.go index b6af39fc35..d5ea84d33e 100644 --- a/base/main_test_cluster.go +++ b/base/main_test_cluster.go @@ -11,6 +11,7 @@ package base import ( "context" "fmt" + "log" "strings" "time" @@ -18,11 +19,14 @@ import ( ) // firstServerVersionToSupportMobileXDCR this is the first server version to support Mobile XDCR feature -var firstServerVersionToSupportMobileXDCR = &ComparableBuildVersion{ - epoch: 0, - major: 7, - minor: 6, - patch: 2, +var firstServerVersionToSupportMobileXDCR *ComparableBuildVersion + +func init() { + var err error + firstServerVersionToSupportMobileXDCR, err = NewComparableBuildVersionFromString("7.6.4@5074") + if err != nil { + log.Fatalf("Couldn't parse firstServerVersionToSupportMobileXDCR: %v", err) + } } type clusterLogFunc func(ctx context.Context, format string, args ...interface{}) @@ -216,9 +220,13 @@ func (c *tbpCluster) mobileXDCRCompatible(ctx context.Context) (bool, error) { return false, nil } - // take server version, server version will be the first 5 character of version string - // in the form of x.x.x - vrs := c.version[:5] + // string is x.y.z-aaaa-enterprise or x.y.z-aaaa-community + // convert to a comparable string that Sync Gateway understands x.y.z@aaaa + components := strings.Split(c.version, "-") + vrs := components[0] + if len(components) > 1 { + vrs += "@" + components[1] + } // convert the above string into a comparable string version, err := NewComparableBuildVersionFromString(vrs) diff --git a/base/stats.go b/base/stats.go index e1f0c4b1d1..e2e2155d45 100644 --- a/base/stats.go +++ b/base/stats.go @@ -88,6 +88,7 @@ const ( StatAddedVersion3dot2dot0 = "3.2.0" StatAddedVersion3dot2dot1 = "3.2.1" StatAddedVersion3dot3dot0 = "3.3.0" + StatAddedVersion4dot0dot0 = "4.0.0" StatDeprecatedVersionNotDeprecated = "" StatDeprecatedVersion3dot2dot0 = "3.2.0" diff --git a/base/util.go b/base/util.go index 99170291df..ac08f71f5f 100644 --- a/base/util.go +++ b/base/util.go @@ -14,6 +14,7 @@ import ( "crypto/rand" "crypto/sha1" "crypto/tls" + "encoding/base64" "encoding/binary" "encoding/hex" "encoding/json" @@ -1019,6 +1020,59 @@ func HexCasToUint64(cas string) uint64 { return binary.LittleEndian.Uint64(casBytes[0:8]) } +// HexCasToUint64ForDelta will convert hex cas to uint64 accounting for any stripped zeros in delta calculation +func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { + var decoded []byte + + // as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length + if len(casByte)%2 != 0 { + casByte = append(casByte, '0') + } + + // create byte array for decoding into + decodedLen := hex.DecodedLen(len(casByte)) + // binary.LittleEndian.Uint64 expects length 8 byte array, if larger we should error, if smaller + // (because of stripped 0's then we should make it length 8). + if decodedLen > 8 { + return 0, fmt.Errorf("corrupt hex value, decoded length larger than expected") + } + if decodedLen < 8 { + // can be less than 8 given we have stripped the 0's for some values, in this case we need to ensure large eniough + decoded = make([]byte, 8) + } else { + decoded = make([]byte, decodedLen) + } + + if _, err := hex.Decode(decoded, casByte); err != nil { + return 0, err + } + res := binary.LittleEndian.Uint64(decoded) + return res, nil +} + +// Uint64ToLittleEndianHexAndStripZeros will convert a uint64 type to little endian hex, stripping any zeros off the end +// + stripping 0x from start +func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string { + hexCas := Uint64CASToLittleEndianHex(cas) + + i := len(hexCas) - 1 + for i > 2 && hexCas[i] == '0' { + i-- + } + // strip 0x from start + return string(hexCas[2 : i+1]) +} + +func HexToBase64(s string) ([]byte, error) { + decoded := make([]byte, hex.DecodedLen(len(s))) + if _, err := hex.Decode(decoded, []byte(s)); err != nil { + return nil, err + } + encoded := make([]byte, base64.RawStdEncoding.EncodedLen(len(decoded))) + base64.RawStdEncoding.Encode(encoded, decoded) + return encoded, nil +} + func CasToString(cas uint64) string { return string(Uint64CASToLittleEndianHex(cas)) } @@ -1033,6 +1087,17 @@ func Uint64CASToLittleEndianHex(cas uint64) []byte { return encodedArray } +// Converts a string decimal representation ("100") to little endian hex string ("0x64") +func StringDecimalToLittleEndianHex(value string) (string, error) { + intValue, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return "", err + } + hexValue := Uint64CASToLittleEndianHex(intValue) + return string(hexValue), nil + +} + func Crc32cHash(input []byte) uint32 { // crc32.MakeTable already ensures singleton table creation, so shouldn't need to cache. table := crc32.MakeTable(crc32.Castagnoli) diff --git a/base/util_test.go b/base/util_test.go index a767625b95..46e8430067 100644 --- a/base/util_test.go +++ b/base/util_test.go @@ -1735,3 +1735,40 @@ func TestCASToLittleEndianHex(t *testing.T) { littleEndianHex := Uint64CASToLittleEndianHex(casValue) require.Equal(t, expHexValue, string(littleEndianHex)) } + +func TestUint64CASToLittleEndianHexAndStripZeros(t *testing.T) { + hexLE := "0x0000000000000000" + u64 := HexCasToUint64(hexLE) + hexLEStripped := Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err := HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xffffffffffffffff" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xd123456e789a0bcf" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xd123456e78000000" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xa500000000000000" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) +} diff --git a/base/util_testing.go b/base/util_testing.go index 1696ff16e5..0471387be6 100644 --- a/base/util_testing.go +++ b/base/util_testing.go @@ -19,6 +19,7 @@ import ( "io" "io/fs" "log" + "maps" "math/rand" "os" "path/filepath" @@ -210,6 +211,9 @@ func TestUseXattrs() bool { if err != nil { panic(fmt.Sprintf("unable to parse %q value %q: %v", TestEnvSyncGatewayUseXattrs, useXattrs, err)) } + if !val { + panic(fmt.Sprintf("sync gateway %s requires xattrs to be enabled, remove env var %s=%s", ProductVersion, TestEnvSyncGatewayUseXattrs, useXattrs)) + } return val } @@ -783,6 +787,11 @@ func RequireDocNotFoundError(t testing.TB, e error) { require.True(t, IsDocNotFoundError(e), fmt.Sprintf("Expected error to be a doc not found error, but was: %v", e)) } +// RequireXattrNotFoundError asserts that the given error represents an xattr not found error. +func RequireXattrNotFoundError(t testing.TB, e error) { + require.True(t, IsXattrNotFoundError(e), fmt.Sprintf("Expected error to be an xattr not found error, but was: %v", e)) +} + func requireCasMismatchError(t testing.TB, err error) { require.Error(t, err, "Expected an error of type IsCasMismatch %+v\n", err) require.True(t, IsCasMismatch(err), "Expected error of type IsCasMismatch but got %+v\n", err) @@ -970,3 +979,14 @@ func numFilesInDir(t *testing.T, dir string, recursive bool) int { require.NoError(t, err) return numFiles } + +// ResetCBGTCertPools resets the cert pools used for cbgt in a test. +func ResetCBGTCertPools(t *testing.T) { + // CBG-4394: removing root certs for the bucket should be done, but it is keyed based on the bucket UUID, and multiple dbs can use the same bucket + cbgtGlobalsLock.Lock() + defer cbgtGlobalsLock.Unlock() + oldRootCAs := maps.Clone(cbgtRootCertPools) + t.Cleanup(func() { + cbgtRootCertPools = oldRootCAs + }) +} diff --git a/base/version.go b/base/version.go index 6f96e26bdd..c26393e5a1 100644 --- a/base/version.go +++ b/base/version.go @@ -19,8 +19,8 @@ import ( const ( ProductName = "Couchbase Sync Gateway" - ProductAPIVersionMajor = "3" - ProductAPIVersionMinor = "3" + ProductAPIVersionMajor = "4" + ProductAPIVersionMinor = "0" ProductAPIVersion = ProductAPIVersionMajor + "." + ProductAPIVersionMinor ) diff --git a/channels/log_entry.go b/channels/log_entry.go index 4b8d7cd81b..cf51a446b6 100644 --- a/channels/log_entry.go +++ b/channels/log_entry.go @@ -14,7 +14,10 @@ package channels import ( "fmt" + "strconv" "time" + + "github.com/couchbase/sync_gateway/base" ) // Bits in LogEntry.Flags @@ -41,32 +44,36 @@ type LogEntry struct { PrevSequence uint64 // Sequence of previous active revision IsPrincipal bool // Whether the log-entry is a tracking entry for a principal doc CollectionID uint32 // Collection ID + SourceID string // SourceID allocated to the doc's Current Version on the HLV + Version uint64 // Version allocated to the doc's Current Version on the HLV } func (l LogEntry) String() string { return fmt.Sprintf( - "seq: %d docid: %s revid: %s collectionID: %d", + "seq: %d docid: %s revid: %s collectionID: %d source: %s version: %d", l.Sequence, l.DocID, l.RevID, l.CollectionID, + l.SourceID, + l.Version, ) } type ChannelMap map[string]*ChannelRemoval type ChannelRemoval struct { - Seq uint64 `json:"seq,omitempty"` - RevID string `json:"rev"` - Deleted bool `json:"del,omitempty"` + Seq uint64 `json:"seq,omitempty"` + Rev RevAndVersion `json:"rev"` + Deleted bool `json:"del,omitempty"` } -func (channelMap ChannelMap) ChannelsRemovedAtSequence(seq uint64) (ChannelMap, string) { +func (channelMap ChannelMap) ChannelsRemovedAtSequence(seq uint64) (ChannelMap, RevAndVersion) { var channelsRemoved = make(ChannelMap) - var revIdRemoved string + var revIdRemoved RevAndVersion for channel, removal := range channelMap { if removal != nil && removal.Seq == seq { channelsRemoved[channel] = removal - revIdRemoved = removal.RevID // Will be the same RevID for each removal + revIdRemoved = removal.Rev // Will be the same Rev for each removal } } return channelsRemoved, revIdRemoved @@ -81,3 +88,48 @@ func (channelMap ChannelMap) KeySet() []string { } return result } + +// RevAndVersion is used to store both revTreeID and currentVersion in a single property, for backwards compatibility +// with existing indexes using rev. When only RevTreeID is specified, is marshalled/unmarshalled as a string. Otherwise +// marshalled normally. +type RevAndVersion struct { + RevTreeID string `json:"rev,omitempty"` + CurrentSource string `json:"src,omitempty"` + CurrentVersion string `json:"ver,omitempty"` // Version needs to be hex string here to support macro expansion when writing to _sync.rev +} + +// RevAndVersionJSON aliases RevAndVersion to support conditional unmarshalling from either string (revTreeID) or +// map (RevAndVersion) representations +type RevAndVersionJSON RevAndVersion + +// Marshals RevAndVersion as simple string when only RevTreeID is specified - otherwise performs standard +// marshalling +func (rv RevAndVersion) MarshalJSON() (data []byte, err error) { + + if rv.CurrentSource == "" { + return base.JSONMarshal(rv.RevTreeID) + } + return base.JSONMarshal(RevAndVersionJSON(rv)) +} + +// Unmarshals either from string (legacy, revID only) or standard RevAndVersion unmarshalling. +func (rv *RevAndVersion) UnmarshalJSON(data []byte) error { + + if len(data) == 0 { + return nil + } + switch data[0] { + case '"': + return base.JSONUnmarshal(data, &rv.RevTreeID) + case '{': + return base.JSONUnmarshal(data, (*RevAndVersionJSON)(rv)) + default: + return fmt.Errorf("unrecognized JSON format for RevAndVersion: %s", data) + } +} + +// CV returns ver@src in big endian format 1@cbl for CBL format. +func (rv RevAndVersion) CV() string { + // this should match db.Version.String() + return strconv.FormatUint(base.HexCasToUint64(rv.CurrentVersion), 16) + "@" + rv.CurrentSource +} diff --git a/db/access_test.go b/db/access_test.go index 9b7e9f682e..f48f874d3d 100644 --- a/db/access_test.go +++ b/db/access_test.go @@ -43,7 +43,7 @@ func TestDynamicChannelGrant(t *testing.T) { // Create a document in channel chan1 doc1Body := Body{"channel": "chan1", "greeting": "hello"} - _, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc1", doc1Body, []string{"1-a"}, false) + _, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc1", doc1Body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // Verify user cannot access document @@ -53,7 +53,7 @@ func TestDynamicChannelGrant(t *testing.T) { // Write access granting document grantingBody := Body{"type": "setaccess", "owner": "user1", "channel": "chan1"} - _, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant1", grantingBody, []string{"1-a"}, false) + _, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant1", grantingBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // Verify reloaded user can access document @@ -65,12 +65,12 @@ func TestDynamicChannelGrant(t *testing.T) { // Create a document in channel chan2 doc2Body := Body{"channel": "chan2", "greeting": "hello"} - _, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc2", doc2Body, []string{"1-a"}, false) + _, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc2", doc2Body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // Write access granting document for chan2 (tests invalidation when channels/inval_seq exists) grantingBody = Body{"type": "setaccess", "owner": "user1", "channel": "chan2"} - _, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant2", grantingBody, []string{"1-a"}, false) + _, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant2", grantingBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // Verify user can now access both documents diff --git a/db/active_replicator.go b/db/active_replicator.go index 7cae4bfa2c..778a1e31b3 100644 --- a/db/active_replicator.go +++ b/db/active_replicator.go @@ -208,8 +208,13 @@ func connect(arc *activeReplicatorCommon, idSuffix string) (blipSender *blip.Sen arc.replicationStats.NumConnectAttempts.Add(1) var originPatterns []string // no origin headers for ISGR + // NewSGBlipContext doesn't set cancellation context - active replication cancellation on db close is handled independently - blipContext, err := NewSGBlipContext(arc.ctx, arc.config.ID+idSuffix, originPatterns, nil) + // TODO: CBG-3661 ActiveReplicator subprotocol versions + // - make this configurable for testing mixed-version replications + // - if unspecified, default to v2 and v3 until VV is supported with ISGR, then also include v4 + protocols := []string{CBMobileReplicationV3.SubprotocolString(), CBMobileReplicationV2.SubprotocolString()} + blipContext, err := NewSGBlipContextWithProtocols(arc.ctx, arc.config.ID+idSuffix, originPatterns, protocols, nil) if err != nil { return nil, nil, err } diff --git a/db/attachment.go b/db/attachment.go index b60baab22c..eb7fcf90fc 100644 --- a/db/attachment.go +++ b/db/attachment.go @@ -293,7 +293,7 @@ func (c *DatabaseCollection) ForEachStubAttachment(body Body, minRevpos int, doc return base.HTTPErrorf(http.StatusBadRequest, "Invalid attachment") } if meta["data"] == nil { - if revpos, ok := base.ToInt64(meta["revpos"]); revpos < int64(minRevpos) || !ok { + if revpos, ok := base.ToInt64(meta["revpos"]); revpos < int64(minRevpos) || (!ok && minRevpos > 0) { continue } digest, ok := meta["digest"].(string) diff --git a/db/attachment_compaction.go b/db/attachment_compaction.go index 0d54835ee1..57b9ccb642 100644 --- a/db/attachment_compaction.go +++ b/db/attachment_compaction.go @@ -193,9 +193,9 @@ type AttachmentsMetaMap struct { Attachments map[string]AttachmentsMeta `json:"_attachments"` } -// AttachmentCompactionData struct to unmarshal a document sync data into in order to process attachments during mark +// AttachmentCompactionSyncData struct to unmarshal a document sync data into in order to process attachments during mark // phase. Contains only what is necessary -type AttachmentCompactionData struct { +type AttachmentCompactionSyncData struct { Attachments map[string]AttachmentsMeta `json:"attachments"` Flags uint8 `json:"flags"` History struct { @@ -204,29 +204,42 @@ type AttachmentCompactionData struct { } `json:"history"` } -// getAttachmentSyncData takes the data type and data from the DCP feed and will return a AttachmentCompactionData +// AttachmentCompactionGlobalSyncData is to unmarshal a documents global xattr in order to process attachments during mark phase. +type AttachmentCompactionGlobalSyncData struct { + Attachments map[string]AttachmentsMeta `json:"attachments_meta"` +} + +// getAttachmentSyncData takes the data type and data from the DCP feed and will return a AttachmentCompactionSyncData // struct containing data needed to process attachments on a document. -func getAttachmentSyncData(dataType uint8, data []byte) (*AttachmentCompactionData, error) { - var attachmentData *AttachmentCompactionData +func getAttachmentSyncData(dataType uint8, data []byte) (*AttachmentCompactionSyncData, error) { + var attachmentSyncData *AttachmentCompactionSyncData + var attachmentGlobalSyncData AttachmentCompactionGlobalSyncData var documentBody []byte if dataType&base.MemcachedDataTypeXattr != 0 { - body, xattrs, err := sgbucket.DecodeValueWithXattrs([]string{base.SyncXattrName}, data) + body, xattrs, err := sgbucket.DecodeValueWithXattrs([]string{base.SyncXattrName, base.GlobalXattrName}, data) if err != nil { if errors.Is(err, sgbucket.ErrXattrInvalidLen) { return nil, nil } return nil, fmt.Errorf("Could not parse DCP attachment sync data: %w", err) } - err = base.JSONUnmarshal(xattrs[base.SyncXattrName], &attachmentData) + err = base.JSONUnmarshal(xattrs[base.SyncXattrName], &attachmentSyncData) if err != nil { return nil, err } + if xattrs[base.GlobalXattrName] != nil && attachmentSyncData.Attachments == nil { + err = base.JSONUnmarshal(xattrs[base.GlobalXattrName], &attachmentGlobalSyncData) + if err != nil { + return nil, err + } + attachmentSyncData.Attachments = attachmentGlobalSyncData.Attachments + } documentBody = body } else { type AttachmentDataSync struct { - AttachmentData AttachmentCompactionData `json:"_sync"` + AttachmentData AttachmentCompactionSyncData `json:"_sync"` } var attachmentDataSync AttachmentDataSync err := base.JSONUnmarshal(data, &attachmentDataSync) @@ -235,21 +248,21 @@ func getAttachmentSyncData(dataType uint8, data []byte) (*AttachmentCompactionDa } documentBody = data - attachmentData = &attachmentDataSync.AttachmentData + attachmentSyncData = &attachmentDataSync.AttachmentData } // If we've not yet found any attachments have a last effort attempt to grab it from the body for pre-2.5 documents - if len(attachmentData.Attachments) == 0 { + if len(attachmentSyncData.Attachments) == 0 { attachmentMetaMap, err := checkForInlineAttachments(documentBody) if err != nil { return nil, err } if attachmentMetaMap != nil { - attachmentData.Attachments = attachmentMetaMap.Attachments + attachmentSyncData.Attachments = attachmentMetaMap.Attachments } } - return attachmentData, nil + return attachmentSyncData, nil } // checkForInlineAttachments will scan a body for "_attachments" for pre-2.5 attachments and will return any attachments diff --git a/db/attachment_test.go b/db/attachment_test.go index 2395913924..78d7759fe9 100644 --- a/db/attachment_test.go +++ b/db/attachment_test.go @@ -49,11 +49,12 @@ func TestBackupOldRevisionWithAttachments(t *testing.T) { require.NoError(t, base.JSONUnmarshal([]byte(rev1Data), &rev1Body)) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - rev1ID, _, err := collection.Put(ctx, docID, rev1Body) + rev1ID, docRev1, err := collection.Put(ctx, docID, rev1Body) require.NoError(t, err) assert.Equal(t, "1-12ff9ce1dd501524378fe092ce9aee8f", rev1ID) - rev1OldBody, err := collection.getOldRevisionJSON(ctx, docID, rev1ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + rev1OldBody, err := collection.getOldRevisionJSON(ctx, docID, base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) if deltasEnabled && xattrsEnabled { require.NoError(t, err) assert.Contains(t, string(rev1OldBody), "hello.txt") @@ -67,17 +68,18 @@ func TestBackupOldRevisionWithAttachments(t *testing.T) { var rev2Body Body rev2Data := `{"test": true, "updated": true, "_attachments": {"hello.txt": {"stub": true, "revpos": 1}}}` require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body)) - _, _, err = collection.PutExistingRevWithBody(ctx, docID, rev2Body, []string{"2-abc", rev1ID}, true) + docRev2, _, err := collection.PutExistingRevWithBody(ctx, docID, rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) - rev2ID := "2-abc" // now in any case - we'll have rev 1 backed up - rev1OldBody, err = collection.getOldRevisionJSON(ctx, docID, rev1ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + rev1OldBody, err = collection.getOldRevisionJSON(ctx, docID, base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) require.NoError(t, err) assert.Contains(t, string(rev1OldBody), "hello.txt") // and rev 2 should be present only for the xattrs and deltas case - rev2OldBody, err := collection.getOldRevisionJSON(ctx, docID, rev2ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + rev2OldBody, err := collection.getOldRevisionJSON(ctx, docID, base.Crc32cHashString([]byte(docRev2.HLV.GetCurrentVersionString()))) if deltasEnabled && xattrsEnabled { require.NoError(t, err) assert.Contains(t, string(rev2OldBody), "hello.txt") @@ -101,7 +103,7 @@ func TestAttachments(t *testing.T) { assert.NoError(t, base.JSONUnmarshal([]byte(rev1input), &body)) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - revid, _, err := collection.Put(ctx, "doc1", body) + revid, docRev1, err := collection.Put(ctx, "doc1", body) rev1id := revid assert.NoError(t, err, "Couldn't create document") @@ -189,12 +191,13 @@ func TestAttachments(t *testing.T) { assert.Equal(t, float64(2), bye["revpos"]) log.Printf("Expire body of rev 1, then add a child...") // test fix of #498 - err = collection.dataStore.Delete(oldRevisionKey("doc1", rev1id)) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.dataStore.Delete(oldRevisionKey("doc1", base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString())))) assert.NoError(t, err, "Couldn't compact old revision") rev2Bstr := `{"_attachments": {"bye.txt": {"stub":true,"revpos":1,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}, "_rev": "2-f000"}` var body2B Body assert.NoError(t, base.JSONUnmarshal([]byte(rev2Bstr), &body2B)) - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body2B, []string{"2-f000", rev1id}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body2B, []string{"2-f000", rev1id}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't update document") } @@ -278,7 +281,7 @@ func TestAttachmentCASRetryAfterNewAttachment(t *testing.T) { rev2Data := `{"prop1":"value2", "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body)) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) log.Printf("Done creating rev 2 for key %s", key) @@ -309,7 +312,7 @@ func TestAttachmentCASRetryAfterNewAttachment(t *testing.T) { var rev3Body Body rev3Data := `{"prop1":"value3", "_attachments": {"hello.txt": {"revpos":2,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` require.NoError(t, base.JSONUnmarshal([]byte(rev3Data), &rev3Body)) - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) log.Printf("rev 3 done") @@ -341,7 +344,7 @@ func TestAttachmentCASRetryDuringNewAttachment(t *testing.T) { rev2Data := `{"prop1":"value2"}` require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body)) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) log.Printf("Done creating rev 2 for key %s", key) @@ -372,7 +375,7 @@ func TestAttachmentCASRetryDuringNewAttachment(t *testing.T) { var rev3Body Body rev3Data := `{"prop1":"value3", "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` require.NoError(t, base.JSONUnmarshal([]byte(rev3Data), &rev3Body)) - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) log.Printf("rev 3 done") @@ -561,7 +564,7 @@ func TestRetrieveAncestorAttachments(t *testing.T) { // Create document (rev 1) text := `{"key": "value", "version": "1a"}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) - doc, revID, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false) + doc, revID, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) @@ -569,49 +572,49 @@ func TestRetrieveAncestorAttachments(t *testing.T) { text = `{"key": "value", "version": "2a", "_attachments": {"att1.txt": {"data": "YXR0MS50eHQ="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "3a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-a", "2-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "4a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "5a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-a", "4-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-a", "4-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "6a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"6-a", "5-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"6-a", "5-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "3b", "type": "pruned"}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) text = `{"key": "value", "version": "3b", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}` assert.NoError(t, base.JSONUnmarshal([]byte(text), &body)) body[BodyRev] = revID - doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false) + doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") log.Printf("doc: %v", doc) } diff --git a/db/background_mgr.go b/db/background_mgr.go index e37e66545b..0640250b22 100644 --- a/db/background_mgr.go +++ b/db/background_mgr.go @@ -87,7 +87,7 @@ type BackgroundManagerStatus struct { } // BackgroundManagerProcessI is an interface satisfied by any of the background processes -// Examples of this: ReSync, Compaction +// Examples of this: ReSync, Compaction, Attachment Migration type BackgroundManagerProcessI interface { Init(ctx context.Context, options map[string]interface{}, clusterStatus []byte) error Run(ctx context.Context, options map[string]interface{}, persistClusterStatusCallback updateStatusCallbackFunc, terminator *base.SafeTerminator) error diff --git a/db/background_mgr_attachment_compaction.go b/db/background_mgr_attachment_compaction.go index c3ba2d3005..ac134a56a8 100644 --- a/db/background_mgr_attachment_compaction.go +++ b/db/background_mgr_attachment_compaction.go @@ -102,23 +102,6 @@ func (a *AttachmentCompactionManager) Init(ctx context.Context, options map[stri return newRunInit() } -func (a *AttachmentCompactionManager) PurgeDCPMetadata(ctx context.Context, datastore base.DataStore, database *Database, metadataKeyPrefix string) error { - - bucket, err := base.AsGocbV2Bucket(database.Bucket) - if err != nil { - return err - } - numVbuckets, err := bucket.GetMaxVbno() - if err != nil { - return err - } - - metadata := base.NewDCPMetadataCS(ctx, datastore, numVbuckets, base.DefaultNumWorkers, metadataKeyPrefix) - base.InfofCtx(ctx, base.KeyDCP, "purging persisted dcp metadata for attachment compaction run %s", a.CompactID) - metadata.Purge(ctx, base.DefaultNumWorkers) - return nil -} - func (a *AttachmentCompactionManager) Run(ctx context.Context, options map[string]interface{}, persistClusterStatusCallback updateStatusCallbackFunc, terminator *base.SafeTerminator) error { database := options["database"].(*Database) @@ -204,7 +187,8 @@ func (a *AttachmentCompactionManager) handleAttachmentCompactionRollbackError(ct if errors.As(err, &rollbackErr) || errors.Is(err, base.ErrVbUUIDMismatch) { base.InfofCtx(ctx, base.KeyDCP, "rollback indicated on %s phase of attachment compaction, resetting the task", phase) // to rollback any phase for attachment compaction we need to purge all persisted dcp metadata - err = a.PurgeDCPMetadata(ctx, dataStore, database, keyPrefix) + base.InfofCtx(ctx, base.KeyDCP, "Purging invalid checkpoints for background task run %s", a.CompactID) + err = PurgeDCPCheckpoints(ctx, database.DatabaseContext, keyPrefix, a.CompactID) if err != nil { base.WarnfCtx(ctx, "error occurred during purging of dcp metadata: %s", err) return false, err diff --git a/db/background_mgr_attachment_migration.go b/db/background_mgr_attachment_migration.go new file mode 100644 index 0000000000..66db30ecb1 --- /dev/null +++ b/db/background_mgr_attachment_migration.go @@ -0,0 +1,372 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package db + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/google/uuid" +) + +type AttachmentMigrationManager struct { + DocsProcessed base.AtomicInt + DocsChanged base.AtomicInt + MigrationID string + CollectionIDs []uint32 + databaseCtx *DatabaseContext + lock sync.RWMutex +} + +var _ BackgroundManagerProcessI = &AttachmentMigrationManager{} + +const MetaVersionValue = "4.0.0" // Meta version to set in syncInfo document upon completion of attachment migration for collection + +func NewAttachmentMigrationManager(database *DatabaseContext) *BackgroundManager { + metadataStore := database.MetadataStore + metaKeys := database.MetadataKeys + return &BackgroundManager{ + name: "attachment_migration", + Process: &AttachmentMigrationManager{ + databaseCtx: database, + }, + clusterAwareOptions: &ClusterAwareBackgroundManagerOptions{ + metadataStore: metadataStore, + metaKeys: metaKeys, + processSuffix: "attachment_migration", + }, + terminator: base.NewSafeTerminator(), + } +} + +func (a *AttachmentMigrationManager) Init(ctx context.Context, options map[string]interface{}, clusterStatus []byte) error { + newRunInit := func() error { + uniqueUUID, err := uuid.NewRandom() + if err != nil { + return err + } + + a.MigrationID = uniqueUUID.String() + base.InfofCtx(ctx, base.KeyAll, "Attachment Migration: Starting new migration run with migration ID: %s", a.MigrationID) + return nil + } + + if clusterStatus != nil { + var statusDoc AttachmentMigrationManagerStatusDoc + err := base.JSONUnmarshal(clusterStatus, &statusDoc) + + reset, _ := options["reset"].(bool) + if reset { + base.InfofCtx(ctx, base.KeyAll, "Attachment Migration: Resetting migration process. Will not resume any partially completed process") + } + + // If the previous run completed, or there was an error during unmarshalling the status we will start the + // process from scratch with a new migration ID. Otherwise, we should resume with the migration ID, stats specified in the doc. + if statusDoc.State == BackgroundProcessStateCompleted || err != nil || reset { + return newRunInit() + } + a.MigrationID = statusDoc.MigrationID + a.SetStatus(statusDoc.DocsChanged, statusDoc.DocsProcessed) + a.SetCollectionIDs(statusDoc.CollectionIDs) + + base.InfofCtx(ctx, base.KeyAll, "Attachment Migration: Resuming migration with migration ID: %s, %d already processed", a.MigrationID, a.DocsProcessed.Value()) + + return nil + } + + return newRunInit() +} + +func (a *AttachmentMigrationManager) Run(ctx context.Context, options map[string]interface{}, persistClusterStatusCallback updateStatusCallbackFunc, terminator *base.SafeTerminator) error { + db := a.databaseCtx + migrationLoggingID := "Migration: " + a.MigrationID + + persistClusterStatus := func() { + err := persistClusterStatusCallback(ctx) + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to persist latest cluster status for attachment migration: %v", migrationLoggingID, err) + } + } + defer persistClusterStatus() + + var processFailure error + failProcess := func(err error, format string, args ...interface{}) bool { + processFailure = err + terminator.Close() + base.WarnfCtx(ctx, format, args...) + return false + } + + callback := func(event sgbucket.FeedEvent) bool { + docID := string(event.Key) + collection := db.CollectionByID[event.CollectionID] + base.TracefCtx(ctx, base.KeyAll, "[%s] Received DCP event %d for doc %v", migrationLoggingID, event.Opcode, base.UD(docID)) + + // Ignore documents without xattrs, to avoid processing unnecessary documents + if event.DataType&base.MemcachedDataTypeXattr == 0 { + return true + } + + // Don't want to process raw binary docs + // The binary check should suffice but for additional safety also check for empty bodies + if event.DataType == base.MemcachedDataTypeRaw || len(event.Value) == 0 { + return true + } + + // We only want to process full docs. Not any sync docs. + if strings.HasPrefix(docID, base.SyncDocPrefix) { + return true + } + + a.DocsProcessed.Add(1) + syncData, _, _, err := UnmarshalDocumentSyncDataFromFeed(event.Value, event.DataType, collection.userXattrKey(), false) + if err != nil { + failProcess(err, "[%s] error unmarshaling document %s: %v, stopping attachment migration.", migrationLoggingID, base.UD(docID), err) + } + + if syncData == nil || syncData.Attachments == nil { + // no attachments to migrate + return true + } + + collCtx := collection.AddCollectionContext(ctx) + collWithUser := &DatabaseCollectionWithUser{ + DatabaseCollection: collection, + } + // xattr migration to take place + err = collWithUser.MigrateAttachmentMetadata(collCtx, docID, event.Cas, syncData) + if err != nil { + failProcess(err, "[%s] error migrating document attachment metadata for doc: %s: %v", migrationLoggingID, base.UD(docID), err) + } + a.DocsChanged.Add(1) + return true + } + + bucket, err := base.AsGocbV2Bucket(db.Bucket) + if err != nil { + return err + } + + currCollectionIDs, err := getCollectionIDsForMigration(db) + if err != nil { + return err + } + dcpFeedKey := GenerateAttachmentMigrationDCPStreamName(a.MigrationID) + dcpPrefix := db.MetadataKeys.DCPCheckpointPrefix(db.Options.GroupID) + + // check for mismatch in collection id's between current collections on the db and prev run + checkpointPrefix := fmt.Sprintf("%s:%v", dcpPrefix, dcpFeedKey) + err = a.resetDCPMetadataIfNeeded(ctx, db, checkpointPrefix, currCollectionIDs) + if err != nil { + return err + } + + a.SetCollectionIDs(currCollectionIDs) + dcpOptions := getMigrationDCPClientOptions(currCollectionIDs, db.Options.GroupID, dcpPrefix) + dcpClient, err := base.NewDCPClient(ctx, dcpFeedKey, callback, *dcpOptions, bucket) + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to create attachment migration DCP client: %v", migrationLoggingID, err) + return err + } + base.DebugfCtx(ctx, base.KeyAll, "[%s] Starting DCP feed %q for attachment migration", migrationLoggingID, dcpFeedKey) + + doneChan, err := dcpClient.Start() + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to start attachment migration DCP feed: %v", migrationLoggingID, err) + _ = dcpClient.Close() + return err + } + base.TracefCtx(ctx, base.KeyAll, "[%s] DCP client started for Attachment Migration.", migrationLoggingID) + + select { + case <-doneChan: + err = dcpClient.Close() + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to close attachment migration DCP client after attachment migration process was finished %v", migrationLoggingID, err) + } + if processFailure != nil { + return processFailure + } + updatedDsNames := make(map[base.ScopeAndCollectionName]struct{}, len(db.CollectionByID)) + // set sync info metadata version + for _, collectionID := range currCollectionIDs { + dbc := db.CollectionByID[collectionID] + if err := base.SetSyncInfoMetaVersion(dbc.dataStore, MetaVersionValue); err != nil { + base.WarnfCtx(ctx, "[%s] Completed attachment migration, but unable to update syncInfo for collection %s: %v", migrationLoggingID, dbc.Name, err) + return err + } + updatedDsNames[base.ScopeAndCollectionName{Scope: dbc.ScopeName, Collection: dbc.Name}] = struct{}{} + } + collectionsRequiringMigration := make([]base.ScopeAndCollectionName, 0) + for _, dsName := range db.RequireAttachmentMigration { + _, ok := updatedDsNames[dsName] + if !ok { + collectionsRequiringMigration = append(collectionsRequiringMigration, dsName) + } + } + db.RequireAttachmentMigration = collectionsRequiringMigration + + base.InfofCtx(ctx, base.KeyAll, "[%s] Finished migrating attachment metadata from sync data to global sync data. %d/%d docs changed", migrationLoggingID, a.DocsChanged.Value(), a.DocsProcessed.Value()) + case <-terminator.Done(): + err = dcpClient.Close() + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to close attachment migration DCP client after attachment migration process was terminated %v", migrationLoggingID, err) + return err + } + if processFailure != nil { + return processFailure + } + err = <-doneChan + if err != nil { + return err + } + base.InfofCtx(ctx, base.KeyAll, "[%s] Attachment Migration was terminated. Docs changed: %d Docs Processed: %d", migrationLoggingID, a.DocsChanged.Value(), a.DocsProcessed.Value()) + } + return nil +} + +func (a *AttachmentMigrationManager) SetStatus(docChanged, docProcessed int64) { + + a.DocsChanged.Set(docChanged) + a.DocsProcessed.Set(docProcessed) +} + +func (a *AttachmentMigrationManager) SetCollectionIDs(collectionID []uint32) { + a.lock.Lock() + defer a.lock.Unlock() + + a.CollectionIDs = collectionID +} + +func (a *AttachmentMigrationManager) ResetStatus() { + a.lock.Lock() + defer a.lock.Unlock() + + a.DocsProcessed.Set(0) + a.DocsChanged.Set(0) + a.CollectionIDs = nil +} + +func (a *AttachmentMigrationManager) GetProcessStatus(status BackgroundManagerStatus) ([]byte, []byte, error) { + a.lock.RLock() + defer a.lock.RUnlock() + + response := AttachmentMigrationManagerResponse{ + BackgroundManagerStatus: status, + MigrationID: a.MigrationID, + DocsChanged: a.DocsChanged.Value(), + DocsProcessed: a.DocsProcessed.Value(), + } + + meta := AttachmentMigrationMeta{ + CollectionIDs: a.CollectionIDs, + } + + statusJSON, err := base.JSONMarshal(response) + if err != nil { + return nil, nil, err + } + metaJSON, err := base.JSONMarshal(meta) + if err != nil { + return nil, nil, err + } + + return statusJSON, metaJSON, err +} + +func getMigrationDCPClientOptions(collectionIDs []uint32, groupID, prefix string) *base.DCPClientOptions { + clientOptions := &base.DCPClientOptions{ + OneShot: true, + FailOnRollback: false, + MetadataStoreType: base.DCPMetadataStoreCS, + GroupID: groupID, + CollectionIDs: collectionIDs, + CheckpointPrefix: prefix, + } + return clientOptions +} + +type AttachmentMigrationManagerResponse struct { + BackgroundManagerStatus + MigrationID string `json:"migration_id"` + DocsChanged int64 `json:"docs_changed"` + DocsProcessed int64 `json:"docs_processed"` +} + +type AttachmentMigrationMeta struct { + CollectionIDs []uint32 `json:"collection_ids"` +} + +type AttachmentMigrationManagerStatusDoc struct { + AttachmentMigrationManagerResponse `json:"status"` + AttachmentMigrationMeta `json:"meta"` +} + +// GenerateAttachmentMigrationDCPStreamName returns the DCP stream name for a resync. +func GenerateAttachmentMigrationDCPStreamName(migrationID string) string { + return fmt.Sprintf( + "sg-%v:att_migration:%v", + base.ProductAPIVersion, + migrationID) +} + +// resetDCPMetadataIfNeeded will check for mismatch between current collectionIDs and collectionIDs on previous run +func (a *AttachmentMigrationManager) resetDCPMetadataIfNeeded(ctx context.Context, database *DatabaseContext, metadataKeyPrefix string, collectionIDs []uint32) error { + // if we are on our first run, no collections will be defined on the manager yet + if len(a.CollectionIDs) == 0 { + return nil + } + if len(a.CollectionIDs) != len(collectionIDs) { + base.InfofCtx(ctx, base.KeyDCP, "Purging invalid checkpoints for background task run %s", a.MigrationID) + err := PurgeDCPCheckpoints(ctx, database, metadataKeyPrefix, a.MigrationID) + if err != nil { + return err + } + return nil + } + slices.Sort(collectionIDs) + slices.Sort(a.CollectionIDs) + purgeNeeded := slices.Compare(collectionIDs, a.CollectionIDs) + if purgeNeeded != 0 { + base.InfofCtx(ctx, base.KeyDCP, "Purging invalid checkpoints for background task run %s", a.MigrationID) + err := PurgeDCPCheckpoints(ctx, database, metadataKeyPrefix, a.MigrationID) + if err != nil { + return err + } + } + return nil +} + +// getCollectionIDsForMigration will get all collection IDs required for DCP client on migration run +func getCollectionIDsForMigration(db *DatabaseContext) ([]uint32, error) { + collectionIDs := make([]uint32, 0) + + // if all collections are included in RequireAttachmentMigration then we need to run against all collections, + // if no collections are specified in RequireAttachmentMigration, run against all collections. This is to support job + // being triggered by rest api (even after job was previously completed) + if len(db.RequireAttachmentMigration) == 0 { + // get all collection IDs + collectionIDs = db.GetCollectionIDs() + } else { + // iterate through and grab collectionIDs we need + for _, v := range db.RequireAttachmentMigration { + collection, err := db.GetDatabaseCollection(v.ScopeName(), v.CollectionName()) + if err != nil { + return nil, base.RedactErrorf("failed to find ID for collection %s.%s", base.MD(v.ScopeName()), base.MD(v.CollectionName())) + } + collectionIDs = append(collectionIDs, collection.GetCollectionID()) + } + } + return collectionIDs, nil +} diff --git a/db/background_mgr_attachment_migration_test.go b/db/background_mgr_attachment_migration_test.go new file mode 100644 index 0000000000..47c58f1193 --- /dev/null +++ b/db/background_mgr_attachment_migration_test.go @@ -0,0 +1,260 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package db + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/couchbase/sync_gateway/base" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAttachmentMigrationTaskMixMigratedAndNonMigratedDocs(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create some docs with attachments defined + for i := 0; i < 10; i++ { + docBody := Body{ + "value": 1234, + BodyAttachments: map[string]interface{}{"myatt": map[string]interface{}{"content_type": "text/plain", "data": "SGVsbG8gV29ybGQh"}}, + } + key := fmt.Sprintf("%s_%d", t.Name(), i) + _, doc, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + assert.NotNil(t, doc.SyncData.Attachments) + } + + // Move some subset of the documents attachment metadata from global sync to sync data + for j := 0; j < 5; j++ { + key := fmt.Sprintf("%s_%d", t.Name(), j) + value, xattrs, cas, err := collection.dataStore.GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok := xattrs[base.SyncXattrName] + assert.True(t, ok) + globalXattr, ok := xattrs[base.GlobalXattrName] + assert.True(t, ok) + + var attachs GlobalSyncData + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, true, collection.dataStore) + } + + attachMigrationMgr := NewAttachmentMigrationManager(db.DatabaseContext) + require.NotNil(t, attachMigrationMgr) + + err := attachMigrationMgr.Start(ctx, nil) + require.NoError(t, err) + + // wait for task to complete + RequireBackgroundManagerState(t, ctx, attachMigrationMgr, BackgroundProcessStateCompleted) + + // assert that the subset (5) of the docs were changed, all created docs were processed (10) + stats := getAttachmentMigrationStats(t, attachMigrationMgr.Process) + assert.Equal(t, int64(10), stats.DocsProcessed) + assert.Equal(t, int64(5), stats.DocsChanged) + + // assert that the sync info metadata version doc has been written to the database collection + AssertSyncInfoMetaVersion(t, collection.dataStore) + +} + +func getAttachmentMigrationStats(t *testing.T, migrationManager BackgroundManagerProcessI) AttachmentMigrationManagerResponse { + var resp AttachmentMigrationManagerResponse + rawStatus, _, err := migrationManager.GetProcessStatus(BackgroundManagerStatus{}) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(rawStatus, &resp)) + return resp +} + +func TestAttachmentMigrationManagerResumeStoppedMigration(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + base.LongRunningTest(t) + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create some docs with attachments defined, a large number is needed to allow us to stop the migration midway + // through without it completing first + for i := 0; i < 4000; i++ { + docBody := Body{ + "value": 1234, + BodyAttachments: map[string]interface{}{"myatt": map[string]interface{}{"content_type": "text/plain", "data": "SGVsbG8gV29ybGQh"}}, + } + key := fmt.Sprintf("%s_%d", t.Name(), i) + _, doc, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + require.NotNil(t, doc.SyncData.Attachments) + } + attachMigrationMgr := NewAttachmentMigrationManager(db.DatabaseContext) + require.NotNil(t, attachMigrationMgr) + + err := attachMigrationMgr.Start(ctx, nil) + require.NoError(t, err) + + // Attempt to Stop Process + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + for { + stats := getAttachmentMigrationStats(t, attachMigrationMgr.Process) + if stats.DocsProcessed >= 200 { + err = attachMigrationMgr.Stop() + require.NoError(t, err) + break + } + time.Sleep(1 * time.Microsecond) + } + }() + + RequireBackgroundManagerState(t, ctx, attachMigrationMgr, BackgroundProcessStateStopped) + + stats := getAttachmentMigrationStats(t, attachMigrationMgr.Process) + require.Less(t, stats.DocsProcessed, int64(4000)) + + // assert that the sync info metadata version is not present + var syncInfo base.SyncInfo + _, err = collection.dataStore.Get(base.SGSyncInfo, &syncInfo) + require.Error(t, err) + + // Resume process + err = attachMigrationMgr.Start(ctx, nil) + require.NoError(t, err) + + RequireBackgroundManagerState(t, ctx, attachMigrationMgr, BackgroundProcessStateCompleted) + + stats = getAttachmentMigrationStats(t, attachMigrationMgr.Process) + require.GreaterOrEqual(t, stats.DocsProcessed, int64(4000)) + + // assert that the sync info metadata version doc has been written to the database collection + AssertSyncInfoMetaVersion(t, collection.dataStore) +} + +func TestAttachmentMigrationManagerNoDocsToMigrate(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create a doc with no attachments defined but through sync gateway, so it will have sync data + docBody := Body{ + "value": "doc", + } + key := fmt.Sprintf("%s_%d", t.Name(), 1) + _, _, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + + // add new doc with no sync data (SDK write, no import) + key = fmt.Sprintf("%s_%d", t.Name(), 2) + _, err = collection.dataStore.Add(key, 0, []byte(`{"test":"doc"}`)) + require.NoError(t, err) + + attachMigrationMgr := NewAttachmentMigrationManager(db.DatabaseContext) + require.NotNil(t, attachMigrationMgr) + + err = attachMigrationMgr.Start(ctx, nil) + require.NoError(t, err) + + // wait for task to complete + RequireBackgroundManagerState(t, ctx, attachMigrationMgr, BackgroundProcessStateCompleted) + + // assert that the two added docs above were processed but not changed + stats := getAttachmentMigrationStats(t, attachMigrationMgr.Process) + // no docs should be changed, only one has xattr defined thus should only have one of the two docs processed + assert.Equal(t, int64(1), stats.DocsProcessed) + assert.Equal(t, int64(0), stats.DocsChanged) + + // assert that the sync info metadata version doc has been written to the database collection + AssertSyncInfoMetaVersion(t, collection.dataStore) +} + +func TestMigrationManagerDocWithSyncAndGlobalAttachmentMetadata(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + docBody := Body{ + "value": 1234, + BodyAttachments: map[string]interface{}{"myatt": map[string]interface{}{"content_type": "text/plain", "data": "SGVsbG8gV29ybGQh"}}, + } + key := t.Name() + _, _, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + + xattrs, cas, err := collection.dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + require.Contains(t, xattrs, base.SyncXattrName) + + var syncData SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) + // define some attachment meta on sync data + syncData.Attachments = AttachmentsMeta{} + att := map[string]interface{}{ + "stub": true, + } + syncData.Attachments["someAtt.txt"] = att + + updateXattrs := map[string][]byte{ + base.SyncXattrName: base.MustJSONMarshal(t, syncData), + } + _, err = collection.dataStore.UpdateXattrs(ctx, key, 0, cas, updateXattrs, DefaultMutateInOpts()) + require.NoError(t, err) + + attachMigrationMgr := NewAttachmentMigrationManager(db.DatabaseContext) + require.NotNil(t, attachMigrationMgr) + + err = attachMigrationMgr.Start(ctx, nil) + require.NoError(t, err) + + // wait for task to complete + RequireBackgroundManagerState(t, ctx, attachMigrationMgr, BackgroundProcessStateCompleted) + + // assert that the two added docs above were processed but not changed + stats := getAttachmentMigrationStats(t, attachMigrationMgr.Process) + assert.Equal(t, int64(1), stats.DocsProcessed) + assert.Equal(t, int64(1), stats.DocsChanged) + + // assert that the sync info metadata version doc has been written to the database collection + AssertSyncInfoMetaVersion(t, collection.dataStore) + + xattrs, _, err = collection.dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + require.Contains(t, xattrs, base.SyncXattrName) + + var globalSync GlobalSyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalSync)) + syncData = SyncData{} + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) + + require.NotNil(t, globalSync.GlobalAttachments) + assert.NotNil(t, globalSync.GlobalAttachments["someAtt.txt"]) + assert.NotNil(t, globalSync.GlobalAttachments["myatt"]) + assert.Nil(t, syncData.Attachments) +} diff --git a/db/background_mgr_resync.go b/db/background_mgr_resync.go index 36dc9720e5..09c98a8393 100644 --- a/db/background_mgr_resync.go +++ b/db/background_mgr_resync.go @@ -87,7 +87,7 @@ func (r *ResyncManager) Run(ctx context.Context, options map[string]interface{}, return err } if regenerateSequences { - err := base.SetSyncInfo(dbc.dataStore, dbc.dbCtx.Options.MetadataID) + err := base.SetSyncInfoMetadataID(dbc.dataStore, dbc.dbCtx.Options.MetadataID) if err != nil { base.InfofCtx(ctx, base.KeyAll, "Failed to updateSyncInfo after resync: %v", err) } diff --git a/db/background_mgr_resync_dcp.go b/db/background_mgr_resync_dcp.go index 3faf60f1f1..3336b27644 100644 --- a/db/background_mgr_resync_dcp.go +++ b/db/background_mgr_resync_dcp.go @@ -234,7 +234,7 @@ func (r *ResyncManagerDCP) Run(ctx context.Context, options map[string]interface if !ok { base.WarnfCtx(ctx, "[%s] Completed resync, but unable to update syncInfo for collection %v (not found)", resyncLoggingID, collectionID) } - if err := base.SetSyncInfo(dbc.dataStore, db.DatabaseContext.Options.MetadataID); err != nil { + if err := base.SetSyncInfoMetadataID(dbc.dataStore, db.DatabaseContext.Options.MetadataID); err != nil { base.WarnfCtx(ctx, "[%s] Completed resync, but unable to update syncInfo for collection %v: %v", resyncLoggingID, collectionID, err) } updatedDsNames[base.ScopeAndCollectionName{Scope: dbc.ScopeName, Collection: dbc.Name}] = struct{}{} diff --git a/db/background_mgr_resync_dcp_test.go b/db/background_mgr_resync_dcp_test.go index 5764bd0751..69b48a0fd7 100644 --- a/db/background_mgr_resync_dcp_test.go +++ b/db/background_mgr_resync_dcp_test.go @@ -498,15 +498,15 @@ function sync(doc, oldDoc){ syncData, mou, cas = getSyncAndMou(t, collection, "sgWrite") require.NotNil(t, syncData) require.NotNil(t, mou) - require.Equal(t, base.CasToString(sgWriteCas), mou.PreviousCAS) - require.Equal(t, base.CasToString(cas), mou.CAS) + require.Equal(t, base.CasToString(sgWriteCas), mou.PreviousHexCAS) + require.Equal(t, base.CasToString(cas), mou.HexCAS) syncData, mou, cas = getSyncAndMou(t, collection, "sdkWrite") require.NotNil(t, syncData) require.NotNil(t, mou) - require.Equal(t, initialSDKMou.PreviousCAS, mou.PreviousCAS) - require.NotEqual(t, initialSDKMou.CAS, mou.CAS) - require.Equal(t, base.CasToString(cas), mou.CAS) + require.Equal(t, initialSDKMou.PreviousHexCAS, mou.PreviousHexCAS) + require.NotEqual(t, initialSDKMou.HexCAS, mou.HexCAS) + require.Equal(t, base.CasToString(cas), mou.HexCAS) // Run resync a second time with a new sync function. mou.cas should be updated, mou.pCas should not change. syncFn = ` @@ -519,15 +519,15 @@ function sync(doc, oldDoc){ syncData, mou, cas = getSyncAndMou(t, collection, "sgWrite") require.NotNil(t, syncData) require.NotNil(t, mou) - require.Equal(t, base.CasToString(sgWriteCas), mou.PreviousCAS) - require.Equal(t, base.CasToString(cas), mou.CAS) + require.Equal(t, base.CasToString(sgWriteCas), mou.PreviousHexCAS) + require.Equal(t, base.CasToString(cas), mou.HexCAS) syncData, mou, cas = getSyncAndMou(t, collection, "sdkWrite") require.NotNil(t, syncData) require.NotNil(t, mou) - require.Equal(t, initialSDKMou.PreviousCAS, mou.PreviousCAS) - require.NotEqual(t, initialSDKMou.CAS, mou.CAS) - require.Equal(t, base.CasToString(cas), mou.CAS) + require.Equal(t, initialSDKMou.PreviousHexCAS, mou.PreviousHexCAS) + require.NotEqual(t, initialSDKMou.HexCAS, mou.HexCAS) + require.Equal(t, base.CasToString(cas), mou.HexCAS) } func runResync(t *testing.T, ctx context.Context, db *Database, collection *DatabaseCollectionWithUser, syncFn string) (stats ResyncManagerResponseDCP) { diff --git a/db/blip_handler.go b/db/blip_handler.go index a192e2d762..e38ff755ab 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -507,12 +507,19 @@ func (bh *blipHandler) sendChanges(sender *blip.Sender, opts *sendChangesOptions } } - for _, item := range change.Changes { - changeRow := bh.buildChangesRow(change, item["rev"]) - pendingChanges = append(pendingChanges, changeRow) - if err := sendPendingChangesAt(opts.batchSize); err != nil { - return err + // if V3 and below populate change row with rev id + if bh.activeCBMobileSubprotocol <= CBMobileReplicationV3 { + for _, item := range change.Changes { + changeRow := bh.buildChangesRow(change, item["rev"]) + pendingChanges = append(pendingChanges, changeRow) } + } else { + changeRow := bh.buildChangesRow(change, change.CurrentVersion.String()) + pendingChanges = append(pendingChanges, changeRow) + } + + if err := sendPendingChangesAt(opts.batchSize); err != nil { + return err } } } @@ -765,7 +772,8 @@ func (bh *blipHandler) handleChanges(rq *blip.Message) error { } output.Write([]byte("]")) response := rq.Response() - if bh.sgCanUseDeltas { + // Disable delta sync for protocol versions < 4, CBG-3748 (backwards compatibility for revID delta sync) + if bh.sgCanUseDeltas && bh.useHLV() { base.DebugfCtx(bh.loggingCtx, base.KeyAll, "Setting deltas=true property on handleChanges response") response.Properties[ChangesResponseDeltas] = trueProperty bh.replicationStats.HandleChangesDeltaRequestedCount.Add(int64(nRequested)) @@ -814,15 +822,30 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error { defer func() { bh.replicationStats.HandleChangesTime.Add(time.Since(startTime).Nanoseconds()) }() + changesContainLegacyRevs := false // keep track if proposed changes have legacy revs for delta sync purposes + versionVectorProtocol := bh.useHLV() for i, change := range changeList { docID := change[0].(string) - revID := change[1].(string) + rev := change[1].(string) // rev can represent a RevTree ID or HLV current version parentRevID := "" if len(change) > 2 { parentRevID = change[2].(string) } - status, currentRev := bh.collection.CheckProposedRev(bh.loggingCtx, docID, revID, parentRevID) + var status ProposedRevStatus + var currentRev string + + changeIsVector := false + if versionVectorProtocol { + // only check if rev is vector in VV replication mode + changeIsVector = strings.Contains(rev, "@") + } + if versionVectorProtocol && changeIsVector { + status, currentRev = bh.collection.CheckProposedVersion(bh.loggingCtx, docID, rev, parentRevID) + } else { + changesContainLegacyRevs = true + status, currentRev = bh.collection.CheckProposedRev(bh.loggingCtx, docID, rev, parentRevID) + } if status == ProposedRev_OK_IsNew { // Remember that the doc doesn't exist locally, in order to optimize the upcoming Put: bh.collectionCtx.notePendingInsertion(docID) @@ -852,7 +875,8 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error { } output.Write([]byte("]")) response := rq.Response() - if bh.sgCanUseDeltas { + // Disable delta sync for protocol versions < 4 or changes batches that have legacy revs in them, CBG-3748 (backwards compatibility for revID delta sync) + if bh.sgCanUseDeltas && bh.useHLV() && !changesContainLegacyRevs { base.DebugfCtx(bh.loggingCtx, base.KeyAll, "Setting deltas=true property on proposeChanges response") response.Properties[ChangesResponseDeltas] = trueProperty } @@ -866,35 +890,40 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error { func (bsc *BlipSyncContext) sendRevAsDelta(ctx context.Context, sender *blip.Sender, docID, revID string, deltaSrcRevID string, seq SequenceID, knownRevs map[string]bool, maxHistory int, handleChangesResponseCollection *DatabaseCollectionWithUser, collectionIdx *int) error { bsc.replicationStats.SendRevDeltaRequestedCount.Add(1) - revDelta, redactedRev, err := handleChangesResponseCollection.GetDelta(ctx, docID, deltaSrcRevID, revID) + revDelta, redactedRev, err := handleChangesResponseCollection.GetDelta(ctx, docID, deltaSrcRevID, revID, bsc.useHLV()) if err == ErrForbidden { // nolint: gocritic // can't convert if/else if to switch since base.IsFleeceDeltaError is not switchable return err } else if base.IsFleeceDeltaError(err) { // Something went wrong in the diffing library. We want to know about this! base.WarnfCtx(ctx, "Falling back to full body replication. Error generating delta from %s to %s for key %s - err: %v", deltaSrcRevID, revID, base.UD(docID), err) - return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx) + return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx, false) } else if err == base.ErrDeltaSourceIsTombstone { base.TracefCtx(ctx, base.KeySync, "Falling back to full body replication. Delta source %s is tombstone. Unable to generate delta to %s for key %s", deltaSrcRevID, revID, base.UD(docID)) - return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx) + return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx, false) } else if err != nil { base.DebugfCtx(ctx, base.KeySync, "Falling back to full body replication. Couldn't get delta from %s to %s for key %s - err: %v", deltaSrcRevID, revID, base.UD(docID), err) - return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx) + return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx, false) } if redactedRev != nil { - history := toHistory(redactedRev.History, knownRevs, maxHistory) + var history []string + if !bsc.useHLV() { + history = toHistory(redactedRev.History, knownRevs, maxHistory) + } else { + history = append(history, redactedRev.hlvHistory) + } properties := blipRevMessageProperties(history, redactedRev.Deleted, seq, "") return bsc.sendRevisionWithProperties(ctx, sender, docID, revID, collectionIdx, redactedRev.BodyBytes, nil, properties, seq, nil) } if revDelta == nil { base.DebugfCtx(ctx, base.KeySync, "Falling back to full body replication. Couldn't get delta from %s to %s for key %s", deltaSrcRevID, revID, base.UD(docID)) - return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx) + return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx, false) } resendFullRevisionFunc := func() error { base.InfofCtx(ctx, base.KeySync, "Resending revision as full body. Peer couldn't process delta %s from %s to %s for key %s", base.UD(revDelta.DeltaBytes), deltaSrcRevID, revID, base.UD(docID)) - return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx) + return bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseCollection, collectionIdx, false) } base.TracefCtx(ctx, base.KeySync, "docID: %s - delta: %v", base.UD(docID), base.UD(string(revDelta.DeltaBytes))) @@ -964,6 +993,10 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } }() + if bh.useHLV() && bh.conflictResolver != nil { + return base.HTTPErrorf(http.StatusNotImplemented, "conflict resolver handling (ISGR) not yet implemented for v4 protocol") + } + // throttle concurrent revs if cap(bh.inFlightRevsThrottle) > 0 { select { @@ -982,13 +1015,13 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // Doc metadata comes from the BLIP message metadata, not magic document properties: docID, found := revMessage.ID() - revID, rfound := revMessage.Rev() + rev, rfound := revMessage.Rev() if !found || !rfound { - return base.HTTPErrorf(http.StatusBadRequest, "Missing docID or revID") + return base.HTTPErrorf(http.StatusBadRequest, "Missing docID or rev") } if bh.readOnly { - return base.HTTPErrorf(http.StatusForbidden, "Replication context is read-only, docID: %s, revID:%s", docID, revID) + return base.HTTPErrorf(http.StatusForbidden, "Replication context is read-only, docID: %s, rev:%s", docID, rev) } base.DebugfCtx(bh.loggingCtx, base.KeySyncMsg, "#%d: Type:%s %s", bh.serialNumber, rq.Profile(), revMessage.String()) @@ -1008,7 +1041,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err return err } if removed, ok := body[BodyRemoved].(bool); ok && removed { - base.InfofCtx(bh.loggingCtx, base.KeySync, "Purging doc %v - removed at rev %v", base.UD(docID), revID) + base.InfofCtx(bh.loggingCtx, base.KeySync, "Purging doc %v - removed at rev %v", base.UD(docID), rev) if err := bh.collection.Purge(bh.loggingCtx, docID, true); err != nil { return err } @@ -1020,7 +1053,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err if err != nil { base.WarnfCtx(bh.loggingCtx, "Unable to parse sequence %q from rev message: %v - not tracking for checkpointing", seqStr, err) } else { - bh.collectionCtx.sgr2PullProcessedSeqCallback(&seq, IDAndRev{DocID: docID, RevID: revID}) + bh.collectionCtx.sgr2PullProcessedSeqCallback(&seq, IDAndRev{DocID: docID, RevID: rev}) } } return nil @@ -1028,9 +1061,34 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } newDoc := &Document{ - ID: docID, - RevID: revID, + ID: docID, + } + + var history []string + historyStr := rq.Properties[RevMessageHistory] + var incomingHLV *HybridLogicalVector + var legacyRevList []string + // Build history/HLV + changeIsVector := strings.Contains(rev, "@") + if !bh.useHLV() || !changeIsVector { + newDoc.RevID = rev + history = []string{rev} + if historyStr != "" { + history = append(history, strings.Split(historyStr, ",")...) + } + } else { + versionVectorStr := rev + if historyStr != "" { + versionVectorStr += ";" + historyStr + } + incomingHLV, legacyRevList, err = extractHLVFromBlipMessage(versionVectorStr) + if err != nil { + base.InfofCtx(bh.loggingCtx, base.KeySync, "Error parsing hlv while processing rev for doc %v. HLV:%v Error: %v", base.UD(docID), versionVectorStr, err) + return base.HTTPErrorf(http.StatusUnprocessableEntity, "error extracting hlv from blip message") + } + newDoc.HLV = incomingHLV } + newDoc.UpdateBodyBytes(bodyBytes) injectedAttachmentsForDelta := false @@ -1049,7 +1107,16 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // while retrieving deltaSrcRevID. Couchbase Lite replication guarantees client has access to deltaSrcRevID, // due to no-conflict write restriction, but we still need to enforce security here to prevent leaking data about previous // revisions to malicious actors (in the scenario where that user has write but not read access). - deltaSrcRev, err := bh.collection.GetRev(bh.loggingCtx, docID, deltaSrcRevID, false, nil) + var deltaSrcRev DocumentRevision + if bh.useHLV() { + deltaSrcVersion, parseErr := ParseVersion(deltaSrcRevID) + if parseErr != nil { + return base.HTTPErrorf(http.StatusUnprocessableEntity, "Unable to parse version for delta source for doc %s, error: %v", base.UD(docID), err) + } + deltaSrcRev, err = bh.collection.GetCV(bh.loggingCtx, docID, &deltaSrcVersion) + } else { + deltaSrcRev, err = bh.collection.GetRev(bh.loggingCtx, docID, deltaSrcRevID, false, nil) + } if err != nil { return base.HTTPErrorf(http.StatusUnprocessableEntity, "Can't fetch doc %s for deltaSrc=%s %v", base.UD(docID), deltaSrcRevID, err) } @@ -1075,7 +1142,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // err should only ever be a FleeceDeltaError here - but to be defensive, handle other errors too (e.g. somehow reaching this code in a CE build) if err != nil { // Something went wrong in the diffing library. We want to know about this! - base.WarnfCtx(bh.loggingCtx, "Error patching deltaSrc %s with %s for doc %s with delta - err: %v", deltaSrcRevID, revID, base.UD(docID), err) + base.WarnfCtx(bh.loggingCtx, "Error patching deltaSrc %s with %s for doc %s with delta - err: %v", deltaSrcRevID, rev, base.UD(docID), err) return base.HTTPErrorf(http.StatusUnprocessableEntity, "Error patching deltaSrc with delta: %s", err) } @@ -1113,34 +1180,34 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } } - history := []string{revID} - if historyStr := rq.Properties[RevMessageHistory]; historyStr != "" { - history = append(history, strings.Split(historyStr, ",")...) - } - var rawBucketDoc *sgbucket.BucketDocument - // Pull out attachments + // Attachment processing if injectedAttachmentsForDelta || bytes.Contains(bodyBytes, []byte(BodyAttachments)) { + body := newDoc.Body(bh.loggingCtx) var currentBucketDoc *Document - // Look at attachments with revpos > the last common ancestor's - minRevpos := 1 - if len(history) > 0 { - currentDoc, rawDoc, err := bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, DocUnmarshalSync) - // If we're able to obtain current doc data then we should use the common ancestor generation++ for min revpos - // as we will already have any attachments on the common ancestor so don't need to ask for them. - // Otherwise we'll have to go as far back as we can in the doc history and choose the last entry in there. - if err == nil { - commonAncestor := currentDoc.History.findAncestorFromSet(currentDoc.CurrentRev, history) - minRevpos, _ = ParseRevID(bh.loggingCtx, commonAncestor) - minRevpos++ - rawBucketDoc = rawDoc - currentBucketDoc = currentDoc - } else { - minRevpos, _ = ParseRevID(bh.loggingCtx, history[len(history)-1]) + minRevpos := 0 + if historyStr != "" { + // fetch current bucket doc. Treats error as not found + currentBucketDoc, rawBucketDoc, _ = bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, DocUnmarshalSync) + + // For revtree clients, can use revPos as an optimization. HLV always compares incoming + // attachments with current attachments on the document + if !bh.useHLV() { + // Look at attachments with revpos > the last common ancestor's + // If we're able to obtain current doc data then we should use the common ancestor generation++ for min revpos + // as we will already have any attachments on the common ancestor so don't need to ask for them. + // Otherwise we'll have to go as far back as we can in the doc history and choose the last entry in there. + if currentBucketDoc != nil { + commonAncestor := currentBucketDoc.History.findAncestorFromSet(currentBucketDoc.CurrentRev, history) + minRevpos, _ = ParseRevID(bh.loggingCtx, commonAncestor) + minRevpos++ + } else { + minRevpos, _ = ParseRevID(bh.loggingCtx, history[len(history)-1]) + } } } @@ -1158,7 +1225,9 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err if !ok { // If we don't have this attachment already, ensure incoming revpos is greater than minRevPos, otherwise // update to ensure it's fetched and uploaded - bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, revID) + if minRevpos > 0 { + bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, rev) + } continue } @@ -1198,7 +1267,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // digest is different we need to override the revpos and set it to the current revision to ensure // the attachment is requested and stored if int(incomingAttachmentRevpos) <= minRevpos && currentAttachmentDigest != incomingAttachmentDigest { - bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, revID) + bodyAtts[name].(map[string]interface{})["revpos"], _ = ParseRevID(bh.loggingCtx, rev) } } @@ -1206,7 +1275,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } if err := bh.downloadOrVerifyAttachments(rq.Sender, body, minRevpos, docID, currentDigests); err != nil { - base.ErrorfCtx(bh.loggingCtx, "Error during downloadOrVerifyAttachments for doc %s/%s: %v", base.UD(docID), revID, err) + base.ErrorfCtx(bh.loggingCtx, "Error during downloadOrVerifyAttachments for doc %s/%s: %v", base.UD(docID), rev, err) return err } @@ -1216,7 +1285,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err } if rawBucketDoc == nil && bh.collectionCtx.checkPendingInsertion(docID) { - // At the time we handled the `propseChanges` request, there was no doc with this docID + // At the time we handled the `proposeChanges` request, there was no doc with this docID // in the bucket. As an optimization, tell PutExistingRev to assume the doc still doesn't // exist and bypass getting it from the bucket during the save. If we're wrong, the save // will fail with a CAS mismatch and the retry will fetch the existing doc. @@ -1229,10 +1298,12 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // If the doc is a tombstone we want to allow conflicts when running SGR2 // bh.conflictResolver != nil represents an active SGR2 and BLIPClientTypeSGR2 represents a passive SGR2 forceAllowConflictingTombstone := newDoc.Deleted && (bh.conflictResolver != nil || bh.clientType == BLIPClientTypeSGR2) - if bh.conflictResolver != nil { - _, _, err = bh.collection.PutExistingRevWithConflictResolution(bh.loggingCtx, newDoc, history, true, bh.conflictResolver, forceAllowConflictingTombstone, rawBucketDoc) + if bh.useHLV() && changeIsVector { + _, _, _, err = bh.collection.PutExistingCurrentVersion(bh.loggingCtx, newDoc, incomingHLV, rawBucketDoc, legacyRevList) + } else if bh.conflictResolver != nil { + _, _, err = bh.collection.PutExistingRevWithConflictResolution(bh.loggingCtx, newDoc, history, true, bh.conflictResolver, forceAllowConflictingTombstone, rawBucketDoc, ExistingVersionWithUpdateToHLV) } else { - _, _, err = bh.collection.PutExistingRev(bh.loggingCtx, newDoc, history, revNoConflicts, forceAllowConflictingTombstone, rawBucketDoc) + _, _, err = bh.collection.PutExistingRev(bh.loggingCtx, newDoc, history, revNoConflicts, forceAllowConflictingTombstone, rawBucketDoc, ExistingVersionWithUpdateToHLV) } if err != nil { return err @@ -1244,7 +1315,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err if err != nil { base.WarnfCtx(bh.loggingCtx, "Unable to parse sequence %q from rev message: %v - not tracking for checkpointing", seqProperty, err) } else { - bh.collectionCtx.sgr2PullProcessedSeqCallback(&seq, IDAndRev{DocID: docID, RevID: revID}) + bh.collectionCtx.sgr2PullProcessedSeqCallback(&seq, IDAndRev{DocID: docID, RevID: rev}) } } diff --git a/db/blip_sync_context.go b/db/blip_sync_context.go index cb39b0c34b..ce764c823a 100644 --- a/db/blip_sync_context.go +++ b/db/blip_sync_context.go @@ -340,13 +340,15 @@ func (bsc *BlipSyncContext) handleChangesResponse(ctx context.Context, sender *b if err != nil { return err } + versionVectorProtocol := bsc.useHLV() for i, knownRevsArrayInterface := range answer { seq := changeArray[i][0].(SequenceID) docID := changeArray[i][1].(string) - revID := changeArray[i][2].(string) + rev := changeArray[i][2].(string) if knownRevsArray, ok := knownRevsArrayInterface.([]interface{}); ok { + legacyRev := false deltaSrcRevID := "" knownRevs := knownRevsByDoc[docID] if knownRevs == nil { @@ -354,15 +356,38 @@ func (bsc *BlipSyncContext) handleChangesResponse(ctx context.Context, sender *b knownRevsByDoc[docID] = knownRevs } - // The first element of the knownRevsArray returned from CBL is the parent revision to use as deltaSrc + // The first element of the knownRevsArray returned from CBL is the parent revision to use as deltaSrc for + // revtree clients. For HLV clients, use the cv as deltaSrc if bsc.useDeltas && len(knownRevsArray) > 0 { if revID, ok := knownRevsArray[0].(string); ok { - deltaSrcRevID = revID + if versionVectorProtocol { + msgHLV, _, err := extractHLVFromBlipMessage(revID) + if err != nil { + base.DebugfCtx(ctx, base.KeySync, "Invalid known rev format for hlv on doc: %s falling back to full body replication.", base.UD(docID)) + deltaSrcRevID = "" // will force falling back to full body replication below + } else { + deltaSrcRevID = msgHLV.GetCurrentVersionString() + } + } else { + deltaSrcRevID = revID + } } } for _, rev := range knownRevsArray { if revID, ok := rev.(string); ok { + msgHLV, _, err := extractHLVFromBlipMessage(revID) + if err != nil { + // assume we have received legacy rev if we cannot parse hlv from known revs, and we are in vv replication + if versionVectorProtocol { + legacyRev = true + } + } else { + // extract cv as string + revID = msgHLV.GetCurrentVersionString() + } + // we can assume here that if we fail to parse hlv, we have received a rev id in known revs. If we don't fail to parse hlv + // then we have extracted cv from it and can assign the cv string to known revs here knownRevs[revID] = true } else { base.ErrorfCtx(ctx, "Invalid response to 'changes' message") @@ -371,10 +396,12 @@ func (bsc *BlipSyncContext) handleChangesResponse(ctx context.Context, sender *b } var err error - if deltaSrcRevID != "" { - err = bsc.sendRevAsDelta(ctx, sender, docID, revID, deltaSrcRevID, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx) + + // fallback to sending full revisions for non hlv aware peers, CBG-3748 + if deltaSrcRevID != "" && bsc.useHLV() { + err = bsc.sendRevAsDelta(ctx, sender, docID, rev, deltaSrcRevID, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx) } else { - err = bsc.sendRevision(ctx, sender, docID, revID, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx) + err = bsc.sendRevision(ctx, sender, docID, rev, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx, legacyRev) } if err != nil { return err @@ -386,7 +413,7 @@ func (bsc *BlipSyncContext) handleChangesResponse(ctx context.Context, sender *b sentSeqs = append(sentSeqs, seq) } } else { - base.DebugfCtx(bsc.loggingCtx, base.KeySync, "Peer didn't want revision %s / %s (seq:%v)", base.UD(docID), revID, seq) + base.DebugfCtx(bsc.loggingCtx, base.KeySync, "Peer didn't want revision %s / %s (seq:%v)", base.UD(docID), rev, seq) if collectionCtx.sgr2PushAlreadyKnownSeqsCallback != nil { alreadyKnownSeqs = append(alreadyKnownSeqs, seq) } @@ -566,12 +593,23 @@ func (bsc *BlipSyncContext) setUseDeltas(clientCanUseDeltas bool) { func (bsc *BlipSyncContext) sendDelta(ctx context.Context, sender *blip.Sender, docID string, collectionIdx *int, deltaSrcRevID string, revDelta *RevisionDelta, seq SequenceID, resendFullRevisionFunc func() error) error { - properties := blipRevMessageProperties(revDelta.RevisionHistory, revDelta.ToDeleted, seq, "") + var history []string + if bsc.useHLV() { + history = append(history, revDelta.HlvHistory) + } else { + history = revDelta.RevisionHistory + } + properties := blipRevMessageProperties(history, revDelta.ToDeleted, seq, "") properties[RevMessageDeltaSrc] = deltaSrcRevID base.DebugfCtx(ctx, base.KeySync, "Sending rev %q %s as delta. DeltaSrc:%s", base.UD(docID), revDelta.ToRevID, deltaSrcRevID) - return bsc.sendRevisionWithProperties(ctx, sender, docID, revDelta.ToRevID, collectionIdx, revDelta.DeltaBytes, revDelta.AttachmentStorageMeta, - properties, seq, resendFullRevisionFunc) + if bsc.useHLV() { + return bsc.sendRevisionWithProperties(ctx, sender, docID, revDelta.ToCV, collectionIdx, revDelta.DeltaBytes, revDelta.AttachmentStorageMeta, + properties, seq, resendFullRevisionFunc) + } else { + return bsc.sendRevisionWithProperties(ctx, sender, docID, revDelta.ToRevID, collectionIdx, revDelta.DeltaBytes, revDelta.AttachmentStorageMeta, + properties, seq, resendFullRevisionFunc) + } } // sendBLIPMessage is a simple wrapper around all sent BLIP messages @@ -621,8 +659,20 @@ func (bsc *BlipSyncContext) sendNoRev(sender *blip.Sender, docID, revID string, } // Pushes a revision body to the client -func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sender, docID, revID string, seq SequenceID, knownRevs map[string]bool, maxHistory int, handleChangesResponseCollection *DatabaseCollectionWithUser, collectionIdx *int) error { - rev, originalErr := handleChangesResponseCollection.GetRev(ctx, docID, revID, true, nil) +func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sender, docID, revID string, seq SequenceID, knownRevs map[string]bool, maxHistory int, handleChangesResponseCollection *DatabaseCollectionWithUser, collectionIdx *int, legacyRev bool) error { + + var originalErr error + var docRev DocumentRevision + if !bsc.useHLV() { + docRev, originalErr = handleChangesResponseCollection.GetRev(ctx, docID, revID, true, nil) + } else { + // extract CV string rev representation + version, vrsErr := ParseVersion(revID) + if vrsErr != nil { + return vrsErr + } + docRev, originalErr = handleChangesResponseCollection.GetCV(bsc.loggingCtx, docID, &version) + } // set if we find an alternative revision to send in the event the originally requested rev is unavailable var replacedRevID string @@ -656,37 +706,37 @@ func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sende replacedRevID = revID revID = replacementRev.RevID - rev = replacementRev + docRev = replacementRev } else if originalErr != nil { return fmt.Errorf("failed to GetRev for doc %s with rev %s: %w", base.UD(docID).Redact(), base.MD(revID).Redact(), originalErr) } - base.TracefCtx(ctx, base.KeySync, "sendRevision, rev attachments for %s/%s are %v", base.UD(docID), revID, base.UD(rev.Attachments)) - attachmentStorageMeta := ToAttachmentStorageMeta(rev.Attachments) + base.TracefCtx(ctx, base.KeySync, "sendRevision, rev attachments for %s/%s are %v", base.UD(docID), revID, base.UD(docRev.Attachments)) + attachmentStorageMeta := ToAttachmentStorageMeta(docRev.Attachments) var bodyBytes []byte if base.IsEnterpriseEdition() { // Still need to stamp _attachments into BLIP messages - if len(rev.Attachments) > 0 { - DeleteAttachmentVersion(rev.Attachments) + if len(docRev.Attachments) > 0 { + DeleteAttachmentVersion(docRev.Attachments) var err error - bodyBytes, err = base.InjectJSONProperties(rev.BodyBytes, base.KVPair{Key: BodyAttachments, Val: rev.Attachments}) + bodyBytes, err = base.InjectJSONProperties(docRev.BodyBytes, base.KVPair{Key: BodyAttachments, Val: docRev.Attachments}) if err != nil { return err } } else { - bodyBytes = rev.BodyBytes + bodyBytes = docRev.BodyBytes } } else { - body, err := rev.Body() + body, err := docRev.Body() if err != nil { base.DebugfCtx(ctx, base.KeySync, "Sending norev %q %s due to unavailable revision body: %v", base.UD(docID), revID, err) return bsc.sendNoRev(sender, docID, revID, collectionIdx, seq, err) } // Still need to stamp _attachments into BLIP messages - if len(rev.Attachments) > 0 { - DeleteAttachmentVersion(rev.Attachments) - body[BodyAttachments] = rev.Attachments + if len(docRev.Attachments) > 0 { + DeleteAttachmentVersion(docRev.Attachments) + body[BodyAttachments] = docRev.Attachments } bodyBytes, err = base.JSONMarshalCanonical(body) @@ -699,9 +749,22 @@ func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sende if replacedRevID != "" { bsc.replicationStats.SendReplacementRevCount.Add(1) } + var history []string + if !bsc.useHLV() { + history = toHistory(docRev.History, knownRevs, maxHistory) + } else { + if docRev.hlvHistory != "" { + history = append(history, docRev.hlvHistory) + } + } + if legacyRev { + // append current revID and rest of rev tree after hlv history + revTreeHistory := toHistory(docRev.History, knownRevs, maxHistory) + history = append(history, docRev.RevID) + history = append(history, revTreeHistory...) + } - history := toHistory(rev.History, knownRevs, maxHistory) - properties := blipRevMessageProperties(history, rev.Deleted, seq, replacedRevID) + properties := blipRevMessageProperties(history, docRev.Deleted, seq, replacedRevID) if base.LogDebugEnabled(ctx, base.KeySync) { replacedRevMsg := "" if replacedRevID != "" { @@ -773,3 +836,7 @@ func (bsc *BlipSyncContext) reportStats(updateImmediately bool) { bsc.stats.lastReportTime.Store(currentTime) } + +func (bsc *BlipSyncContext) useHLV() bool { + return bsc.activeCBMobileSubprotocol >= CBMobileReplicationV4 +} diff --git a/db/change_cache.go b/db/change_cache.go index bbeecdcbee..d3265a9cbc 100644 --- a/db/change_cache.go +++ b/db/change_cache.go @@ -129,6 +129,14 @@ func (entry *LogEntry) IsUnusedRange() bool { return entry.DocID == "" && entry.EndSequence > 0 } +func (entry *LogEntry) SetRevAndVersion(rv channels.RevAndVersion) { + entry.RevID = rv.RevTreeID + if rv.CurrentSource != "" { + entry.SourceID = rv.CurrentSource + entry.Version = base.HexCasToUint64(rv.CurrentVersion) + } +} + type LogEntries []*LogEntry // A priority-queue of LogEntries, kept ordered by increasing sequence #. @@ -485,9 +493,11 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { // if the doc was removed from one or more channels at this sequence // Set the removed flag and removed channel set on the LogEntry - if channelRemovals, atRevId := syncData.Channels.ChannelsRemovedAtSequence(seq); len(channelRemovals) > 0 { + if channelRemovals, atRev := syncData.Channels.ChannelsRemovedAtSequence(seq); len(channelRemovals) > 0 { change.DocID = docID - change.RevID = atRevId + change.RevID = atRev.RevTreeID + change.SourceID = atRev.CurrentSource + change.Version = base.HexCasToUint64(atRev.CurrentVersion) change.Channels = channelRemovals } @@ -499,8 +509,9 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { // Now add the entry for the new doc revision: if len(rawUserXattr) > 0 { - collection.revisionCache.Remove(docID, syncData.CurrentRev) + collection.revisionCache.RemoveWithRev(docID, syncData.CurrentRev) } + change := &LogEntry{ Sequence: syncData.Sequence, DocID: docID, @@ -511,6 +522,10 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { Channels: syncData.Channels, CollectionID: event.CollectionID, } + if syncData.HLV != nil { + change.SourceID = syncData.HLV.SourceID + change.Version = syncData.HLV.Version + } millisecondLatency := int(feedLatency / time.Millisecond) diff --git a/db/change_cache_test.go b/db/change_cache_test.go index 0cd1ad5dd1..d2bdbb22cd 100644 --- a/db/change_cache_test.go +++ b/db/change_cache_test.go @@ -74,6 +74,24 @@ func logEntry(seq uint64, docid string, revid string, channelNames []string, col return entry } +func testLogEntryWithCV(seq uint64, docid string, revid string, channelNames []string, collectionID uint32, sourceID string, version uint64) *LogEntry { + entry := &LogEntry{ + Sequence: seq, + DocID: docid, + RevID: revid, + TimeReceived: time.Now(), + CollectionID: collectionID, + SourceID: sourceID, + Version: version, + } + channelMap := make(channels.ChannelMap) + for _, channelName := range channelNames { + channelMap[channelName] = nil + } + entry.Channels = channelMap + return entry +} + func TestLateSequenceHandling(t *testing.T) { context, ctx := setupTestDBWithCacheOptions(t, DefaultCacheOptions()) @@ -1439,7 +1457,7 @@ func TestLateArrivingSequenceTriggersOnChange(t *testing.T) { } var doc1DCPBytes []byte if base.TestUseXattrs() { - body, syncXattr, _, err := doc1.MarshalWithXattrs() + body, syncXattr, _, _, _, err := doc1.MarshalWithXattrs() require.NoError(t, err) doc1DCPBytes = sgbucket.EncodeValueWithXattrs(body, sgbucket.Xattr{Name: base.SyncXattrName, Value: syncXattr}) } else { @@ -1464,7 +1482,7 @@ func TestLateArrivingSequenceTriggersOnChange(t *testing.T) { var dataType sgbucket.FeedDataType = base.MemcachedDataTypeJSON if base.TestUseXattrs() { dataType |= base.MemcachedDataTypeXattr - body, syncXattr, _, err := doc2.MarshalWithXattrs() + body, syncXattr, _, _, _, err := doc2.MarshalWithXattrs() require.NoError(t, err) doc2DCPBytes = sgbucket.EncodeValueWithXattrs(body, sgbucket.Xattr{Name: base.SyncXattrName, Value: syncXattr}) } else { diff --git a/db/changes.go b/db/changes.go index 3cd7ab397e..e8598d0983 100644 --- a/db/changes.go +++ b/db/changes.go @@ -44,19 +44,20 @@ type ChangesOptions struct { // A changes entry; Database.GetChanges returns an array of these. // Marshals into the standard CouchDB _changes format. type ChangeEntry struct { - Seq SequenceID `json:"seq"` - ID string `json:"id"` - Deleted bool `json:"deleted,omitempty"` - Removed base.Set `json:"removed,omitempty"` - Doc json.RawMessage `json:"doc,omitempty"` - Changes []ChangeRev `json:"changes"` - Err error `json:"err,omitempty"` // Used to notify feed consumer of errors - allRemoved bool // Flag to track whether an entry is a removal in all channels visible to the user. - branched bool - backfill backfillFlag // Flag used to identify non-client entries used for backfill synchronization (di only) - principalDoc bool // Used to indicate _user/_role docs - Revoked bool `json:"revoked,omitempty"` - collectionID uint32 + Seq SequenceID `json:"seq"` + ID string `json:"id"` + Deleted bool `json:"deleted,omitempty"` + Removed base.Set `json:"removed,omitempty"` + Doc json.RawMessage `json:"doc,omitempty"` + Changes []ChangeRev `json:"changes"` + Err error `json:"err,omitempty"` // Used to notify feed consumer of errors + allRemoved bool // Flag to track whether an entry is a removal in all channels visible to the user. + branched bool + backfill backfillFlag // Flag used to identify non-client entries used for backfill synchronization (di only) + principalDoc bool // Used to indicate _user/_role docs + Revoked bool `json:"revoked,omitempty"` + collectionID uint32 + CurrentVersion *Version `json:"-"` // the current version of the change entry. (Not marshalled, pending REST support for cv) } const ( @@ -503,6 +504,12 @@ func makeChangeEntry(logEntry *LogEntry, seqID SequenceID, channel channels.ID) principalDoc: logEntry.IsPrincipal, collectionID: logEntry.CollectionID, } + // populate CurrentVersion entry if log entry has sourceID and Version populated + // This allows current version to be nil in event of CV not being populated on log entry + // allowing omitempty to work as expected + if logEntry.SourceID != "" { + change.CurrentVersion = &Version{SourceID: logEntry.SourceID, Value: logEntry.Version} + } if logEntry.Flags&channels.Removed != 0 { change.Removed = base.SetOf(channel.Name) } @@ -1308,6 +1315,12 @@ func createChangesEntry(ctx context.Context, docid string, db *DatabaseCollectio row.Seq = SequenceID{Seq: populatedDoc.Sequence} row.SetBranched((populatedDoc.Flags & channels.Branched) != 0) + if populatedDoc.HLV != nil { + cv := Version{} + cv.SourceID, cv.Value = populatedDoc.HLV.GetCurrentVersion() + row.CurrentVersion = &cv + } + var removedChannels []string userCanSeeDocChannel := false diff --git a/db/changes_test.go b/db/changes_test.go index 4396e251b5..0447893c0f 100644 --- a/db/changes_test.go +++ b/db/changes_test.go @@ -253,7 +253,7 @@ func TestDocDeletionFromChannelCoalescedRemoved(t *testing.T) { sync["recent_sequences"] = []uint64{1, 2, 3} cm := make(channels.ChannelMap) - cm["A"] = &channels.ChannelRemoval{Seq: 2, RevID: "2-e99405a23fa102238fa8c3fd499b15bc"} + cm["A"] = &channels.ChannelRemoval{Seq: 2, Rev: channels.RevAndVersion{RevTreeID: "2-e99405a23fa102238fa8c3fd499b15bc"}} sync["channels"] = cm history := sync["history"].(map[string]interface{}) @@ -285,6 +285,39 @@ func TestDocDeletionFromChannelCoalescedRemoved(t *testing.T) { printChanges(changes) } +func TestCVPopulationOnChangeEntry(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collectionID := collection.GetCollectionID() + bucketUUID := db.EncodedSourceID + + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) + + authenticator := db.Authenticator(base.TestCtx(t)) + user, err := authenticator.NewUser("alice", "letmein", channels.BaseSetOf(t, "A")) + require.NoError(t, err) + require.NoError(t, authenticator.Save(user)) + + collection.user, _ = authenticator.GetUser("alice") + + // Make channel active + _, err = db.channelCache.GetChanges(ctx, channels.NewID("A", collectionID), getChangesOptionsWithZeroSeq(t)) + require.NoError(t, err) + + _, doc, err := collection.Put(ctx, "doc1", Body{"channels": []string{"A"}}) + require.NoError(t, err) + + require.NoError(t, collection.WaitForPendingChanges(base.TestCtx(t))) + + changes := getChanges(t, collection, base.SetOf("A"), getChangesOptionsWithZeroSeq(t)) + require.NoError(t, err) + + assert.Equal(t, doc.ID, changes[0].ID) + assert.Equal(t, bucketUUID, changes[0].CurrentVersion.SourceID) + assert.Equal(t, doc.Cas, changes[0].CurrentVersion.Value) +} + func TestDocDeletionFromChannelCoalesced(t *testing.T) { if base.TestUseXattrs() { t.Skip("This test is known to be failing against couchbase server with XATTRS enabled. Same error as TestDocDeletionFromChannelCoalescedRemoved") @@ -385,7 +418,7 @@ func TestActiveOnlyCacheUpdate(t *testing.T) { // Tombstone 5 documents for i := 2; i <= 6; i++ { key := fmt.Sprintf("%s_%d", t.Name(), i) - _, err = collection.DeleteDoc(ctx, key, revId) + _, _, err = collection.DeleteDoc(ctx, key, revId) require.NoError(t, err, "Couldn't delete document") } @@ -468,14 +501,14 @@ func BenchmarkChangesFeedDocUnmarshalling(b *testing.B) { // Create child rev 1 docBody["child"] = "A" - _, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-A", revId}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-A", revId}, false, ExistingVersionWithUpdateToHLV) if err != nil { b.Fatalf("Error creating child1 rev: %v", err) } // Create child rev 2 docBody["child"] = "B" - _, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-B", revId}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-B", revId}, false, ExistingVersionWithUpdateToHLV) if err != nil { b.Fatalf("Error creating child2 rev: %v", err) } @@ -533,5 +566,45 @@ func TestChangesOptionsStringer(t *testing.T) { expectedFields = append(expectedFields, field.Name) } require.ElementsMatch(t, expectedFields, stringerFields) +} + +// TestCurrentVersionPopulationOnChannelCache: +// - Make channel active on cache +// - Add a doc that is assigned this channel +// - Get the sync data of that doc to assert against the HLV defined on it +// - Wait for the channel cache to be populated with this doc write +// - Assert the CV in the entry fetched from channel cache matches the sync data CV and the bucket UUID on the database context +func TestCurrentVersionPopulationOnChannelCache(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyImport, base.KeyDCP, base.KeyCache, base.KeyHTTP) + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collectionID := collection.GetCollectionID() + bucketUUID := db.EncodedSourceID + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) + // Make channel active + _, err := db.channelCache.GetChanges(ctx, channels.NewID("ABC", collectionID), getChangesOptionsWithZeroSeq(t)) + require.NoError(t, err) + + // Put a doc that gets assigned a CV to populate the channel cache with + _, _, err = collection.Put(ctx, "doc1", Body{"channels": []string{"ABC"}}) + require.NoError(t, err) + err = collection.WaitForPendingChanges(base.TestCtx(t)) + require.NoError(t, err) + + syncData, err := collection.GetDocSyncData(ctx, "doc1") + require.NoError(t, err) + + // get entry of above doc from channel cache + entries, err := db.channelCache.GetChanges(ctx, channels.NewID("ABC", collectionID), getChangesOptionsWithZeroSeq(t)) + require.NoError(t, err) + require.NotNil(t, entries) + + // assert that the source and version has been populated with the channel cache entry for the doc + assert.Equal(t, "doc1", entries[0].DocID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), entries[0].Version) + assert.Equal(t, bucketUUID, entries[0].SourceID) + assert.Equal(t, syncData.HLV.SourceID, entries[0].SourceID) + assert.Equal(t, syncData.HLV.Version, entries[0].Version) } diff --git a/db/changes_view.go b/db/changes_view.go index 2bbe671bfc..4d4e36f03f 100644 --- a/db/changes_view.go +++ b/db/changes_view.go @@ -18,6 +18,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/channels" ) // One "changes" row in a channelsViewResult @@ -25,7 +26,7 @@ type channelsViewRow struct { ID string Key []interface{} // Actually [channelName, sequence] Value struct { - Rev string + Rev channels.RevAndVersion Flags uint8 } } @@ -42,13 +43,12 @@ func nextChannelViewEntry(ctx context.Context, results sgbucket.QueryResultItera entry := &LogEntry{ Sequence: uint64(viewRow.Key[1].(float64)), DocID: viewRow.ID, - RevID: viewRow.Value.Rev, Flags: viewRow.Value.Flags, TimeReceived: time.Now(), CollectionID: collectionID, } + entry.SetRevAndVersion(viewRow.Value.Rev) return entry, true - } func nextChannelQueryEntry(ctx context.Context, results sgbucket.QueryResultIterator, collectionID uint32) (*LogEntry, bool) { @@ -61,14 +61,16 @@ func nextChannelQueryEntry(ctx context.Context, results sgbucket.QueryResultIter entry := &LogEntry{ Sequence: queryRow.Sequence, DocID: queryRow.Id, - RevID: queryRow.Rev, Flags: queryRow.Flags, TimeReceived: time.Now(), CollectionID: collectionID, } + entry.SetRevAndVersion(queryRow.Rev) - if queryRow.RemovalRev != "" { - entry.RevID = queryRow.RemovalRev + if queryRow.RemovalRev != nil { + entry.RevID = queryRow.RemovalRev.RevTreeID + entry.Version = base.HexCasToUint64(queryRow.RemovalRev.CurrentVersion) + entry.SourceID = queryRow.RemovalRev.CurrentSource if queryRow.RemovalDel { entry.SetDeleted() } diff --git a/db/channel_cache_single_test.go b/db/channel_cache_single_test.go index 7b2c883b92..d0431ab4c9 100644 --- a/db/channel_cache_single_test.go +++ b/db/channel_cache_single_test.go @@ -951,6 +951,23 @@ func verifyChannelDocIDs(entries []*LogEntry, docIDs []string) bool { return true } +type cvValues struct { + source string + version uint64 +} + +func verifyCVEntries(entries []*LogEntry, cvs []cvValues) bool { + for index, cv := range cvs { + if entries[index].SourceID != cv.source { + return false + } + if entries[index].Version != cv.version { + return false + } + } + return true +} + func writeEntries(entries []*LogEntry) { for index, entry := range entries { log.Printf("%d:seq=%d, docID=%s, revID=%s", index, entry.Sequence, entry.DocID, entry.RevID) diff --git a/db/channel_cache_test.go b/db/channel_cache_test.go index 4637539848..19b7b9329a 100644 --- a/db/channel_cache_test.go +++ b/db/channel_cache_test.go @@ -53,6 +53,58 @@ func TestChannelCacheMaxSize(t *testing.T) { assert.Equal(t, 4, int(maxEntries)) } +// TestChannelCacheCurrentVersion: +// - Makes channel channels active for channels used in test by requesting changes on each channel +// - Add 4 docs to the channel cache with CV defined in the log entry +// - Get changes for each channel in question and assert that the CV is populated in each entry expected +func TestChannelCacheCurrentVersion(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + cache := db.changeCache.getChannelCache() + + collectionID := GetSingleDatabaseCollection(t, db.DatabaseContext).GetCollectionID() + + // Make channels active + _, err := cache.GetChanges(ctx, channels.NewID("chanA", collectionID), getChangesOptionsWithCtxOnly(t)) + require.NoError(t, err) + _, err = cache.GetChanges(ctx, channels.NewID("chanB", collectionID), getChangesOptionsWithCtxOnly(t)) + require.NoError(t, err) + _, err = cache.GetChanges(ctx, channels.NewID("chanC", collectionID), getChangesOptionsWithCtxOnly(t)) + require.NoError(t, err) + _, err = cache.GetChanges(ctx, channels.NewID("chanD", collectionID), getChangesOptionsWithCtxOnly(t)) + require.NoError(t, err) + + cache.AddToCache(ctx, testLogEntryWithCV(1, "doc1", "1-a", []string{"chanB", "chanC", "chanD"}, collectionID, "test1", 123)) + cache.AddToCache(ctx, testLogEntryWithCV(2, "doc2", "1-a", []string{"chanB", "chanC", "chanD"}, collectionID, "test2", 1234)) + cache.AddToCache(ctx, testLogEntryWithCV(3, "doc3", "1-a", []string{"chanC", "chanD"}, collectionID, "test3", 12345)) + cache.AddToCache(ctx, testLogEntryWithCV(4, "doc4", "1-a", []string{"chanC"}, collectionID, "test4", 123456)) + + // assert on channel cache entries for 'chanC' + entriesChanC, err := cache.GetChanges(ctx, channels.NewID("chanC", collectionID), getChangesOptionsWithZeroSeq(t)) + assert.NoError(t, err) + require.Len(t, entriesChanC, 4) + assert.True(t, verifyChannelSequences(entriesChanC, []uint64{1, 2, 3, 4})) + assert.True(t, verifyChannelDocIDs(entriesChanC, []string{"doc1", "doc2", "doc3", "doc4"})) + assert.True(t, verifyCVEntries(entriesChanC, []cvValues{{source: "test1", version: 123}, {source: "test2", version: 1234}, {source: "test3", version: 12345}, {source: "test4", version: 123456}})) + + // assert on channel cache entries for 'chanD' + entriesChanD, err := cache.GetChanges(ctx, channels.NewID("chanD", collectionID), getChangesOptionsWithZeroSeq(t)) + assert.NoError(t, err) + require.Len(t, entriesChanD, 3) + assert.True(t, verifyChannelSequences(entriesChanD, []uint64{1, 2, 3})) + assert.True(t, verifyChannelDocIDs(entriesChanD, []string{"doc1", "doc2", "doc3"})) + assert.True(t, verifyCVEntries(entriesChanD, []cvValues{{source: "test1", version: 123}, {source: "test2", version: 1234}, {source: "test3", version: 12345}})) + + // assert on channel cache entries for 'chanB' + entriesChanB, err := cache.GetChanges(ctx, channels.NewID("chanB", collectionID), getChangesOptionsWithZeroSeq(t)) + assert.NoError(t, err) + require.Len(t, entriesChanB, 2) + assert.True(t, verifyChannelSequences(entriesChanB, []uint64{1, 2})) + assert.True(t, verifyChannelDocIDs(entriesChanB, []string{"doc1", "doc2"})) + assert.True(t, verifyCVEntries(entriesChanB, []cvValues{{source: "test1", version: 123}, {source: "test2", version: 1234}})) +} + func getCacheUtilization(stats *base.CacheStats) (active, tombstones, removals int) { active = int(stats.ChannelCacheRevsActive.Value()) tombstones = int(stats.ChannelCacheRevsTombstone.Value()) diff --git a/db/crud.go b/db/crud.go index fc34b736c8..98dcee3752 100644 --- a/db/crud.go +++ b/db/crud.go @@ -60,7 +60,7 @@ func (c *DatabaseCollection) GetDocumentWithRaw(ctx context.Context, docid strin return nil, nil, base.HTTPErrorf(400, "Invalid doc ID") } if c.UseXattrs() { - doc, rawBucketDoc, err = c.GetDocWithXattr(ctx, key, unmarshalLevel) + doc, rawBucketDoc, err = c.GetDocWithXattrs(ctx, key, unmarshalLevel) if err != nil { return nil, nil, err } @@ -114,10 +114,10 @@ func (c *DatabaseCollection) GetDocumentWithRaw(ctx context.Context, docid strin return doc, rawBucketDoc, nil } -func (c *DatabaseCollection) GetDocWithXattr(ctx context.Context, key string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, rawBucketDoc *sgbucket.BucketDocument, err error) { +func (c *DatabaseCollection) GetDocWithXattrs(ctx context.Context, key string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, rawBucketDoc *sgbucket.BucketDocument, err error) { rawBucketDoc = &sgbucket.BucketDocument{} var getErr error - rawBucketDoc.Body, rawBucketDoc.Xattrs, rawBucketDoc.Cas, getErr = c.dataStore.GetWithXattrs(ctx, key, c.syncAndUserXattrKeys()) + rawBucketDoc.Body, rawBucketDoc.Xattrs, rawBucketDoc.Cas, getErr = c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncAndUserXattrKeys()) if getErr != nil { return nil, nil, getErr } @@ -143,7 +143,7 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( if c.UseXattrs() { // Retrieve doc and xattr from bucket, unmarshal only xattr. // Triggers on-demand import when document xattr doesn't match cas. - rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncAndUserXattrKeys()) + rawDoc, xattrs, cas, getErr := c.dataStore.GetWithXattrs(ctx, key, c.syncGlobalSyncAndUserXattrKeys()) if getErr != nil { return emptySyncData, getErr } @@ -190,6 +190,12 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( } +// unmarshalDocumentWithXattrs populates individual xattrs on unmarshalDocumentWithXattrs from a provided xattrs map +func (db *DatabaseCollection) unmarshalDocumentWithXattrs(ctx context.Context, docid string, data []byte, xattrs map[string][]byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { + return unmarshalDocumentWithXattrs(ctx, docid, data, xattrs[base.SyncXattrName], xattrs[base.VvXattrName], xattrs[base.MouXattrName], xattrs[db.userXattrKey()], xattrs[base.VirtualXattrRevSeqNo], xattrs[base.GlobalXattrName], cas, unmarshalLevel) + +} + // This gets *just* the Sync Metadata (_sync field) rather than the entire doc, for efficiency // reasons. Unlike GetDocSyncData it does not check for on-demand import; this means it does not // need to read the doc body from the bucket. @@ -197,7 +203,7 @@ func (db *DatabaseCollection) GetDocSyncDataNoImport(ctx context.Context, docid if db.UseXattrs() { var xattrs map[string][]byte var cas uint64 - xattrs, cas, err = db.dataStore.GetXattrs(ctx, docid, []string{base.SyncXattrName}) + xattrs, cas, err = db.dataStore.GetXattrs(ctx, docid, []string{base.SyncXattrName, base.VvXattrName}) if err == nil { var doc *Document doc, err = db.unmarshalDocumentWithXattrs(ctx, docid, nil, xattrs, cas, level) @@ -232,14 +238,22 @@ func (db *DatabaseCollection) GetDocSyncDataNoImport(ctx context.Context, docid return } -// OnDemandImportForGet. Attempts to import the doc based on the provided id, contents and cas. ImportDocRaw does cas retry handling +// OnDemandImportForGet. Attempts to import the doc based on the provided id, contents and cas. ImportDocRaw does cas retry handling // if the document gets updated after the initial retrieval attempt that triggered this. func (c *DatabaseCollection) OnDemandImportForGet(ctx context.Context, docid string, rawDoc []byte, xattrs map[string][]byte, cas uint64) (docOut *Document, err error) { isDelete := rawDoc == nil importDb := DatabaseCollectionWithUser{DatabaseCollection: c, user: nil} var importErr error - docOut, importErr = importDb.ImportDocRaw(ctx, docid, rawDoc, xattrs, isDelete, cas, nil, ImportOnDemand) + importOpts := importDocOptions{ + isDelete: isDelete, + mode: ImportOnDemand, + revSeqNo: 0, // pending work in CBG-4203 + expiry: nil, + } + + // RevSeqNo is 0 here pending work in CBG-4203 + docOut, importErr = importDb.ImportDocRaw(ctx, docid, rawDoc, xattrs, importOpts, cas) if importErr == base.ErrImportCancelledFilter { // If the import was cancelled due to filter, treat as 404 not imported @@ -273,12 +287,25 @@ func (db *DatabaseCollectionWithUser) Get1xRevBody(ctx context.Context, docid, r maxHistory = math.MaxInt32 } - return db.Get1xRevBodyWithHistory(ctx, docid, revid, maxHistory, nil, attachmentsSince, false) + return db.Get1xRevBodyWithHistory(ctx, docid, revid, Get1xRevBodyOptions{ + MaxHistory: maxHistory, + HistoryFrom: nil, + AttachmentsSince: attachmentsSince, + ShowExp: false, + }) +} + +type Get1xRevBodyOptions struct { + MaxHistory int + HistoryFrom []string + AttachmentsSince []string + ShowExp bool + ShowCV bool } // Retrieves rev with request history specified as collection of revids (historyFrom) -func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revid string, maxHistory int, historyFrom []string, attachmentsSince []string, showExp bool) (Body, error) { - rev, err := db.getRev(ctx, docid, revid, maxHistory, historyFrom) +func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revtreeid string, opts Get1xRevBodyOptions) (Body, error) { + rev, err := db.getRev(ctx, docid, revtreeid, opts.MaxHistory, opts.HistoryFrom) if err != nil { return nil, err } @@ -286,14 +313,14 @@ func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Contex // RequestedHistory is the _revisions returned in the body. Avoids mutating revision.History, in case it's needed // during attachment processing below requestedHistory := rev.History - if maxHistory == 0 { + if opts.MaxHistory == 0 { requestedHistory = nil } if requestedHistory != nil { - _, requestedHistory = trimEncodedRevisionsToAncestor(ctx, requestedHistory, historyFrom, maxHistory) + _, requestedHistory = trimEncodedRevisionsToAncestor(ctx, requestedHistory, opts.HistoryFrom, opts.MaxHistory) } - return rev.Mutable1xBody(ctx, db, requestedHistory, attachmentsSince, showExp) + return rev.Mutable1xBody(ctx, db, requestedHistory, opts.AttachmentsSince, opts.ShowExp, opts.ShowCV) } // Underlying revision retrieval used by Get1xRevBody, Get1xRevBodyWithHistory, GetRevCopy. @@ -309,19 +336,34 @@ func (db *DatabaseCollectionWithUser) getRev(ctx context.Context, docid, revid s if revid != "" { // Get a specific revision body and history from the revision cache // (which will load them if necessary, by calling revCacheLoader, above) - revision, err = db.revisionCache.Get(ctx, docid, revid, RevCacheOmitDelta) + revision, err = db.revisionCache.GetWithRev(ctx, docid, revid, RevCacheOmitDelta) } else { // No rev ID given, so load active revision revision, err = db.revisionCache.GetActive(ctx, docid) } - if err != nil { return DocumentRevision{}, err } + return db.documentRevisionForRequest(ctx, docid, revision, &revid, nil, maxHistory, historyFrom) +} + +// documentRevisionForRequest processes the given DocumentRevision and returns a version of it for a given client request, depending on access, deleted, etc. +func (db *DatabaseCollectionWithUser) documentRevisionForRequest(ctx context.Context, docID string, revision DocumentRevision, revID *string, cv *Version, maxHistory int, historyFrom []string) (DocumentRevision, error) { + // ensure only one of cv or revID is specified + if cv != nil && revID != nil { + return DocumentRevision{}, fmt.Errorf("must have one of cv or revID in documentRevisionForRequest (had cv=%v revID=%v)", cv, revID) + } + var requestedVersion string + if revID != nil { + requestedVersion = *revID + } else if cv != nil { + requestedVersion = cv.String() + } + if revision.BodyBytes == nil { if db.ForceAPIForbiddenErrors() { - base.InfofCtx(ctx, base.KeyCRUD, "Doc: %s %s is missing", base.UD(docid), base.MD(revid)) + base.InfofCtx(ctx, base.KeyCRUD, "Doc: %s %s is missing", base.UD(docID), base.MD(requestedVersion)) return DocumentRevision{}, ErrForbidden } return DocumentRevision{}, ErrMissing @@ -340,16 +382,17 @@ func (db *DatabaseCollectionWithUser) getRev(ctx context.Context, docid, revid s _, requestedHistory = trimEncodedRevisionsToAncestor(ctx, requestedHistory, historyFrom, maxHistory) } - isAuthorized, redactedRev := db.authorizeUserForChannels(docid, revision.RevID, revision.Channels, revision.Deleted, requestedHistory) + isAuthorized, redactedRevision := db.authorizeUserForChannels(docID, revision.RevID, cv, revision.Channels, revision.Deleted, requestedHistory) if !isAuthorized { - if revid == "" { + // client just wanted active revision, not a specific one + if requestedVersion == "" { return DocumentRevision{}, ErrForbidden } if db.ForceAPIForbiddenErrors() { - base.InfofCtx(ctx, base.KeyCRUD, "Not authorized to view doc: %s %s", base.UD(docid), base.MD(revid)) + base.InfofCtx(ctx, base.KeyCRUD, "Not authorized to view doc: %s %s", base.UD(docID), base.MD(requestedVersion)) return DocumentRevision{}, ErrForbidden } - return redactedRev, nil + return redactedRevision, nil } // If the revision is a removal cache entry (no body), but the user has access to that removal, then just @@ -358,22 +401,50 @@ func (db *DatabaseCollectionWithUser) getRev(ctx context.Context, docid, revid s return DocumentRevision{}, ErrMissing } - if revision.Deleted && revid == "" { + if revision.Deleted && requestedVersion == "" { return DocumentRevision{}, ErrDeleted } return revision, nil } +func (db *DatabaseCollectionWithUser) GetCV(ctx context.Context, docid string, cv *Version) (revision DocumentRevision, err error) { + if cv != nil { + revision, err = db.revisionCache.GetWithCV(ctx, docid, cv, RevCacheOmitDelta) + } else { + revision, err = db.revisionCache.GetActive(ctx, docid) + } + if err != nil { + return DocumentRevision{}, err + } + + return db.documentRevisionForRequest(ctx, docid, revision, nil, cv, 0, nil) +} + // GetDelta attempts to return the delta between fromRevId and toRevId. If the delta can't be generated, // returns nil. -func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromRevID, toRevID string) (delta *RevisionDelta, redactedRev *DocumentRevision, err error) { +func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromRev, toRev string, useCVRevCache bool) (delta *RevisionDelta, redactedRev *DocumentRevision, err error) { - if docID == "" || fromRevID == "" || toRevID == "" { + if docID == "" || fromRev == "" || toRev == "" { return nil, nil, nil } - - fromRevision, err := db.revisionCache.Get(ctx, docID, fromRevID, RevCacheIncludeDelta) + var fromRevision DocumentRevision + var fromRevVrs Version + if useCVRevCache { + fromRevVrs, err = ParseVersion(fromRev) + if err != nil { + return nil, nil, err + } + fromRevision, err = db.revisionCache.GetWithCV(ctx, docID, &fromRevVrs, RevCacheIncludeDelta) + if err != nil { + return nil, nil, err + } + } else { + fromRevision, err = db.revisionCache.GetWithRev(ctx, docID, fromRev, RevCacheIncludeDelta) + if err != nil { + return nil, nil, err + } + } // If the fromRevision is a removal cache entry (no body), but the user has access to that removal, then just // return 404 missing to indicate that the body of the revision is no longer available. @@ -394,9 +465,9 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR // If delta is found, check whether it is a delta for the toRevID we want if fromRevision.Delta != nil { - if fromRevision.Delta.ToRevID == toRevID { + if fromRevision.Delta.ToCV == toRev || fromRevision.Delta.ToRevID == toRev { - isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRevID, fromRevision.Delta.ToChannels, fromRevision.Delta.ToDeleted, encodeRevisions(ctx, docID, fromRevision.Delta.RevisionHistory)) + isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRev, fromRevision.CV, fromRevision.Delta.ToChannels, fromRevision.Delta.ToDeleted, encodeRevisions(ctx, docID, fromRevision.Delta.RevisionHistory)) if !isAuthorized { return nil, &redactedBody, nil } @@ -411,15 +482,26 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR // Delta is unavailable, but the body is available. if fromRevision.BodyBytes != nil { - // db.DbStats.StatsDeltaSync().Add(base.StatKeyDeltaCacheMisses, 1) db.dbStats().DeltaSync().DeltaCacheMiss.Add(1) - toRevision, err := db.revisionCache.Get(ctx, docID, toRevID, RevCacheIncludeDelta) - if err != nil { - return nil, nil, err + var toRevision DocumentRevision + if useCVRevCache { + cv, err := ParseVersion(toRev) + if err != nil { + return nil, nil, err + } + toRevision, err = db.revisionCache.GetWithCV(ctx, docID, &cv, RevCacheIncludeDelta) + if err != nil { + return nil, nil, err + } + } else { + toRevision, err = db.revisionCache.GetWithRev(ctx, docID, toRev, RevCacheIncludeDelta) + if err != nil { + return nil, nil, err + } } deleted := toRevision.Deleted - isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRevID, toRevision.Channels, deleted, toRevision.History) + isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRev, toRevision.CV, toRevision.Channels, deleted, toRevision.History) if !isAuthorized { return nil, &redactedBody, nil } @@ -430,8 +512,12 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR // If the revision we're generating a delta to is a tombstone, mark it as such and don't bother generating a delta if deleted { - revCacheDelta := newRevCacheDelta([]byte(base.EmptyDocument), fromRevID, toRevision, deleted, nil) - db.revisionCache.UpdateDelta(ctx, docID, fromRevID, revCacheDelta) + revCacheDelta := newRevCacheDelta([]byte(base.EmptyDocument), fromRev, toRevision, deleted, nil) + if useCVRevCache { + db.revisionCache.UpdateDeltaCV(ctx, docID, &fromRevVrs, revCacheDelta) + } else { + db.revisionCache.UpdateDelta(ctx, docID, fromRev, revCacheDelta) + } return &revCacheDelta, nil, nil } @@ -468,17 +554,21 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR if err != nil { return nil, nil, err } - revCacheDelta := newRevCacheDelta(deltaBytes, fromRevID, toRevision, deleted, toRevAttStorageMeta) + revCacheDelta := newRevCacheDelta(deltaBytes, fromRev, toRevision, deleted, toRevAttStorageMeta) // Write the newly calculated delta back into the cache before returning - db.revisionCache.UpdateDelta(ctx, docID, fromRevID, revCacheDelta) + if useCVRevCache { + db.revisionCache.UpdateDeltaCV(ctx, docID, &fromRevVrs, revCacheDelta) + } else { + db.revisionCache.UpdateDelta(ctx, docID, fromRev, revCacheDelta) + } return &revCacheDelta, nil, nil } return nil, nil, nil } -func (col *DatabaseCollectionWithUser) authorizeUserForChannels(docID, revID string, channels base.Set, isDeleted bool, history Revisions) (isAuthorized bool, redactedRev DocumentRevision) { +func (col *DatabaseCollectionWithUser) authorizeUserForChannels(docID, revID string, cv *Version, channels base.Set, isDeleted bool, history Revisions) (isAuthorized bool, redactedRev DocumentRevision) { if col.user != nil { if err := col.user.AuthorizeAnyCollectionChannel(col.ScopeName, col.Name, channels); err != nil { @@ -490,6 +580,7 @@ func (col *DatabaseCollectionWithUser) authorizeUserForChannels(docID, revID str RevID: revID, History: history, Deleted: isDeleted, + CV: cv, } if isDeleted { // Deletions are denoted by the deleted message property during 2.x replication @@ -788,19 +879,13 @@ func (db *DatabaseCollectionWithUser) getAvailableRevAttachments(ctx context.Con // Moves a revision's ancestor's body out of the document object and into a separate db doc. func (db *DatabaseCollectionWithUser) backupAncestorRevs(ctx context.Context, doc *Document, newDoc *Document) { - newBodyBytes, err := newDoc.BodyBytes(ctx) - if err != nil { - base.WarnfCtx(ctx, "Error getting body bytes when backing up ancestor revs") - return - } // Find an ancestor that still has JSON in the document: var json []byte ancestorRevId := newDoc.RevID for { if ancestorRevId = doc.History.getParent(ancestorRevId); ancestorRevId == "" { - // No ancestors with JSON found. Check if we need to back up current rev for delta sync, then return - db.backupRevisionJSON(ctx, doc.ID, newDoc.RevID, "", newBodyBytes, nil, doc.Attachments) + // No ancestors with JSON found. Return early return } else if json = doc.getRevisionBodyJSON(ctx, ancestorRevId, db.RevisionBodyLoader); json != nil { break @@ -808,7 +893,7 @@ func (db *DatabaseCollectionWithUser) backupAncestorRevs(ctx context.Context, do } // Back up the revision JSON as a separate doc in the bucket: - db.backupRevisionJSON(ctx, doc.ID, newDoc.RevID, ancestorRevId, newBodyBytes, json, doc.Attachments) + db.backupRevisionJSON(ctx, doc.ID, doc.HLV.GetCurrentVersionString(), json) // Nil out the ancestor rev's body in the document struct: if ancestorRevId == doc.CurrentRev { @@ -832,7 +917,13 @@ func (db *DatabaseCollectionWithUser) OnDemandImportForWrite(ctx context.Context // Use an admin-scoped database for import importDb := DatabaseCollectionWithUser{DatabaseCollection: db.DatabaseCollection, user: nil} - importedDoc, importErr := importDb.ImportDoc(ctx, docid, doc, isDelete, nil, ImportOnDemand) // nolint:staticcheck + importOpts := importDocOptions{ + expiry: nil, + mode: ImportOnDemand, + isDelete: isDelete, + revSeqNo: 0, // pending work in CBG-4203 + } + importedDoc, importErr := importDb.ImportDoc(ctx, docid, doc, importOpts) // nolint:staticcheck if importErr == base.ErrImportCancelledFilter { // Document exists, but existing doc wasn't imported based on import filter. Treat write as insert @@ -845,6 +936,96 @@ func (db *DatabaseCollectionWithUser) OnDemandImportForWrite(ctx context.Context return nil } +// updateHLV updates the HLV in the sync data appropriately based on what type of document update event we are encountering. mouMatch represents if the _mou.cas == doc.cas +func (db *DatabaseCollectionWithUser) updateHLV(ctx context.Context, d *Document, docUpdateEvent DocUpdateType, mouMatch bool) (*Document, error) { + + hasHLV := d.HLV != nil + if d.HLV == nil { + d.HLV = &HybridLogicalVector{} + base.DebugfCtx(ctx, base.KeyVV, "No existing HLV for doc %s", base.UD(d.ID)) + } else { + base.DebugfCtx(ctx, base.KeyVV, "Existing HLV for doc %s before modification %+v", base.UD(d.ID), d.HLV) + } + switch docUpdateEvent { + case ExistingVersion: + // preserve any other logic on the HLV that has been done by the client, only update to cvCAS will be needed + d.HLV.CurrentVersionCAS = expandMacroCASValueUint64 + case Import: + // Do not update HLV if the current document version (cas) is already included in the existing HLV, as either: + // 1. _vv.cvCAS == document.cas (current mutation is already present as cv), or + // 2. _mou.cas == document.cas (current mutation is already present as cv, and was imported on a different cluster) + + cvCASMatch := hasHLV && d.HLV.CurrentVersionCAS == d.Cas + if !hasHLV || (!cvCASMatch && !mouMatch) { + // Otherwise this is an SDK mutation made by the local cluster that should be added to HLV. + newVVEntry := Version{} + newVVEntry.SourceID = db.dbCtx.EncodedSourceID + newVVEntry.Value = d.Cas + err := d.SyncData.HLV.AddVersion(newVVEntry) + if err != nil { + return nil, err + } + d.HLV.CurrentVersionCAS = d.Cas + base.DebugfCtx(ctx, base.KeyVV, "Adding new version to HLV due to import for doc %s, updated HLV %+v", base.UD(d.ID), d.HLV) + } else { + base.DebugfCtx(ctx, base.KeyVV, "Not updating HLV to _mou.cas == doc.cas for doc %s, extant HLV %+v", base.UD(d.ID), d.HLV) + } + case NewVersion, ExistingVersionWithUpdateToHLV: + // add a new entry to the version vector + newVVEntry := Version{} + newVVEntry.SourceID = db.dbCtx.EncodedSourceID + newVVEntry.Value = expandMacroCASValueUint64 + err := d.SyncData.HLV.AddVersion(newVVEntry) + if err != nil { + return nil, err + } + // update the cvCAS on the SGWrite event too + d.HLV.CurrentVersionCAS = expandMacroCASValueUint64 + } + return d, nil +} + +// MigrateAttachmentMetadata will move any attachment metadata defined in sync data to global sync xattr +func (c *DatabaseCollectionWithUser) MigrateAttachmentMetadata(ctx context.Context, docID string, cas uint64, syncData *SyncData) error { + xattrs, _, err := c.dataStore.GetXattrs(ctx, docID, []string{base.GlobalXattrName}) + if err != nil && !base.IsXattrNotFoundError(err) { + return err + } + var globalData GlobalSyncData + if xattrs[base.GlobalXattrName] != nil { + // we have a global xattr to preserve + err := base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalData) + if err != nil { + return base.RedactErrorf("Failed to Unmarshal global sync data when attempting to migrate sync data attachments to global xattr with id: %s. Error: %v", base.UD(docID), err) + } + // add the sync data attachment metadata to global xattr + for i, v := range syncData.Attachments { + globalData.GlobalAttachments[i] = v + } + } else { + globalData.GlobalAttachments = syncData.Attachments + } + globalXattr, err := base.JSONMarshal(globalData) + if err != nil { + return base.RedactErrorf("Failed to Marshal global sync data when attempting to migrate sync data attachments to global xattr with id: %s. Error: %v", base.UD(docID), err) + } + syncData.Attachments = nil + rawSyncXattr, err := base.JSONMarshal(*syncData) + if err != nil { + return base.RedactErrorf("Failed to Marshal sync data when attempting to migrate sync data attachments to global xattr with id: %s. Error: %v", base.UD(docID), err) + } + + // build macro expansion for sync data. This will avoid the update to xattrs causing an extra import event (i.e. sync cas will be == to doc cas) + opts := &sgbucket.MutateInOptions{} + spec := macroExpandSpec(base.SyncXattrName) + opts.MacroExpansion = spec + opts.PreserveExpiry = true // if doc has expiry, we should preserve this + + updatedXattr := map[string][]byte{base.SyncXattrName: rawSyncXattr, base.GlobalXattrName: globalXattr} + _, err = c.dataStore.UpdateXattrs(ctx, docID, 0, cas, updatedXattr, opts) + return err +} + // Updates or creates a document. // The new body's BodyRev property must match the current revision's, if any. func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, body Body) (newRevID string, doc *Document, err error) { @@ -884,8 +1065,11 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod return "", nil, err } + docUpdateEvent := NewVersion allowImport := db.UseXattrs() - doc, newRevID, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &expiry, nil, nil, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + + doc, newRevID, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &expiry, nil, docUpdateEvent, nil, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + var isSgWrite bool var crc32Match bool @@ -992,9 +1176,157 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod return newRevID, doc, err } +func (db *DatabaseCollectionWithUser) PutExistingCurrentVersion(ctx context.Context, newDoc *Document, newDocHLV *HybridLogicalVector, existingDoc *sgbucket.BucketDocument, revTreeHistory []string) (doc *Document, cv *Version, newRevID string, err error) { + var matchRev string + if existingDoc != nil { + doc, unmarshalErr := db.unmarshalDocumentWithXattrs(ctx, newDoc.ID, existingDoc.Body, existingDoc.Xattrs, existingDoc.Cas, DocUnmarshalRev) + if unmarshalErr != nil { + return nil, nil, "", base.HTTPErrorf(http.StatusBadRequest, "Error unmarshaling existing doc") + } + matchRev = doc.CurrentRev + } + generation, _ := ParseRevID(ctx, matchRev) + if generation < 0 { + return nil, nil, "", base.HTTPErrorf(http.StatusBadRequest, "Invalid revision ID") + } + generation++ //nolint + + docUpdateEvent := ExistingVersion + allowImport := db.UseXattrs() + doc, newRevID, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &newDoc.DocExpiry, nil, docUpdateEvent, existingDoc, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + // (Be careful: this block can be invoked multiple times if there are races!) + + var isSgWrite bool + var crc32Match bool + + // Is this doc an sgWrite? + if doc != nil { + isSgWrite, crc32Match, _ = doc.IsSGWrite(ctx, nil) + if crc32Match { + db.dbStats().Database().Crc32MatchCount.Add(1) + } + } + + // If the existing doc isn't an SG write, import prior to updating + if doc != nil && !isSgWrite && db.UseXattrs() { + err := db.OnDemandImportForWrite(ctx, newDoc.ID, doc, newDoc.Deleted) + if err != nil { + return nil, nil, false, nil, err + } + } + + // set up revTreeID for backward compatibility + var previousRevTreeID string + var prevGeneration int + var newGeneration int + if len(revTreeHistory) == 0 { + previousRevTreeID = doc.CurrentRev + prevGeneration, _ = ParseRevID(ctx, previousRevTreeID) + newGeneration = prevGeneration + 1 + } else { + previousRevTreeID = revTreeHistory[0] + prevGeneration, _ = ParseRevID(ctx, previousRevTreeID) + newGeneration = prevGeneration + 1 + } + + // Conflict check here + // if doc has no HLV defined this is a new doc we haven't seen before, skip conflict check + if doc.HLV == nil { + doc.HLV = NewHybridLogicalVector() + addNewerVersionsErr := doc.HLV.AddNewerVersions(newDocHLV) + if addNewerVersionsErr != nil { + return nil, nil, false, nil, addNewerVersionsErr + } + } else { + if doc.HLV.isDominating(newDocHLV) { + base.DebugfCtx(ctx, base.KeyCRUD, "PutExistingCurrentVersion(%q): No new versions to add", base.UD(newDoc.ID)) + return nil, nil, false, nil, base.ErrUpdateCancel // No new revisions to add + } + if newDocHLV.isDominating(doc.HLV) { + // update hlv for all newer incoming source version pairs + addNewerVersionsErr := doc.HLV.AddNewerVersions(newDocHLV) + if addNewerVersionsErr != nil { + return nil, nil, false, nil, addNewerVersionsErr + } + } else { + base.InfofCtx(ctx, base.KeyCRUD, "conflict detected between the two HLV's for doc %s, incoming version %s, local version %s", base.UD(doc.ID), newDocHLV.GetCurrentVersionString(), doc.HLV.GetCurrentVersionString()) + // cancel rest of update, HLV needs to be sent back to client with merge versions populated + return nil, nil, false, nil, base.HTTPErrorf(http.StatusConflict, "Document revision conflict") + } + } + // populate merge versions + if newDocHLV.MergeVersions != nil { + doc.HLV.MergeVersions = newDocHLV.MergeVersions + } + // rev tree conflict check if we have rev tree history to check against + currentRevIndex := len(revTreeHistory) + parent := "" + if currentRevIndex > 0 { + for i, revid := range revTreeHistory { + if doc.History.contains(revid) { + currentRevIndex = i + parent = revid + break + } + } + // conflict check on rev tree history + if db.IsIllegalConflict(ctx, doc, parent, newDoc.Deleted, true, revTreeHistory) { + return nil, nil, false, nil, base.HTTPErrorf(http.StatusConflict, "Document revision conflict") + } + } + // Add all the new revisions to the rev tree: + for i := currentRevIndex - 1; i >= 0; i-- { + err := doc.History.addRevision(newDoc.ID, + RevInfo{ + ID: revTreeHistory[i], + Parent: parent, + Deleted: i == 0 && newDoc.Deleted}) + + if err != nil { + return nil, nil, false, nil, err + } + parent = revTreeHistory[i] + } + + // Process the attachments, replacing bodies with digests. + newAttachments, err := db.storeAttachments(ctx, doc, newDoc.DocAttachments, newGeneration, previousRevTreeID, nil) + if err != nil { + return nil, nil, false, nil, err + } + + // generate rev id for new arriving doc + strippedBody, _ := stripInternalProperties(newDoc._body) + encoding, err := base.JSONMarshalCanonical(strippedBody) + if err != nil { + return nil, nil, false, nil, err + } + newRev := CreateRevIDWithBytes(newGeneration, previousRevTreeID, encoding) + + if err := doc.History.addRevision(newDoc.ID, RevInfo{ID: newRev, Parent: previousRevTreeID, Deleted: newDoc.Deleted}); err != nil { + base.InfofCtx(ctx, base.KeyCRUD, "Failed to add revision ID: %s, for doc: %s, error: %v", newRev, base.UD(newDoc.ID), err) + return nil, nil, false, nil, base.ErrRevTreeAddRevFailure + } + + newDoc.RevID = newRev + + return newDoc, newAttachments, false, nil, nil + }) + + if doc != nil && doc.HLV != nil { + if cv == nil { + cv = &Version{} + } + source, version := doc.HLV.GetCurrentVersion() + cv.SourceID = source + cv.Value = version + } + + return doc, cv, newRevID, err +} + // Adds an existing revision to a document along with its history (list of rev IDs.) -func (db *DatabaseCollectionWithUser) PutExistingRev(ctx context.Context, newDoc *Document, docHistory []string, noConflicts bool, forceAllConflicts bool, existingDoc *sgbucket.BucketDocument) (doc *Document, newRevID string, err error) { - return db.PutExistingRevWithConflictResolution(ctx, newDoc, docHistory, noConflicts, nil, forceAllConflicts, existingDoc) +func (db *DatabaseCollectionWithUser) PutExistingRev(ctx context.Context, newDoc *Document, docHistory []string, noConflicts bool, forceAllConflicts bool, existingDoc *sgbucket.BucketDocument, docUpdateEvent DocUpdateType) (doc *Document, newRevID string, err error) { + return db.PutExistingRevWithConflictResolution(ctx, newDoc, docHistory, noConflicts, nil, forceAllConflicts, existingDoc, docUpdateEvent) } // PutExistingRevWithConflictResolution Adds an existing revision to a document along with its history (list of rev IDs.) @@ -1002,7 +1334,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRev(ctx context.Context, newDoc // 1. If noConflicts == false, the revision will be added to the rev tree as a conflict // 2. If noConflicts == true and a conflictResolverFunc is not provided, a 409 conflict error will be returned // 3. If noConflicts == true and a conflictResolverFunc is provided, conflicts will be resolved and the result added to the document. -func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx context.Context, newDoc *Document, docHistory []string, noConflicts bool, conflictResolver *ConflictResolver, forceAllowConflictingTombstone bool, existingDoc *sgbucket.BucketDocument) (doc *Document, newRevID string, err error) { +func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx context.Context, newDoc *Document, docHistory []string, noConflicts bool, conflictResolver *ConflictResolver, forceAllowConflictingTombstone bool, existingDoc *sgbucket.BucketDocument, docUpdateEvent DocUpdateType) (doc *Document, newRevID string, err error) { newRev := docHistory[0] generation, _ := ParseRevID(ctx, newRev) if generation < 0 { @@ -1010,7 +1342,8 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c } allowImport := db.UseXattrs() - doc, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &newDoc.DocExpiry, nil, existingDoc, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + doc, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, allowImport, &newDoc.DocExpiry, nil, docUpdateEvent, existingDoc, false, func(doc *Document) (resultDoc *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + // (Be careful: this block can be invoked multiple times if there are races!) var isSgWrite bool @@ -1109,7 +1442,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c return doc, newRev, err } -func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context, docid string, body Body, docHistory []string, noConflicts bool) (doc *Document, newRev string, err error) { +func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context, docid string, body Body, docHistory []string, noConflicts bool, docUpdateEvent DocUpdateType) (doc *Document, newRev string, err error) { err = validateAPIDocUpdate(body) if err != nil { return nil, "", err @@ -1134,7 +1467,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context newDoc.UpdateBody(body) - doc, newRevID, putExistingRevErr := db.PutExistingRev(ctx, newDoc, docHistory, noConflicts, false, nil) + doc, newRevID, putExistingRevErr := db.PutExistingRev(ctx, newDoc, docHistory, noConflicts, false, nil, docUpdateEvent) if putExistingRevErr != nil { return nil, "", putExistingRevErr @@ -1548,7 +1881,7 @@ func (db *DatabaseCollectionWithUser) storeOldBodyInRevTreeAndUpdateCurrent(ctx // Store the new revision body into the doc: doc.setRevisionBody(ctx, newRevID, newDoc, db.AllowExternalRevBodyStorage(), newDocHasAttachments) doc.SyncData.Attachments = newDoc.DocAttachments - doc.metadataOnlyUpdate = newDoc.metadataOnlyUpdate + doc.MetadataOnlyUpdate = newDoc.MetadataOnlyUpdate if doc.CurrentRev == newRevID { doc.NewestRev = "" @@ -1559,7 +1892,7 @@ func (db *DatabaseCollectionWithUser) storeOldBodyInRevTreeAndUpdateCurrent(ctx if doc.CurrentRev != prevCurrentRev { doc.promoteNonWinningRevisionBody(ctx, doc.CurrentRev, db.RevisionBodyLoader) // If the update resulted in promoting a previous non-winning revision body to winning, this isn't a metadata only update. - doc.metadataOnlyUpdate = nil + doc.MetadataOnlyUpdate = nil } } } @@ -1812,13 +2145,41 @@ func (db *DatabaseCollectionWithUser) IsIllegalConflict(ctx context.Context, doc return true } -func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, docExists bool, doc *Document, allowImport bool, previousDocSequenceIn uint64, unusedSequences []uint64, callback updateAndReturnDocCallback, expiry *uint32) (retSyncFuncExpiry *uint32, retNewRevID string, retStoredDoc *Document, retOldBodyJSON string, retUnusedSequences []uint64, changedAccessPrincipals []string, changedRoleAccessUsers []string, createNewRevIDSkipped bool, err error) { +func (col *DatabaseCollectionWithUser) documentUpdateFunc( + ctx context.Context, + docExists bool, + doc *Document, + allowImport bool, + previousDocSequenceIn uint64, + unusedSequences []uint64, + callback updateAndReturnDocCallback, + expiry *uint32, + docUpdateEvent DocUpdateType, +) ( + retSyncFuncExpiry *uint32, + retNewRevID string, + retStoredDoc *Document, + retOldBodyJSON string, + retUnusedSequences []uint64, + changedAccessPrincipals []string, + changedRoleAccessUsers []string, + createNewRevIDSkipped bool, + revokedChannelsRequiringExpansion []string, + err error) { err = validateExistingDoc(doc, allowImport, docExists) if err != nil { return } + // compute mouMatch before the callback modifies doc.MetadataOnlyUpdate + mouMatch := false + if doc.MetadataOnlyUpdate != nil && doc.MetadataOnlyUpdate.CAS() == doc.Cas { + mouMatch = doc.MetadataOnlyUpdate.CAS() == doc.Cas + base.DebugfCtx(ctx, base.KeyVV, "updateDoc(%q): _mou:%+v Metadata-only update match:%t", base.UD(doc.ID), doc.MetadataOnlyUpdate, mouMatch) + } else { + base.DebugfCtx(ctx, base.KeyVV, "updateDoc(%q): has no _mou", base.UD(doc.ID)) + } // Invoke the callback to update the document and with a new revision body to be used by the Sync Function: newDoc, newAttachments, createNewRevIDSkipped, updatedExpiry, err := callback(doc) if err != nil { @@ -1875,6 +2236,14 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d return } + // The callback has updated the HLV for mutations coming from CBL. Update the HLV so that the current version is set before + // we call updateChannels, which needs to set the current version for removals + // update the HLV values + doc, err = col.updateHLV(ctx, doc, docUpdateEvent, mouMatch) + if err != nil { + return + } + if doc.CurrentRev != prevCurrentRev || createNewRevIDSkipped { // Most of the time this update will change the doc's current rev. (The exception is // if the new rev is a conflict that doesn't win the revid comparison.) If so, we @@ -1886,7 +2255,7 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d return } } - _, err = doc.updateChannels(ctx, channelSet) + _, revokedChannelsRequiringExpansion, err = doc.updateChannels(ctx, channelSet) if err != nil { return } @@ -1911,7 +2280,7 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d doc.ClusterUUID = col.serverUUID() doc.TimeSaved = time.Now() - return updatedExpiry, newRevID, newDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, err + return updatedExpiry, newRevID, newDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, revokedChannelsRequiringExpansion, err } // Function type for the callback passed into updateAndReturnDoc @@ -1922,7 +2291,8 @@ type updateAndReturnDocCallback func(*Document) (resultDoc *Document, resultAtta // 2. Specify the existing document body/xattr/cas, to avoid initial retrieval of the doc in cases that the current contents are already known (e.g. import). // On cas failure, the document will still be reloaded from the bucket as usual. // 3. If isImport=true, document body will not be updated - only metadata xattr(s) -func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, docid string, allowImport bool, expiry *uint32, opts *sgbucket.MutateInOptions, existingDoc *sgbucket.BucketDocument, isImport bool, callback updateAndReturnDocCallback) (doc *Document, newRevID string, err error) { + +func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, docid string, allowImport bool, expiry *uint32, opts *sgbucket.MutateInOptions, docUpdateEvent DocUpdateType, existingDoc *sgbucket.BucketDocument, isImport bool, callback updateAndReturnDocCallback) (doc *Document, newRevID string, err error) { key := realDocID(docid) if key == "" { return nil, "", base.HTTPErrorf(400, "Invalid doc ID") @@ -1962,8 +2332,9 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do base.ErrorfCtx(ctx, "Error retrieving previous leaf attachments of doc: %s, Error: %v", base.UD(docid), err) } prevCurrentRev = doc.CurrentRev + isNewDocCreation = currentValue == nil - syncFuncExpiry, newRevID, storedDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, err = db.documentUpdateFunc(ctx, !isNewDocCreation, doc, allowImport, docSequence, unusedSequences, callback, expiry) + syncFuncExpiry, newRevID, storedDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, _, err = db.documentUpdateFunc(ctx, !isNewDocCreation, doc, allowImport, docSequence, unusedSequences, callback, expiry, docUpdateEvent) if err != nil { return } @@ -1998,7 +2369,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do if expiry != nil { initialExpiry = *expiry } - casOut, err = db.dataStore.WriteUpdateWithXattrs(ctx, key, db.syncMouAndUserXattrKeys(), initialExpiry, existingDoc, opts, func(currentValue []byte, currentXattrs map[string][]byte, cas uint64) (updatedDoc sgbucket.UpdatedDoc, err error) { + casOut, err = db.dataStore.WriteUpdateWithXattrs(ctx, key, db.syncGlobalSyncMouRevSeqNoAndUserXattrKeys(), initialExpiry, existingDoc, opts, func(currentValue []byte, currentXattrs map[string][]byte, cas uint64) (updatedDoc sgbucket.UpdatedDoc, err error) { // Be careful: this block can be invoked multiple times if there are races! if doc, err = db.unmarshalDocumentWithXattrs(ctx, docid, currentValue, currentXattrs, cas, DocUnmarshalAll); err != nil { return @@ -2018,7 +2389,8 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do } isNewDocCreation = currentValue == nil - updatedDoc.Expiry, newRevID, storedDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, err = db.documentUpdateFunc(ctx, !isNewDocCreation, doc, allowImport, docSequence, unusedSequences, callback, expiry) + var revokedChannelsRequiringExpansion []string + updatedDoc.Expiry, newRevID, storedDoc, oldBodyJSON, unusedSequences, changedAccessPrincipals, changedRoleAccessUsers, createNewRevIDSkipped, revokedChannelsRequiringExpansion, err = db.documentUpdateFunc(ctx, !isNewDocCreation, doc, allowImport, docSequence, unusedSequences, callback, expiry, docUpdateEvent) if err != nil { return } @@ -2034,10 +2406,15 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do return } + // update the mutate in options based on the above logic + updatedDoc.Spec = doc.SyncData.HLV.computeMacroExpansions() + + updatedDoc.Spec = appendRevocationMacroExpansions(updatedDoc.Spec, revokedChannelsRequiringExpansion) + updatedDoc.IsTombstone = currentRevFromHistory.Deleted - if doc.metadataOnlyUpdate != nil { - if doc.metadataOnlyUpdate.CAS != "" { - updatedDoc.Spec = append(updatedDoc.Spec, sgbucket.NewMacroExpansionSpec(xattrMouCasPath(), sgbucket.MacroCas)) + if doc.MetadataOnlyUpdate != nil { + if doc.MetadataOnlyUpdate.HexCAS != "" { + updatedDoc.Spec = append(updatedDoc.Spec, sgbucket.NewMacroExpansionSpec(XattrMouCasPath(), sgbucket.MacroCas)) } } else { if currentXattrs[base.MouXattrName] != nil && !isNewDocCreation { @@ -2047,16 +2424,29 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do // Return the new raw document value for the bucket to store. doc.SetCrc32cUserXattrHash() - var rawSyncXattr, rawMouXattr, rawDocBody []byte - rawDocBody, rawSyncXattr, rawMouXattr, err = doc.MarshalWithXattrs() + + var rawSyncXattr, rawMouXattr, rawVvXattr, rawGlobalSync, rawDocBody []byte + rawDocBody, rawSyncXattr, rawVvXattr, rawMouXattr, rawGlobalSync, err = doc.MarshalWithXattrs() + if err != nil { + return updatedDoc, err + } + if len(rawDocBody) > 0 { updatedDoc.Doc = rawDocBody docBytes = len(updatedDoc.Doc) } - updatedDoc.Xattrs = map[string][]byte{base.SyncXattrName: rawSyncXattr} + + updatedDoc.Xattrs = map[string][]byte{base.SyncXattrName: rawSyncXattr, base.VvXattrName: rawVvXattr} if rawMouXattr != nil && db.useMou() { updatedDoc.Xattrs[base.MouXattrName] = rawMouXattr } + if rawGlobalSync != nil { + updatedDoc.Xattrs[base.GlobalXattrName] = rawGlobalSync + } else { + if currentXattrs[base.GlobalXattrName] != nil && !isNewDocCreation { + updatedDoc.XattrsToDelete = append(updatedDoc.XattrsToDelete, base.GlobalXattrName) + } + } // Warn when sync data is larger than a configured threshold if db.unsupportedOptions() != nil && db.unsupportedOptions().WarningThresholds != nil { @@ -2071,7 +2461,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do // Prior to saving doc, remove the revision in cache if createNewRevIDSkipped { - db.revisionCache.Remove(doc.ID, doc.CurrentRev) + db.revisionCache.RemoveWithRev(doc.ID, doc.CurrentRev) } base.DebugfCtx(ctx, base.KeyCRUD, "Saving doc (seq: #%d, id: %v rev: %v)", doc.Sequence, base.UD(doc.ID), doc.CurrentRev) @@ -2086,9 +2476,11 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do } else if doc != nil { // Update the in-memory CAS values to match macro-expanded values doc.Cas = casOut - if doc.metadataOnlyUpdate != nil && doc.metadataOnlyUpdate.CAS == expandMacroCASValue { - doc.metadataOnlyUpdate.CAS = base.CasToString(casOut) + if doc.MetadataOnlyUpdate != nil && doc.MetadataOnlyUpdate.HexCAS == expandMacroCASValueString { + doc.MetadataOnlyUpdate.HexCAS = base.CasToString(casOut) } + // update the doc's HLV defined post macro expansion + doc = db.postWriteUpdateHLV(ctx, doc, casOut) } } @@ -2110,6 +2502,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do } } + // ErrUpdateCancel is returned when the incoming revision is already known if err == base.ErrUpdateCancel { return nil, "", nil } else if err != nil { @@ -2164,6 +2557,8 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do Attachments: doc.Attachments, Expiry: doc.Expiry, Deleted: doc.History[newRevID].Deleted, + hlvHistory: doc.HLV.ToHistoryForHLV(), + CV: &Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}, } if createNewRevIDSkipped { @@ -2238,6 +2633,37 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do return doc, newRevID, nil } +func (db *DatabaseCollectionWithUser) postWriteUpdateHLV(ctx context.Context, doc *Document, casOut uint64) *Document { + if doc.HLV == nil { + return doc + } + if doc.HLV.Version == expandMacroCASValueUint64 { + doc.HLV.Version = casOut + } + if doc.HLV.CurrentVersionCAS == expandMacroCASValueUint64 { + doc.HLV.CurrentVersionCAS = casOut + } + // backup new revision to the bucket now we have a doc assigned a CV (post macro expansion) for delta generation purposes + backupRev := db.deltaSyncEnabled() && db.deltaSyncRevMaxAgeSeconds() != 0 + if db.UseXattrs() && backupRev { + var newBodyWithAtts = doc._rawBody + if len(doc.Attachments) > 0 { + var err error + newBodyWithAtts, err = base.InjectJSONProperties(doc._rawBody, base.KVPair{ + Key: BodyAttachments, + Val: doc.Attachments, + }) + if err != nil { + base.WarnfCtx(ctx, "Unable to marshal new revision body during backupRevisionJSON: doc=%q rev=%q cv=%q err=%v ", base.UD(doc.ID), doc.CurrentRev, doc.HLV.GetCurrentVersionString(), err) + return doc + } + } + revHash := base.Crc32cHashString([]byte(doc.HLV.GetCurrentVersionString())) + _ = db.setOldRevisionJSON(ctx, doc.ID, revHash, newBodyWithAtts, db.deltaSyncRevMaxAgeSeconds()) + } + return doc +} + // getAttachmentIDsForLeafRevisions returns a map of attachment docids with values of attachment names. func getAttachmentIDsForLeafRevisions(ctx context.Context, db *DatabaseCollectionWithUser, doc *Document, newRevID string) (map[string][]string, error) { leafAttachments := make(map[string][]string) @@ -2392,10 +2818,10 @@ func (db *DatabaseCollectionWithUser) Post(ctx context.Context, body Body) (doci } // Deletes a document, by adding a new revision whose _deleted property is true. -func (db *DatabaseCollectionWithUser) DeleteDoc(ctx context.Context, docid string, revid string) (string, error) { +func (db *DatabaseCollectionWithUser) DeleteDoc(ctx context.Context, docid string, revid string) (string, *Document, error) { body := Body{BodyDeleted: true, BodyRev: revid} - newRevID, _, err := db.Put(ctx, docid, body) - return newRevID, err + newRevID, doc, err := db.Put(ctx, docid, body) + return newRevID, doc, err } // Purges a document from the bucket (no tombstone) @@ -2652,7 +3078,7 @@ func (c *DatabaseCollection) checkForUpgrade(ctx context.Context, key string, un return nil, nil } - doc, rawDocument, err := c.GetDocWithXattr(ctx, key, unmarshalLevel) + doc, rawDocument, err := c.GetDocWithXattrs(ctx, key, unmarshalLevel) if err != nil || doc == nil || !doc.HasValidSyncData() { return nil, nil } @@ -2753,11 +3179,63 @@ func (db *DatabaseCollectionWithUser) CheckProposedRev(ctx context.Context, doci } } -const ( - xattrMacroCas = "cas" // standard _sync property name for CAS - xattrMacroValueCrc32c = "value_crc32c" // standard _sync property name for crc32c +// CheckProposedVersion - given DocID and a version in string form, check whether it can be added without conflict. +func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, docid, proposedVersionStr string, previousVersionStr string) (status ProposedRevStatus, currentVersion string) { - expandMacroCASValue = "expand" // static value that indicates that a CAS macro expansion should be applied to a property + proposedVersion, err := ParseVersion(proposedVersionStr) + if err != nil { + base.WarnfCtx(ctx, "Couldn't parse proposed version for doc %q / %q: %v", base.UD(docid), proposedVersionStr, err) + return ProposedRev_Error, "" + } + + var previousVersion Version + if previousVersionStr != "" { + var err error + previousVersion, err = ParseVersion(previousVersionStr) + if err != nil { + base.WarnfCtx(ctx, "Couldn't parse previous version for doc %q / %q: %v", base.UD(docid), previousVersionStr, err) + return ProposedRev_Error, "" + } + } + + localDocCV := Version{} + doc, err := db.GetDocSyncDataNoImport(ctx, docid, DocUnmarshalNoHistory) + if doc.HLV != nil { + localDocCV.SourceID, localDocCV.Value = doc.HLV.GetCurrentVersion() + } + if err != nil { + if !base.IsDocNotFoundError(err) && !errors.Is(err, base.ErrXattrNotFound) { + base.WarnfCtx(ctx, "CheckProposedRev(%q) --> %T %v", base.UD(docid), err, err) + return ProposedRev_Error, "" + } + // New document not found on server + return ProposedRev_OK_IsNew, "" + } else if localDocCV == previousVersion { + // Non-conflicting update, client's previous version is server's CV + return ProposedRev_OK, "" + } else if doc.HLV.DominatesSource(proposedVersion) { + // SGW already has this version + return ProposedRev_Exists, "" + } else if localDocCV.SourceID == proposedVersion.SourceID && localDocCV.Value < proposedVersion.Value { + // previousVersion didn't match, but proposed version and server CV have matching source, and proposed version is newer + return ProposedRev_OK, "" + } else { + // Conflict, return the current cv. This may be a false positive conflict if the client has replicated + // the server cv via a different peer. Client is responsible for performing this check based on the + // returned localDocCV + return ProposedRev_Conflict, localDocCV.String() + } +} + +const ( + xattrMacroCas = "cas" // SyncData.Cas + xattrMacroValueCrc32c = "value_crc32c" // SyncData.Crc32c + xattrMacroCurrentRevVersion = "rev.ver" // SyncDataJSON.RevAndVersion.CurrentVersion + versionVectorVrsMacro = "ver" // PersistedHybridLogicalVector.Version + versionVectorCVCASMacro = "cvCas" // PersistedHybridLogicalVector.CurrentVersionCAS + + expandMacroCASValueUint64 = math.MaxUint64 // static value that indicates that a CAS macro expansion should be applied to a property + expandMacroCASValueString = "expand" ) func macroExpandSpec(xattrName string) []sgbucket.MacroExpansionSpec { @@ -2777,6 +3255,23 @@ func xattrCrc32cPath(xattrKey string) string { return xattrKey + "." + xattrMacroValueCrc32c } -func xattrMouCasPath() string { +// XattrMouCasPath returns the xattr path for the CAS value for expansion, _mou.cas +func XattrMouCasPath() string { return base.MouXattrName + "." + xattrMacroCas } + +func xattrCurrentRevVersionPath(xattrKey string) string { + return xattrKey + "." + xattrMacroCurrentRevVersion +} + +func xattrCurrentVersionPath(xattrKey string) string { + return xattrKey + "." + versionVectorVrsMacro +} + +func xattrCurrentVersionCASPath(xattrKey string) string { + return xattrKey + "." + versionVectorCVCASMacro +} + +func xattrRevokedChannelVersionPath(xattrKey string, channelName string) string { + return xattrKey + ".channels." + channelName + "." + xattrMacroCurrentRevVersion +} diff --git a/db/crud_test.go b/db/crud_test.go index 46818cf362..503c5aa572 100644 --- a/db/crud_test.go +++ b/db/crud_test.go @@ -14,11 +14,13 @@ import ( "context" "encoding/json" "log" + "reflect" "testing" "time" sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/channels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -78,7 +80,7 @@ func TestRevisionCacheLoad(t *testing.T) { // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Flush the cache @@ -119,7 +121,7 @@ func TestHasAttachmentsFlag(t *testing.T) { // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create rev 2-a @@ -130,7 +132,7 @@ func TestHasAttachmentsFlag(t *testing.T) { rev2a_body := unmarshalBody(t, `{"_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`) rev2a_body["key1"] = prop_1000_bytes rev2a_body["version"] = "2a" - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2a_body[BodyId] = doc.ID rev2a_body[BodyRev] = newRev assert.NoError(t, err, "add 2-a") @@ -156,7 +158,7 @@ func TestHasAttachmentsFlag(t *testing.T) { rev2b_body := unmarshalBody(t, `{"_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`) rev2b_body["key1"] = prop_1000_bytes rev2b_body["version"] = "2b" - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2b_body[BodyId] = doc.ID rev2b_body[BodyRev] = newRev assert.NoError(t, err, "add 2-b") @@ -243,18 +245,18 @@ func TestHasAttachmentsFlagForLegacyAttachments(t *testing.T) { require.NoError(t, err) // Get the existing bucket doc - _, existingBucketDoc, err := collection.GetDocWithXattr(ctx, docID, DocUnmarshalAll) + _, existingBucketDoc, err := collection.GetDocWithXattrs(ctx, docID, DocUnmarshalAll) require.NoError(t, err) // Migrate document metadata from document body to system xattr. - _, _, err = collection.migrateMetadata(ctx, docID, body, existingBucketDoc, nil) + _, _, err = collection.migrateMetadata(ctx, docID, existingBucketDoc, nil) require.NoError(t, err) } // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create rev 2-a with legacy attachment. @@ -283,7 +285,7 @@ func TestHasAttachmentsFlagForLegacyAttachments(t *testing.T) { rev2b_body := Body{} rev2b_body["key1"] = prop_1000_bytes rev2b_body["version"] = "2b" - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2b_body[BodyId] = doc.ID rev2b_body[BodyRev] = newRev assert.NoError(t, err, "add 2-b") @@ -318,7 +320,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create rev 2-a @@ -329,7 +331,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev2a_body := Body{} rev2a_body["key1"] = prop_1000_bytes rev2a_body["version"] = "2a" - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2a_body[BodyId] = doc.ID rev2a_body[BodyRev] = newRev assert.NoError(t, err, "add 2-a") @@ -348,7 +350,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev2b_body := Body{} rev2b_body["key1"] = prop_1000_bytes rev2b_body["version"] = "2b" - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2b_body[BodyId] = doc.ID rev2b_body[BodyRev] = newRev assert.NoError(t, err, "add 2-b") @@ -391,7 +393,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev3b_body := Body{} rev3b_body["version"] = "3b" rev3b_body[BodyDeleted] = true - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b"}, false, ExistingVersionWithUpdateToHLV) rev3b_body[BodyId] = doc.ID rev3b_body[BodyRev] = newRev rev3b_body[BodyDeleted] = true @@ -428,7 +430,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev2c_body := Body{} rev2c_body["key1"] = prop_1000_bytes rev2c_body["version"] = "2c" - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2c_body, []string{"2-c", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2c_body, []string{"2-c", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2c_body[BodyId] = doc.ID rev2c_body[BodyRev] = newRev assert.NoError(t, err, "add 2-c") @@ -450,7 +452,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev3c_body["version"] = "3c" rev3c_body["key1"] = prop_1000_bytes rev3c_body[BodyDeleted] = true - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-c"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-c"}, false, ExistingVersionWithUpdateToHLV) rev3c_body[BodyId] = doc.ID rev3c_body[BodyRev] = newRev rev3c_body[BodyDeleted] = true @@ -479,7 +481,7 @@ func TestRevisionStorageConflictAndTombstones(t *testing.T) { rev3a_body := Body{} rev3a_body["key1"] = prop_1000_bytes rev3a_body["version"] = "3a" - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2c_body, []string{"3-a", "2-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2c_body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-a") revTree, err = getRevTreeList(ctx, collection.dataStore, "doc1", db.UseXattrs()) @@ -502,7 +504,7 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { // Create rev 2-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create rev 2-a @@ -513,7 +515,7 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { rev2a_body := Body{} rev2a_body["key1"] = prop_1000_bytes rev2a_body["version"] = "2a" - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2a_body[BodyId] = doc.ID rev2a_body[BodyRev] = newRev assert.NoError(t, err, "add 2-a") @@ -532,7 +534,7 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { rev2b_body := Body{} rev2b_body["key1"] = prop_1000_bytes rev2b_body["version"] = "2b" - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2b_body[BodyId] = doc.ID rev2b_body[BodyRev] = newRev assert.NoError(t, err, "add 2-b") @@ -577,7 +579,7 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { rev3b_body["version"] = "3b" rev3b_body["key1"] = prop_1000_bytes rev3b_body[BodyDeleted] = true - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b"}, false, ExistingVersionWithUpdateToHLV) rev3b_body[BodyId] = doc.ID rev3b_body[BodyRev] = newRev rev3b_body[BodyDeleted] = true @@ -612,17 +614,17 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { activeRevBody := Body{} activeRevBody["version"] = "...a" activeRevBody["key1"] = prop_1000_bytes - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"3-a", "2-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"4-a", "3-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"4-a", "3-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 4-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"5-a", "4-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"5-a", "4-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 5-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"6-a", "5-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"6-a", "5-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 6-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"7-a", "6-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"7-a", "6-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 7-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"8-a", "7-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"8-a", "7-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 8-a") // Verify that 3-b is still present at this point @@ -631,7 +633,7 @@ func TestRevisionStoragePruneTombstone(t *testing.T) { assert.NoError(t, err, "Rev 3-b should still exist") // Add one more rev that triggers pruning since gen(9-3) > revsLimit - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"9-a", "8-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", activeRevBody, []string{"9-a", "8-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 9-a") // Verify that 3-b has been pruned @@ -660,7 +662,7 @@ func TestOldRevisionStorage(t *testing.T) { // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "version": "1a", "large": prop_1000_bytes} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 1-a") // Create rev 2-a @@ -669,7 +671,7 @@ func TestOldRevisionStorage(t *testing.T) { // 2-a log.Printf("Create rev 2-a") rev2a_body := Body{"key1": "value2", "version": "2a", "large": prop_1000_bytes} - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") rev2a_body[BodyId] = doc.ID rev2a_body[BodyRev] = newRev @@ -689,7 +691,7 @@ func TestOldRevisionStorage(t *testing.T) { // 3-a log.Printf("Create rev 3-a") rev3a_body := Body{"key1": "value2", "version": "3a", "large": prop_1000_bytes} - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3a_body, []string{"3-a", "2-a", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3a_body, []string{"3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 3-a") rev3a_body[BodyId] = doc.ID rev3a_body[BodyRev] = newRev @@ -708,7 +710,7 @@ func TestOldRevisionStorage(t *testing.T) { // 3-a log.Printf("Create rev 2-b") rev2b_body := Body{"key1": "value2", "version": "2b", "large": prop_1000_bytes} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 2-b") // Retrieve the document: @@ -731,7 +733,7 @@ func TestOldRevisionStorage(t *testing.T) { // 6-a log.Printf("Create rev 6-a") rev6a_body := Body{"key1": "value2", "version": "6a", "large": prop_1000_bytes} - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev6a_body, []string{"6-a", "5-a", "4-a", "3-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev6a_body, []string{"6-a", "5-a", "4-a", "3-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 6-a") rev6a_body[BodyId] = doc.ID rev6a_body[BodyRev] = newRev @@ -756,7 +758,7 @@ func TestOldRevisionStorage(t *testing.T) { // 6-a log.Printf("Create rev 3-b") rev3b_body := Body{"key1": "value2", "version": "3b", "large": prop_1000_bytes} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 3-b") // Same again and again @@ -775,12 +777,12 @@ func TestOldRevisionStorage(t *testing.T) { log.Printf("Create rev 3-c") rev3c_body := Body{"key1": "value2", "version": "3c", "large": prop_1000_bytes} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 3-c") log.Printf("Create rev 3-d") rev3d_body := Body{"key1": "value2", "version": "3d", "large": prop_1000_bytes} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3d_body, []string{"3-d", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3d_body, []string{"3-d", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 3-d") // Create new winning revision on 'b' branch. Triggers movement of 6-a to inline storage. Force cas retry, check document contents @@ -799,7 +801,7 @@ func TestOldRevisionStorage(t *testing.T) { // 7-b log.Printf("Create rev 7-b") rev7b_body := Body{"key1": "value2", "version": "7b", "large": prop_1000_bytes} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev7b_body, []string{"7-b", "6-b", "5-b", "4-b", "3-b"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev7b_body, []string{"7-b", "6-b", "5-b", "4-b", "3-b"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "add 7-b") } @@ -820,7 +822,7 @@ func TestOldRevisionStorageError(t *testing.T) { // Create rev 1-a log.Printf("Create rev 1-a") body := Body{"key1": "value1", "v": "1a"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create rev 2-a @@ -829,7 +831,7 @@ func TestOldRevisionStorageError(t *testing.T) { // 2-a log.Printf("Create rev 2-a") rev2a_body := Body{"key1": "value2", "v": "2a"} - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2a_body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2a_body[BodyId] = doc.ID rev2a_body[BodyRev] = newRev assert.NoError(t, err, "add 2-a") @@ -848,7 +850,7 @@ func TestOldRevisionStorageError(t *testing.T) { // 3-a log.Printf("Create rev 3-a") rev3a_body := Body{"key1": "value2", "v": "3a"} - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3a_body, []string{"3-a", "2-a", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3a_body, []string{"3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev3a_body[BodyId] = doc.ID rev3a_body[BodyRev] = newRev assert.NoError(t, err, "add 3-a") @@ -861,7 +863,7 @@ func TestOldRevisionStorageError(t *testing.T) { // 3-a log.Printf("Create rev 2-b") rev2b_body := Body{"key1": "value2", "v": "2b"} - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev2b_body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) rev2b_body[BodyId] = doc.ID rev2b_body[BodyRev] = newRev assert.NoError(t, err, "add 2-b") @@ -886,7 +888,7 @@ func TestOldRevisionStorageError(t *testing.T) { // 6-a log.Printf("Create rev 6-a") rev6a_body := Body{"key1": "value2", "v": "6a"} - doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev6a_body, []string{"6-a", "5-a", "4-a", "3-a"}, false) + doc, newRev, err = collection.PutExistingRevWithBody(ctx, "doc1", rev6a_body, []string{"6-a", "5-a", "4-a", "3-a"}, false, ExistingVersionWithUpdateToHLV) rev6a_body[BodyId] = doc.ID rev6a_body[BodyRev] = newRev assert.NoError(t, err, "add 6-a") @@ -912,7 +914,7 @@ func TestOldRevisionStorageError(t *testing.T) { // 6-a log.Printf("Create rev 3-b") rev3b_body := Body{"key1": "value2", "v": "3b"} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3b_body, []string{"3-b", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-b") // Same again @@ -932,7 +934,7 @@ func TestOldRevisionStorageError(t *testing.T) { log.Printf("Create rev 3-c") rev3c_body := Body{"key1": "value2", "v": "3c"} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-c") } @@ -949,7 +951,7 @@ func TestLargeSequence(t *testing.T) { // Write a doc via SG body := Body{"key1": "largeSeqTest"} - _, _, err := collection.PutExistingRevWithBody(ctx, "largeSeqDoc", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "largeSeqDoc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add largeSeqDoc") syncData, err := collection.GetDocSyncData(ctx, "largeSeqDoc") @@ -1024,7 +1026,7 @@ func TestMalformedRevisionStorageRecovery(t *testing.T) { // 6-a log.Printf("Attempt to create rev 3-c") rev3c_body := Body{"key1": "value2", "v": "3c"} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev3c_body, []string{"3-c", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-c") } @@ -1036,16 +1038,16 @@ func BenchmarkDatabaseGet1xRev(b *testing.B) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, b, db) body := Body{"foo": "bar", "rev": "1-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) largeDoc := make([]byte, 1000000) longBody := Body{"val": string(largeDoc), "rev": "1-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", longBody, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", longBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) var shortWithAttachmentsDataBody Body shortWithAttachmentsData := `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}, "rev":"1-a"}` _ = base.JSONUnmarshal([]byte(shortWithAttachmentsData), &shortWithAttachmentsDataBody) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", shortWithAttachmentsDataBody, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", shortWithAttachmentsDataBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) b.Run("ShortLatest", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -1064,9 +1066,9 @@ func BenchmarkDatabaseGet1xRev(b *testing.B) { }) updateBody := Body{"rev": "2-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", updateBody, []string{"2-a", "1-a"}, false) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", updateBody, []string{"2-a", "1-a"}, false) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", updateBody, []string{"2-a", "1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) b.Run("ShortOld", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -1093,16 +1095,16 @@ func BenchmarkDatabaseGetRev(b *testing.B) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, b, db) body := Body{"foo": "bar", "rev": "1-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) largeDoc := make([]byte, 1000000) longBody := Body{"val": string(largeDoc), "rev": "1-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", longBody, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", longBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) var shortWithAttachmentsDataBody Body shortWithAttachmentsData := `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}, "rev":"1-a"}` _ = base.JSONUnmarshal([]byte(shortWithAttachmentsData), &shortWithAttachmentsDataBody) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", shortWithAttachmentsDataBody, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", shortWithAttachmentsDataBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) b.Run("ShortLatest", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -1121,9 +1123,9 @@ func BenchmarkDatabaseGetRev(b *testing.B) { }) updateBody := Body{"rev": "2-a"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", updateBody, []string{"2-a", "1-a"}, false) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", updateBody, []string{"2-a", "1-a"}, false) - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", updateBody, []string{"2-a", "1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc2", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc3", updateBody, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) b.Run("ShortOld", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -1151,7 +1153,7 @@ func BenchmarkHandleRevDelta(b *testing.B) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, b, db) body := Body{"foo": "bar"} - _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, _ = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) getDelta := func(newDoc *Document) { deltaSrcRev, _ := collection.GetRev(ctx, "doc1", "1-a", false, nil) @@ -1194,24 +1196,25 @@ func BenchmarkHandleRevDelta(b *testing.B) { } func TestGetAvailableRevAttachments(t *testing.T) { + t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)") db, ctx := setupTestDB(t) defer db.Close(ctx) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) // Create the very first revision of the document with attachment; let's call this as rev 1-a payload := `{"sku":"6213100","_attachments":{"camera.txt":{"data":"Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}` - _, rev, err := collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"1-a"}, false) + _, rev, err := collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") ancestor := rev // Ancestor revision // Create the second revision of the document with attachment reference; payload = `{"sku":"6213101","_attachments":{"camera.txt":{"stub":true,"revpos":1}}}` - _, rev, err = collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"2-a", "1-a"}, false) + _, rev, err = collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) parent := rev // Immediate ancestor or parent revision assert.NoError(t, err, "Couldn't create document") payload = `{"sku":"6213102","_attachments":{"camera.txt":{"stub":true,"revpos":1}}}` - doc, _, err := collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"3-a", "2-a"}, false) + doc, _, err := collection.PutExistingRevWithBody(ctx, "camera", unmarshalBody(t, payload), []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") // Get available attachments by immediate ancestor revision or parent revision @@ -1238,11 +1241,11 @@ func TestGet1xRevAndChannels(t *testing.T) { docId := "dd6d2dcc679d12b9430a9787bab45b33" payload := `{"sku":"6213100","_attachments":{"camera.txt":{"data":"Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}` - doc1, rev1, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"1-a"}, false) + doc1, rev1, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") payload = `{"sku":"6213101","_attachments":{"lens.txt":{"data":"Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}` - doc2, rev2, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"2-a", "1-a"}, false) + doc2, rev2, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") // Get the 1x revision from document with list revision enabled @@ -1270,7 +1273,7 @@ func TestGet1xRevAndChannels(t *testing.T) { assert.Equal(t, []interface{}{"a"}, revisions[RevisionsIds]) // Delete the document, creating tombstone revision rev3 - rev3, err := collection.DeleteDoc(ctx, docId, rev2) + rev3, _, err := collection.DeleteDoc(ctx, docId, rev2) require.NoError(t, err) bodyBytes, removed, err = collection.get1xRevFromDoc(ctx, doc2, rev3, true) assert.False(t, removed) @@ -1301,7 +1304,7 @@ func TestGet1xRevFromDoc(t *testing.T) { // Create the first revision of the document docId := "356779a9a1696714480f57fa3fb66d4c" payload := `{"city":"Los Angeles"}` - doc, rev1, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"1-a"}, false) + doc, rev1, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") assert.NotEmpty(t, doc, "Document shouldn't be empty") assert.Equal(t, "1-a", rev1, "Provided input revision ID should be returned") @@ -1324,7 +1327,7 @@ func TestGet1xRevFromDoc(t *testing.T) { // Create the second revision of the document payload = `{"city":"Hollywood"}` - doc, rev2, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"2-a", "1-a"}, false) + doc, rev2, err := collection.PutExistingRevWithBody(ctx, docId, unmarshalBody(t, payload), []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't create document") assert.NotEmpty(t, doc, "Document shouldn't be empty") assert.Equal(t, "2-a", rev2, "Provided input revision ID should be returned") @@ -1761,3 +1764,381 @@ func TestReleaseSequenceOnDocWriteFailure(t *testing.T) { assert.Equal(t, int64(1), db.DbStats.Database().SequenceReleasedCount.Value()) }, time.Second*10, time.Millisecond*100) } + +// TestPutExistingCurrentVersion: +// - Put a document in a db +// - Assert on the update to HLV after that PUT +// - Construct a HLV to represent the doc created locally being updated on a client +// - Call PutExistingCurrentVersion simulating doc update arriving over replicator +// - Assert that the doc's HLV in the bucket has been updated correctly with the CV, PV and cvCAS +func TestPutExistingCurrentVersion(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + bucketUUID := db.EncodedSourceID + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create a new doc + key := "doc1" + body := Body{"key1": "value1"} + + rev, _, err := collection.Put(ctx, key, body) + require.NoError(t, err) + + // assert on HLV on that above PUT + syncData, err := collection.GetDocSyncData(ctx, "doc1") + assert.NoError(t, err) + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // store the cas version allocated to the above doc creation for creation of incoming HLV later in test + originalDocVersion := syncData.HLV.Version + + // PUT an update to the above doc + body = Body{"key1": "value11"} + body[BodyRev] = rev + _, _, err = collection.Put(ctx, key, body) + require.NoError(t, err) + + // grab the new version for the above update to assert against later in test + syncData, err = collection.GetDocSyncData(ctx, "doc1") + assert.NoError(t, err) + docUpdateVersion := syncData.HLV.Version + docUpdateVersionInt := docUpdateVersion + + // construct a mock doc update coming over a replicator + body = Body{"key1": "value2"} + newDoc := createTestDocument(key, "", body, false, 0) + + // Simulate a conflicting doc update happening from a client that + // has only replicated the initial version of the document + pv := make(HLVVersions) + pv[syncData.HLV.SourceID] = originalDocVersion + + // create a version larger than the allocated version above + incomingVersion := docUpdateVersionInt + 10 + incomingHLV := &HybridLogicalVector{ + SourceID: "test", + Version: incomingVersion, + PreviousVersions: pv, + } + + doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil, nil) + assertHTTPError(t, err, 409) + require.Nil(t, doc) + require.Nil(t, cv) + + // Update the client's HLV to include the latest SGW version. + incomingHLV.PreviousVersions[syncData.HLV.SourceID] = docUpdateVersion + // TODO: because currentRev isn't being updated, storeOldBodyInRevTreeAndUpdateCurrent isn't + // updating the document body. Need to review whether it makes sense to keep using + // storeOldBodyInRevTreeAndUpdateCurrent, or if this needs a larger overhaul to support VV + doc, cv, _, err = collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil, nil) + require.NoError(t, err) + assert.Equal(t, "test", cv.SourceID) + assert.Equal(t, incomingVersion, cv.Value) + assert.Equal(t, []byte(`{"key1":"value2"}`), doc._rawBody) + + // assert on the sync data from the above update to the doc + // CV should be equal to CV of update on client but the cvCAS should be updated with the new update and + // PV should contain the old CV pair + syncData, err = collection.GetDocSyncData(ctx, "doc1") + assert.NoError(t, err) + + assert.Equal(t, "test", syncData.HLV.SourceID) + assert.Equal(t, incomingVersion, syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + // update the pv map so we can assert we have correct pv map in HLV + pv[bucketUUID] = docUpdateVersion + assert.True(t, reflect.DeepEqual(syncData.HLV.PreviousVersions, pv)) + assert.Equal(t, "3-60b024c44c283b369116c2c2570e8088", syncData.CurrentRev) + + // Attempt to push the same client update, validate server rejects as an already known version and cancels the update. + // This case doesn't return error, verify that SyncData hasn't been changed. + _, _, _, err = collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil, nil) + require.NoError(t, err) + syncData2, err := collection.GetDocSyncData(ctx, "doc1") + require.NoError(t, err) + require.Equal(t, syncData.TimeSaved, syncData2.TimeSaved) + require.Equal(t, syncData.CurrentRev, syncData2.CurrentRev) + +} + +// TestPutExistingCurrentVersionWithConflict: +// - Put a document in a db +// - Assert on the update to HLV after that PUT +// - Construct a HLV to represent the doc created locally being updated on a client +// - Call PutExistingCurrentVersion simulating doc update arriving over replicator +// - Assert conflict between the local HLV for the doc and the incoming mutation is correctly identified +// - Assert that the doc's HLV in the bucket hasn't been updated +func TestPutExistingCurrentVersionWithConflict(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + bucketUUID := db.EncodedSourceID + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create a new doc + key := "doc1" + body := Body{"key1": "value1"} + + _, _, err := collection.Put(ctx, key, body) + require.NoError(t, err) + + // assert on the HLV values after the above creation of the doc + syncData, err := collection.GetDocSyncData(ctx, "doc1") + assert.NoError(t, err) + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // create a new doc update to simulate a doc update arriving over replicator from, client + body = Body{"key1": "value2"} + newDoc := createTestDocument(key, "", body, false, 0) + incomingHLV := &HybridLogicalVector{ + SourceID: "test", + Version: 1234, + } + + // assert that a conflict is correctly identified and the doc and cv are nil + doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil, nil) + assertHTTPError(t, err, 409) + require.Nil(t, doc) + require.Nil(t, cv) + + // assert persisted doc hlv hasn't been updated + syncData, err = collection.GetDocSyncData(ctx, "doc1") + assert.NoError(t, err) + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) +} + +// TestPutExistingCurrentVersionWithNoExistingDoc: +// - Purpose of this test is to test PutExistingRevWithBody code pathway where an +// existing doc is not provided from the bucket into the function simulating a new, not seen +// before doc entering this code path +func TestPutExistingCurrentVersionWithNoExistingDoc(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + bucketUUID := db.BucketUUID + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // construct a mock doc update coming over a replicator + body := Body{"key1": "value2"} + newDoc := createTestDocument("doc2", "", body, false, 0) + + // construct a HLV that simulates a doc update happening on a client + // this means moving the current source version pair to PV and adding new sourceID and version pair to CV + pv := make(HLVVersions) + pv[bucketUUID] = uint64(2) + // create a version larger than the allocated version above + incomingVersion := uint64(2 + 10) + incomingHLV := &HybridLogicalVector{ + SourceID: "test", + Version: incomingVersion, + PreviousVersions: pv, + } + // call PutExistingCurrentVersion with empty existing doc + doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, &sgbucket.BucketDocument{}, nil) + require.NoError(t, err) + assert.NotNil(t, doc) + // assert on returned CV value + assert.Equal(t, "test", cv.SourceID) + assert.Equal(t, incomingVersion, cv.Value) + assert.Equal(t, []byte(`{"key1":"value2"}`), doc._rawBody) + + // assert on the sync data from the above update to the doc + // CV should be equal to CV of update on client but the cvCAS should be updated with the new update and + // PV should contain the old CV pair + syncData, err := collection.GetDocSyncData(ctx, "doc2") + assert.NoError(t, err) + assert.Equal(t, "test", syncData.HLV.SourceID) + assert.Equal(t, incomingVersion, syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + // update the pv map so we can assert we have correct pv map in HLV + assert.True(t, reflect.DeepEqual(syncData.HLV.PreviousVersions, pv)) + assert.Equal(t, "1-3a208ea66e84121b528f05b5457d1134", syncData.CurrentRev) +} + +// TestGetCVWithDocResidentInCache: +// - Two test cases, one with doc a user will have access to, one without +// - Purpose is to have a doc that is resident in rev cache and use the GetCV function to retrieve these docs +// - Assert that the doc the user has access to is corrected fetched +// - Assert the doc the user doesn't have access to is fetched but correctly redacted +func TestGetCVWithDocResidentInCache(t *testing.T) { + const docID = "doc1" + + testCases := []struct { + name string + docChannels []string + access bool + }{ + { + name: "getCVWithUserAccess", + docChannels: []string{"A"}, + access: true, + }, + { + name: "getCVWithoutUserAccess", + docChannels: []string{"B"}, + access: false, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) + + // Create a user with access to channel A + authenticator := db.Authenticator(base.TestCtx(t)) + user, err := authenticator.NewUser("alice", "letmein", channels.BaseSetOf(t, "A")) + require.NoError(t, err) + require.NoError(t, authenticator.Save(user)) + collection.user, err = authenticator.GetUser("alice") + require.NoError(t, err) + + // create doc with the channels for the test case + docBody := Body{"channels": testCase.docChannels} + rev, doc, err := collection.Put(ctx, docID, docBody) + require.NoError(t, err) + + vrs := doc.HLV.Version + src := doc.HLV.SourceID + sv := &Version{Value: vrs, SourceID: src} + revision, err := collection.GetCV(ctx, docID, sv) + require.NoError(t, err) + if testCase.access { + assert.Equal(t, rev, revision.RevID) + assert.Equal(t, sv, revision.CV) + assert.Equal(t, docID, revision.DocID) + assert.Equal(t, []byte(`{"channels":["A"]}`), revision.BodyBytes) + } else { + assert.Equal(t, rev, revision.RevID) + assert.Equal(t, sv, revision.CV) + assert.Equal(t, docID, revision.DocID) + assert.Equal(t, []byte(RemovedRedactedDocument), revision.BodyBytes) + } + }) + } +} + +// TestGetByCVForDocNotResidentInCache: +// - Setup db with rev cache size of 1 +// - Put two docs forcing eviction of the first doc +// - Use GetCV function to fetch the first doc, forcing the rev cache to load the doc from bucket +// - Assert the doc revision fetched is correct to the first doc we created +func TestGetByCVForDocNotResidentInCache(t *testing.T) { + t.Skip("") + + db, ctx := SetupTestDBWithOptions(t, DatabaseContextOptions{ + RevisionCacheOptions: &RevisionCacheOptions{ + MaxItemCount: 1, + }, + }) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) + + // Create a user with access to channel A + authenticator := db.Authenticator(base.TestCtx(t)) + user, err := authenticator.NewUser("alice", "letmein", channels.BaseSetOf(t, "A")) + require.NoError(t, err) + require.NoError(t, authenticator.Save(user)) + collection.user, err = authenticator.GetUser("alice") + require.NoError(t, err) + + const ( + doc1ID = "doc1" + doc2ID = "doc2" + ) + + revBody := Body{"channels": []string{"A"}} + rev, doc, err := collection.Put(ctx, doc1ID, revBody) + require.NoError(t, err) + + // put another doc that should evict first doc from cache + _, _, err = collection.Put(ctx, doc2ID, revBody) + require.NoError(t, err) + + // get by CV should force a load from bucket and have a cache miss + vrs := doc.HLV.Version + src := doc.HLV.SourceID + sv := &Version{Value: vrs, SourceID: src} + revision, err := collection.GetCV(ctx, doc1ID, sv) + require.NoError(t, err) + + // assert the fetched doc is the first doc we added and assert that we did in fact get cache miss + assert.Equal(t, int64(1), db.DbStats.Cache().RevisionCacheMisses.Value()) + assert.Equal(t, rev, revision.RevID) + assert.Equal(t, sv, revision.CV) + assert.Equal(t, doc1ID, revision.DocID) + assert.Equal(t, []byte(`{"channels":["A"]}`), revision.BodyBytes) +} + +// TestGetCVActivePathway: +// - Two test cases, one with doc a user will have access to, one without +// - Purpose is top specify nil CV to the GetCV function to force the GetActive code pathway +// - Assert doc that is created is fetched correctly when user has access to doc +// - Assert that correct error is returned when user has no access to the doc +func TestGetCVActivePathway(t *testing.T) { + const docID = "doc1" + + testCases := []struct { + name string + docChannels []string + access bool + }{ + { + name: "activeFetchWithUserAccess", + docChannels: []string{"A"}, + access: true, + }, + { + name: "activeFetchWithoutUserAccess", + docChannels: []string{"B"}, + access: false, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) + + // Create a user with access to channel A + authenticator := db.Authenticator(base.TestCtx(t)) + user, err := authenticator.NewUser("alice", "letmein", channels.BaseSetOf(t, "A")) + require.NoError(t, err) + require.NoError(t, authenticator.Save(user)) + collection.user, err = authenticator.GetUser("alice") + require.NoError(t, err) + + // test get active path by specifying nil cv + revBody := Body{"channels": testCase.docChannels} + rev, doc, err := collection.Put(ctx, docID, revBody) + require.NoError(t, err) + revision, err := collection.GetCV(ctx, docID, nil) + + if testCase.access == true { + require.NoError(t, err) + vrs := doc.HLV.Version + src := doc.HLV.SourceID + sv := &Version{Value: vrs, SourceID: src} + assert.Equal(t, rev, revision.RevID) + assert.Equal(t, sv, revision.CV) + assert.Equal(t, docID, revision.DocID) + assert.Equal(t, []byte(`{"channels":["A"]}`), revision.BodyBytes) + } else { + require.Error(t, err) + assert.ErrorContains(t, err, ErrForbidden.Error()) + assert.Equal(t, DocumentRevision{}, revision) + } + }) + } +} diff --git a/db/database.go b/db/database.go index 25dff0ed8b..af2bdb7ec5 100644 --- a/db/database.go +++ b/db/database.go @@ -25,6 +25,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" pkgerrors "github.com/pkg/errors" + "golang.org/x/exp/maps" ) const ( @@ -48,6 +49,15 @@ const ( DBCompactRunning ) +const ( + Import DocUpdateType = iota + NewVersion + ExistingVersion + ExistingVersionWithUpdateToHLV +) + +type DocUpdateType uint32 + const ( DefaultRevsLimitNoConflicts = 50 DefaultRevsLimitConflicts = 100 @@ -92,6 +102,8 @@ type DatabaseContext struct { MetadataStore base.DataStore // Storage for database metadata (anything that isn't an end-user's/customer's documents) Bucket base.Bucket // Storage BucketSpec base.BucketSpec // The BucketSpec + BucketUUID string // The bucket UUID for the bucket the database is created against + EncodedSourceID string // The md5 hash of bucket UUID + cluster UUID for the bucket/cluster the database is created against but encoded in base64 BucketLock sync.RWMutex // Control Access to the underlying bucket object mutationListener changeListener // Caching feed listener ImportListener *importListener // Import feed listener @@ -110,6 +122,7 @@ type DatabaseContext struct { ResyncManager *BackgroundManager TombstoneCompactionManager *BackgroundManager AttachmentCompactionManager *BackgroundManager + AttachmentMigrationManager *BackgroundManager ExitChanges chan struct{} // Active _changes feeds on the DB will close when this channel is closed OIDCProviders auth.OIDCProviderMap // OIDC clients LocalJWTProviders auth.LocalJWTProviderMap @@ -134,6 +147,7 @@ type DatabaseContext struct { CollectionNames map[string]map[string]struct{} // Map of scope, collection names MetadataKeys *base.MetadataKeys // Factory to generate metadata document keys RequireResync base.ScopeAndCollectionNames // Collections requiring resync before database can go online + RequireAttachmentMigration base.ScopeAndCollectionNames // Collections that require the attachment migration background task to run against CORS *auth.CORSConfig // CORS configuration EnableMou bool // Write _mou xattr when performing metadata-only update. Set based on bucket capability on connect WasInitializedSynchronously bool // true if the database was initialized synchronously @@ -339,8 +353,8 @@ func ConnectToBucket(ctx context.Context, spec base.BucketSpec, failFast bool) ( return ibucket.(base.Bucket), nil } -// Returns Couchbase Server Cluster UUID on a timeout. If running against walrus, do return an empty string. -func getServerUUID(ctx context.Context, bucket base.Bucket) (string, error) { +// GetServerUUID returns Couchbase Server Cluster UUID on a timeout. If running against rosmar, do return an empty string. +func GetServerUUID(ctx context.Context, bucket base.Bucket) (string, error) { gocbV2Bucket, err := base.AsGocbV2Bucket(bucket) if err != nil { return "", nil @@ -379,7 +393,7 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket, return nil, err } - serverUUID, err := getServerUUID(ctx, bucket) + serverUUID, err := GetServerUUID(ctx, bucket) if err != nil { return nil, err } @@ -397,6 +411,15 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket, metadataStore = bucket.DefaultDataStore() } + bucketUUID, err := bucket.UUID() + if err != nil { + return nil, err + } + sourceID, err := CreateEncodedSourceID(bucketUUID, serverUUID) + if err != nil { + return nil, err + } + // Register the cbgt pindex type for the configGroup RegisterImportPindexImpl(ctx, options.GroupID) @@ -405,6 +428,8 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket, UUID: cbgt.NewUUID(), MetadataStore: metadataStore, Bucket: bucket, + BucketUUID: bucketUUID, + EncodedSourceID: sourceID, StartTime: time.Now(), autoImport: autoImport, Options: options, @@ -666,6 +691,14 @@ func (context *DatabaseContext) stopBackgroundManagers() []*BackgroundManager { } } + if context.AttachmentMigrationManager != nil { + if !isBackgroundManagerStopped(context.AttachmentMigrationManager.GetRunState()) { + if err := context.AttachmentMigrationManager.Stop(); err == nil { + bgManagers = append(bgManagers, context.AttachmentMigrationManager) + } + } + } + return bgManagers } @@ -937,6 +970,7 @@ type IDRevAndSequence struct { DocID string RevID string Sequence uint64 + CV string } // The ForEachDocID options for limiting query results @@ -972,13 +1006,15 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal var found bool var docid, revid string var seq uint64 + var cv string var channels []string if c.useViews() { var viewRow AllDocsViewQueryRow found = results.Next(ctx, &viewRow) if found { docid = viewRow.Key - revid = viewRow.Value.RevID + revid = viewRow.Value.RevID.RevTreeID + cv = viewRow.Value.RevID.CV() seq = viewRow.Value.Sequence channels = viewRow.Value.Channels } @@ -986,7 +1022,8 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal found = results.Next(ctx, &queryRow) if found { docid = queryRow.Id - revid = queryRow.RevID + revid = queryRow.RevID.RevTreeID + cv = queryRow.RevID.CV() seq = queryRow.Sequence channels = make([]string, 0) // Query returns all channels, but we only want to return active channels @@ -1001,7 +1038,7 @@ func (c *DatabaseCollection) processForEachDocIDResults(ctx context.Context, cal break } - if ok, err := callback(IDRevAndSequence{docid, revid, seq}, channels); ok { + if ok, err := callback(IDRevAndSequence{DocID: docid, RevID: revid, Sequence: seq, CV: cv}, channels); ok { count++ } else if err != nil { return err @@ -1804,7 +1841,7 @@ func (db *DatabaseCollectionWithUser) getResyncedDocument(ctx context.Context, d forceUpdate = true } - changedChannels, err := doc.updateChannels(ctx, channels) + changedChannels, _, err := doc.updateChannels(ctx, channels) changed = len(doc.Access.updateAccess(ctx, doc, access)) + len(doc.RoleAccess.updateAccess(ctx, doc, roles)) + len(changedChannels) @@ -1850,32 +1887,35 @@ func (db *DatabaseCollectionWithUser) resyncDocument(ctx context.Context, docid, } doc.SetCrc32cUserXattrHash() - // Update metadataOnlyUpdate based on previous Cas, metadataOnlyUpdate + // Update MetadataOnlyUpdate based on previous Cas, MetadataOnlyUpdate if db.useMou() { - doc.metadataOnlyUpdate = computeMetadataOnlyUpdate(doc.Cas, doc.metadataOnlyUpdate) + doc.MetadataOnlyUpdate = computeMetadataOnlyUpdate(doc.Cas, doc.RevSeqNo, doc.MetadataOnlyUpdate) } - _, rawXattr, rawMouXattr, err := updatedDoc.MarshalWithXattrs() + _, rawSyncXattr, rawVvXattr, rawMouXattr, rawGlobalXattr, err := updatedDoc.MarshalWithXattrs() updatedDoc := sgbucket.UpdatedDoc{ Doc: nil, // Resync does not require document body update Xattrs: map[string][]byte{ - base.SyncXattrName: rawXattr, + base.SyncXattrName: rawSyncXattr, + base.VvXattrName: rawVvXattr, }, Expiry: updatedExpiry, } if db.useMou() { updatedDoc.Xattrs[base.MouXattrName] = rawMouXattr - if doc.metadataOnlyUpdate.CAS == expandMacroCASValue { - updatedDoc.Spec = append(updatedDoc.Spec, sgbucket.NewMacroExpansionSpec(xattrMouCasPath(), sgbucket.MacroCas)) + if doc.MetadataOnlyUpdate.HexCAS == expandMacroCASValueString { + updatedDoc.Spec = append(updatedDoc.Spec, sgbucket.NewMacroExpansionSpec(XattrMouCasPath(), sgbucket.MacroCas)) } } - + if rawGlobalXattr != nil { + updatedDoc.Xattrs[base.GlobalXattrName] = rawGlobalXattr + } return updatedDoc, err } opts := &sgbucket.MutateInOptions{ MacroExpansion: macroExpandSpec(base.SyncXattrName), } - _, err = db.dataStore.WriteUpdateWithXattrs(ctx, key, db.syncMouAndUserXattrKeys(), 0, nil, opts, writeUpdateFunc) + _, err = db.dataStore.WriteUpdateWithXattrs(ctx, key, db.syncGlobalSyncMouRevSeqNoAndUserXattrKeys(), 0, nil, opts, writeUpdateFunc) } else { _, err = db.dataStore.Update(key, 0, func(currentValue []byte) ([]byte, *uint32, bool, error) { // Be careful: this block can be invoked multiple times if there are races! @@ -2454,6 +2494,16 @@ func (db *DatabaseContext) StartOnlineProcesses(ctx context.Context) (returnedEr } db.backgroundTasks = append(db.backgroundTasks, bgtSyncTime) + db.AttachmentMigrationManager = NewAttachmentMigrationManager(db) + // if we have collections requiring migration, run the job + if len(db.RequireAttachmentMigration) > 0 && !db.BucketSpec.IsWalrusBucket() { + err := db.AttachmentMigrationManager.Start(ctx, nil) + if err != nil { + base.WarnfCtx(ctx, "Error trying to migrate attachments for %s with error: %v", db.Name, err) + } + base.DebugfCtx(ctx, base.KeyAll, "Migrating attachment metadata automatically to Sync Gateway 4.0+ for collections %v", db.RequireAttachmentMigration) + } + if err := base.RequireNoBucketTTL(ctx, db.Bucket); err != nil { return err } @@ -2544,3 +2594,26 @@ func (db *Database) DataStoreNames() base.ScopeAndCollectionNames { } return names } + +// GetCollectionIDs will return all collection IDs for all collections configured on the database +func (db *DatabaseContext) GetCollectionIDs() []uint32 { + return maps.Keys(db.CollectionByID) +} + +// PurgeDCPCheckpoints will purge all DCP metadata from previous run in the bucket, used to reset dcp client to 0 +func PurgeDCPCheckpoints(ctx context.Context, database *DatabaseContext, checkpointPrefix string, taskID string) error { + + bucket, err := base.AsGocbV2Bucket(database.Bucket) + if err != nil { + return err + } + numVbuckets, err := bucket.GetMaxVbno() + if err != nil { + return err + } + + datastore := database.MetadataStore + metadata := base.NewDCPMetadataCS(ctx, datastore, numVbuckets, base.DefaultNumWorkers, checkpointPrefix) + metadata.Purge(ctx, base.DefaultNumWorkers) + return nil +} diff --git a/db/database_collection.go b/db/database_collection.go index 495865f8bb..8cc316c95f 100644 --- a/db/database_collection.go +++ b/db/database_collection.go @@ -237,9 +237,9 @@ func (c *DatabaseCollection) unsupportedOptions() *UnsupportedOptions { return c.dbCtx.Options.UnsupportedOptions } -// syncAndUserXattrKeys returns the xattr keys for the user and sync xattrs. -func (c *DatabaseCollection) syncAndUserXattrKeys() []string { - xattrKeys := []string{base.SyncXattrName} +// syncGlobalSyncAndUserXattrKeys returns the xattr keys for the user and sync xattrs. +func (c *DatabaseCollection) syncGlobalSyncAndUserXattrKeys() []string { + xattrKeys := []string{base.SyncXattrName, base.VvXattrName, base.GlobalXattrName} userXattrKey := c.userXattrKey() if userXattrKey != "" { xattrKeys = append(xattrKeys, userXattrKey) @@ -247,11 +247,11 @@ func (c *DatabaseCollection) syncAndUserXattrKeys() []string { return xattrKeys } -// syncMouAndUserXattrKeys returns the xattr keys for the user, mou and sync xattrs. -func (c *DatabaseCollection) syncMouAndUserXattrKeys() []string { - xattrKeys := []string{base.SyncXattrName} +// syncGlobalSyncMouRevSeqNoAndUserXattrKeys returns the xattr keys for the user, mou, revSeqNo and sync xattrs. +func (c *DatabaseCollection) syncGlobalSyncMouRevSeqNoAndUserXattrKeys() []string { + xattrKeys := []string{base.SyncXattrName, base.VvXattrName} if c.useMou() { - xattrKeys = append(xattrKeys, base.MouXattrName) + xattrKeys = append(xattrKeys, base.MouXattrName, base.VirtualXattrRevSeqNo, base.GlobalXattrName) } userXattrKey := c.userXattrKey() if userXattrKey != "" { diff --git a/db/database_test.go b/db/database_test.go index af2dd75bd5..0fc82f77ec 100644 --- a/db/database_test.go +++ b/db/database_test.go @@ -297,7 +297,7 @@ func TestDatabase(t *testing.T) { body["key2"] = int64(4444) history := []string{"4-four", "3-three", "2-488724414d0ed6b398d6d2aeb228d797", "1-cb0c9a22be0e5a1b01084ec019defa81"} - doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", body, history, false) + doc, newRev, err := collection.PutExistingRevWithBody(ctx, "doc1", body, history, false, ExistingVersionWithUpdateToHLV) body[BodyId] = doc.ID body[BodyRev] = newRev assert.NoError(t, err, "PutExistingRev failed") @@ -310,6 +310,139 @@ func TestDatabase(t *testing.T) { } +// TestCheckProposedVersion ensures that a given CV will return the appropriate status based on the information present in the HLV. +func TestCheckProposedVersion(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + // create a doc + body := Body{"key1": "value1", "key2": 1234} + _, doc, err := collection.Put(ctx, "doc1", body) + require.NoError(t, err) + cvSource, cvValue := doc.HLV.GetCurrentVersion() + currentVersion := Version{cvSource, cvValue} + + testCases := []struct { + name string + newVersion Version + previousVersion *Version + expectedStatus ProposedRevStatus + expectedRev string + }{ + { + // proposed version matches the current server version + // Already known + name: "version exists", + newVersion: currentVersion, + previousVersion: nil, + expectedStatus: ProposedRev_Exists, + expectedRev: "", + }, + { + // proposed version is newer than server cv (same source), and previousVersion matches server cv + // Not a conflict + name: "new version,same source,prev matches", + newVersion: Version{cvSource, incrementCas(cvValue, 100)}, + previousVersion: ¤tVersion, + expectedStatus: ProposedRev_OK, + expectedRev: "", + }, + { + // proposed version is newer than server cv (same source), and previousVersion is not specified. + // Not a conflict, even without previousVersion, because of source match + name: "new version,same source,prev not specified", + newVersion: Version{cvSource, incrementCas(cvValue, 100)}, + previousVersion: nil, + expectedStatus: ProposedRev_OK, + expectedRev: "", + }, + { + // proposed version is from a source not present in server HLV, and previousVersion matches server cv + // Not a conflict, due to previousVersion match + name: "new version,new source,prev matches", + newVersion: Version{"other", incrementCas(cvValue, 100)}, + previousVersion: ¤tVersion, + expectedStatus: ProposedRev_OK, + expectedRev: "", + }, + { + // proposed version is newer than server cv (same source), but previousVersion does not match server cv. + // Not a conflict, regardless of previousVersion mismatch, because of source match between proposed + // version and cv + name: "new version,prev mismatch,new matches cv", + newVersion: Version{cvSource, incrementCas(cvValue, 100)}, + previousVersion: &Version{"other", incrementCas(cvValue, 50)}, + expectedStatus: ProposedRev_OK, + expectedRev: "", + }, + { + // proposed version is already known, source matches cv + name: "proposed version already known, no prev version", + newVersion: Version{cvSource, incrementCas(cvValue, -100)}, + expectedStatus: ProposedRev_Exists, + expectedRev: "", + }, + { + // conflict - previous version is older than CV + name: "conflict,same source,server updated", + newVersion: Version{"other", incrementCas(cvValue, -100)}, + previousVersion: &Version{cvSource, incrementCas(cvValue, -50)}, + expectedStatus: ProposedRev_Conflict, + expectedRev: Version{cvSource, cvValue}.String(), + }, + { + // conflict - previous version is older than CV + name: "conflict,new source,server updated", + newVersion: Version{"other", incrementCas(cvValue, 100)}, + previousVersion: &Version{"other", incrementCas(cvValue, -50)}, + expectedStatus: ProposedRev_Conflict, + expectedRev: Version{cvSource, cvValue}.String(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + previousVersionStr := "" + if tc.previousVersion != nil { + previousVersionStr = tc.previousVersion.String() + } + status, rev := collection.CheckProposedVersion(ctx, "doc1", tc.newVersion.String(), previousVersionStr) + assert.Equal(t, tc.expectedStatus, status) + assert.Equal(t, tc.expectedRev, rev) + }) + } + + t.Run("invalid hlv", func(t *testing.T) { + hlvString := "" + status, _ := collection.CheckProposedVersion(ctx, "doc1", hlvString, "") + assert.Equal(t, ProposedRev_Error, status) + }) + + // New doc cases - standard insert + t.Run("new doc", func(t *testing.T) { + newVersion := Version{"other", 100}.String() + status, _ := collection.CheckProposedVersion(ctx, "doc2", newVersion, "") + assert.Equal(t, ProposedRev_OK_IsNew, status) + }) + + // New doc cases - insert with prev version (previous version purged from SGW) + t.Run("new doc with prev version", func(t *testing.T) { + newVersion := Version{"other", 100}.String() + prevVersion := Version{"another other", 50}.String() + status, _ := collection.CheckProposedVersion(ctx, "doc2", newVersion, prevVersion) + assert.Equal(t, ProposedRev_OK_IsNew, status) + }) + +} + +func incrementCas(cas uint64, delta int) (casOut uint64) { + cas = cas + uint64(delta) + return cas +} + func TestGetDeleted(t *testing.T) { db, ctx := setupTestDB(t) @@ -320,7 +453,7 @@ func TestGetDeleted(t *testing.T) { rev1id, _, err := collection.Put(ctx, "doc1", body) assert.NoError(t, err, "Put") - rev2id, err := collection.DeleteDoc(ctx, "doc1", rev1id) + rev2id, _, err := collection.DeleteDoc(ctx, "doc1", rev1id) assert.NoError(t, err, "DeleteDoc") // Get the deleted doc with its history; equivalent to GET with ?revs=true @@ -363,7 +496,7 @@ func TestGetRemovedAsUser(t *testing.T) { "key1": 1234, "channels": []string{"ABC"}, } - rev1id, _, err := collection.Put(ctx, "doc1", rev1body) + rev1id, docRev1, err := collection.Put(ctx, "doc1", rev1body) assert.NoError(t, err, "Put") rev2body := Body{ @@ -371,7 +504,7 @@ func TestGetRemovedAsUser(t *testing.T) { "channels": []string{"NBC"}, BodyRev: rev1id, } - rev2id, _, err := collection.Put(ctx, "doc1", rev2body) + rev2id, docRev2, err := collection.Put(ctx, "doc1", rev2body) assert.NoError(t, err, "Put Rev 2") // Add another revision, so that rev 2 is obsolete @@ -409,7 +542,9 @@ func TestGetRemovedAsUser(t *testing.T) { ShardCount: DefaultRevisionCacheShardCount, } collection.dbCtx.revisionCache = NewShardedLRURevisionCache(cacheOptions, backingStoreMap, cacheHitCounter, cacheMissCounter, cacheNumItems, memoryCacheStat) - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + cv := docRev2.HLV.GetCurrentVersionString() + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(cv))) assert.NoError(t, err, "Purge old revision JSON") // Try again with a user who doesn't have access to this revision @@ -435,7 +570,9 @@ func TestGetRemovedAsUser(t *testing.T) { assert.Equal(t, expectedResult, body) // Ensure revision is unavailable for a non-leaf revision that isn't available via the rev cache, and wasn't a channel removal - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev1id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + cv = docRev1.HLV.GetCurrentVersionString() + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(cv))) assert.NoError(t, err, "Purge old revision JSON") _, err = collection.Get1xRevBody(ctx, "doc1", rev1id, true, nil) @@ -498,7 +635,7 @@ func TestGetRemovalMultiChannel(t *testing.T) { "channels": []string{"ABC"}, BodyRev: rev1ID, } - rev2ID, _, err := collection.Put(ctx, "doc1", rev2Body) + rev2ID, docRev2, err := collection.Put(ctx, "doc1", rev2Body) require.NoError(t, err, "Error creating doc") // Create the third revision of doc1 on channel ABC. @@ -550,7 +687,8 @@ func TestGetRemovalMultiChannel(t *testing.T) { // Flush the revision cache and purge the old revision backup. db.FlushRevisionCacheForTest() - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(docRev2.HLV.GetCurrentVersionString()))) require.NoError(t, err, "Error purging old revision JSON") // Try with a user who has access to this revision. @@ -577,135 +715,212 @@ func TestGetRemovalMultiChannel(t *testing.T) { // Test delta sync behavior when the fromRevision is a channel removal. func TestDeltaSyncWhenFromRevIsChannelRemoval(t *testing.T) { - db, ctx := setupTestDB(t) - defer db.Close(ctx) - collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - - // Create the first revision of doc1. - rev1Body := Body{ - "k1": "v1", - "channels": []string{"ABC", "NBC"}, - } - rev1ID, _, err := collection.Put(ctx, "doc1", rev1Body) - require.NoError(t, err, "Error creating doc") - - // Create the second revision of doc1 on channel ABC as removal from channel NBC. - rev2Body := Body{ - "k2": "v2", - "channels": []string{"ABC"}, - BodyRev: rev1ID, - } - rev2ID, _, err := collection.Put(ctx, "doc1", rev2Body) - require.NoError(t, err, "Error creating doc") - - // Create the third revision of doc1 on channel ABC. - rev3Body := Body{ - "k3": "v3", - "channels": []string{"ABC"}, - BodyRev: rev2ID, + testCases := []struct { + name string + versionVector bool + }{ + // Revs are backed up by hash of CV now, now way to fetch backup revs by revID till CBG-3748 (backwards compatibility for revID) + //{ + // name: "revTree test", + // versionVector: false, + //}, + { + name: "versionVector test", + versionVector: true, + }, } - rev3ID, _, err := collection.Put(ctx, "doc1", rev3Body) - require.NoError(t, err, "Error creating doc") - require.NotEmpty(t, rev3ID, "Error creating doc") - - // Flush the revision cache and purge the old revision backup. - db.FlushRevisionCacheForTest() - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2ID) - require.NoError(t, err, "Error purging old revision JSON") - - // Request delta between rev2ID and rev3ID (toRevision "rev2ID" is channel removal) - // as a user who doesn't have access to the removed revision via any other channel. - authenticator := db.Authenticator(ctx) - user, err := authenticator.NewUser("alice", "pass", base.SetOf("NBC")) - require.NoError(t, err, "Error creating user") - - collection.user = user - require.NoError(t, db.DbStats.InitDeltaSyncStats()) - - delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2ID, rev3ID) - require.Equal(t, base.HTTPErrorf(404, "missing"), err) - assert.Nil(t, delta) - assert.Nil(t, redactedRev) + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - // Request delta between rev2ID and rev3ID (toRevision "rev2ID" is channel removal) - // as a user who has access to the removed revision via another channel. - user, err = authenticator.NewUser("bob", "pass", base.SetOf("ABC")) - require.NoError(t, err, "Error creating user") + // Create the first revision of doc1. + rev1Body := Body{ + "k1": "v1", + "channels": []string{"ABC", "NBC"}, + } + rev1ID, _, err := collection.Put(ctx, "doc1", rev1Body) + require.NoError(t, err, "Error creating doc") + + // Create the second revision of doc1 on channel ABC as removal from channel NBC. + rev2Body := Body{ + "k2": "v2", + "channels": []string{"ABC"}, + BodyRev: rev1ID, + } + rev2ID, docRev2, err := collection.Put(ctx, "doc1", rev2Body) + require.NoError(t, err, "Error creating doc") + + // Create the third revision of doc1 on channel ABC. + rev3Body := Body{ + "k3": "v3", + "channels": []string{"ABC"}, + BodyRev: rev2ID, + } + rev3ID, docRev3, err := collection.Put(ctx, "doc1", rev3Body) + require.NoError(t, err, "Error creating doc") + require.NotEmpty(t, rev3ID, "Error creating doc") + + // Flush the revision cache and purge the old revision backup. + db.FlushRevisionCacheForTest() + if testCase.versionVector { + cvStr := docRev2.HLV.GetCurrentVersionString() + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(cvStr))) + require.NoError(t, err, "Error purging old revision JSON") + } else { + err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2ID) + require.NoError(t, err, "Error purging old revision JSON") + } - collection.user = user - require.NoError(t, db.DbStats.InitDeltaSyncStats()) + // Request delta between rev2ID and rev3ID (toRevision "rev2ID" is channel removal) + // as a user who doesn't have access to the removed revision via any other channel. + authenticator := db.Authenticator(ctx) + user, err := authenticator.NewUser("alice", "pass", base.SetOf("NBC")) + require.NoError(t, err, "Error creating user") + + collection.user = user + require.NoError(t, db.DbStats.InitDeltaSyncStats()) + + if testCase.versionVector { + rev2 := docRev2.HLV.ExtractCurrentVersionFromHLV() + rev3 := docRev3.HLV.ExtractCurrentVersionFromHLV() + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2.String(), rev3.String(), true) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } else { + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2ID, rev3ID, false) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } - delta, redactedRev, err = collection.GetDelta(ctx, "doc1", rev2ID, rev3ID) - require.Equal(t, base.HTTPErrorf(404, "missing"), err) - assert.Nil(t, delta) - assert.Nil(t, redactedRev) + // Request delta between rev2ID and rev3ID (toRevision "rev2ID" is channel removal) + // as a user who has access to the removed revision via another channel. + user, err = authenticator.NewUser("bob", "pass", base.SetOf("ABC")) + require.NoError(t, err, "Error creating user") + + collection.user = user + require.NoError(t, db.DbStats.InitDeltaSyncStats()) + + if testCase.versionVector { + rev2 := docRev2.HLV.ExtractCurrentVersionFromHLV() + rev3 := docRev3.HLV.ExtractCurrentVersionFromHLV() + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2.String(), rev3.String(), true) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } else { + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2ID, rev3ID, false) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } + }) + } } // Test delta sync behavior when the toRevision is a channel removal. func TestDeltaSyncWhenToRevIsChannelRemoval(t *testing.T) { - db, ctx := setupTestDB(t) - defer db.Close(ctx) - collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) - - // Create the first revision of doc1. - rev1Body := Body{ - "k1": "v1", - "channels": []string{"ABC", "NBC"}, + t.Skip("Pending work for channel removal at rev cache CBG-3814") + testCases := []struct { + name string + versionVector bool + }{ + { + name: "revTree test", + versionVector: false, + }, + { + name: "versionVector test", + versionVector: true, + }, } - rev1ID, _, err := collection.Put(ctx, "doc1", rev1Body) - require.NoError(t, err, "Error creating doc") + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) - // Create the second revision of doc1 on channel ABC as removal from channel NBC. - rev2Body := Body{ - "k2": "v2", - "channels": []string{"ABC"}, - BodyRev: rev1ID, - } - rev2ID, _, err := collection.Put(ctx, "doc1", rev2Body) - require.NoError(t, err, "Error creating doc") + // Create the first revision of doc1. + rev1Body := Body{ + "k1": "v1", + "channels": []string{"ABC", "NBC"}, + } + rev1ID, _, err := collection.Put(ctx, "doc1", rev1Body) + require.NoError(t, err, "Error creating doc") + + // Create the second revision of doc1 on channel ABC as removal from channel NBC. + rev2Body := Body{ + "k2": "v2", + "channels": []string{"ABC"}, + BodyRev: rev1ID, + } + rev2ID, docRev2, err := collection.Put(ctx, "doc1", rev2Body) + require.NoError(t, err, "Error creating doc") + + // Create the third revision of doc1 on channel ABC. + rev3Body := Body{ + "k3": "v3", + "channels": []string{"ABC"}, + BodyRev: rev2ID, + } + rev3ID, docRev3, err := collection.Put(ctx, "doc1", rev3Body) + require.NoError(t, err, "Error creating doc") + require.NotEmpty(t, rev3ID, "Error creating doc") + + // Flush the revision cache and purge the old revision backup. + db.FlushRevisionCacheForTest() + err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2ID) + require.NoError(t, err, "Error purging old revision JSON") + + // Request delta between rev1ID and rev2ID (toRevision "rev2ID" is channel removal) + // as a user who doesn't have access to the removed revision via any other channel. + authenticator := db.Authenticator(ctx) + user, err := authenticator.NewUser("alice", "pass", base.SetOf("NBC")) + require.NoError(t, err, "Error creating user") + + collection.user = user + require.NoError(t, db.DbStats.InitDeltaSyncStats()) + + if testCase.versionVector { + rev2 := docRev2.HLV.ExtractCurrentVersionFromHLV() + rev3 := docRev3.HLV.ExtractCurrentVersionFromHLV() + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2.String(), rev3.String(), true) + require.NoError(t, err) + assert.Nil(t, delta) + assert.Equal(t, `{"_removed":true}`, string(redactedRev.BodyBytes)) + } else { + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev1ID, rev2ID, false) + require.NoError(t, err) + assert.Nil(t, delta) + assert.Equal(t, `{"_removed":true}`, string(redactedRev.BodyBytes)) + } - // Create the third revision of doc1 on channel ABC. - rev3Body := Body{ - "k3": "v3", - "channels": []string{"ABC"}, - BodyRev: rev2ID, + // Request delta between rev1ID and rev2ID (toRevision "rev2ID" is channel removal) + // as a user who has access to the removed revision via another channel. + user, err = authenticator.NewUser("bob", "pass", base.SetOf("ABC")) + require.NoError(t, err, "Error creating user") + + collection.user = user + require.NoError(t, db.DbStats.InitDeltaSyncStats()) + if testCase.versionVector { + rev2 := docRev2.HLV.ExtractCurrentVersionFromHLV() + rev3 := docRev3.HLV.ExtractCurrentVersionFromHLV() + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev2.String(), rev3.String(), true) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } else { + delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev1ID, rev2ID, false) + require.Equal(t, base.HTTPErrorf(404, "missing"), err) + assert.Nil(t, delta) + assert.Nil(t, redactedRev) + } + }) } - rev3ID, _, err := collection.Put(ctx, "doc1", rev3Body) - require.NoError(t, err, "Error creating doc") - require.NotEmpty(t, rev3ID, "Error creating doc") - - // Flush the revision cache and purge the old revision backup. - db.FlushRevisionCacheForTest() - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2ID) - require.NoError(t, err, "Error purging old revision JSON") - - // Request delta between rev1ID and rev2ID (toRevision "rev2ID" is channel removal) - // as a user who doesn't have access to the removed revision via any other channel. - authenticator := db.Authenticator(ctx) - user, err := authenticator.NewUser("alice", "pass", base.SetOf("NBC")) - require.NoError(t, err, "Error creating user") - - collection.user = user - require.NoError(t, db.DbStats.InitDeltaSyncStats()) - - delta, redactedRev, err := collection.GetDelta(ctx, "doc1", rev1ID, rev2ID) - require.NoError(t, err) - assert.Nil(t, delta) - assert.Equal(t, `{"_removed":true}`, string(redactedRev.BodyBytes)) - - // Request delta between rev1ID and rev2ID (toRevision "rev2ID" is channel removal) - // as a user who has access to the removed revision via another channel. - user, err = authenticator.NewUser("bob", "pass", base.SetOf("ABC")) - require.NoError(t, err, "Error creating user") - - collection.user = user - require.NoError(t, db.DbStats.InitDeltaSyncStats()) - - delta, redactedRev, err = collection.GetDelta(ctx, "doc1", rev1ID, rev2ID) - require.Equal(t, base.HTTPErrorf(404, "missing"), err) - assert.Nil(t, delta) - assert.Nil(t, redactedRev) } // Test retrieval of a channel removal revision, when the revision is not otherwise available @@ -720,7 +935,7 @@ func TestGetRemoved(t *testing.T) { "key1": 1234, "channels": []string{"ABC"}, } - rev1id, _, err := collection.Put(ctx, "doc1", rev1body) + rev1id, docRev1, err := collection.Put(ctx, "doc1", rev1body) assert.NoError(t, err, "Put") rev2body := Body{ @@ -728,7 +943,7 @@ func TestGetRemoved(t *testing.T) { "channels": []string{"NBC"}, BodyRev: rev1id, } - rev2id, _, err := collection.Put(ctx, "doc1", rev2body) + rev2id, docRev2, err := collection.Put(ctx, "doc1", rev2body) assert.NoError(t, err, "Put Rev 2") // Add another revision, so that rev 2 is obsolete @@ -766,7 +981,8 @@ func TestGetRemoved(t *testing.T) { ShardCount: DefaultRevisionCacheShardCount, } collection.dbCtx.revisionCache = NewShardedLRURevisionCache(cacheOptions, backingStoreMap, cacheHitCounter, cacheMissCounter, cacheNumItems, memoryCacheStat) - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(docRev2.HLV.GetCurrentVersionString()))) assert.NoError(t, err, "Purge old revision JSON") // Get the removal revision with its history; equivalent to GET with ?revs=true @@ -775,7 +991,8 @@ func TestGetRemoved(t *testing.T) { require.Nil(t, body) // Ensure revision is unavailable for a non-leaf revision that isn't available via the rev cache, and wasn't a channel removal - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev1id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) assert.NoError(t, err, "Purge old revision JSON") _, err = collection.Get1xRevBody(ctx, "doc1", rev1id, true, nil) @@ -794,7 +1011,7 @@ func TestGetRemovedAndDeleted(t *testing.T) { "key1": 1234, "channels": []string{"ABC"}, } - rev1id, _, err := collection.Put(ctx, "doc1", rev1body) + rev1id, docRev1, err := collection.Put(ctx, "doc1", rev1body) assert.NoError(t, err, "Put") rev2body := Body{ @@ -802,7 +1019,7 @@ func TestGetRemovedAndDeleted(t *testing.T) { BodyDeleted: true, BodyRev: rev1id, } - rev2id, _, err := collection.Put(ctx, "doc1", rev2body) + rev2id, docRev2, err := collection.Put(ctx, "doc1", rev2body) assert.NoError(t, err, "Put Rev 2") // Add another revision, so that rev 2 is obsolete @@ -840,7 +1057,8 @@ func TestGetRemovedAndDeleted(t *testing.T) { ShardCount: DefaultRevisionCacheShardCount, } collection.dbCtx.revisionCache = NewShardedLRURevisionCache(cacheOptions, backingStoreMap, cacheHitCounter, cacheMissCounter, cacheNumItems, memoryCacheStats) - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev2id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(docRev2.HLV.GetCurrentVersionString()))) assert.NoError(t, err, "Purge old revision JSON") // Get the deleted doc with its history; equivalent to GET with ?revs=true @@ -849,7 +1067,8 @@ func TestGetRemovedAndDeleted(t *testing.T) { require.Nil(t, body) // Ensure revision is unavailable for a non-leaf revision that isn't available via the rev cache, and wasn't a channel removal - err = collection.PurgeOldRevisionJSON(ctx, "doc1", rev1id) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + err = collection.PurgeOldRevisionJSON(ctx, "doc1", base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) assert.NoError(t, err, "Purge old revision JSON") _, err = collection.Get1xRevBody(ctx, "doc1", rev1id, true, nil) @@ -922,7 +1141,7 @@ func TestAllDocsOnly(t *testing.T) { } // Now delete one document and try again: - _, err = collection.DeleteDoc(ctx, ids[23].DocID, ids[23].RevID) + _, _, err = collection.DeleteDoc(ctx, ids[23].DocID, ids[23].RevID) assert.NoError(t, err, "Couldn't delete doc 23") alldocs, err = allDocIDs(ctx, collection.DatabaseCollection) @@ -1038,18 +1257,18 @@ func TestRepeatedConflict(t *testing.T) { // Create rev 1 of "doc": body := Body{"n": 1, "channels": []string{"all", "1"}} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create two conflicting changes: body["n"] = 2 body["channels"] = []string{"all", "2b"} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") body["n"] = 3 body["channels"] = []string{"all", "2a"} - _, newRev, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false) + _, newRev, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") // Get the _rev that was set in the body by PutExistingRevWithBody() and make assertions on it @@ -1058,7 +1277,7 @@ func TestRepeatedConflict(t *testing.T) { // Remove the _rev key from the body, and call PutExistingRevWithBody() again, which should re-add it delete(body, BodyRev) - _, newRev, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false) + _, newRev, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err) // The _rev should pass the same assertions as before, since PutExistingRevWithBody() should re-add it @@ -1072,6 +1291,7 @@ func TestConflicts(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close(ctx) collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + bucketUUID := db.EncodedSourceID collection.ChannelMapper = channels.NewChannelMapper(ctx, channels.DocChannelsSyncFunction, db.Options.JavascriptTimeout) @@ -1086,7 +1306,7 @@ func TestConflicts(t *testing.T) { // Create rev 1 of "doc": body := Body{"n": 1, "channels": []string{"all", "1"}} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Wait for rev to be cached @@ -1099,11 +1319,11 @@ func TestConflicts(t *testing.T) { // Create two conflicting changes: body["n"] = 2 body["channels"] = []string{"all", "2b"} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") body["n"] = 3 body["channels"] = []string{"all", "2a"} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") cacheWaiter.Add(2) @@ -1143,19 +1363,23 @@ func TestConflicts(t *testing.T) { Conflicts: true, ChangesCtx: base.TestCtx(t), } + changes := getChanges(t, collection, channels.BaseSetOf(t, "all"), options) + source, version := collection.GetDocumentCurrentVersion(t, "doc") + assert.Len(t, changes, 1) assert.Equal(t, &ChangeEntry{ - Seq: SequenceID{Seq: 3}, - ID: "doc", - Changes: []ChangeRev{{"rev": "2-b"}, {"rev": "2-a"}}, - branched: true, - collectionID: collectionID, + Seq: SequenceID{Seq: 3}, + ID: "doc", + Changes: []ChangeRev{{"rev": "2-b"}, {"rev": "2-a"}}, + branched: true, + collectionID: collectionID, + CurrentVersion: &Version{SourceID: source, Value: version}, }, changes[0], ) // Delete 2-b; verify this makes 2-a current: - rev3, err := collection.DeleteDoc(ctx, "doc", "2-b") + rev3, _, err := collection.DeleteDoc(ctx, "doc", "2-b") assert.NoError(t, err, "delete 2-b") rawBody, _, _ = collection.dataStore.GetRaw("doc") @@ -1180,11 +1404,12 @@ func TestConflicts(t *testing.T) { changes = getChanges(t, collection, channels.BaseSetOf(t, "all"), options) assert.Len(t, changes, 1) assert.Equal(t, &ChangeEntry{ - Seq: SequenceID{Seq: 4}, - ID: "doc", - Changes: []ChangeRev{{"rev": "2-a"}, {"rev": rev3}}, - branched: true, - collectionID: collectionID, + Seq: SequenceID{Seq: 4}, + ID: "doc", + Changes: []ChangeRev{{"rev": "2-a"}, {"rev": rev3}}, + branched: true, + collectionID: collectionID, + CurrentVersion: &Version{SourceID: bucketUUID, Value: doc.Cas}, }, changes[0]) } @@ -1229,55 +1454,55 @@ func TestNoConflictsMode(t *testing.T) { // Create revs 1 and 2 of "doc": body := Body{"n": 1, "channels": []string{"all", "1"}} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") body["n"] = 2 - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") // Try to create a conflict branching from rev 1: - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assertHTTPError(t, err, 409) // Try to create a conflict with no common ancestor: - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-c", "1-c"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-c", "1-c"}, false, ExistingVersionWithUpdateToHLV) assertHTTPError(t, err, 409) // Try to create a conflict with a longer history: - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-d", "3-d", "2-d", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-d", "3-d", "2-d", "1-a"}, false, ExistingVersionWithUpdateToHLV) assertHTTPError(t, err, 409) // Try to create a conflict with no history: - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-e"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-e"}, false, ExistingVersionWithUpdateToHLV) assertHTTPError(t, err, 409) // Create a non-conflict with a longer history, ending in a deletion: body[BodyDeleted] = true - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 4-a") delete(body, BodyDeleted) // Try to resurrect the document with a conflicting branch - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-f", "3-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-f", "3-a"}, false, ExistingVersionWithUpdateToHLV) assertHTTPError(t, err, 409) // Resurrect the tombstoned document with a disconnected branch): - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-f"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-f"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-f") // Tombstone the resurrected branch body[BodyDeleted] = true - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-f", "1-f"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-f", "1-f"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-f") delete(body, BodyDeleted) // Resurrect the tombstoned document with a valid history (descendents of leaf) - _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-f", "4-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-f", "4-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 5-f") delete(body, BodyDeleted) // Create a new document with a longer history: - _, _, err = collection.PutExistingRevWithBody(ctx, "COD", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "COD", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add COD") delete(body, BodyDeleted) @@ -1305,34 +1530,34 @@ func TestAllowConflictsFalseTombstoneExistingConflict(t *testing.T) { // Create documents with multiple non-deleted branches log.Printf("Creating docs") body := Body{"n": 1} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create two conflicting changes: body["n"] = 2 - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") body["n"] = 3 - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") // Set AllowConflicts to false db.Options.AllowConflicts = base.BoolPtr(false) // Attempt to tombstone a non-leaf node of a conflicted document - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-c", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-c", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.True(t, err != nil, "expected error tombstoning non-leaf") // Tombstone the non-winning branch of a conflicted document @@ -1382,27 +1607,27 @@ func TestAllowConflictsFalseTombstoneExistingConflictNewEditsFalse(t *testing.T) // Create documents with multiple non-deleted branches log.Printf("Creating docs") body := Body{"n": 1} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 1-a") // Create two conflicting changes: body["n"] = 2 - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-b") body["n"] = 3 - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 2-a") // Set AllowConflicts to false @@ -1411,12 +1636,12 @@ func TestAllowConflictsFalseTombstoneExistingConflictNewEditsFalse(t *testing.T) // Attempt to tombstone a non-leaf node of a conflicted document body[BodyDeleted] = true - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-c", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-c", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.True(t, err != nil, "expected error tombstoning non-leaf") // Tombstone the non-winning branch of a conflicted document body[BodyDeleted] = true - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-a (tombstone)") doc, err := collection.GetDocument(ctx, "doc1", DocUnmarshalAll) assert.NoError(t, err, "Retrieve doc post-tombstone") @@ -1424,7 +1649,7 @@ func TestAllowConflictsFalseTombstoneExistingConflictNewEditsFalse(t *testing.T) // Tombstone the winning branch of a conflicted document body[BodyDeleted] = true - _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"3-b", "2-b"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc2", body, []string{"3-b", "2-b"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-b (tombstone)") doc, err = collection.GetDocument(ctx, "doc2", DocUnmarshalAll) assert.NoError(t, err, "Retrieve doc post-tombstone") @@ -1433,7 +1658,7 @@ func TestAllowConflictsFalseTombstoneExistingConflictNewEditsFalse(t *testing.T) // Set revs_limit=1, then tombstone non-winning branch of a conflicted document. Validate retrieval still works. body[BodyDeleted] = true db.RevsLimit = uint32(1) - _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"3-a", "2-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc3", body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "add 3-a (tombstone)") doc, err = collection.GetDocument(ctx, "doc3", DocUnmarshalAll) assert.NoError(t, err, "Retrieve doc post-tombstone") @@ -1469,7 +1694,7 @@ func TestSyncFnOnPush(t *testing.T) { body["channels"] = "clibup" history := []string{"4-four", "3-three", "2-488724414d0ed6b398d6d2aeb228d797", rev1id} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, history, false) + newDoc, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, history, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "PutExistingRev failed") // Check that the doc has the correct channel (test for issue #300) @@ -1477,7 +1702,7 @@ func TestSyncFnOnPush(t *testing.T) { require.NoError(t, err) assert.Equal(t, channels.ChannelMap{ "clibup": nil, - "public": &channels.ChannelRemoval{Seq: 2, RevID: "4-four"}, + "public": &channels.ChannelRemoval{Seq: 2, Rev: channels.RevAndVersion{RevTreeID: "4-four", CurrentSource: newDoc.HLV.SourceID, CurrentVersion: base.CasToString(newDoc.HLV.Version)}}, }, doc.Channels) assert.Equal(t, base.SetOf("clibup"), doc.History["4-four"].Channels) @@ -1835,6 +2060,164 @@ func TestChannelView(t *testing.T) { log.Printf("View Query returned entry (%d): %v", i, entry) } assert.Len(t, entries, 1) + require.Equal(t, "doc1", entries[0].DocID) + collection.RequireCurrentVersion(t, "doc1", entries[0].SourceID, entries[0].Version) +} + +func TestChannelQuery(t *testing.T) { + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + _, err := collection.UpdateSyncFun(ctx, `function(doc, oldDoc) { + channel(doc.channels); + }`) + require.NoError(t, err) + + // Create doc + body := Body{"key1": "value1", "key2": 1234, "channels": "ABC"} + rev1ID, _, err := collection.Put(ctx, "doc1", body) + require.NoError(t, err, "Couldn't create doc1") + + // Create a doc to test removal handling. Needs three revisions so that the removal rev (2) isn't + // the current revision + removedDocID := "removed_doc" + removedDocRev1, rev1, err := collection.Put(ctx, removedDocID, body) + require.NoError(t, err, "Couldn't create removed_doc") + + updatedChannelBody := Body{"_rev": removedDocRev1, "key1": "value1", "key2": 1234, "channels": "DEF"} + removalRev, rev2, err := collection.Put(ctx, removedDocID, updatedChannelBody) + require.NoError(t, err, "Couldn't update removed_doc") + + updatedChannelBody = Body{"_rev": removalRev, "key1": "value1", "key2": 2345, "channels": "DEF"} + _, rev3, err := collection.Put(ctx, removedDocID, updatedChannelBody) + require.NoError(t, err, "Couldn't update removed_doc") + + log.Printf("versions: [%v %v %v]", rev1.HLV.Version, rev2.HLV.Version, rev3.HLV.Version) + + // TODO: check the case where the channel is removed with a putExistingRev mutation + + var entries LogEntries + + // Test query retrieval via star channel and named channel (queries use different indexes) + testCases := []struct { + testName string + channelName string + expectedRev channels.RevAndVersion + }{ + { + testName: "star channel", + channelName: "*", + expectedRev: rev3.GetRevAndVersion(), + }, + { + testName: "named channel", + channelName: "ABC", + expectedRev: rev2.GetRevAndVersion(), + }, + } + + for _, testCase := range testCases { + t.Run(testCase.testName, func(t *testing.T) { + entries, err = collection.getChangesInChannelFromQuery(ctx, testCase.channelName, 0, 100, 0, false) + require.NoError(t, err) + + for i, entry := range entries { + log.Printf("Channel Query returned entry (%d): %v", i, entry) + } + require.Len(t, entries, 2) + require.Equal(t, "doc1", entries[0].DocID) + require.Equal(t, rev1ID, entries[0].RevID) + collection.RequireCurrentVersion(t, "doc1", entries[0].SourceID, entries[0].Version) + + removedDocEntry := entries[1] + require.Equal(t, removedDocID, removedDocEntry.DocID) + + log.Printf("removedDocEntry Version: %v", removedDocEntry.Version) + require.Equal(t, testCase.expectedRev.RevTreeID, removedDocEntry.RevID) + require.Equal(t, testCase.expectedRev.CurrentSource, removedDocEntry.SourceID) + require.Equal(t, base.HexCasToUint64(testCase.expectedRev.CurrentVersion), removedDocEntry.Version) + }) + } + +} + +// TestChannelQueryRevocation ensures that the correct rev (revTreeID and cv) is returned by the channel query. +func TestChannelQueryRevocation(t *testing.T) { + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + _, err := collection.UpdateSyncFun(ctx, `function(doc, oldDoc) { + channel(doc.channels); + }`) + require.NoError(t, err) + + // Create doc with three channels (ABC, DEF, GHI) + docID := "removalTestDoc" + body := Body{"key1": "value1", "key2": 1234, "channels": []string{"ABC", "DEF", "GHI"}} + rev1ID, _, err := collection.Put(ctx, docID, body) + require.NoError(t, err, "Couldn't create document") + + // Update the doc with a simple PUT to remove channel ABC + updatedChannelBody := Body{"_rev": rev1ID, "key1": "value1", "key2": 1234, "channels": []string{"DEF", "GHI"}} + _, rev2, err := collection.Put(ctx, docID, updatedChannelBody) + require.NoError(t, err, "Couldn't update document via Put") + + // Update the doc with PutExistingCurrentVersion to remove channel DEF + /* TODO: requires fix to HLV conflict detection + updatedChannelBody = Body{"_rev": rev2ID, "key1": "value1", "key2": 2345, "channels": "GHI"} + existingDoc, err := collection.GetDocument(ctx, docID, DocUnmarshalAll) + require.NoError(t, err) + cblVersion := Version{SourceID: "CBLSource", Value: existingDoc.HLV.Version + 10} + hlvErr := existingDoc.HLV.AddVersion(cblVersion) + require.NoError(t, hlvErr) + existingDoc.UpdateBody(updatedChannelBody) + rev3, _, _, err := collection.PutExistingCurrentVersion(ctx, existingDoc, *existingDoc.HLV, nil) + require.NoError(t, err, "Couldn't update document via PutExistingCurrentVersion") + + */ + + var entries LogEntries + + // Test query retrieval via star channel and named channel (queries use different indexes) + testCases := []struct { + testName string + channelName string + expectedRev channels.RevAndVersion + }{ + { + testName: "removal by SGW write", + channelName: "ABC", + expectedRev: rev2.GetRevAndVersion(), + }, + /* + { + testName: "removal by CBL write", + channelName: "DEF", + expectedRev: rev3.GetRevAndVersion(), + }, + */ + } + + for _, testCase := range testCases { + t.Run(testCase.testName, func(t *testing.T) { + entries, err = collection.getChangesInChannelFromQuery(ctx, testCase.channelName, 0, 100, 0, false) + require.NoError(t, err) + + for i, entry := range entries { + log.Printf("Channel Query returned entry (%d): %v", i, entry) + } + require.Len(t, entries, 1) + removedDocEntry := entries[0] + require.Equal(t, docID, removedDocEntry.DocID) + + log.Printf("removedDocEntry Version: %v", removedDocEntry.Version) + require.Equal(t, testCase.expectedRev.RevTreeID, removedDocEntry.RevID) + require.Equal(t, testCase.expectedRev.CurrentSource, removedDocEntry.SourceID) + require.Equal(t, base.HexCasToUint64(testCase.expectedRev.CurrentVersion), removedDocEntry.Version) + }) + } } @@ -2158,7 +2541,7 @@ func TestConcurrentPushSameNewNonWinningRevision(t *testing.T) { enableCallback = false body := Body{"name": "Emily", "age": 20} collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b", "2-b", "1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 3-b") } } @@ -2173,29 +2556,29 @@ func TestConcurrentPushSameNewNonWinningRevision(t *testing.T) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) body := Body{"name": "Olivia", "age": 80} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 1-a") body = Body{"name": "Harry", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-a") body = Body{"name": "Amelia", "age": 20} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 3-a") body = Body{"name": "Charlie", "age": 10} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 4-a") body = Body{"name": "Noah", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-b") enableCallback = true body = Body{"name": "Emily", "age": 20} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 3-b") doc, err := collection.GetDocument(ctx, "doc1", DocUnmarshalAll) @@ -2216,7 +2599,7 @@ func TestConcurrentPushSameTombstoneWinningRevision(t *testing.T) { enableCallback = false body := Body{"name": "Charlie", "age": 10, BodyDeleted: true} collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't add revision 4-a (tombstone)") } } @@ -2231,19 +2614,19 @@ func TestConcurrentPushSameTombstoneWinningRevision(t *testing.T) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) body := Body{"name": "Olivia", "age": 80} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 1-a") body = Body{"name": "Harry", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-a") body = Body{"name": "Amelia", "age": 20} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 3-a") body = Body{"name": "Noah", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-b") doc, err := collection.GetDocument(ctx, "doc1", DocUnmarshalAll) @@ -2253,7 +2636,7 @@ func TestConcurrentPushSameTombstoneWinningRevision(t *testing.T) { enableCallback = true body = Body{"name": "Charlie", "age": 10, BodyDeleted: true} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't add revision 4-a (tombstone)") doc, err = collection.GetDocument(ctx, "doc1", DocUnmarshalAll) @@ -2274,7 +2657,7 @@ func TestConcurrentPushDifferentUpdateNonWinningRevision(t *testing.T) { enableCallback = false body := Body{"name": "Joshua", "age": 11} collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b1", "2-b", "1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b1", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't add revision 3-b1") } } @@ -2289,29 +2672,29 @@ func TestConcurrentPushDifferentUpdateNonWinningRevision(t *testing.T) { collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) body := Body{"name": "Olivia", "age": 80} - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 1-a") body = Body{"name": "Harry", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-a") body = Body{"name": "Amelia", "age": 20} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 3-a") body = Body{"name": "Charlie", "age": 10} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"4-a", "3-a", "2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 4-a") body = Body{"name": "Noah", "age": 40} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Adding revision 2-b") enableCallback = true body = Body{"name": "Liam", "age": 12} - _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b2", "2-b", "1-a"}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-b2", "2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Couldn't add revision 3-b2") doc, err := collection.GetDocument(ctx, "doc1", DocUnmarshalAll) @@ -2345,7 +2728,7 @@ func TestIncreasingRecentSequences(t *testing.T) { enableCallback = false // Write a doc collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-abc", revid}, true) + _, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"2-abc", revid}, true, ExistingVersionWithUpdateToHLV) assert.NoError(t, err) } } @@ -2362,7 +2745,7 @@ func TestIncreasingRecentSequences(t *testing.T) { assert.NoError(t, err) enableCallback = true - doc, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-abc", "2-abc", revid}, true) + doc, _, err := collection.PutExistingRevWithBody(ctx, "doc1", body, []string{"3-abc", "2-abc", revid}, true, ExistingVersionWithUpdateToHLV) assert.NoError(t, err) assert.True(t, sort.IsSorted(base.SortedUint64Slice(doc.SyncData.RecentSequences))) @@ -2457,7 +2840,7 @@ func TestDeleteWithNoTombstoneCreationSupport(t *testing.T) { assert.NoError(t, err) var doc Body - var xattr Body + var xattr SyncData var xattrs map[string][]byte // Ensure document has been added @@ -2471,8 +2854,8 @@ func TestDeleteWithNoTombstoneCreationSupport(t *testing.T) { assert.Equal(t, int64(1), db.DbStats.SharedBucketImport().ImportCount.Value()) assert.Nil(t, doc) - assert.Equal(t, "1-2cac91faf7b3f5e5fd56ff377bdb5466", xattr["rev"]) - assert.Equal(t, float64(2), xattr["sequence"]) + assert.Equal(t, "1-2cac91faf7b3f5e5fd56ff377bdb5466", xattr.CurrentRev) + assert.Equal(t, uint64(2), xattr.Sequence) } func TestResyncUpdateAllDocChannels(t *testing.T) { @@ -2529,7 +2912,7 @@ func TestTombstoneCompactionStopWithManager(t *testing.T) { docID := fmt.Sprintf("doc%d", i) rev, _, err := collection.Put(ctx, docID, Body{}) assert.NoError(t, err) - _, err = collection.DeleteDoc(ctx, docID, rev) + _, _, err = collection.DeleteDoc(ctx, docID, rev) assert.NoError(t, err) } @@ -2808,72 +3191,62 @@ func Test_invalidateAllPrincipalsCache(t *testing.T) { } func Test_resyncDocument(t *testing.T) { - testCases := []struct { - useXattr bool - }{ - {useXattr: true}, - {useXattr: false}, + if !base.TestUseXattrs() { + t.Skip("Walrus doesn't support xattr") } + db, ctx := setupTestDB(t) + defer db.Close(ctx) - for _, testCase := range testCases { - t.Run(fmt.Sprintf("Test_resyncDocument with useXattr: %t", testCase.useXattr), func(t *testing.T) { - if !base.TestUseXattrs() && testCase.useXattr { - t.Skip("Don't run xattr tests on non xattr tests") - } - db, ctx := setupTestDB(t) - defer db.Close(ctx) - - db.Options.EnableXattr = testCase.useXattr - db.Options.QueryPaginationLimit = 100 - collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + db.Options.EnableXattr = true + db.Options.QueryPaginationLimit = 100 + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) - syncFn := ` + syncFn := ` function sync(doc, oldDoc){ channel("channel." + "ABC"); } ` - _, err := collection.UpdateSyncFun(ctx, syncFn) - require.NoError(t, err) + _, err := collection.UpdateSyncFun(ctx, syncFn) + require.NoError(t, err) - docID := uuid.NewString() + docID := uuid.NewString() - updateBody := make(map[string]interface{}) - updateBody["val"] = "value" - _, doc, err := collection.Put(ctx, docID, updateBody) - require.NoError(t, err) - assert.NotNil(t, doc) + updateBody := make(map[string]interface{}) + updateBody["val"] = "value" + _, doc, err := collection.Put(ctx, docID, updateBody) + require.NoError(t, err) + assert.NotNil(t, doc) - syncFn = ` + syncFn = ` function sync(doc, oldDoc){ channel("channel." + "ABC12332423234"); } ` - _, err = collection.UpdateSyncFun(ctx, syncFn) - require.NoError(t, err) + _, err = collection.UpdateSyncFun(ctx, syncFn) + require.NoError(t, err) - _, _, err = collection.resyncDocument(ctx, docID, realDocID(docID), false, []uint64{10}) - require.NoError(t, err) - err = collection.WaitForPendingChanges(ctx) - require.NoError(t, err) + _, _, err = collection.resyncDocument(ctx, docID, realDocID(docID), false, []uint64{10}) + require.NoError(t, err) + err = collection.WaitForPendingChanges(ctx) + require.NoError(t, err) - syncData, err := collection.GetDocSyncData(ctx, docID) - assert.NoError(t, err) + syncData, err := collection.GetDocSyncData(ctx, docID) + assert.NoError(t, err) - assert.Len(t, syncData.ChannelSet, 2) - assert.Len(t, syncData.Channels, 2) - found := false + assert.Len(t, syncData.ChannelSet, 2) + assert.Len(t, syncData.Channels, 2) + found := false - for _, chSet := range syncData.ChannelSet { - if chSet.Name == "channel.ABC12332423234" { - found = true - break - } - } - - assert.True(t, found) - assert.Equal(t, 2, int(db.DbStats.Database().SyncFunctionCount.Value())) - }) + for _, chSet := range syncData.ChannelSet { + if chSet.Name == "channel.ABC12332423234" { + found = true + break + } } + + assert.True(t, found) + assert.Equal(t, 2, int(db.DbStats.Database().SyncFunctionCount.Value())) + } func Test_getUpdatedDocument(t *testing.T) { @@ -2960,7 +3333,7 @@ func TestImportCompactPanic(t *testing.T) { // Create a document, then delete it, to create a tombstone rev, doc, err := collection.Put(ctx, "test", Body{}) require.NoError(t, err) - _, err = collection.DeleteDoc(ctx, doc.ID, rev) + _, _, err = collection.DeleteDoc(ctx, doc.ID, rev) require.NoError(t, err) require.NoError(t, collection.WaitForPendingChanges(ctx)) @@ -3346,7 +3719,7 @@ func TestInject1xBodyProperties(t *testing.T) { var rev2Body Body rev2Data := `{"key":"value", "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body)) - _, rev2ID, err := collection.PutExistingRevWithBody(ctx, "doc", rev2Body, []string{"2-abc", rev1ID}, true) + _, rev2ID, err := collection.PutExistingRevWithBody(ctx, "doc", rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) docRev, err := collection.GetRev(ctx, "doc", rev2ID, true, nil) @@ -3401,3 +3774,206 @@ func TestDatabaseCloseIdempotent(t *testing.T) { defer db.BucketLock.Unlock() db._stopOnlineProcesses(ctx) } + +// TestSettingSyncInfo: +// - Purpose of the test is to call both SetSyncInfoMetaVersion + SetSyncInfoMetadataID in different orders +// asserting that the operations preserve the metadataID/metaVersion +// - Permutations include doc being created if it doesn't exist, one element being updated and preserving the other +// elements if it exists and +func TestSettingSyncInfo(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, _ := GetSingleDatabaseCollectionWithUser(ctx, t, db) + ds := collection.GetCollectionDatastore() + + require.NoError(t, base.SetSyncInfoMetaVersion(ds, "1")) + require.NoError(t, base.SetSyncInfoMetadataID(ds, "someID")) + + // assert that after above operations meta version is preserved after setting of metadataID + var syncInfo base.SyncInfo + _, err := ds.Get(base.SGSyncInfo, &syncInfo) + require.NoError(t, err) + assert.Equal(t, "1", syncInfo.MetaDataVersion) + assert.Equal(t, "someID", syncInfo.MetadataID) + + // remove sync info to test another permutation + require.NoError(t, ds.Delete(base.SGSyncInfo)) + + require.NoError(t, base.SetSyncInfoMetadataID(ds, "someID")) + require.NoError(t, base.SetSyncInfoMetaVersion(ds, "1")) + + // assert that after above operations metadataID is preserved after setting of metaVersion + syncInfo = base.SyncInfo{} + _, err = ds.Get(base.SGSyncInfo, &syncInfo) + require.NoError(t, err) + assert.Equal(t, "1", syncInfo.MetaDataVersion) + assert.Equal(t, "someID", syncInfo.MetadataID) + + // test updating each element in sync info now both elements are defined + require.NoError(t, base.SetSyncInfoMetaVersion(ds, "4")) + _, err = ds.Get(base.SGSyncInfo, &syncInfo) + require.NoError(t, err) + assert.Equal(t, "4", syncInfo.MetaDataVersion) + assert.Equal(t, "someID", syncInfo.MetadataID) + + require.NoError(t, base.SetSyncInfoMetadataID(ds, "test")) + _, err = ds.Get(base.SGSyncInfo, &syncInfo) + require.NoError(t, err) + assert.Equal(t, "4", syncInfo.MetaDataVersion) + assert.Equal(t, "test", syncInfo.MetadataID) +} + +// TestRequireMigration: +// - Purpose is to test code pathways inside the InitSyncInfo function will return requires attachment migration +// as expected. +func TestRequireMigration(t *testing.T) { + type testCase struct { + name string + initialMetaID string + newMetadataID string + metaVersion string + requireMigration bool + } + testCases := []testCase{ + { + name: "sync info in bucket with metadataID set", + initialMetaID: "someID", + requireMigration: true, + }, + { + name: "sync info in bucket with metadataID set, set newMetadataID", + initialMetaID: "someID", + newMetadataID: "testID", + requireMigration: true, + }, + { + name: "correct metaversion already defined, no metadata ID to set", + metaVersion: "4.0.0", + requireMigration: false, + }, + { + name: "correct metaversion already defined, metadata ID to set", + metaVersion: "4.0.0", + newMetadataID: "someID", + requireMigration: false, + }, + { + name: "old metaversion defined, metadata ID to set", + metaVersion: "3.0.0", + newMetadataID: "someID", + requireMigration: true, + }, + { + name: "old metaversion defined, no metadata ID to set", + metaVersion: "3.0.0", + requireMigration: true, + }, + } + ctx := base.TestCtx(t) + tb := base.GetTestBucket(t) + defer tb.Close(ctx) + ds := tb.GetSingleDataStore() + for _, testcase := range testCases { + t.Run(testcase.name, func(t *testing.T) { + if testcase.initialMetaID != "" { + require.NoError(t, base.SetSyncInfoMetadataID(ds, testcase.initialMetaID)) + } + if testcase.metaVersion != "" { + require.NoError(t, base.SetSyncInfoMetaVersion(ds, testcase.metaVersion)) + } + + _, requireMigration, err := base.InitSyncInfo(ctx, ds, testcase.newMetadataID) + require.NoError(t, err) + if testcase.requireMigration { + assert.True(t, requireMigration) + } else { + assert.False(t, requireMigration) + } + + // cleanup bucket + require.NoError(t, ds.Delete(base.SGSyncInfo)) + }) + } +} + +// TestInitSyncInfoRequireMigrationEmptyBucket: +// - Empty bucket call InitSyncInfo with metadata ID defined, assert require migration is returned +// - Cleanup bucket +// - Call InitSyncInfo with metadata ID not defined, assert require migration is not returned +func TestInitSyncInfoRequireMigrationEmptyBucket(t *testing.T) { + ctx := base.TestCtx(t) + tb := base.GetTestBucket(t) + defer tb.Close(ctx) + ds := tb.GetSingleDataStore() + + // test no sync info in bucket and set metadataID, returns requireMigration + _, requireMigration, err := base.InitSyncInfo(ctx, ds, "someID") + require.NoError(t, err) + assert.True(t, requireMigration) + + // delete the doc, test no sync info in bucket returns requireMigration + require.NoError(t, ds.Delete(base.SGSyncInfo)) + _, requireMigration, err = base.InitSyncInfo(ctx, ds, "") + require.NoError(t, err) + assert.True(t, requireMigration) +} + +// TestInitSyncInfoMetaVersionComparison: +// - Test requireMigration is true for metaVersion == 4.0.0 and > 4.0.0 +// - Test requireMigration is true for non-existent metaVersion +func TestInitSyncInfoMetaVersionComparison(t *testing.T) { + type testCase struct { + name string + metadataID string + metaVersion string + } + testCases := []testCase{ + { + name: "requireMigration for sync info with no meta version defined", + metadataID: "someID", + }, + { + name: "test requireMigration for metaVersion == 4.0.0", + metadataID: "someID", + metaVersion: "4.0.0", + }, + { + name: "test we return true for metaVersion minor version > 4.0.0", + metadataID: "someID", + metaVersion: "4.1.0", + }, + { + name: "test we return true for metaVersion patch version > 4.0.0", + metadataID: "someID", + metaVersion: "4.0.1", + }, + { + name: "test we return true for metaVersion major version > 4.0.0", + metadataID: "someID", + metaVersion: "5.0.0", + }, + } + ctx := base.TestCtx(t) + tb := base.GetTestBucket(t) + defer tb.Close(ctx) + ds := tb.GetSingleDataStore() + for _, testcase := range testCases { + t.Run(testcase.name, func(t *testing.T) { + // set sync info with no metaversion + require.NoError(t, base.SetSyncInfoMetadataID(ds, testcase.metadataID)) + + if testcase.metaVersion == "" { + _, requireMigration, err := base.InitSyncInfo(ctx, ds, "someID") + require.NoError(t, err) + assert.True(t, requireMigration) + } else { + require.NoError(t, base.SetSyncInfoMetaVersion(ds, testcase.metaVersion)) + _, requireMigration, err := base.InitSyncInfo(ctx, ds, "someID") + require.NoError(t, err) + assert.False(t, requireMigration) + } + // cleanup bucket + require.NoError(t, ds.Delete(base.SGSyncInfo)) + }) + } +} diff --git a/db/document.go b/db/document.go index 85056bb40a..a8a14071b9 100644 --- a/db/document.go +++ b/db/document.go @@ -16,6 +16,7 @@ import ( "errors" "fmt" "math" + "strconv" "time" sgbucket "github.com/couchbase/sg-bucket" @@ -34,11 +35,11 @@ const DocumentHistoryMaxEntriesPerChannel = 5 type DocumentUnmarshalLevel uint8 const ( - DocUnmarshalAll = DocumentUnmarshalLevel(iota) // Unmarshals sync metadata and body - DocUnmarshalSync // Unmarshals all sync metadata - DocUnmarshalNoHistory // Unmarshals sync metadata excluding history - DocUnmarshalHistory // Unmarshals history + rev + CAS only - DocUnmarshalRev // Unmarshals rev + CAS only + DocUnmarshalAll = DocumentUnmarshalLevel(iota) // Unmarshals metadata and body + DocUnmarshalSync // Unmarshals metadata + DocUnmarshalNoHistory // Unmarshals metadata excluding revtree history + DocUnmarshalHistory // Unmarshals revtree history + rev + CAS only + DocUnmarshalRev // Unmarshals revTreeID + CAS only (no HLV) DocUnmarshalCAS // Unmarshals CAS (for import check) only DocUnmarshalNone // No unmarshalling (skips import/upgrade check) ) @@ -62,30 +63,47 @@ type ChannelSetEntry struct { Compacted bool `json:"compacted,omitempty"` } +// MetadataOnlyUpdate represents a cas value of a document modification if it only updated xattrs and not the document body. The previous cas and revSeqNo are stored as the version of the document before any metadata was modified. This is serialized as _mou. type MetadataOnlyUpdate struct { - CAS string `json:"cas,omitempty"` - PreviousCAS string `json:"pCas,omitempty"` + HexCAS string `json:"cas,omitempty"` // 0x0 hex value from Couchbase Server + PreviousHexCAS string `json:"pCas,omitempty"` // 0x0 hex value from Couchbase Server + PreviousRevSeqNo uint64 `json:"pRev,omitempty"` +} + +func (m *MetadataOnlyUpdate) String() string { + return fmt.Sprintf("{CAS:%d PreviousCAS:%d PreviousRevSeqNo:%d}", m.CAS(), m.PreviousCAS(), m.PreviousRevSeqNo) +} + +// CAS returns the CAS value as a uint64 +func (m *MetadataOnlyUpdate) CAS() uint64 { + return base.HexCasToUint64(m.HexCAS) +} + +// PreviousCAS returns the previous CAS value as a uint64 +func (m *MetadataOnlyUpdate) PreviousCAS() uint64 { + return base.HexCasToUint64(m.PreviousHexCAS) } // The sync-gateway metadata stored in the "_sync" property of a Couchbase document. type SyncData struct { - CurrentRev string `json:"rev"` - NewestRev string `json:"new_rev,omitempty"` // Newest rev, if different from CurrentRev - Flags uint8 `json:"flags,omitempty"` - Sequence uint64 `json:"sequence,omitempty"` - UnusedSequences []uint64 `json:"unused_sequences,omitempty"` // unused sequences due to update conflicts/CAS retry - RecentSequences []uint64 `json:"recent_sequences,omitempty"` // recent sequences for this doc - used in server dedup handling - Channels channels.ChannelMap `json:"channels,omitempty"` - Access UserAccessMap `json:"access,omitempty"` - RoleAccess UserAccessMap `json:"role_access,omitempty"` - Expiry *time.Time `json:"exp,omitempty"` // Document expiry. Information only - actual expiry/delete handling is done by bucket storage. Needs to be pointer for omitempty to work (see https://github.com/golang/go/issues/4357) - Cas string `json:"cas"` // String representation of a cas value, populated via macro expansion - Crc32c string `json:"value_crc32c"` // String representation of crc32c hash of doc body, populated via macro expansion - Crc32cUserXattr string `json:"user_xattr_value_crc32c,omitempty"` // String representation of crc32c hash of user xattr - TombstonedAt int64 `json:"tombstoned_at,omitempty"` // Time the document was tombstoned. Used for view compaction - Attachments AttachmentsMeta `json:"attachments,omitempty"` - ChannelSet []ChannelSetEntry `json:"channel_set"` - ChannelSetHistory []ChannelSetEntry `json:"channel_set_history"` + CurrentRev string `json:"-"` // CurrentRev. Persisted as RevAndVersion in SyncDataJSON + NewestRev string `json:"new_rev,omitempty"` // Newest rev, if different from CurrentRev + Flags uint8 `json:"flags,omitempty"` + Sequence uint64 `json:"sequence,omitempty"` + UnusedSequences []uint64 `json:"unused_sequences,omitempty"` // unused sequences due to update conflicts/CAS retry + RecentSequences []uint64 `json:"recent_sequences,omitempty"` // recent sequences for this doc - used in server dedup handling + Channels channels.ChannelMap `json:"channels,omitempty"` + Access UserAccessMap `json:"access,omitempty"` + RoleAccess UserAccessMap `json:"role_access,omitempty"` + Expiry *time.Time `json:"exp,omitempty"` // Document expiry. Information only - actual expiry/delete handling is done by bucket storage. Needs to be pointer for omitempty to work (see https://github.com/golang/go/issues/4357) + Cas string `json:"cas"` // String representation of a cas value, populated via macro expansion + Crc32c string `json:"value_crc32c"` // String representation of crc32c hash of doc body, populated via macro expansion + Crc32cUserXattr string `json:"user_xattr_value_crc32c,omitempty"` // String representation of crc32c hash of user xattr + TombstonedAt int64 `json:"tombstoned_at,omitempty"` // Time the document was tombstoned. Used for view compaction + Attachments AttachmentsMeta `json:"attachments,omitempty"` + ChannelSet []ChannelSetEntry `json:"channel_set"` + ChannelSetHistory []ChannelSetEntry `json:"channel_set_history"` + HLV *HybridLogicalVector `json:"-"` // Marshalled/Unmarshalled separately from SyncData for storage in _vv, see MarshalWithXattrs/UnmarshalWithXattrs // Only used for performance metrics: TimeSaved time.Time `json:"time_saved,omitempty"` // Timestamp of save. @@ -101,6 +119,17 @@ type SyncData struct { removedRevisionBodyKeys map[string]string // keys of non-winning revisions that have been removed (and so may require deletion), indexed by revID } +// determine set of current channels based on removal entries. +func (sd *SyncData) getCurrentChannels() base.Set { + ch := base.SetOf() + for channelName, channelRemoval := range sd.Channels { + if channelRemoval == nil || channelRemoval.Seq == 0 { + ch.Add(channelName) + } + } + return ch +} + func (sd *SyncData) HashRedact(salt string) SyncData { // Creating a new SyncData with the redacted info. We copy all the information which stays the same and create new @@ -174,18 +203,24 @@ func (sd *SyncData) HashRedact(salt string) SyncData { // Document doesn't do any locking - document instances aren't intended to be shared across multiple goroutines. type Document struct { SyncData // Sync metadata + GlobalSyncData // Global sync metadata, this will hold non cluster specific sync metadata to be copied by XDCR _body Body // Marshalled document body. Unmarshalled lazily - should be accessed using Body() _rawBody []byte // Raw document body, as retrieved from the bucket. Marshaled lazily - should be accessed using BodyBytes() ID string `json:"-"` // Doc id. (We're already using a custom MarshalJSON for *document that's based on body, so the json:"-" probably isn't needed here) Cas uint64 // Document cas rawUserXattr []byte // Raw user xattr as retrieved from the bucket - metadataOnlyUpdate *MetadataOnlyUpdate // Contents of _mou xattr, marshalled/unmarshalled with document from xattrs + MetadataOnlyUpdate *MetadataOnlyUpdate // Contents of _mou xattr, marshalled/unmarshalled with document from xattrs Deleted bool DocExpiry uint32 RevID string DocAttachments AttachmentsMeta inlineSyncData bool + RevSeqNo uint64 // Server rev seq no for a document +} + +type GlobalSyncData struct { + GlobalAttachments AttachmentsMeta `json:"attachments_meta,omitempty"` } type historyOnlySyncData struct { @@ -195,7 +230,7 @@ type historyOnlySyncData struct { type revOnlySyncData struct { casOnlySyncData - CurrentRev string `json:"rev"` + CurrentRev channels.RevAndVersion `json:"rev"` } type casOnlySyncData struct { @@ -395,27 +430,21 @@ func unmarshalDocument(docid string, data []byte) (*Document, error) { return doc, nil } -// unmarshalDocumentWithXattrs populates individual xattrs on unmarshalDocumentWithXattrs from a provided xattrs map -func (db *DatabaseCollection) unmarshalDocumentWithXattrs(ctx context.Context, docid string, data []byte, xattrs map[string][]byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { - return unmarshalDocumentWithXattrs(ctx, docid, data, xattrs[base.SyncXattrName], xattrs[base.MouXattrName], xattrs[db.userXattrKey()], cas, unmarshalLevel) - -} - -func unmarshalDocumentWithXattrs(ctx context.Context, docid string, data []byte, syncXattrData, mouXattrData, userXattrData []byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { +func unmarshalDocumentWithXattrs(ctx context.Context, docid string, data, syncXattrData, hlvXattrData, mouXattrData, userXattrData, virtualXattr []byte, globalSyncData []byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { - if syncXattrData == nil || len(syncXattrData) == 0 { + if len(syncXattrData) == 0 && len(hlvXattrData) == 0 { // If no xattr data, unmarshal as standard doc doc, err = unmarshalDocument(docid, data) } else { doc = NewDocument(docid) - err = doc.UnmarshalWithXattr(ctx, data, syncXattrData, unmarshalLevel) + err = doc.UnmarshalWithXattrs(ctx, data, syncXattrData, hlvXattrData, virtualXattr, globalSyncData, unmarshalLevel) } if err != nil { return nil, err } if len(mouXattrData) > 0 { - if err := base.JSONUnmarshal(mouXattrData, &doc.metadataOnlyUpdate); err != nil { + if err := base.JSONUnmarshal(mouXattrData, &doc.MetadataOnlyUpdate); err != nil { base.WarnfCtx(ctx, "Failed to unmarshal mouXattr for key %v, mou will be ignored. Err: %v mou:%s", base.UD(docid), err, mouXattrData) } } @@ -450,14 +479,14 @@ func UnmarshalDocumentSyncData(data []byte, needHistory bool) (*SyncData, error) // TODO: Using a pool of unmarshal workers may help prevent memory spikes under load func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey string, needHistory bool) (result *SyncData, rawBody []byte, rawXattrs map[string][]byte, err error) { - var body []byte - var xattrValues map[string][]byte // If xattr datatype flag is set, data includes both xattrs and document body. Check for presence of sync xattr. // Note that there could be a non-sync xattr present + var xattrValues map[string][]byte + var hlv *HybridLogicalVector if dataType&base.MemcachedDataTypeXattr != 0 { - xattrKeys := []string{base.SyncXattrName, base.MouXattrName} + xattrKeys := []string{base.SyncXattrName, base.MouXattrName, base.VvXattrName, base.GlobalXattrName} if userXattrKey != "" { xattrKeys = append(xattrKeys, userXattrKey) } @@ -466,19 +495,32 @@ func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey return nil, nil, nil, err } - rawSyncXattr, _ := xattrValues[base.SyncXattrName] // If the sync xattr is present, use that to build SyncData - if len(rawSyncXattr) > 0 { + syncXattr, ok := xattrValues[base.SyncXattrName] + + if vvXattr, ok := xattrValues[base.VvXattrName]; ok { + err = base.JSONUnmarshal(vvXattr, &hlv) + if err != nil { + return nil, nil, nil, fmt.Errorf("error unmarshalling HLV: %w", err) + } + } + + if ok && len(syncXattr) > 0 { result = &SyncData{} if needHistory { result.History = make(RevTree) } - err = base.JSONUnmarshal(rawSyncXattr, result) + err = base.JSONUnmarshal(syncXattr, result) if err != nil { - return nil, nil, nil, fmt.Errorf("Found _sync xattr (%q), but could not unmarshal: %w", syncXattr, err) + return nil, nil, nil, fmt.Errorf("Found _sync xattr (%q), but could not unmarshal: %w", string(syncXattr), err) + } + + if hlv != nil { + result.HLV = hlv } return result, body, xattrValues, nil } + } else { // Xattr flag not set - data is just the document body body = data @@ -488,6 +530,15 @@ func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey if len(body) != 0 { result, err = UnmarshalDocumentSyncData(body, needHistory) } + + // If no sync data was found but HLV was present, initialize empty sync data + if result == nil && hlv != nil { + result = &SyncData{} + } + // If HLV was found, add to sync data + if hlv != nil { + result.HLV = hlv + } return result, body, xattrValues, err } @@ -503,7 +554,7 @@ func UnmarshalDocumentFromFeed(ctx context.Context, docid string, cas uint64, da if err != nil { return nil, err } - return unmarshalDocumentWithXattrs(ctx, docid, body, xattrs[base.SyncXattrName], xattrs[base.MouXattrName], xattrs[userXattrKey], cas, DocUnmarshalAll) + return unmarshalDocumentWithXattrs(ctx, docid, body, xattrs[base.SyncXattrName], xattrs[base.VvXattrName], xattrs[base.MouXattrName], xattrs[userXattrKey], xattrs[base.VirtualXattrRevSeqNo], nil, cas, DocUnmarshalAll) } func (doc *SyncData) HasValidSyncData() bool { @@ -887,7 +938,7 @@ func (doc *Document) addToChannelSetHistory(channelName string, historyEntry Cha // Updates the Channels property of a document object with current & past channels. // Returns the set of channels that have changed (document joined or left in this revision) -func (doc *Document) updateChannels(ctx context.Context, newChannels base.Set) (changedChannels base.Set, err error) { +func (doc *Document) updateChannels(ctx context.Context, newChannels base.Set) (changedChannels base.Set, revokedChannelsRequiringExpansion []string, err error) { var changed []string oldChannels := doc.Channels if oldChannels == nil { @@ -896,14 +947,19 @@ func (doc *Document) updateChannels(ctx context.Context, newChannels base.Set) ( } else { // Mark every no-longer-current channel as unsubscribed: curSequence := doc.Sequence + curRevAndVersion := doc.GetRevAndVersion() for channel, removal := range oldChannels { if removal == nil && !newChannels.Contains(channel) { oldChannels[channel] = &channels.ChannelRemoval{ Seq: curSequence, - RevID: doc.CurrentRev, + Rev: curRevAndVersion, Deleted: doc.hasFlag(channels.Deleted)} doc.updateChannelHistory(channel, curSequence, false) changed = append(changed, channel) + // If the current version requires macro expansion, new removal in channel map will also require macro expansion + if doc.HLV != nil && doc.HLV.Version == expandMacroCASValueUint64 { + revokedChannelsRequiringExpansion = append(revokedChannelsRequiringExpansion, channel) + } } } } @@ -932,7 +988,7 @@ func (doc *Document) IsChannelRemoval(ctx context.Context, revID string) (bodyBy // Iterate over the document's channel history, looking for channels that were removed at revID. If found, also identify whether the removal was a tombstone. for channel, removal := range doc.Channels { - if removal != nil && removal.RevID == revID { + if removal != nil && removal.Rev.RevTreeID == revID { removedChannels[channel] = struct{}{} if removal.Deleted == true { isDelete = true @@ -1056,11 +1112,12 @@ func (doc *Document) MarshalJSON() (data []byte, err error) { return data, err } -// UnmarshalWithXattr unmarshals the provided raw document and xattr bytes. The provided DocumentUnmarshalLevel +// UnmarshalWithXattrs unmarshals the provided raw document and xattr bytes when present. The provided DocumentUnmarshalLevel // (unmarshalLevel) specifies how much of the provided document/xattr needs to be initially unmarshalled. If // unmarshalLevel is anything less than the full document + metadata, the raw data is retained for subsequent // lazy unmarshalling as needed. -func (doc *Document) UnmarshalWithXattr(ctx context.Context, data []byte, xdata []byte, unmarshalLevel DocumentUnmarshalLevel) error { +// Must handle cases where document body and hlvXattrData are present without syncXattrData for all DocumentUnmarshalLevel +func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrData, hlvXattrData, virtualXattr []byte, globalSyncData []byte, unmarshalLevel DocumentUnmarshalLevel) error { if doc.ID == "" { base.WarnfCtx(ctx, "Attempted to unmarshal document without ID set") return errors.New("Document was unmarshalled without ID set") @@ -1068,65 +1125,120 @@ func (doc *Document) UnmarshalWithXattr(ctx context.Context, data []byte, xdata switch unmarshalLevel { case DocUnmarshalAll, DocUnmarshalSync: - // Unmarshal full document and/or sync metadata + // Unmarshal full document and/or sync metadata. Documents written by XDCR may have HLV but no sync data doc.SyncData = SyncData{History: make(RevTree)} - unmarshalErr := base.JSONUnmarshal(xdata, &doc.SyncData) - if unmarshalErr != nil { - return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattr() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), unmarshalErr)) + if syncXattrData != nil { + unmarshalErr := base.JSONUnmarshal(syncXattrData, &doc.SyncData) + if unmarshalErr != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), unmarshalErr)) + } + } + if hlvXattrData != nil { + // parse the raw bytes of the hlv and convert deltas back to full values in memory + err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) + if err != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) + } + } + if virtualXattr != nil { + var revSeqNo string + err := base.JSONUnmarshal(virtualXattr, &revSeqNo) + if err != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal doc virtual revSeqNo xattr during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) + } + if revSeqNo != "" { + revNo, err := strconv.ParseUint(revSeqNo, 10, 64) + if err != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed convert rev seq number %q during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", revSeqNo, base.UD(doc.ID), err)) + } + doc.RevSeqNo = revNo + } + } + if len(globalSyncData) > 0 { + if err := base.JSONUnmarshal(globalSyncData, &doc.GlobalSyncData); err != nil { + base.WarnfCtx(ctx, "Failed to unmarshal globalSync xattr for key %v, globalSync will be ignored. Err: %v globalSync:%s", base.UD(doc.ID), err, globalSyncData) + } + doc.SyncData.Attachments = doc.GlobalSyncData.GlobalAttachments } doc._rawBody = data // Unmarshal body if requested and present if unmarshalLevel == DocUnmarshalAll && len(data) > 0 { return doc._body.Unmarshal(data) } - case DocUnmarshalNoHistory: // Unmarshal sync metadata only, excluding history doc.SyncData = SyncData{} - unmarshalErr := base.JSONUnmarshal(xdata, &doc.SyncData) - if unmarshalErr != nil { - return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattr() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), unmarshalErr)) + if syncXattrData != nil { + unmarshalErr := base.JSONUnmarshal(syncXattrData, &doc.SyncData) + if unmarshalErr != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), unmarshalErr)) + } + } + if hlvXattrData != nil { + // parse the raw bytes of the hlv and convert deltas back to full values in memory + err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) + if err != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), err)) + } + } + if len(globalSyncData) > 0 { + if err := base.JSONUnmarshal(globalSyncData, &doc.GlobalSyncData); err != nil { + base.WarnfCtx(ctx, "Failed to unmarshal globalSync xattr for key %v, globalSync will be ignored. Err: %v globalSync:%s", base.UD(doc.ID), err, globalSyncData) + } + doc.SyncData.Attachments = doc.GlobalSyncData.GlobalAttachments } doc._rawBody = data case DocUnmarshalHistory: - historyOnlyMeta := historyOnlySyncData{History: make(RevTree)} - unmarshalErr := base.JSONUnmarshal(xdata, &historyOnlyMeta) - if unmarshalErr != nil { - return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattr() doc with id: %s (DocUnmarshalHistory). Error: %v", base.UD(doc.ID), unmarshalErr)) - } - doc.SyncData = SyncData{ - CurrentRev: historyOnlyMeta.CurrentRev, - History: historyOnlyMeta.History, - Cas: historyOnlyMeta.Cas, + if syncXattrData != nil { + historyOnlyMeta := historyOnlySyncData{History: make(RevTree)} + unmarshalErr := base.JSONUnmarshal(syncXattrData, &historyOnlyMeta) + if unmarshalErr != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattrs() doc with id: %s (DocUnmarshalHistory). Error: %v", base.UD(doc.ID), unmarshalErr)) + } + doc.SyncData = SyncData{ + CurrentRev: historyOnlyMeta.CurrentRev.RevTreeID, + History: historyOnlyMeta.History, + Cas: historyOnlyMeta.Cas, + } + } else { + doc.SyncData = SyncData{} } doc._rawBody = data case DocUnmarshalRev: // Unmarshal only rev and cas from sync metadata - var revOnlyMeta revOnlySyncData - unmarshalErr := base.JSONUnmarshal(xdata, &revOnlyMeta) - if unmarshalErr != nil { - return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattr() doc with id: %s (DocUnmarshalRev). Error: %v", base.UD(doc.ID), unmarshalErr)) - } - doc.SyncData = SyncData{ - CurrentRev: revOnlyMeta.CurrentRev, - Cas: revOnlyMeta.Cas, + if syncXattrData != nil { + var revOnlyMeta revOnlySyncData + unmarshalErr := base.JSONUnmarshal(syncXattrData, &revOnlyMeta) + if unmarshalErr != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattrs() doc with id: %s (DocUnmarshalRev). Error: %v", base.UD(doc.ID), unmarshalErr)) + } + doc.SyncData = SyncData{ + CurrentRev: revOnlyMeta.CurrentRev.RevTreeID, + Cas: revOnlyMeta.Cas, + } + } else { + doc.SyncData = SyncData{} } doc._rawBody = data case DocUnmarshalCAS: // Unmarshal only cas from sync metadata - var casOnlyMeta casOnlySyncData - unmarshalErr := base.JSONUnmarshal(xdata, &casOnlyMeta) - if unmarshalErr != nil { - return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattr() doc with id: %s (DocUnmarshalCAS). Error: %v", base.UD(doc.ID), unmarshalErr)) - } - doc.SyncData = SyncData{ - Cas: casOnlyMeta.Cas, + if syncXattrData != nil { + var casOnlyMeta casOnlySyncData + unmarshalErr := base.JSONUnmarshal(syncXattrData, &casOnlyMeta) + if unmarshalErr != nil { + return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalWithXattrs() doc with id: %s (DocUnmarshalCAS). Error: %v", base.UD(doc.ID), unmarshalErr)) + } + doc.SyncData = SyncData{ + Cas: casOnlyMeta.Cas, + } + } else { + doc.SyncData = SyncData{} } doc._rawBody = data } // If there's no body, but there is an xattr, set deleted flag and initialize an empty body - if len(data) == 0 && len(xdata) > 0 { + if len(data) == 0 && len(syncXattrData) > 0 { doc._body = Body{} doc._rawBody = []byte(base.EmptyDocument) doc.Deleted = true @@ -1134,7 +1246,8 @@ func (doc *Document) UnmarshalWithXattr(ctx context.Context, data []byte, xdata return nil } -func (doc *Document) MarshalWithXattrs() (data []byte, syncXattr []byte, mouXattr []byte, err error) { +// MarshalWithXattrs marshals the Document into body, and sync, vv and mou xattrs for persistence. +func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, globalXattr []byte, err error) { // Grab the rawBody if it's already marshalled, otherwise unmarshal the body if doc._rawBody != nil { if !doc.IsDeleted() { @@ -1151,40 +1264,124 @@ func (doc *Document) MarshalWithXattrs() (data []byte, syncXattr []byte, mouXatt if !deleted { data, err = base.JSONMarshal(body) if err != nil { - return nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc body with id: %s. Error: %v", base.UD(doc.ID), err)) + return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc body with id: %s. Error: %v", base.UD(doc.ID), err)) } } } } + if doc.SyncData.HLV != nil { + vvXattr, err = base.JSONMarshal(doc.SyncData.HLV) + if err != nil { + return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) + } + } + // assign any attachments we have stored in document sync data to global sync data + // then nil the sync data attachments to prevent marshalling of it + doc.GlobalSyncData.GlobalAttachments = doc.Attachments + doc.Attachments = nil syncXattr, err = base.JSONMarshal(doc.SyncData) if err != nil { - return nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc SyncData with id: %s. Error: %v", base.UD(doc.ID), err)) + return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc SyncData with id: %s. Error: %v", base.UD(doc.ID), err)) } - if doc.metadataOnlyUpdate != nil { - mouXattr, err = base.JSONMarshal(doc.metadataOnlyUpdate) + if doc.MetadataOnlyUpdate != nil { + mouXattr, err = base.JSONMarshal(doc.MetadataOnlyUpdate) if err != nil { - return nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc MouData with id: %s. Error: %v", base.UD(doc.ID), err)) + return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc MouData with id: %s. Error: %v", base.UD(doc.ID), err)) } } + // marshal global xattrs if there are attachments defined + if len(doc.GlobalSyncData.GlobalAttachments) > 0 { + globalXattr, err = base.JSONMarshal(doc.GlobalSyncData) + if err != nil { + return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc GlobalXattr with id: %s. Error: %v", base.UD(doc.ID), err)) + } + // restore attachment meta to sync data post global xattr construction + doc.Attachments = make(AttachmentsMeta) + doc.Attachments = doc.GlobalSyncData.GlobalAttachments + } - return data, syncXattr, mouXattr, nil + return data, syncXattr, vvXattr, mouXattr, globalXattr, nil } // computeMetadataOnlyUpdate computes a new metadataOnlyUpdate based on the existing document's CAS and metadataOnlyUpdate -func computeMetadataOnlyUpdate(currentCas uint64, currentMou *MetadataOnlyUpdate) *MetadataOnlyUpdate { +func computeMetadataOnlyUpdate(currentCas uint64, revNo uint64, currentMou *MetadataOnlyUpdate) *MetadataOnlyUpdate { var prevCas string currentCasString := base.CasToString(currentCas) - if currentMou != nil && currentCasString == currentMou.CAS { - prevCas = currentMou.PreviousCAS + if currentMou != nil && currentCasString == currentMou.HexCAS { + prevCas = currentMou.PreviousHexCAS } else { prevCas = currentCasString } metadataOnlyUpdate := &MetadataOnlyUpdate{ - CAS: expandMacroCASValue, // when non-empty, this is replaced with cas macro expansion - PreviousCAS: prevCas, + HexCAS: expandMacroCASValueString, // when non-empty, this is replaced with cas macro expansion + PreviousHexCAS: prevCas, + PreviousRevSeqNo: revNo, } return metadataOnlyUpdate } + +// HasCurrentVersion Compares the specified CV with the fetched documents CV, returns error on mismatch between the two +func (d *Document) HasCurrentVersion(ctx context.Context, cv Version) error { + if d.HLV == nil { + return base.RedactErrorf("no HLV present in fetched doc %s", base.UD(d.ID)) + } + + // fetch the current version for the loaded doc and compare against the CV specified in the IDandCV key + fetchedDocSource, fetchedDocVersion := d.HLV.GetCurrentVersion() + if fetchedDocSource != cv.SourceID || fetchedDocVersion != cv.Value { + base.DebugfCtx(ctx, base.KeyCRUD, "mismatch between specified current version and fetched document current version for doc %s", base.UD(d.ID)) + // return not found as specified cv does not match fetched doc cv + return base.ErrNotFound + } + return nil +} + +// SyncDataAlias is an alias for SyncData that doesn't define custom MarshalJSON/UnmarshalJSON +type SyncDataAlias SyncData + +// SyncDataJSON is the persisted form of SyncData, with RevAndVersion populated at marshal time +type SyncDataJSON struct { + *SyncDataAlias + RevAndVersion channels.RevAndVersion `json:"rev"` +} + +// MarshalJSON populates RevAndVersion using CurrentRev and the HLV (current) source and version. +// Marshals using SyncDataAlias to avoid recursion, and SyncDataJSON to add the combined RevAndVersion. +func (s SyncData) MarshalJSON() (data []byte, err error) { + + var sdj SyncDataJSON + var sd SyncDataAlias + sd = (SyncDataAlias)(s) + sdj.SyncDataAlias = &sd + sdj.RevAndVersion = s.GetRevAndVersion() + return base.JSONMarshal(sdj) +} + +// UnmarshalJSON unmarshals using SyncDataJSON, then sets currentRev on SyncData based on the value in RevAndVersion. +// The HLV's current version stored in RevAndVersion is ignored at unmarshal time - the value in the HLV is the source +// of truth. +func (s *SyncData) UnmarshalJSON(data []byte) error { + + var sdj *SyncDataJSON + err := base.JSONUnmarshal(data, &sdj) + if err != nil { + return err + } + if sdj.SyncDataAlias != nil { + *s = SyncData(*sdj.SyncDataAlias) + s.CurrentRev = sdj.RevAndVersion.RevTreeID + } + return nil +} + +func (s *SyncData) GetRevAndVersion() (rav channels.RevAndVersion) { + rav.RevTreeID = s.CurrentRev + if s.HLV != nil { + rav.CurrentSource = s.HLV.SourceID + rav.CurrentVersion = string(base.Uint64CASToLittleEndianHex(s.HLV.Version)) + } + return rav +} diff --git a/db/document_test.go b/db/document_test.go index 027a08bda7..baf4bbb230 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -14,6 +14,7 @@ import ( "bytes" "encoding/binary" "log" + "reflect" "testing" sgbucket "github.com/couchbase/sg-bucket" @@ -136,7 +137,7 @@ func BenchmarkDocUnmarshal(b *testing.B) { b.Run(bm.name, func(b *testing.B) { ctx := base.TestCtx(b) for i := 0; i < b.N; i++ { - _, _ = unmarshalDocumentWithXattrs(ctx, "doc_1k", doc1k_body, doc1k_meta, nil, nil, 1, bm.unmarshalLevel) + _, _ = unmarshalDocumentWithXattrs(ctx, "doc_1k", doc1k_body, doc1k_meta, nil, nil, nil, nil, nil, 1, bm.unmarshalLevel) } }) } @@ -191,6 +192,198 @@ func BenchmarkUnmarshalBody(b *testing.B) { } } +const doc_meta_no_vv = `{ + "rev": "3-89758294abc63157354c2b08547c2d21", + "sequence": 7, + "recent_sequences": [ + 5, + 6, + 7 + ], + "history": { + "revs": [ + "1-fc591a068c153d6c3d26023d0d93dcc1", + "2-0eab03571bc55510c8fc4bfac9fe4412", + "3-89758294abc63157354c2b08547c2d21" + ], + "parents": [ + -1, + 0, + 1 + ], + "channels": [ + [ + "ABC", + "DEF" + ], + [ + "ABC", + "DEF", + "GHI" + ], + [ + "ABC", + "GHI" + ] + ] + }, + "channels": { + "ABC": null, + "DEF": { + "seq": 7, + "rev": "3-89758294abc63157354c2b08547c2d21" + }, + "GHI": null + }, + "cas": "", + "time_saved": "2017-10-25T12:45:29.622450174-07:00" + }` + +const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", + "mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] +}` + +func TestParseVersionVectorSyncData(t *testing.T) { + mv := make(HLVVersions) + pv := make(HLVVersions) + mv["s_LhRPsa7CpjEvP5zeXTXEBA"] = 1628620455147864000 + mv["s_NqiIe0LekFPLeX4JvTO6Iw"] = 1628620458747363292 + pv["s_YZvBpEaztom9z5V/hDoeIw"] = 1628620455135215600 + + ctx := base.TestCtx(t) + + sync_meta := []byte(doc_meta_no_vv) + vv_meta := []byte(doc_meta_vv) + doc, err := unmarshalDocumentWithXattrs(ctx, "doc_1k", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalNoHistory) + require.NoError(t, err) + + vrsCAS := uint64(123456) + // assert on doc version vector values + assert.Equal(t, vrsCAS, doc.SyncData.HLV.CurrentVersionCAS) + assert.Equal(t, vrsCAS, doc.SyncData.HLV.Version) + assert.Equal(t, "cb06dc003846116d9b66d2ab23887a96", doc.SyncData.HLV.SourceID) + assert.True(t, reflect.DeepEqual(mv, doc.SyncData.HLV.MergeVersions)) + assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) + + doc, err = unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll) + require.NoError(t, err) + + // assert on doc version vector values + assert.Equal(t, vrsCAS, doc.SyncData.HLV.CurrentVersionCAS) + assert.Equal(t, vrsCAS, doc.SyncData.HLV.Version) + assert.Equal(t, "cb06dc003846116d9b66d2ab23887a96", doc.SyncData.HLV.SourceID) + assert.True(t, reflect.DeepEqual(mv, doc.SyncData.HLV.MergeVersions)) + assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) + + doc, err = unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalNoHistory) + require.NoError(t, err) + + // assert on doc version vector values + assert.Equal(t, vrsCAS, doc.SyncData.HLV.CurrentVersionCAS) + assert.Equal(t, vrsCAS, doc.SyncData.HLV.Version) + assert.Equal(t, "cb06dc003846116d9b66d2ab23887a96", doc.SyncData.HLV.SourceID) + assert.True(t, reflect.DeepEqual(mv, doc.SyncData.HLV.MergeVersions)) + assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) +} + +const doc_meta_vv_corrupt = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", + "mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd61c008cd61c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] +}` + +func TestParseVersionVectorCorruptDelta(t *testing.T) { + + ctx := base.TestCtx(t) + + sync_meta := []byte(doc_meta_no_vv) + vv_meta := []byte(doc_meta_vv_corrupt) + _, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll) + require.Error(t, err) + +} + +// TestRevAndVersion tests marshalling and unmarshalling rev and current version +func TestRevAndVersion(t *testing.T) { + + ctx := base.TestCtx(t) + testCases := []struct { + testName string + revTreeID string + source string + version string + }{ + { + testName: "rev_and_version", + revTreeID: "1-abc", + source: "source1", + version: "0x0100000000000000", + }, + { + testName: "both_empty", + revTreeID: "", + source: "", + version: "0", + }, + { + testName: "revTreeID_only", + revTreeID: "1-abc", + source: "", + version: "0", + }, + { + testName: "currentVersion_only", + revTreeID: "", + source: "source1", + version: "0x0100000000000000", + }, + } + + var expectedSequence = uint64(100) + for _, test := range testCases { + t.Run(test.testName, func(t *testing.T) { + syncData := &SyncData{ + CurrentRev: test.revTreeID, + Sequence: expectedSequence, + } + if test.source != "" { + syncData.HLV = &HybridLogicalVector{ + SourceID: test.source, + Version: base.HexCasToUint64(test.version), + } + } + // SyncData test + marshalledSyncData, err := base.JSONMarshal(syncData) + require.NoError(t, err) + log.Printf("marshalled:%s", marshalledSyncData) + + // Document test + document := NewDocument("docID") + document.SyncData.CurrentRev = test.revTreeID + document.SyncData.Sequence = expectedSequence + document.SyncData.HLV = &HybridLogicalVector{ + SourceID: test.source, + Version: base.HexCasToUint64(test.version), + } + + marshalledDoc, marshalledXattr, marshalledVvXattr, _, _, err := document.MarshalWithXattrs() + require.NoError(t, err) + + newDocument := NewDocument("docID") + err = newDocument.UnmarshalWithXattrs(ctx, marshalledDoc, marshalledXattr, marshalledVvXattr, nil, nil, DocUnmarshalAll) + require.NoError(t, err) + require.Equal(t, test.revTreeID, newDocument.CurrentRev) + require.Equal(t, expectedSequence, newDocument.Sequence) + if test.source != "" { + require.NotNil(t, newDocument.HLV) + require.Equal(t, test.source, newDocument.HLV.SourceID) + require.Equal(t, base.HexCasToUint64(test.version), newDocument.HLV.Version) + } + //require.Equal(t, test.expectedCombinedVersion, newDocument.RevAndVersion) + }) + } +} + func TestParseDocumentCas(t *testing.T) { syncData := &SyncData{} syncData.Cas = "0x00002ade734fb714" @@ -320,7 +513,7 @@ func TestInvalidXattrStreamEmptyBody(t *testing.T) { emptyBody := []byte{} // DecodeValueWithXattrs is the underlying function - body, xattrs, err := sgbucket.DecodeValueWithXattrs([]string{"_sync"}, inputStream) + body, xattrs, err := sgbucket.DecodeValueWithXattrs([]string{base.SyncXattrName}, inputStream) require.NoError(t, err) require.Equal(t, emptyBody, body) require.Empty(t, xattrs) @@ -354,3 +547,128 @@ func getSingleXattrDCPBytes() []byte { dcpBody = append(dcpBody, body...) return dcpBody } + +const syncDataWithAttachment = `{ + "attachments": { + "bye.txt": { + "digest": "sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc=", + "length": 19, + "revpos": 1, + "stub": true, + "ver": 2 + }, + "hello.txt": { + "digest": "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", + "length": 11, + "revpos": 1, + "stub": true, + "ver": 2 + } + }, + "cas": "0x0000d2ba4104f217", + "channel_set": [ + { + "name": "sg_test_0", + "start": 1 + } + ], + "channel_set_history": null, + "channels": { + "sg_test_0": null + }, + "cluster_uuid": "6eca6cdd1ffcd7b2b7ea07039e68a774", + "history": { + "channels": [ + [ + "sg_test_0" + ] + ], + "parents": [ + -1 + ], + "revs": [ + "1-ca9ad22802b66f662ff171f226211d5c" + ] + }, + "recent_sequences": [ + 1 + ], + "rev": { + "rev": "1-ca9ad22802b66f662ff171f226211d5c", + "src": "RS1pdSMRlrNr0Ns0oOfc8A", + "ver": "0x0000d2ba4104f217" + }, + "sequence": 1, + "time_saved": "2024-09-04T11:38:05.093225+01:00", + "value_crc32c": "0x297bd0aa" + }` + +const globalXattr = `{ + "attachments_meta": { + "bye.txt": { + "digest": "sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc=", + "length": 19, + "revpos": 1, + "stub": true, + "ver": 2 + }, + "hello.txt": { + "digest": "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", + "length": 11, + "revpos": 1, + "stub": true, + "ver": 2 + } + } + }` + +// TestAttachmentReadStoredInXattr tests reads legacy format for attachments being stored in sync data xattr as well as +// testing the new location for attachments in global xattr +func TestAttachmentReadStoredInXattr(t *testing.T) { + ctx := base.TestCtx(t) + + // unmarshal attachments on sync data + testSync := []byte(syncDataWithAttachment) + doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, testSync, nil, nil, nil, nil, nil, 1, DocUnmarshalSync) + require.NoError(t, err) + + // assert on attachments + atts := doc.Attachments + assert.Len(t, atts, 2) + hello := atts["hello.txt"].(map[string]interface{}) + assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) + assert.Equal(t, float64(11), hello["length"]) + assert.Equal(t, float64(1), hello["revpos"]) + assert.Equal(t, float64(2), hello["ver"]) + assert.True(t, hello["stub"].(bool)) + + bye := atts["bye.txt"].(map[string]interface{}) + assert.Equal(t, "sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc=", bye["digest"]) + assert.Equal(t, float64(19), bye["length"]) + assert.Equal(t, float64(1), bye["revpos"]) + assert.Equal(t, float64(2), bye["ver"]) + assert.True(t, bye["stub"].(bool)) + + // unmarshal attachments on global data + testGlobal := []byte(globalXattr) + sync_meta_no_attachments := []byte(doc_meta_no_vv) + doc, err = unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta_no_attachments, nil, nil, nil, nil, testGlobal, 1, DocUnmarshalSync) + require.NoError(t, err) + + // assert on attachments + atts = doc.Attachments + assert.Len(t, atts, 2) + hello = atts["hello.txt"].(map[string]interface{}) + assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) + assert.Equal(t, float64(11), hello["length"]) + assert.Equal(t, float64(1), hello["revpos"]) + assert.Equal(t, float64(2), hello["ver"]) + assert.True(t, hello["stub"].(bool)) + + bye = atts["bye.txt"].(map[string]interface{}) + assert.Equal(t, "sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc=", bye["digest"]) + assert.Equal(t, float64(19), bye["length"]) + assert.Equal(t, float64(1), bye["revpos"]) + assert.Equal(t, float64(2), bye["ver"]) + assert.True(t, bye["stub"].(bool)) +} diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index b930859428..ae4820a17f 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -9,79 +9,246 @@ package db import ( + "crypto/md5" + "encoding/base64" + "encoding/hex" "fmt" + "sort" + "strconv" + "strings" + sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" ) -type HybridLogicalVector struct { - CurrentVersionCAS uint64 // current version cas (or cvCAS) stores the current CAS at the time of replication - SourceID string // source bucket uuid of where this entry originated from - Version uint64 // current cas of the current version on the version vector - MergeVersions map[string]uint64 // map of merge versions for fast efficient lookup - PreviousVersions map[string]uint64 // map of previous versions for fast efficient lookup +type HLVVersions map[string]uint64 // map of source ID to version uint64 version value + +// Version is representative of a single entry in a HybridLogicalVector. +type Version struct { + // SourceID is an ID representing the source of the value (e.g. Couchbase Lite ID) + SourceID string `json:"source_id"` + // Value is a Hybrid Logical Clock value (In Couchbase Server, CAS is a HLC) + Value uint64 `json:"version"` +} + +// VersionsDeltas will be sorted by version, first entry will be fill version then after that will be calculated deltas +type VersionsDeltas []Version + +func (vde VersionsDeltas) Len() int { return len(vde) } + +func (vde VersionsDeltas) Swap(i, j int) { + vde[i], vde[j] = vde[j], vde[i] +} + +func (vde VersionsDeltas) Less(i, j int) bool { + if vde[i].Value == vde[j].Value { + return false + } + return vde[i].Value < vde[j].Value +} + +// VersionDeltas calculate the deltas of input map +func VersionDeltas(versions map[string]uint64) VersionsDeltas { + if versions == nil { + return nil + } + + vdm := make(VersionsDeltas, 0, len(versions)) + for src, vrs := range versions { + vdm = append(vdm, CreateVersion(src, vrs)) + } + + // return early for single entry + if len(vdm) == 1 { + return vdm + } + + // sort the list + sort.Sort(vdm) + + // traverse in reverse order and calculate delta between versions, leaving the first element as is + for i := len(vdm) - 1; i >= 1; i-- { + vdm[i].Value = vdm[i].Value - vdm[i-1].Value + } + return vdm +} + +// VersionsToDeltas will calculate deltas from the input map (pv or mv). Then will return the deltas in persisted format +func VersionsToDeltas(m map[string]uint64) []string { + if len(m) == 0 { + return nil + } + + var vrsList []string + deltas := VersionDeltas(m) + for _, delta := range deltas { + listItem := delta.StringForVersionDelta() + vrsList = append(vrsList, listItem) + } + + return vrsList +} + +// PersistedDeltasToMap converts the list of deltas in pv or mv from the bucket back from deltas into full versions in map format +func PersistedDeltasToMap(vvList []string) (map[string]uint64, error) { + vv := make(map[string]uint64) + if len(vvList) == 0 { + return vv, nil + } + + var lastEntryVersion uint64 + for _, v := range vvList { + timestampString, sourceBase64, found := strings.Cut(v, "@") + if !found { + return nil, fmt.Errorf("Malformed version string %s, delimiter not found", v) + } + ver, err := base.HexCasToUint64ForDelta([]byte(timestampString)) + if err != nil { + return nil, err + } + lastEntryVersion = ver + lastEntryVersion + vv[sourceBase64] = lastEntryVersion + } + return vv, nil +} + +// CreateVersion creates an encoded sourceID and version pair +func CreateVersion(source string, version uint64) Version { + return Version{ + SourceID: source, + Value: version, + } +} + +// ParseVersion will parse source version pair from string format +func ParseVersion(versionString string) (version Version, err error) { + timestampString, sourceBase64, found := strings.Cut(versionString, "@") + if !found { + return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString) + } + version.SourceID = sourceBase64 + // remove any leading whitespace, this should be addressed in CBG-3662 + if len(timestampString) > 0 && timestampString[0] == ' ' { + timestampString = timestampString[1:] + } + vrs, err := strconv.ParseUint(timestampString, 16, 64) + if err != nil { + return version, err + } + version.Value = vrs + return version, nil } -// CurrentVersionVector is a structure used to add a new sourceID:CAS entry to a HLV -type CurrentVersionVector struct { - VersionCAS uint64 - SourceID string +// String returns a version/sourceID pair in CBL string format. This does not match the format serialized on CBS, which will be in 0x0 format. +func (v Version) String() string { + return strconv.FormatUint(v.Value, 16) + "@" + v.SourceID } -type PersistedHybridLogicalVector struct { - CurrentVersionCAS string `json:"cvCas,omitempty"` - SourceID string `json:"src,omitempty"` - Version string `json:"vrs,omitempty"` - MergeVersions map[string]string `json:"mv,omitempty"` - PreviousVersions map[string]string `json:"pv,omitempty"` +// StringForVersionDelta will take a version struct and convert the value to delta format +// (encoding it to LE hex, stripping any 0's off the end and stripping leading 0x) +func (v Version) StringForVersionDelta() string { + encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(v.Value) + return encodedVal + "@" + v.SourceID } -type PersistedVersionVector struct { - PersistedHybridLogicalVector `json:"_vv"` +// ExtractCurrentVersionFromHLV will take the current version form the HLV struct and return it in the Version struct +func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { + src, vrs := hlv.GetCurrentVersion() + currVersion := CreateVersion(src, vrs) + return &currVersion +} + +// HybridLogicalVector is the in memory format for the hLv. +type HybridLogicalVector struct { + CurrentVersionCAS uint64 // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication + SourceID string // source bucket uuid in (base64 encoded format) of where this entry originated from + Version uint64 // current cas in little endian hex format of the current version on the version vector + MergeVersions HLVVersions // map of merge versions for fast efficient lookup + PreviousVersions HLVVersions // map of previous versions for fast efficient lookup } -// NewHybridLogicalVector returns a HybridLogicalVector struct with maps initialised in the struct -func NewHybridLogicalVector() HybridLogicalVector { - return HybridLogicalVector{ - PreviousVersions: make(map[string]uint64), - MergeVersions: make(map[string]uint64), +// NewHybridLogicalVector returns an initialised HybridLogicalVector. +func NewHybridLogicalVector() *HybridLogicalVector { + return &HybridLogicalVector{ + PreviousVersions: make(HLVVersions), + MergeVersions: make(HLVVersions), } } -// GetCurrentVersion return the current version vector from the HLV in memory +// GetCurrentVersion returns the current version from the HLV in memory. func (hlv *HybridLogicalVector) GetCurrentVersion() (string, uint64) { return hlv.SourceID, hlv.Version } -// IsInConflict tests to see if in memory HLV is conflicting with another HLV -func (hlv *HybridLogicalVector) IsInConflict(otherVector HybridLogicalVector) bool { - // test if either HLV(A) or HLV(B) are dominating over each other. If so they are not in conflict - if hlv.isDominating(otherVector) || otherVector.isDominating(*hlv) { +// GetCurrentVersion returns the current version in transport format +func (hlv *HybridLogicalVector) GetCurrentVersionString() string { + if hlv == nil || hlv.SourceID == "" { + return "" + } + version := Version{ + SourceID: hlv.SourceID, + Value: hlv.Version, + } + return version.String() +} + +// IsVersionKnown checks to see whether the HLV already contains a Version for the provided +// source with a matching or newer value +func (hlv *HybridLogicalVector) DominatesSource(version Version) bool { + existingValueForSource, found := hlv.GetValue(version.SourceID) + if !found { return false } - // if the version vectors aren't dominating over one another then conflict is present - return true + return existingValueForSource >= version.Value + } -// AddVersion adds a version vector to the in memory representation of a HLV and moves current version vector to -// previous versions on the HLV if needed -func (hlv *HybridLogicalVector) AddVersion(newVersion CurrentVersionVector) error { - if newVersion.VersionCAS < hlv.Version { - return fmt.Errorf("attempting to add new version vector entry with a CAS that is less than the current version CAS value") +// AddVersion adds newVersion to the in memory representation of the HLV. +func (hlv *HybridLogicalVector) AddVersion(newVersion Version) error { + var newVersionCAS uint64 + hlvVersionCAS := hlv.Version + if newVersion.Value != expandMacroCASValueUint64 { + newVersionCAS = newVersion.Value + } + // check if this is the first time we're adding a source - version pair + if hlv.SourceID == "" { + hlv.Version = newVersion.Value + hlv.SourceID = newVersion.SourceID + return nil } // if new entry has the same source we simple just update the version if newVersion.SourceID == hlv.SourceID { - hlv.Version = newVersion.VersionCAS + if newVersion.Value != expandMacroCASValueUint64 && newVersionCAS < hlvVersionCAS { + return fmt.Errorf("attempting to add new version vector entry with a CAS that is less than the current version CAS value for the same source. Current cas: %d new cas %d", hlv.Version, newVersion.Value) + } + hlv.Version = newVersion.Value return nil } // if we get here this is a new version from a different sourceID thus need to move current sourceID to previous versions and update current version - hlv.PreviousVersions[hlv.SourceID] = hlv.Version - hlv.Version = newVersion.VersionCAS + if hlv.PreviousVersions == nil { + hlv.PreviousVersions = make(HLVVersions) + } + // we need to check if source ID already exists in PV, if so we need to ensure we are only updating with the + // sourceID-version pair if incoming version is greater than version already there + if currPVVersion, ok := hlv.PreviousVersions[hlv.SourceID]; ok { + // if we get here source ID exists in PV, only replace version if it is less than the incoming version + currPVVersionCAS := currPVVersion + if currPVVersionCAS < hlvVersionCAS { + hlv.PreviousVersions[hlv.SourceID] = hlv.Version + } else { + return fmt.Errorf("local hlv has current source in previous version with version greater than current version. Current CAS: %d, PV CAS %d", hlv.Version, currPVVersion) + } + } else { + // source doesn't exist in PV so add + hlv.PreviousVersions[hlv.SourceID] = hlv.Version + } + hlv.Version = newVersion.Value hlv.SourceID = newVersion.SourceID return nil } -// Remove removes a vector from previous versions section of in memory HLV +// Remove removes a source from previous versions of the HLV. +// TODO: Does this need to remove source from current version as well? Merge Versions? func (hlv *HybridLogicalVector) Remove(source string) error { // if entry is not found in previous versions we return error if hlv.PreviousVersions[source] == 0 { @@ -91,168 +258,365 @@ func (hlv *HybridLogicalVector) Remove(source string) error { return nil } -// isDominating tests if in memory HLV is dominating over another -func (hlv *HybridLogicalVector) isDominating(otherVector HybridLogicalVector) bool { - // Dominating Criteria: - // HLV A dominates HLV B if source(A) == source(B) and version(A) > version(B) - // If there is an entry in pv(B) for A's current source and version(A) > B's version for that pv entry then A is dominating - // if there is an entry in mv(B) for A's current source and version(A) > B's version for that pv entry then A is dominating - - // Grab the latest CAS version for HLV(A)'s sourceID in HLV(B), if HLV(A) version CAS is > HLV(B)'s then it is dominating - // If 0 CAS is returned then the sourceID does not exist on HLV(B) - if latestCAS := otherVector.GetVersion(hlv.SourceID); latestCAS != 0 && hlv.Version > latestCAS { - return true - } - // HLV A is not dominating over HLV B - return false +// isDominating tests if in memory HLV is dominating over another. +// If HLV A dominates CV of HLV B, it can be assumed to dominate the entire HLV, since +// CV dominates PV for a given HLV. Given this, it's sufficient to check whether HLV A +// has a version for HLV B's current source that's greater than or equal to HLV B's current version. +func (hlv *HybridLogicalVector) isDominating(otherVector *HybridLogicalVector) bool { + return hlv.DominatesSource(Version{otherVector.SourceID, otherVector.Version}) } -// isEqual tests if in memory HLV is equal to another -func (hlv *HybridLogicalVector) isEqual(otherVector HybridLogicalVector) bool { - // if in HLV(A) sourceID the same as HLV(B) sourceID and HLV(A) CAS is equal to HLV(B) CAS then the two HLV's are equal - if hlv.SourceID == otherVector.SourceID && hlv.Version == otherVector.Version { - return true +// GetVersion returns the latest decoded CAS value in the HLV for a given sourceID +func (hlv *HybridLogicalVector) GetValue(sourceID string) (uint64, bool) { + if sourceID == "" { + return 0, false + } + var latestVersion uint64 + if sourceID == hlv.SourceID { + latestVersion = hlv.Version } - // if the HLV(A) merge versions isn't empty and HLV(B) merge versions isn't empty AND if - // merge versions between the two HLV's are the same, they are equal - if len(hlv.MergeVersions) != 0 && len(otherVector.MergeVersions) != 0 { - if hlv.equalMergeVectors(otherVector) { - return true + if pvEntry, ok := hlv.PreviousVersions[sourceID]; ok { + entry := pvEntry + if entry > latestVersion { + latestVersion = entry } } - if len(hlv.PreviousVersions) != 0 && len(otherVector.PreviousVersions) != 0 { - if hlv.equalPreviousVectors(otherVector) { - return true + if mvEntry, ok := hlv.MergeVersions[sourceID]; ok { + entry := mvEntry + if entry > latestVersion { + latestVersion = entry } } - // they aren't equal - return false + // if we have 0 cas value, there is no entry for this source ID in the HLV + if latestVersion == 0 { + return latestVersion, false + } + return latestVersion, true } -// equalMergeVectors tests if two merge vectors between HLV's are equal or not -func (hlv *HybridLogicalVector) equalMergeVectors(otherVector HybridLogicalVector) bool { - if len(hlv.MergeVersions) != len(otherVector.MergeVersions) { - return false +// AddNewerVersions will take a hlv and add any newer source/version pairs found across CV and PV found in the other HLV taken as parameter +// when both HLV +func (hlv *HybridLogicalVector) AddNewerVersions(otherVector *HybridLogicalVector) error { + + // create current version for incoming vector and attempt to add it to the local HLV, AddVersion will handle if attempting to add older + // version than local HLVs CV pair + otherVectorCV := Version{SourceID: otherVector.SourceID, Value: otherVector.Version} + err := hlv.AddVersion(otherVectorCV) + if err != nil { + return err } - for k, v := range hlv.MergeVersions { - if v != otherVector.MergeVersions[k] { - return false + + if otherVector.PreviousVersions != nil || len(otherVector.PreviousVersions) != 0 { + // Iterate through incoming vector previous versions, update with the version from other vector + // for source if the local version for that source is lower + for i, v := range otherVector.PreviousVersions { + if hlv.PreviousVersions[i] == 0 { + hlv.setPreviousVersion(i, v) + } else { + // if we get here then there is entry for this source in PV so we must check if its newer or not + otherHLVPVValue := v + localHLVPVValue := hlv.PreviousVersions[i] + if localHLVPVValue < otherHLVPVValue { + hlv.setPreviousVersion(i, v) + } + } } } - return true + // if current source exists in PV, delete it. + if _, ok := hlv.PreviousVersions[hlv.SourceID]; ok { + delete(hlv.PreviousVersions, hlv.SourceID) + } + return nil } -// equalPreviousVectors tests if two previous versions vectors between two HLV's are equal or not -func (hlv *HybridLogicalVector) equalPreviousVectors(otherVector HybridLogicalVector) bool { - if len(hlv.PreviousVersions) != len(otherVector.PreviousVersions) { - return false +// computeMacroExpansions returns the mutate in spec needed for the document update based off the outcome in updateHLV +func (hlv *HybridLogicalVector) computeMacroExpansions() []sgbucket.MacroExpansionSpec { + var outputSpec []sgbucket.MacroExpansionSpec + if hlv.Version == expandMacroCASValueUint64 { + spec := sgbucket.NewMacroExpansionSpec(xattrCurrentVersionPath(base.VvXattrName), sgbucket.MacroCas) + outputSpec = append(outputSpec, spec) + // If version is being expanded, we need to also specify the macro expansion for the expanded rev property + currentRevSpec := sgbucket.NewMacroExpansionSpec(xattrCurrentRevVersionPath(base.SyncXattrName), sgbucket.MacroCas) + outputSpec = append(outputSpec, currentRevSpec) } - for k, v := range hlv.PreviousVersions { - if v != otherVector.PreviousVersions[k] { - return false - } + if hlv.CurrentVersionCAS == expandMacroCASValueUint64 { + spec := sgbucket.NewMacroExpansionSpec(xattrCurrentVersionCASPath(base.VvXattrName), sgbucket.MacroCas) + outputSpec = append(outputSpec, spec) } - return true + return outputSpec } -// GetVersion returns the latest CAS value in the HLV for a given sourceID, if the sourceID is not present in the HLV it will return 0 CAS value -func (hlv *HybridLogicalVector) GetVersion(sourceID string) uint64 { - var latestVersion uint64 - if sourceID == hlv.SourceID { - latestVersion = hlv.Version +// setPreviousVersion will take a source/version pair and add it to the HLV previous versions map +func (hlv *HybridLogicalVector) setPreviousVersion(source string, version uint64) { + if hlv.PreviousVersions == nil { + hlv.PreviousVersions = make(HLVVersions) } - if pvEntry := hlv.PreviousVersions[sourceID]; pvEntry > latestVersion { - latestVersion = pvEntry + hlv.PreviousVersions[source] = version +} + +func (hlv *HybridLogicalVector) IsVersionKnown(otherVersion Version) bool { + value, found := hlv.GetValue(otherVersion.SourceID) + if !found { + return false } - if mvEntry := hlv.MergeVersions[sourceID]; mvEntry > latestVersion { - latestVersion = mvEntry + return value >= otherVersion.Value +} + +// toHistoryForHLV formats blip History property for V4 replication and above +func (hlv *HybridLogicalVector) ToHistoryForHLV() string { + // take pv and mv from hlv if defined and add to history + var s strings.Builder + // Merge versions must be defined first if they exist + if hlv.MergeVersions != nil { + // We need to keep track of where we are in the map, so we don't add a trailing ',' to end of string + itemNo := 1 + for key, value := range hlv.MergeVersions { + vrs := Version{SourceID: key, Value: value} + s.WriteString(vrs.String()) + if itemNo < len(hlv.MergeVersions) { + s.WriteString(",") + } + itemNo++ + } + } + if hlv.PreviousVersions != nil { + // We need to keep track of where we are in the map, so we don't add a trailing ',' to end of string + itemNo := 1 + // only need ';' if we have MV and PV both defined + if len(hlv.MergeVersions) > 0 && len(hlv.PreviousVersions) > 0 { + s.WriteString(";") + } + for key, value := range hlv.PreviousVersions { + vrs := Version{SourceID: key, Value: value} + s.WriteString(vrs.String()) + if itemNo < len(hlv.PreviousVersions) { + s.WriteString(",") + } + itemNo++ + } } - return latestVersion + return s.String() } -func (hlv *HybridLogicalVector) MarshalJSON() ([]byte, error) { +// appendRevocationMacroExpansions adds macro expansions for the channel map. Not strictly an HLV operation +// but putting the function here as it's required when the HLV's current version is being macro expanded +func appendRevocationMacroExpansions(currentSpec []sgbucket.MacroExpansionSpec, channelNames []string) (updatedSpec []sgbucket.MacroExpansionSpec) { + for _, channelName := range channelNames { + spec := sgbucket.NewMacroExpansionSpec(xattrRevokedChannelVersionPath(base.SyncXattrName, channelName), sgbucket.MacroCas) + currentSpec = append(currentSpec, spec) + } + return currentSpec + +} - persistedHLV, err := hlv.convertHLVToPersistedFormat() +// extractHLVFromBlipMessage extracts the full HLV a string in the format seen over Blip +// blip string may be the following formats +// 1. cv only: cv +// 2. cv and pv: cv;pv +// 3. cv, pv, and mv: cv;mv;pv +// +// Function will return list of revIDs if legacy rev ID was found in the HLV history section (PV) +// +// TODO: CBG-3662 - Optimise once we've settled on and tested the format with CBL +func extractHLVFromBlipMessage(versionVectorStr string) (*HybridLogicalVector, []string, error) { + hlv := &HybridLogicalVector{} + + vectorFields := strings.Split(versionVectorStr, ";") + vectorLength := len(vectorFields) + if (vectorLength == 1 && vectorFields[0] == "") || vectorLength > 3 { + return &HybridLogicalVector{}, nil, fmt.Errorf("invalid hlv in changes message received") + } + + // add current version (should always be present) + cvStr := vectorFields[0] + version := strings.Split(cvStr, "@") + if len(version) < 2 { + return &HybridLogicalVector{}, nil, fmt.Errorf("invalid version in changes message received") + } + + vrs, err := strconv.ParseUint(version[0], 16, 64) + if err != nil { + return &HybridLogicalVector{}, nil, err + } + err = hlv.AddVersion(Version{SourceID: version[1], Value: vrs}) if err != nil { - return nil, err + return &HybridLogicalVector{}, nil, err } - return base.JSONMarshal(*persistedHLV) + switch vectorLength { + case 1: + // cv only + return hlv, nil, nil + case 2: + // only cv and pv present + sourceVersionListPV, legacyRev, err := parseVectorValues(vectorFields[1]) + if err != nil { + return &HybridLogicalVector{}, nil, err + } + hlv.PreviousVersions = make(HLVVersions) + for _, v := range sourceVersionListPV { + hlv.PreviousVersions[v.SourceID] = v.Value + } + return hlv, legacyRev, nil + case 3: + // cv, mv and pv present + sourceVersionListPV, legacyRev, err := parseVectorValues(vectorFields[2]) + hlv.PreviousVersions = make(HLVVersions) + if err != nil { + return &HybridLogicalVector{}, nil, err + } + for _, pv := range sourceVersionListPV { + hlv.PreviousVersions[pv.SourceID] = pv.Value + } + + sourceVersionListMV, _, err := parseVectorValues(vectorFields[1]) + hlv.MergeVersions = make(HLVVersions) + if err != nil { + return &HybridLogicalVector{}, nil, err + } + for _, mv := range sourceVersionListMV { + hlv.MergeVersions[mv.SourceID] = mv.Value + } + return hlv, legacyRev, nil + default: + return &HybridLogicalVector{}, nil, fmt.Errorf("invalid hlv in changes message received") + } } -func (hlv *HybridLogicalVector) UnmarshalJSON(inputjson []byte) error { - persistedJSON := PersistedVersionVector{} - err := base.JSONUnmarshal(inputjson, &persistedJSON) - if err != nil { - return err +// parseVectorValues takes an HLV section (cv, pv or mv) in string form and splits into +// source and version pairs. Also returns legacyRev list if legacy revID's are found in the input string. +func parseVectorValues(vectorStr string) (versions []Version, legacyRevList []string, err error) { + versionsStr := strings.Split(vectorStr, ",") + versions = make([]Version, 0, len(versionsStr)) + + for _, v := range versionsStr { + // remove any leading whitespace form the string value + // TODO: Can avoid by restricting spec + if len(v) > 0 && v[0] == ' ' { + v = v[1:] + } + version, err := ParseVersion(v) + if err != nil { + // If v is a legacy rev ID, ignore when constructing the HLV. + if isLegacyRev(v) { + legacyRevList = append(legacyRevList, v) + continue + } + return nil, legacyRevList, err + } + versions = append(versions, version) } - // convert the data to in memory format - hlv.convertPersistedHLVToInMemoryHLV(persistedJSON) - return nil + + return versions, legacyRevList, nil } -func (hlv *HybridLogicalVector) convertHLVToPersistedFormat() (*PersistedVersionVector, error) { - persistedHLV := PersistedVersionVector{} - var cvCasByteArray []byte - if hlv.CurrentVersionCAS != 0 { - cvCasByteArray = base.Uint64CASToLittleEndianHex(hlv.CurrentVersionCAS) +// isLegacyRev returns true if the given string is a revID, false otherwise. Has the same functionality as ParseRevID +// but doesn't warn for malformed revIDs +func isLegacyRev(rev string) bool { + if rev == "" { + return false } - vrsCasByteArray := base.Uint64CASToLittleEndianHex(hlv.Version) - pvPersistedFormat, err := convertMapToPersistedFormat(hlv.PreviousVersions) - if err != nil { - return nil, err + idx := strings.Index(rev, "-") + if idx == -1 { + return false } - mvPersistedFormat, err := convertMapToPersistedFormat(hlv.MergeVersions) + + gen, err := strconv.Atoi(rev[:idx]) if err != nil { - return nil, err + return false + } else if gen < 1 { + return false } + return true +} + +// Helper functions for version source and value encoding +func EncodeSource(source string) string { + return base64.StdEncoding.EncodeToString([]byte(source)) +} - persistedHLV.CurrentVersionCAS = string(cvCasByteArray) - persistedHLV.SourceID = hlv.SourceID - persistedHLV.Version = string(vrsCasByteArray) - persistedHLV.PreviousVersions = pvPersistedFormat - persistedHLV.MergeVersions = mvPersistedFormat - return &persistedHLV, nil +// EncodeValueStr converts a simplified number ("1") to a hex-encoded string +func EncodeValueStr(value string) (string, error) { + return base.StringDecimalToLittleEndianHex(strings.TrimSpace(value)) } -func (hlv *HybridLogicalVector) convertPersistedHLVToInMemoryHLV(persistedJSON PersistedVersionVector) { - hlv.CurrentVersionCAS = base.HexCasToUint64(persistedJSON.CurrentVersionCAS) - hlv.SourceID = persistedJSON.SourceID - // convert the hex cas to uint64 cas - hlv.Version = base.HexCasToUint64(persistedJSON.Version) - // convert the maps form persisted format to the in memory format - hlv.PreviousVersions = convertMapToInMemoryFormat(persistedJSON.PreviousVersions) - hlv.MergeVersions = convertMapToInMemoryFormat(persistedJSON.MergeVersions) +// CreateEncodedSourceID will hash the bucket UUID and cluster UUID using md5 hash function then will base64 encode it +// This function is in sync with xdcr implementation of UUIDstoDocumentSource https://github.com/couchbase/goxdcr/blob/dfba7a5b4251d93db46e2b0b4b55ea014218931b/hlv/hlv.go#L51 +func CreateEncodedSourceID(bucketUUID, clusterUUID string) (string, error) { + md5Hash := md5.Sum([]byte(bucketUUID + clusterUUID)) + hexStr := hex.EncodeToString(md5Hash[:]) + source, err := base.HexToBase64(hexStr) + if err != nil { + return "", err + } + return string(source), nil } -// convertMapToPersistedFormat will convert in memory map of previous versions or merge versions into the persisted format map -func convertMapToPersistedFormat(memoryMap map[string]uint64) (map[string]string, error) { - if memoryMap == nil { - return nil, nil +func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { + type BucketVector struct { + CurrentVersionCAS string `json:"cvCas,omitempty"` + SourceID string `json:"src"` + Version string `json:"ver"` + PV *[]string `json:"pv,omitempty"` + MV *[]string `json:"mv,omitempty"` } - returnedMap := make(map[string]string) - var persistedCAS string - for source, cas := range memoryMap { - casByteArray := base.Uint64CASToLittleEndianHex(cas) - persistedCAS = string(casByteArray) - // remove the leading '0x' from the CAS value - persistedCAS = persistedCAS[2:] - returnedMap[source] = persistedCAS + var cvCas string + var vrsCas string + + var bucketHLV = BucketVector{} + if hlv.CurrentVersionCAS != 0 { + cvCas = base.CasToString(hlv.CurrentVersionCAS) + bucketHLV.CurrentVersionCAS = cvCas + } + vrsCas = base.CasToString(hlv.Version) + bucketHLV.Version = vrsCas + bucketHLV.SourceID = hlv.SourceID + + pvPersistedFormat := VersionsToDeltas(hlv.PreviousVersions) + if len(pvPersistedFormat) > 0 { + bucketHLV.PV = &pvPersistedFormat } - return returnedMap, nil + mvPersistedFormat := VersionsToDeltas(hlv.MergeVersions) + if len(mvPersistedFormat) > 0 { + bucketHLV.MV = &mvPersistedFormat + } + + return base.JSONMarshal(&bucketHLV) } -// convertMapToInMemoryFormat will convert the persisted format map to an in memory format of that map. -// Used for previous versions and merge versions maps on HLV -func convertMapToInMemoryFormat(persistedMap map[string]string) map[string]uint64 { - if persistedMap == nil { - return nil +func (hlv *HybridLogicalVector) UnmarshalJSON(inputjson []byte) error { + type BucketVector struct { + CurrentVersionCAS string `json:"cvCas,omitempty"` + SourceID string `json:"src"` + Version string `json:"ver"` + PV *[]string `json:"pv,omitempty"` + MV *[]string `json:"mv,omitempty"` } - returnedMap := make(map[string]uint64) - // convert each CAS entry from little endian hex to Uint64 - for key, value := range persistedMap { - returnedMap[key] = base.HexCasToUint64(value) + var bucketDeltas BucketVector + err := base.JSONUnmarshal(inputjson, &bucketDeltas) + if err != nil { + return err + } + if bucketDeltas.CurrentVersionCAS != "" { + hlv.CurrentVersionCAS = base.HexCasToUint64(bucketDeltas.CurrentVersionCAS) } - return returnedMap + + hlv.SourceID = bucketDeltas.SourceID + hlv.Version = base.HexCasToUint64(bucketDeltas.Version) + if bucketDeltas.PV != nil { + prevVersion, err := PersistedDeltasToMap(*bucketDeltas.PV) + if err != nil { + return err + } + hlv.PreviousVersions = prevVersion + } + if bucketDeltas.MV != nil { + mergeVersion, err := PersistedDeltasToMap(*bucketDeltas.MV) + if err != nil { + return err + } + hlv.MergeVersions = mergeVersion + } + return nil } diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index 6d436e6417..d0fc27a68b 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -9,11 +9,16 @@ package db import ( + "encoding/base64" + "math/rand/v2" "reflect" "strconv" "strings" "testing" + "time" + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,32 +27,32 @@ import ( // - Tests internal api methods on the HLV work as expected // - Tests methods GetCurrentVersion, AddVersion and Remove func TestInternalHLVFunctions(t *testing.T) { - pv := make(map[string]uint64) - currSourceId := "s_5pRi8Piv1yLcLJ1iVNJIsA" - const currVersion = 12345678 - pv["s_YZvBpEaztom9z5V/hDoeIw"] = 64463204720 + pv := make(HLVVersions) + currSourceId := EncodeSource("5pRi8Piv1yLcLJ1iVNJIsA") + currVersion := uint64(12345678) + pv[EncodeSource("YZvBpEaztom9z5V/hDoeIw")] = 64463204720 - inputHLV := []string{"s_5pRi8Piv1yLcLJ1iVNJIsA@12345678", "s_YZvBpEaztom9z5V/hDoeIw@64463204720", "m_s_NqiIe0LekFPLeX4JvTO6Iw@345454"} + inputHLV := []string{"5pRi8Piv1yLcLJ1iVNJIsA@12345678", "YZvBpEaztom9z5V/hDoeIw@64463204720", "m_NqiIe0LekFPLeX4JvTO6Iw@345454"} hlv := createHLVForTest(t, inputHLV) - const newCAS = 123456789 + newCAS := uint64(123456789) const newSource = "s_testsource" // create a new version vector entry that will error method AddVersion - badNewVector := CurrentVersionVector{ - VersionCAS: 123345, - SourceID: currSourceId, + badNewVector := Version{ + Value: 123345, + SourceID: currSourceId, } // create a new version vector entry that should be added to HLV successfully - newVersionVector := CurrentVersionVector{ - VersionCAS: newCAS, - SourceID: currSourceId, + newVersionVector := Version{ + Value: newCAS, + SourceID: currSourceId, } // Get current version vector, sourceID and CAS pair source, version := hlv.GetCurrentVersion() assert.Equal(t, currSourceId, source) - assert.Equal(t, uint64(currVersion), version) + assert.Equal(t, currVersion, version) // add new version vector with same sourceID as current sourceID and assert it doesn't add to previous versions then restore HLV to previous state require.NoError(t, hlv.AddVersion(newVersionVector)) @@ -62,7 +67,7 @@ func TestInternalHLVFunctions(t *testing.T) { // Add a new version vector pair to the HLV structure and assert that it moves the current version vector pair to the previous versions section newVersionVector.SourceID = newSource require.NoError(t, hlv.AddVersion(newVersionVector)) - assert.Equal(t, uint64(newCAS), hlv.Version) + assert.Equal(t, newCAS, hlv.Version) assert.Equal(t, newSource, hlv.SourceID) assert.True(t, reflect.DeepEqual(hlv.PreviousVersions, pv)) @@ -75,7 +80,7 @@ func TestInternalHLVFunctions(t *testing.T) { } // TestConflictDetectionDominating: -// - Tests two cases where one HLV's is said to be 'dominating' over another and thus not in conflict +// - Tests cases where one HLV's is said to be 'dominating' over another // - Test case 1: where sourceID is the same between HLV's but HLV(A) has higher version CAS than HLV(B) thus A dominates // - Test case 2: where sourceID is different and HLV(A) sourceID is present in HLV(B) PV and HLV(A) has dominating version // - Test case 3: where sourceID is different and HLV(A) sourceID is present in HLV(B) MV and HLV(A) has dominating version @@ -83,149 +88,684 @@ func TestInternalHLVFunctions(t *testing.T) { // - Assert that all scenarios returns false from IsInConflict method, as we have a HLV that is dominating in each case func TestConflictDetectionDominating(t *testing.T) { testCases := []struct { - name string - inputListHLVA []string - inputListHLVB []string + name string + inputListHLVA []string + inputListHLVB []string + expectedResult bool }{ { - name: "Test case 1", - inputListHLVA: []string{"cluster1@20", "cluster2@2"}, - inputListHLVB: []string{"cluster1@10", "cluster2@1"}, + name: "Matching current source, newer version", + inputListHLVA: []string{"cluster1@20", "cluster2@2"}, + inputListHLVB: []string{"cluster1@10", "cluster2@1"}, + expectedResult: true, + }, { + name: "Matching current source and version", + inputListHLVA: []string{"cluster1@20", "cluster2@2"}, + inputListHLVB: []string{"cluster1@20", "cluster2@1"}, + expectedResult: true, }, { - name: "Test case 2", - inputListHLVA: []string{"cluster1@20", "cluster3@3"}, - inputListHLVB: []string{"cluster2@10", "cluster1@15"}, + name: "B CV found in A's PV", + inputListHLVA: []string{"cluster1@20", "cluster2@10"}, + inputListHLVB: []string{"cluster2@10", "cluster1@15"}, + expectedResult: true, }, { - name: "Test case 3", - inputListHLVA: []string{"cluster1@20", "cluster3@3"}, - inputListHLVB: []string{"cluster2@10", "m_cluster1@12", "m_cluster2@11"}, + name: "B CV older than A's PV for same source", + inputListHLVA: []string{"cluster1@20", "cluster2@10"}, + inputListHLVB: []string{"cluster2@10", "cluster1@15"}, + expectedResult: true, }, { - name: "Test case 4", - inputListHLVA: []string{"cluster2@10", "cluster1@15"}, - - inputListHLVB: []string{"cluster1@20", "cluster3@3"}, + name: "Unique sources in A", + inputListHLVA: []string{"cluster1@20", "cluster2@15", "cluster3@3"}, + inputListHLVB: []string{"cluster2@10", "cluster1@10"}, + expectedResult: true, + }, + { + name: "Unique sources in B", + inputListHLVA: []string{"cluster1@20"}, + inputListHLVB: []string{"cluster1@15", "cluster3@3"}, + expectedResult: true, + }, + { + name: "B has newer cv", + inputListHLVA: []string{"cluster1@10"}, + inputListHLVB: []string{"cluster1@15"}, + expectedResult: false, + }, + { + name: "B has newer cv than A pv", + inputListHLVA: []string{"cluster2@20", "cluster1@10"}, + inputListHLVB: []string{"cluster1@15", "cluster2@20"}, + expectedResult: false, + }, + { + name: "B's cv not found in A", + inputListHLVA: []string{"cluster2@20", "cluster1@10"}, + inputListHLVB: []string{"cluster3@5"}, + expectedResult: false, }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { hlvA := createHLVForTest(t, testCase.inputListHLVA) hlvB := createHLVForTest(t, testCase.inputListHLVB) - require.False(t, hlvA.IsInConflict(hlvB)) + require.True(t, hlvA.isDominating(hlvB) == testCase.expectedResult) + }) } } -// TestConflictEqualHLV: -// - Creates two 'equal' HLV's and asserts we identify them as equal -// - Then tests other code path in event source ID differs and current CAS differs but with identical merge versions -// that we identify they are in fact 'equal' -// - Then test the same but for previous versions -func TestConflictEqualHLV(t *testing.T) { - // two vectors with the same sourceID and version pair as the current vector - inputHLVA := []string{"cluster1@10", "cluster2@3"} - inputHLVB := []string{"cluster1@10", "cluster2@4"} - hlvA := createHLVForTest(t, inputHLVA) - hlvB := createHLVForTest(t, inputHLVB) - require.True(t, hlvA.isEqual(hlvB)) - - // test conflict detection with different version CAS but same merge versions - inputHLVA = []string{"cluster2@12", "cluster3@3", "cluster4@2"} - inputHLVB = []string{"cluster1@10", "cluster3@3", "cluster4@2"} - hlvA = createHLVForTest(t, inputHLVA) - hlvB = createHLVForTest(t, inputHLVB) - require.True(t, hlvA.isEqual(hlvB)) - - // test conflict detection with different version CAS but same previous version vectors - inputHLVA = []string{"cluster3@2", "cluster1@3", "cluster2@5"} - hlvA = createHLVForTest(t, inputHLVA) - inputHLVB = []string{"cluster4@7", "cluster1@3", "cluster2@5"} - hlvB = createHLVForTest(t, inputHLVB) - require.True(t, hlvA.isEqual(hlvB)) - - // remove an entry from one of the HLV PVs to assert we get false returned from isEqual - require.NoError(t, hlvA.Remove("cluster1")) - require.False(t, hlvA.isEqual(hlvB)) -} - -// TestConflictExample: -// - Takes example conflict scenario from PRD to see if we correctly identify conflict in that scenario -// - Creates two HLV's similar to ones in example and calls IsInConflict to assert it returns true -func TestConflictExample(t *testing.T) { - input := []string{"cluster1@11", "cluster3@2", "cluster2@4"} - inMemoryHLV := createHLVForTest(t, input) - - input = []string{"cluster2@2", "cluster3@3"} - otherVector := createHLVForTest(t, input) - require.True(t, inMemoryHLV.IsInConflict(otherVector)) -} - // createHLVForTest is a helper function to create a HLV for use in a test. Takes a list of strings in the format of and assumes // first entry is current version. For merge version entries you must specify 'm_' as a prefix to sourceID NOTE: it also sets cvCAS to the current version -func createHLVForTest(tb *testing.T, inputList []string) HybridLogicalVector { +func createHLVForTest(tb *testing.T, inputList []string) *HybridLogicalVector { hlvOutput := NewHybridLogicalVector() // first element will be current version and source pair currentVersionPair := strings.Split(inputList[0], "@") - hlvOutput.SourceID = currentVersionPair[0] - version, err := strconv.Atoi(currentVersionPair[1]) + hlvOutput.SourceID = base64.StdEncoding.EncodeToString([]byte(currentVersionPair[0])) + value, err := strconv.ParseUint(currentVersionPair[1], 10, 64) require.NoError(tb, err) - hlvOutput.Version = uint64(version) - hlvOutput.CurrentVersionCAS = uint64(version) + hlvOutput.Version = value + hlvOutput.CurrentVersionCAS = value // remove current version entry in list now we have parsed it into the HLV inputList = inputList[1:] - for _, value := range inputList { - currentVersionPair = strings.Split(value, "@") - version, err = strconv.Atoi(currentVersionPair[1]) + for _, version := range inputList { + currentVersionPair = strings.Split(version, "@") + value, err = strconv.ParseUint(currentVersionPair[1], 10, 64) require.NoError(tb, err) if strings.HasPrefix(currentVersionPair[0], "m_") { // add entry to merge version removing the leading prefix for sourceID - hlvOutput.MergeVersions[currentVersionPair[0][2:]] = uint64(version) + hlvOutput.MergeVersions[EncodeSource(currentVersionPair[0][2:])] = value } else { - // if its not got the prefix we assume its a previous version entry - hlvOutput.PreviousVersions[currentVersionPair[0]] = uint64(version) + // if it's not got the prefix we assume it's a previous version entry + hlvOutput.PreviousVersions[EncodeSource(currentVersionPair[0])] = value } } return hlvOutput } -// TestHybridLogicalVectorPersistence: -// - Tests the process of constructing in memory HLV and marshaling it to persisted format -// - Asserts on the format -// - Unmarshal the HLV and assert that the process works as expected -func TestHybridLogicalVectorPersistence(t *testing.T) { - // create HLV - inputHLV := []string{"cb06dc003846116d9b66d2ab23887a96@123456", "s_YZvBpEaztom9z5V/hDoeIw@1628620455135215600", "m_s_NqiIe0LekFPLeX4JvTO6Iw@1628620455139868700", - "m_s_LhRPsa7CpjEvP5zeXTXEBA@1628620455147864000"} - inMemoryHLV := createHLVForTest(t, inputHLV) - - // marshal in memory hlv into persisted form - byteArray, err := inMemoryHLV.MarshalJSON() +func TestAddNewerVersionsBetweenTwoVectorsWhenNotInConflict(t *testing.T) { + testCases := []struct { + name string + localInput []string + incomingInput []string + expected []string + }{ + { + name: "testcase1", + localInput: []string{"abc@15"}, + incomingInput: []string{"def@25", "abc@20"}, + expected: []string{"def@25", "abc@20"}, + }, + { + name: "testcase2", + localInput: []string{"abc@15", "def@30"}, + incomingInput: []string{"def@35", "abc@15"}, + expected: []string{"def@35", "abc@15"}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + localHLV := createHLVForTest(t, test.localInput) + incomingHLV := createHLVForTest(t, test.incomingInput) + expectedHLV := createHLVForTest(t, test.expected) + + _ = localHLV.AddNewerVersions(incomingHLV) + // assert on expected values + assert.Equal(t, expectedHLV.SourceID, localHLV.SourceID) + assert.Equal(t, expectedHLV.Version, localHLV.Version) + assert.True(t, reflect.DeepEqual(expectedHLV.PreviousVersions, localHLV.PreviousVersions)) + }) + } +} + +// Tests import of server-side mutations made by HLV-aware and non-HLV-aware peers +func TestHLVImport(t *testing.T) { + + base.SetUpTestLogging(t, base.LevelInfo, base.KeyMigrate, base.KeyImport) + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + type outputData struct { + docID string + preImportHLV *HybridLogicalVector + preImportCas uint64 + preImportMou *MetadataOnlyUpdate + postImportCas uint64 + preImportRevSeqNo uint64 + } + + var standardBody = []byte(`{"prop":"value"}`) + otherSource := "otherSource" + var testCases = []struct { + name string + preFunc func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) + expectedMou func(output *outputData) *MetadataOnlyUpdate + expectedHLV func(output *outputData) *HybridLogicalVector + }{ + { + name: "SDK write, no existing doc", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + _, err := collection.dataStore.WriteCas(docID, 0, 0, standardBody, sgbucket.Raw) + require.NoError(t, err, "write error") + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(output.preImportCas)), + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return &HybridLogicalVector{ + SourceID: db.EncodedSourceID, + Version: output.preImportCas, + CurrentVersionCAS: output.preImportCas, + } + }, + }, + { + name: "SDK write, existing doc", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + _, doc, err := collection.Put(ctx, docID, Body{"foo": "bar"}) + require.NoError(t, err) + _, err = collection.dataStore.WriteCas(docID, 0, doc.Cas, standardBody, sgbucket.Raw) + require.NoError(t, err, "write error") + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(output.preImportCas)), + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return &HybridLogicalVector{ + SourceID: db.EncodedSourceID, + Version: output.preImportCas, + CurrentVersionCAS: output.preImportCas, + } + }, + }, + { + name: "HLV write from without mou", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + hlvHelper := NewHLVAgent(t, collection.dataStore, otherSource, "_vv") + _ = hlvHelper.InsertWithHLV(ctx, docID) + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(output.preImportCas)), + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return &HybridLogicalVector{ + SourceID: EncodeSource(otherSource), + Version: output.preImportCas, + CurrentVersionCAS: output.preImportCas, + } + }, + }, + { + name: "XDCR stamped with _mou", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + hlvHelper := NewHLVAgent(t, collection.dataStore, otherSource, "_vv") + cas := hlvHelper.InsertWithHLV(ctx, docID) + + _, xattrs, _, err := collection.dataStore.GetWithXattrs(ctx, docID, []string{base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + mou := &MetadataOnlyUpdate{ + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(cas)), + PreviousRevSeqNo: RetrieveDocRevSeqNo(t, xattrs[base.VirtualXattrRevSeqNo]), + } + opts := &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec(XattrMouCasPath(), sgbucket.MacroCas), + }, + } + _, err = collection.dataStore.UpdateXattrs(ctx, docID, 0, cas, map[string][]byte{base.MouXattrName: base.MustJSONMarshal(t, mou)}, opts) + require.NoError(t, err) + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: output.preImportMou.PreviousHexCAS, + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return output.preImportHLV + }, + }, + { + name: "invalid _mou, but valid hlv", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + hlvHelper := NewHLVAgent(t, collection.dataStore, otherSource, "_vv") + cas := hlvHelper.InsertWithHLV(ctx, docID) + + _, xattrs, _, err := collection.dataStore.GetWithXattrs(ctx, docID, []string{base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + mou := &MetadataOnlyUpdate{ + HexCAS: "invalid", + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(cas)), + PreviousRevSeqNo: RetrieveDocRevSeqNo(t, xattrs[base.VirtualXattrRevSeqNo]), + } + _, err = collection.dataStore.UpdateXattrs(ctx, docID, 0, cas, map[string][]byte{base.MouXattrName: base.MustJSONMarshal(t, mou)}, nil) + require.NoError(t, err) + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(output.preImportCas)), + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return &HybridLogicalVector{ + SourceID: db.EncodedSourceID, + Version: output.preImportCas, + CurrentVersionCAS: output.preImportCas, + PreviousVersions: map[string]uint64{ + EncodeSource(otherSource): output.preImportHLV.CurrentVersionCAS, + }, + } + }, + }, + { + name: "SDK write with valid _mou, but no HLV", + preFunc: func(t *testing.T, collection *DatabaseCollectionWithUser, docID string) { + cas, err := collection.dataStore.WriteCas(docID, 0, 0, standardBody, sgbucket.Raw) + require.NoError(t, err) + _, xattrs, _, err := collection.dataStore.GetWithXattrs(ctx, docID, []string{base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + + mou := &MetadataOnlyUpdate{ + PreviousHexCAS: string(base.Uint64CASToLittleEndianHex(cas)), + PreviousRevSeqNo: RetrieveDocRevSeqNo(t, xattrs[base.VirtualXattrRevSeqNo]), + } + opts := &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec(XattrMouCasPath(), sgbucket.MacroCas), + }, + } + _, err = collection.dataStore.UpdateXattrs(ctx, docID, 0, cas, map[string][]byte{base.MouXattrName: base.MustJSONMarshal(t, mou)}, opts) + require.NoError(t, err) + }, + expectedMou: func(output *outputData) *MetadataOnlyUpdate { + return &MetadataOnlyUpdate{ + HexCAS: string(base.Uint64CASToLittleEndianHex(output.postImportCas)), + PreviousHexCAS: output.preImportMou.PreviousHexCAS, + PreviousRevSeqNo: output.preImportRevSeqNo, + } + }, + expectedHLV: func(output *outputData) *HybridLogicalVector { + return &HybridLogicalVector{ + SourceID: db.EncodedSourceID, + Version: output.preImportCas, + CurrentVersionCAS: output.preImportCas, + } + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + docID := strings.ToLower(testCase.name) + testCase.preFunc(t, collection, docID) + + xattrNames := []string{base.SyncXattrName, base.VvXattrName, base.MouXattrName, base.VirtualXattrRevSeqNo} + _, existingXattrs, preImportCas, err := collection.dataStore.GetWithXattrs(ctx, docID, xattrNames) + require.NoError(t, err) + revSeqNo := RetrieveDocRevSeqNo(t, existingXattrs[base.VirtualXattrRevSeqNo]) + + var preImportMou *MetadataOnlyUpdate + if mouBytes, ok := existingXattrs[base.MouXattrName]; ok && mouBytes != nil { + require.NoError(t, base.JSONUnmarshal(mouBytes, &preImportMou)) + } + importOpts := importDocOptions{ + isDelete: false, + expiry: nil, + mode: ImportFromFeed, + revSeqNo: revSeqNo, + } + _, err = collection.ImportDocRaw(ctx, docID, standardBody, existingXattrs, importOpts, preImportCas) + require.NoError(t, err, "import error") + + xattrs, finalCas, err := collection.dataStore.GetXattrs(ctx, docID, xattrNames) + require.NoError(t, err) + require.NotEqual(t, preImportCas, finalCas) + + // validate _sync.cas was expanded to document cas + require.Contains(t, xattrs, base.SyncXattrName) + var syncData *SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) + require.Equal(t, finalCas, base.HexCasToUint64(syncData.Cas)) + + output := outputData{ + docID: docID, + preImportCas: preImportCas, + preImportMou: preImportMou, + postImportCas: finalCas, + preImportRevSeqNo: revSeqNo, + } + if existingHLV, ok := existingXattrs[base.VvXattrName]; ok { + + require.NoError(t, base.JSONUnmarshal(existingHLV, &output.preImportHLV)) + } + + if testCase.expectedMou != nil { + require.Contains(t, xattrs, base.MouXattrName) + var mou *MetadataOnlyUpdate + require.NoError(t, base.JSONUnmarshal(xattrs[base.MouXattrName], &mou)) + require.Contains(t, xattrs, base.MouXattrName) + require.Equal(t, *testCase.expectedMou(&output), *mou) + } + var hlv *HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &hlv)) + require.Equal(t, *testCase.expectedHLV(&output), *hlv) + }) + } + +} + +// TestHLVMapToCBLString: +// - Purpose is to test the ability to extract from HLV maps in CBL replication format +// - Three test cases, both MV and PV defined, only PV defined and only MV defined +// - To protect against flake added some splitting of the result string in test case 1 as we cannot guarantee the +// order the string will be made in given map iteration is random +func TestHLVMapToCBLString(t *testing.T) { + + testCases := []struct { + name string + inputHLV []string + expectedStr string + both bool + }{ + { + name: "Both PV and mv", + inputHLV: []string{"cb06dc003846116d9b66d2ab23887a96@123456", "YZvBpEaztom9z5V/hDoeIw@1628620455135215600", "m_NqiIe0LekFPLeX4JvTO6Iw@1628620455139868700", + "m_LhRPsa7CpjEvP5zeXTXEBA@1628620455147864000"}, + expectedStr: "169a05acd68c001c@TnFpSWUwTGVrRlBMZVg0SnZUTzZJdw==,169a05acd705ffc0@TGhSUHNhN0NwakV2UDV6ZVhUWEVCQQ==;169a05acd644fff0@WVp2QnBFYXp0b205ejVWL2hEb2VJdw==", + both: true, + }, + { + name: "Just PV", + inputHLV: []string{"cb06dc003846116d9b66d2ab23887a96@123456", "YZvBpEaztom9z5V/hDoeIw@1628620455135215600"}, + expectedStr: "169a05acd644fff0@WVp2QnBFYXp0b205ejVWL2hEb2VJdw==", + }, + { + name: "Just MV", + inputHLV: []string{"cb06dc003846116d9b66d2ab23887a96@123456", "m_NqiIe0LekFPLeX4JvTO6Iw@1628620455139868700"}, + expectedStr: "169a05acd68c001c@TnFpSWUwTGVrRlBMZVg0SnZUTzZJdw==", + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + hlv := createHLVForTest(t, test.inputHLV) + historyStr := hlv.ToHistoryForHLV() + + if test.both { + initial := strings.Split(historyStr, ";") + mvSide := strings.Split(initial[0], ",") + assert.Contains(t, test.expectedStr, initial[1]) + for _, v := range mvSide { + assert.Contains(t, test.expectedStr, v) + } + } else { + assert.Equal(t, test.expectedStr, historyStr) + } + }) + } +} + +// TestInvalidHLVOverChangesMessage: +// - Test hlv string that has too many sections to it (parts delimited by ;) +// - Test hlv string that is empty +// - Assert that extractHLVFromBlipMessage will return error in both cases +func TestInvalidHLVInBlipMessageForm(t *testing.T) { + hlvStr := "25@def; 22@def,21@eff; 20@abc,18@hij; 222@hiowdwdew, 5555@dhsajidfgd" + + hlv, _, err := extractHLVFromBlipMessage(hlvStr) + require.Error(t, err) + assert.ErrorContains(t, err, "invalid hlv in changes message received") + assert.Equal(t, &HybridLogicalVector{}, hlv) + + hlvStr = "" + hlv, _, err = extractHLVFromBlipMessage(hlvStr) + require.Error(t, err) + assert.ErrorContains(t, err, "invalid hlv in changes message received") + assert.Equal(t, &HybridLogicalVector{}, hlv) +} + +var extractHLVFromBlipMsgBMarkCases = []struct { + name string + hlvString string + expectedHLV []string + mergeVersions bool + previousVersions bool +}{ + { + name: "mv and pv, leading spaces", // with spaces + hlvString: "25@def; 22@def, 21@eff, 500@x, 501@xx, 4000@xxx, 700@y, 701@yy, 702@yyy; 20@abc, 18@hij, 3@x, 4@xx, 5@xxx, 6@xxxx, 7@xxxxx, 3@y, 4@yy, 5@yyy, 6@yyyy, 7@yyyyy, 2@xy, 3@xyy, 4@xxy", // 15 pv 8 mv + expectedHLV: []string{"def@25", "abc@20", "hij@18", "x@3", "xx@4", "xxx@5", "xxxx@6", "xxxxx@7", "y@3", "yy@4", "yyy@5", "yyyy@6", "yyyyy@7", "xy@2", "xyy@3", "xxy@4", "m_def@22", "m_eff@21", "m_x@500", "m_xx@501", "m_xxx@4000", "m_y@700", "m_yy@701", "m_yyy@702"}, + previousVersions: true, + mergeVersions: true, + }, + { + name: "mv and pv, no spaces", // without spaces + hlvString: "25@def;22@def,21@eff,500@x,501@xx,4000@xxx,700@y,701@yy,702@yyy;20@abc,18@hij,3@x,4@xx,5@xxx,6@xxxx,7@xxxxx,3@y,4@yy,5@yyy,6@yyyy,7@yyyyy,2@xy,3@xyy,4@xxy", // 15 pv 8 mv + expectedHLV: []string{"def@25", "abc@20", "hij@18", "x@3", "xx@4", "xxx@5", "xxxx@6", "xxxxx@7", "y@3", "yy@4", "yyy@5", "yyyy@6", "yyyyy@7", "xy@2", "xyy@3", "xxy@4", "m_def@22", "m_eff@21", "m_x@500", "m_xx@501", "m_xxx@4000", "m_y@700", "m_yy@701", "m_yyy@702"}, + previousVersions: true, + mergeVersions: true, + }, + { + name: "pv only", + hlvString: "25@def; 20@abc,18@hij", + expectedHLV: []string{"def@25", "abc@20", "hij@18"}, + previousVersions: true, + }, + { + name: "mv and pv, mixed spacing", + hlvString: "25@def; 22@def,21@eff; 20@abc,18@hij,3@x,4@xx,5@xxx,6@xxxx,7@xxxxx,3@y,4@yy,5@yyy,6@yyyy,7@yyyyy,2@xy,3@xyy,4@xxy", // 15 + expectedHLV: []string{"def@25", "abc@20", "hij@18", "x@3", "xx@4", "xxx@5", "xxxx@6", "xxxxx@7", "y@3", "yy@4", "yyy@5", "yyyy@6", "yyyyy@7", "xy@2", "xyy@3", "xxy@4", "m_def@22", "m_eff@21"}, + mergeVersions: true, + previousVersions: true, + }, + { + name: "cv only", + hlvString: "24@def", + expectedHLV: []string{"def@24"}, + }, + { + name: "cv and mv,base64 encoded", + hlvString: "1@Hell0CA; 1@1Hr0k43xS662TToxODDAxQ", + expectedHLV: []string{"Hell0CA@1", "1Hr0k43xS662TToxODDAxQ@1"}, + previousVersions: true, + }, + { + name: "cv and mv - small", + hlvString: "25@def; 22@def,21@eff; 20@abc,18@hij", + expectedHLV: []string{"def@25", "abc@20", "hij@18", "m_def@22", "m_eff@21"}, + mergeVersions: true, + previousVersions: true, + }, +} + +// TestExtractHLVFromChangesMessage: +// - Test case 1: CV entry and 1 PV entry +// - Test case 2: CV entry and 2 PV entries +// - Test case 3: CV entry, 2 MV entries and 2 PV entries +// - Test case 4: just CV entry +// - Each test case gets run through extractHLVFromBlipMessage and assert that the resulting HLV +// is correct to what is expected +func TestExtractHLVFromChangesMessage(t *testing.T) { + for _, test := range extractHLVFromBlipMsgBMarkCases { + t.Run(test.name, func(t *testing.T) { + expectedVector := createHLVForTest(t, test.expectedHLV) + + // TODO: When CBG-3662 is done, should be able to simplify base64 handling to treat source as a string + // that may represent a base64 encoding + base64EncodedHlvString := EncodeTestHistory(test.hlvString) + hlv, _, err := extractHLVFromBlipMessage(base64EncodedHlvString) + require.NoError(t, err) + + assert.Equal(t, expectedVector.SourceID, hlv.SourceID) + assert.Equal(t, expectedVector.Version, hlv.Version) + if test.previousVersions { + assert.True(t, reflect.DeepEqual(expectedVector.PreviousVersions, hlv.PreviousVersions)) + } + if test.mergeVersions { + assert.True(t, reflect.DeepEqual(expectedVector.MergeVersions, hlv.MergeVersions)) + } + }) + } +} + +func BenchmarkExtractHLVFromBlipMessage(b *testing.B) { + for _, bm := range extractHLVFromBlipMsgBMarkCases { + b.Run(bm.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _, _ = extractHLVFromBlipMessage(bm.hlvString) + } + }) + } +} + +func TestParseCBLVersion(t *testing.T) { + vrsString := "19@YWJj" + + vrs, err := ParseVersion(vrsString) + require.NoError(t, err) + assert.Equal(t, "YWJj", vrs.SourceID) + assert.Equal(t, uint64(25), vrs.Value) + + cblString := vrs.String() + assert.Equal(t, vrsString, cblString) +} + +// TestVersionDeltaCalculation: +// - Create some random versions and assign to a source/version map +// - Convert the map to deltas and assert that first item in list is greater than all other elements +// - Create a test HLV and convert it to persisted format in bytes +// - Convert this back to in memory format, assert each elem of in memory format previous versions map is the same as +// the corresponding element in the original pvMap +// - Do the same for a pv map that will have two entries with the same version value +// - Do the same as above but for nil maps +func TestVersionDeltaCalculation(t *testing.T) { + src1 := "src1" + src2 := "src2" + src3 := "src3" + src4 := "src4" + src5 := "src5" + + timeNow := time.Now().UnixNano() + // make some version deltas + v1 := uint64(timeNow - rand.Int64N(1000000000000)) + v2 := uint64(timeNow - rand.Int64N(1000000000000)) + v3 := uint64(timeNow - rand.Int64N(1000000000000)) + v4 := uint64(timeNow - rand.Int64N(1000000000000)) + v5 := uint64(timeNow - rand.Int64N(1000000000000)) + + // make map of source to version + pvMap := make(HLVVersions) + pvMap[src1] = v1 + pvMap[src2] = v2 + pvMap[src3] = v3 + pvMap[src4] = v4 + pvMap[src5] = v5 + + // convert to version delta map assert that first element is larger than all other elements + deltas := VersionDeltas(pvMap) + assert.Greater(t, deltas[0].Value, deltas[1].Value) + assert.Greater(t, deltas[0].Value, deltas[2].Value) + assert.Greater(t, deltas[0].Value, deltas[3].Value) + assert.Greater(t, deltas[0].Value, deltas[4].Value) + + // create a test hlv + inputHLVA := []string{"cluster3@2"} + hlv := createHLVForTest(t, inputHLVA) + hlv.PreviousVersions = pvMap + expSrc := hlv.SourceID + expVal := hlv.Version + expCas := hlv.CurrentVersionCAS + + // convert hlv to persisted format + vvXattr, err := base.JSONMarshal(&hlv) + require.NoError(t, err) + + // convert the bytes back to an in memory format of hlv + memHLV := NewHybridLogicalVector() + err = base.JSONUnmarshal(vvXattr, &memHLV) require.NoError(t, err) - // convert to string and assert the in memory struct is converted to persisted form correctly - // no guarantee the order of the marshaling of the mv part so just assert on the values - strHLV := string(byteArray) - assert.Contains(t, strHLV, `"cvCas":"0x40e2010000000000`) - assert.Contains(t, strHLV, `"src":"cb06dc003846116d9b66d2ab23887a96"`) - assert.Contains(t, strHLV, `"vrs":"0x40e2010000000000"`) - assert.Contains(t, strHLV, `"s_LhRPsa7CpjEvP5zeXTXEBA":"c0ff05d7ac059a16"`) - assert.Contains(t, strHLV, `"s_NqiIe0LekFPLeX4JvTO6Iw":"1c008cd6ac059a16"`) - assert.Contains(t, strHLV, `"pv":{"s_YZvBpEaztom9z5V/hDoeIw":"f0ff44d6ac059a16"}`) - - // Unmarshal the in memory constructed HLV above - hlvFromPersistance := HybridLogicalVector{} - err = hlvFromPersistance.UnmarshalJSON(byteArray) + assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) + assert.Equal(t, pvMap[src2], memHLV.PreviousVersions[src2]) + assert.Equal(t, pvMap[src3], memHLV.PreviousVersions[src3]) + assert.Equal(t, pvMap[src4], memHLV.PreviousVersions[src4]) + assert.Equal(t, pvMap[src5], memHLV.PreviousVersions[src5]) + + // assert that the other elements are as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.MergeVersions, 0) + + // test hlv with two pv version entries that are equal to each other + hlv = createHLVForTest(t, inputHLVA) + // make src3 have the same version value as src2 + pvMap[src3] = pvMap[src2] + hlv.PreviousVersions = pvMap + + // convert hlv to persisted format + vvXattr, err = base.JSONMarshal(&hlv) + require.NoError(t, err) + + // convert the bytes back to an in memory format of hlv + memHLV = NewHybridLogicalVector() + err = base.JSONUnmarshal(vvXattr, &memHLV) + require.NoError(t, err) + + assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) + assert.Equal(t, pvMap[src2], memHLV.PreviousVersions[src2]) + assert.Equal(t, pvMap[src3], memHLV.PreviousVersions[src3]) + assert.Equal(t, pvMap[src4], memHLV.PreviousVersions[src4]) + assert.Equal(t, pvMap[src5], memHLV.PreviousVersions[src5]) + + // assert that the other elements are as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.MergeVersions, 0) + + // test hlv with nil merge versions and nil previous versions to test panic safe + pvMap = nil + hlv2 := createHLVForTest(t, inputHLVA) + hlv2.PreviousVersions = pvMap + hlv2.MergeVersions = nil + deltas = VersionDeltas(pvMap) + assert.Nil(t, deltas) + + // construct byte array from hlv + vvXattr, err = base.JSONMarshal(&hlv2) + require.NoError(t, err) + // convert the bytes back to an in memory format of hlv + memHLV = &HybridLogicalVector{} + err = base.JSONUnmarshal(vvXattr, &memHLV) require.NoError(t, err) - // assertions on values of unmarshaled HLV - assert.Equal(t, inMemoryHLV.CurrentVersionCAS, hlvFromPersistance.CurrentVersionCAS) - assert.Equal(t, inMemoryHLV.SourceID, hlvFromPersistance.SourceID) - assert.Equal(t, inMemoryHLV.Version, hlvFromPersistance.Version) - assert.Equal(t, inMemoryHLV.PreviousVersions, hlvFromPersistance.PreviousVersions) - assert.Equal(t, inMemoryHLV.MergeVersions, hlvFromPersistance.MergeVersions) + // assert in memory hlv is as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.PreviousVersions, 0) + assert.Len(t, memHLV.MergeVersions, 0) } diff --git a/db/import.go b/db/import.go index 32f1b6737f..e89003b310 100644 --- a/db/import.go +++ b/db/import.go @@ -31,11 +31,18 @@ const ( ImportOnDemand // On-demand import. Reattempt import on cas write failure of the imported doc until either the import succeeds, or existing doc is an SG write. ) +type importDocOptions struct { + expiry *uint32 + isDelete bool + revSeqNo uint64 + mode ImportMode +} + // Imports a document that was written by someone other than sync gateway, given the existing state of the doc in raw bytes -func (db *DatabaseCollectionWithUser) ImportDocRaw(ctx context.Context, docid string, value []byte, xattrs map[string][]byte, isDelete bool, cas uint64, expiry *uint32, mode ImportMode) (docOut *Document, err error) { +func (db *DatabaseCollectionWithUser) ImportDocRaw(ctx context.Context, docid string, value []byte, xattrs map[string][]byte, importOpts importDocOptions, cas uint64) (docOut *Document, err error) { var body Body - if isDelete { + if importOpts.isDelete { body = Body{} } else { err := body.Unmarshal(value) @@ -58,11 +65,11 @@ func (db *DatabaseCollectionWithUser) ImportDocRaw(ctx context.Context, docid st Cas: cas, } - return db.importDoc(ctx, docid, body, expiry, isDelete, existingBucketDoc, mode) + return db.importDoc(ctx, docid, body, importOpts.expiry, importOpts.isDelete, importOpts.revSeqNo, existingBucketDoc, importOpts.mode) } // Import a document, given the existing state of the doc in *document format. -func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid string, existingDoc *Document, isDelete bool, expiry *uint32, mode ImportMode) (docOut *Document, err error) { +func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid string, existingDoc *Document, importOpts importDocOptions) (docOut *Document, err error) { if existingDoc == nil { return nil, base.RedactErrorf("No existing doc present when attempting to import %s", base.UD(docid)) @@ -86,11 +93,11 @@ func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid strin } else { if existingDoc.Deleted { existingBucketDoc.Xattrs[base.SyncXattrName], err = base.JSONMarshal(existingDoc.SyncData) - if err == nil && existingDoc.metadataOnlyUpdate != nil && db.useMou() { - existingBucketDoc.Xattrs[base.MouXattrName], err = base.JSONMarshal(existingDoc.metadataOnlyUpdate) + if err == nil && existingDoc.MetadataOnlyUpdate != nil && db.useMou() { + existingBucketDoc.Xattrs[base.MouXattrName], err = base.JSONMarshal(existingDoc.MetadataOnlyUpdate) } } else { - existingBucketDoc.Body, existingBucketDoc.Xattrs[base.SyncXattrName], existingBucketDoc.Xattrs[base.MouXattrName], err = existingDoc.MarshalWithXattrs() + existingBucketDoc.Body, existingBucketDoc.Xattrs[base.SyncXattrName], existingBucketDoc.Xattrs[base.VvXattrName], existingBucketDoc.Xattrs[base.MouXattrName], existingBucketDoc.Xattrs[base.GlobalXattrName], err = existingDoc.MarshalWithXattrs() } } @@ -98,7 +105,7 @@ func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid strin return nil, err } - return db.importDoc(ctx, docid, existingDoc.Body(ctx), expiry, isDelete, existingBucketDoc, mode) + return db.importDoc(ctx, docid, existingDoc.Body(ctx), importOpts.expiry, importOpts.isDelete, importOpts.revSeqNo, existingBucketDoc, importOpts.mode) } // Import document @@ -108,7 +115,7 @@ func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid strin // isDelete - whether the document to be imported is a delete // existingDoc - bytes/cas/expiry of the document to be imported (including xattr when available) // mode - ImportMode - ImportFromFeed or ImportOnDemand -func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid string, body Body, expiry *uint32, isDelete bool, existingDoc *sgbucket.BucketDocument, mode ImportMode) (docOut *Document, err error) { +func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid string, body Body, expiry *uint32, isDelete bool, revNo uint64, existingDoc *sgbucket.BucketDocument, mode ImportMode) (docOut *Document, err error) { base.DebugfCtx(ctx, base.KeyImport, "Attempting to import doc %q...", base.UD(docid)) importStartTime := time.Now() @@ -146,7 +153,8 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin existingDoc.Expiry = *expiry } - docOut, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, true, expiry, mutationOptions, existingDoc, true, func(doc *Document) (resultDocument *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { + docUpdateEvent := Import + docOut, _, err = db.updateAndReturnDoc(ctx, newDoc.ID, true, expiry, mutationOptions, docUpdateEvent, existingDoc, true, func(doc *Document) (resultDocument *Document, resultAttachmentData updatedAttachments, createNewRevIDSkipped bool, updatedExpiry *uint32, resultErr error) { // Perform cas mismatch check first, as we want to identify cas mismatch before triggering migrate handling. // If there's a cas mismatch, the doc has been updated since the version that triggered the import. Handling depends on import mode. if doc.Cas != existingDoc.Cas { @@ -192,7 +200,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin // If the existing doc is a legacy SG write (_sync in body), check for migrate instead of import. _, ok := body[base.SyncPropertyName] if ok || doc.inlineSyncData { - migratedDoc, requiresImport, migrateErr := db.migrateMetadata(ctx, newDoc.ID, body, existingDoc, mutationOptions) + migratedDoc, requiresImport, migrateErr := db.migrateMetadata(ctx, newDoc.ID, existingDoc, mutationOptions) if migrateErr != nil { return nil, nil, false, updatedExpiry, migrateErr } @@ -329,7 +337,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin // If this is a metadata-only update, set metadataOnlyUpdate based on old doc's cas and mou if metadataOnlyUpdate && db.useMou() { - newDoc.metadataOnlyUpdate = computeMetadataOnlyUpdate(doc.Cas, doc.metadataOnlyUpdate) + newDoc.MetadataOnlyUpdate = computeMetadataOnlyUpdate(doc.Cas, revNo, doc.MetadataOnlyUpdate) } return newDoc, nil, !shouldGenerateNewRev, updatedExpiry, nil @@ -374,7 +382,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin // Migrates document metadata from document body to system xattr. On CAS failure, retrieves current doc body and retries // migration if _sync property exists. If _sync property is not found, returns doc and sets requiresImport to true -func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid string, body Body, existingDoc *sgbucket.BucketDocument, opts *sgbucket.MutateInOptions) (docOut *Document, requiresImport bool, err error) { +func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid string, existingDoc *sgbucket.BucketDocument, opts *sgbucket.MutateInOptions) (docOut *Document, requiresImport bool, err error) { // Unmarshal the existing doc in legacy SG format doc, unmarshalErr := unmarshalDocument(docid, existingDoc.Body) @@ -396,14 +404,21 @@ func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid } // Persist the document in xattr format - value, syncXattrValue, _, marshalErr := doc.MarshalWithXattrs() + value, syncXattr, vvXattr, _, globalXattr, marshalErr := doc.MarshalWithXattrs() if marshalErr != nil { return nil, false, marshalErr } xattrs := map[string][]byte{ - base.SyncXattrName: syncXattrValue, + base.SyncXattrName: syncXattr, + } + if vvXattr != nil { + xattrs[base.VvXattrName] = vvXattr } + if globalXattr != nil { + xattrs[base.GlobalXattrName] = globalXattr + } + var casOut uint64 var writeErr error var xattrsToDelete []string diff --git a/db/import_listener.go b/db/import_listener.go index cc07629d0b..603bf54e83 100644 --- a/db/import_listener.go +++ b/db/import_listener.go @@ -190,13 +190,13 @@ func (il *importListener) ImportFeedEvent(ctx context.Context, collection *Datab } } + docID := string(event.Key) // If syncData is nil, or if this was not an SG write, attempt to import if syncData == nil || !isSGWrite { isDelete := event.Opcode == sgbucket.FeedOpDeletion if isDelete { rawBody = nil } - docID := string(event.Key) // last attempt to exit processing if the importListener has been closed before attempting to write to the bucket select { @@ -205,8 +205,14 @@ func (il *importListener) ImportFeedEvent(ctx context.Context, collection *Datab return default: } + importOpts := importDocOptions{ + isDelete: isDelete, + mode: ImportFromFeed, + expiry: &event.Expiry, + revSeqNo: event.RevNo, + } - _, err := collection.ImportDocRaw(ctx, docID, rawBody, rawXattrs, isDelete, event.Cas, &event.Expiry, ImportFromFeed) + _, err := collection.ImportDocRaw(ctx, docID, rawBody, rawXattrs, importOpts, event.Cas) if err != nil { if err == base.ErrImportCasFailure { base.DebugfCtx(ctx, base.KeyImport, "Not importing mutation - document %s has been subsequently updated and will be imported based on that mutation.", base.UD(docID)) @@ -216,6 +222,13 @@ func (il *importListener) ImportFeedEvent(ctx context.Context, collection *Datab base.DebugfCtx(ctx, base.KeyImport, "Did not import doc %q - external update will not be accessible via Sync Gateway. Reason: %v", base.UD(docID), err) } } + } else if syncData != nil && syncData.Attachments != nil { + base.DebugfCtx(ctx, base.KeyImport, "Attachment metadata found in sync data for doc with id %s, migrating attachment metadata", base.UD(docID)) + // we have attachments to migrate + err := collection.MigrateAttachmentMetadata(ctx, docID, event.Cas, syncData) + if err != nil { + base.WarnfCtx(ctx, "error migrating attachment metadata from sync data to global sync for doc %s. Error: %v", base.UD(docID), err) + } } } diff --git a/db/import_test.go b/db/import_test.go index ae70b182df..8f1ee92475 100644 --- a/db/import_test.go +++ b/db/import_test.go @@ -69,8 +69,8 @@ func TestFeedImport(t *testing.T) { mouXattr, mouOk := xattrs[base.MouXattrName] require.True(t, mouOk) require.NoError(t, base.JSONUnmarshal(mouXattr, &mou)) - require.Equal(t, base.CasToString(writeCas), mou.PreviousCAS) - require.Equal(t, base.CasToString(importCas), mou.CAS) + require.Equal(t, base.CasToString(writeCas), mou.PreviousHexCAS) + require.Equal(t, base.CasToString(importCas), mou.HexCAS) } else { // Expect not found fetching mou xattr require.Error(t, err) @@ -104,11 +104,11 @@ func TestOnDemandImportMou(t *testing.T) { require.NoError(t, err) if db.UseMou() { - require.NotNil(t, doc.metadataOnlyUpdate) - require.Equal(t, base.CasToString(writeCas), doc.metadataOnlyUpdate.PreviousCAS) - require.Equal(t, base.CasToString(doc.Cas), doc.metadataOnlyUpdate.CAS) + require.NotNil(t, doc.MetadataOnlyUpdate) + require.Equal(t, base.CasToString(writeCas), doc.MetadataOnlyUpdate.PreviousHexCAS) + require.Equal(t, base.CasToString(doc.Cas), doc.MetadataOnlyUpdate.HexCAS) } else { - require.Nil(t, doc.metadataOnlyUpdate) + require.Nil(t, doc.MetadataOnlyUpdate) } }) @@ -138,8 +138,8 @@ func TestOnDemandImportMou(t *testing.T) { var mou *MetadataOnlyUpdate require.True(t, mouOk) require.NoError(t, base.JSONUnmarshal(mouXattr, &mou)) - require.Equal(t, base.CasToString(writeCas), mou.PreviousCAS) - require.Equal(t, base.CasToString(importCas), mou.CAS) + require.Equal(t, base.CasToString(writeCas), mou.PreviousHexCAS) + require.Equal(t, base.CasToString(importCas), mou.HexCAS) } else { // expect not found fetching mou xattr require.Error(t, err) @@ -184,7 +184,7 @@ func TestMigrateMetadata(t *testing.T) { assert.NoError(t, err, "Error writing doc w/ expiry") // Get the existing bucket doc - _, existingBucketDoc, err := collection.GetDocWithXattr(ctx, key, DocUnmarshalAll) + _, existingBucketDoc, err := collection.GetDocWithXattrs(ctx, key, DocUnmarshalAll) require.NoError(t, err) // Set the expiry value to a stale value (it's about to be stale, since below it will get updated to a later value) existingBucketDoc.Expiry = uint32(syncMetaExpiry.Unix()) @@ -206,18 +206,70 @@ func TestMigrateMetadata(t *testing.T) { require.NoError(t, err) // Call migrateMeta with stale args that have old stale expiry - _, _, err = collection.migrateMetadata( - ctx, - key, - body, - existingBucketDoc, - &sgbucket.MutateInOptions{PreserveExpiry: false}, - ) + _, _, err = collection.migrateMetadata(ctx, key, existingBucketDoc, &sgbucket.MutateInOptions{PreserveExpiry: false}) assert.True(t, err != nil) assert.True(t, err == base.ErrCasFailureShouldRetry) } +// Tests metadata migration where a document with inline sync data has been replicated by XDCR, so also has an +// existing HLV. Migration should preserve the existing HLV while moving doc._sync to sync xattr +func TestMigrateMetadataWithHLV(t *testing.T) { + + if !base.TestUseXattrs() { + t.Skip("This test only works with XATTRS enabled") + } + + base.SetUpTestLogging(t, base.LevelInfo, base.KeyMigrate, base.KeyImport) + + db, ctx := setupTestDB(t) + defer db.Close(ctx) + + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + key := "TestMigrateMetadata" + bodyBytes := rawDocWithSyncMeta() + body := Body{} + err := body.Unmarshal(bodyBytes) + assert.NoError(t, err, "Error unmarshalling body") + + hlv := &HybridLogicalVector{} + require.NoError(t, hlv.AddVersion(CreateVersion("source123", 100))) + hlv.CurrentVersionCAS = 100 + hlvBytes := base.MustJSONMarshal(t, hlv) + xattrBytes := map[string][]byte{ + base.VvXattrName: hlvBytes, + } + + // Create via the SDK with inline sync metadata and an existing _vv xattr + _, err = collection.dataStore.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrBytes, nil, nil) + require.NoError(t, err) + + // Get the existing bucket doc + _, existingBucketDoc, err := collection.GetDocWithXattrs(ctx, key, DocUnmarshalAll) + require.NoError(t, err) + + // Migrate metadata + _, _, err = collection.migrateMetadata(ctx, key, existingBucketDoc, &sgbucket.MutateInOptions{PreserveExpiry: false}) + require.NoError(t, err) + + // Fetch the existing doc, ensure _vv is preserved + var migratedHLV *HybridLogicalVector + _, migratedBucketDoc, err := collection.GetDocWithXattrs(ctx, key, DocUnmarshalAll) + require.NoError(t, err) + migratedHLVBytes, ok := migratedBucketDoc.Xattrs[base.VvXattrName] + require.True(t, ok) + require.NoError(t, base.JSONUnmarshal(migratedHLVBytes, &migratedHLV)) + require.Equal(t, hlv.Version, migratedHLV.Version) + require.Equal(t, hlv.SourceID, migratedHLV.SourceID) + require.Equal(t, hlv.CurrentVersionCAS, migratedHLV.CurrentVersionCAS) + + migratedSyncXattrBytes, ok := migratedBucketDoc.Xattrs[base.SyncXattrName] + require.True(t, ok) + require.NotZero(t, len(migratedSyncXattrBytes)) + +} + // This invokes db.importDoc() with two different scenarios: // // Scenario 1: normal import @@ -277,7 +329,7 @@ func TestImportWithStaleBucketDocCorrectExpiry(t *testing.T) { assert.NoError(t, err, "Error writing doc w/ expiry") // Get the existing bucket doc - _, existingBucketDoc, err := collection.GetDocWithXattr(ctx, key, DocUnmarshalAll) + _, existingBucketDoc, err := collection.GetDocWithXattrs(ctx, key, DocUnmarshalAll) assert.NoError(t, err, fmt.Sprintf("Error retrieving doc w/ xattr: %v", err)) body = Body{} @@ -305,7 +357,7 @@ func TestImportWithStaleBucketDocCorrectExpiry(t *testing.T) { require.NoError(t, err) // Import the doc (will migrate as part of the import since the doc contains sync meta) - _, errImportDoc := collection.importDoc(ctx, key, body, &expiry, false, existingBucketDoc, ImportOnDemand) + _, errImportDoc := collection.importDoc(ctx, key, body, &expiry, false, 0, existingBucketDoc, ImportOnDemand) assert.NoError(t, errImportDoc, "Unexpected error") // Make sure the doc in the bucket has expected XATTR @@ -445,7 +497,7 @@ func TestImportWithCasFailureUpdate(t *testing.T) { assert.NoError(t, err) // Get the existing bucket doc - _, existingBucketDoc, err = collection.GetDocWithXattr(ctx, testcase.docname, DocUnmarshalAll) + _, existingBucketDoc, err = collection.GetDocWithXattrs(ctx, testcase.docname, DocUnmarshalAll) assert.NoError(t, err, fmt.Sprintf("Error retrieving doc w/ xattr: %v", err)) importD := `{"new":"Val"}` @@ -456,7 +508,7 @@ func TestImportWithCasFailureUpdate(t *testing.T) { runOnce = true // Trigger import - _, err = collection.importDoc(ctx, testcase.docname, bodyD, nil, false, existingBucketDoc, ImportOnDemand) + _, err = collection.importDoc(ctx, testcase.docname, bodyD, nil, false, 0, existingBucketDoc, ImportOnDemand) assert.NoError(t, err) // Check document has the rev and new body @@ -527,7 +579,7 @@ func TestImportNullDoc(t *testing.T) { existingDoc := &sgbucket.BucketDocument{Body: rawNull, Cas: 1} // Import a null document - importedDoc, err := collection.importDoc(ctx, key+"1", body, nil, false, existingDoc, ImportOnDemand) + importedDoc, err := collection.importDoc(ctx, key+"1", body, nil, false, 1, existingDoc, ImportOnDemand) assert.Equal(t, base.ErrEmptyDocument, err) assert.True(t, importedDoc == nil, "Expected no imported doc") } @@ -545,21 +597,26 @@ func TestImportNullDocRaw(t *testing.T) { xattrs := map[string][]byte{ base.SyncXattrName: []byte("{}"), } - importedDoc, err := collection.ImportDocRaw(ctx, "TestImportNullDoc", []byte("null"), xattrs, false, 1, &exp, ImportFromFeed) + importOpts := importDocOptions{ + isDelete: false, + expiry: &exp, + revSeqNo: 1, + mode: ImportFromFeed, + } + importedDoc, err := collection.ImportDocRaw(ctx, "TestImportNullDoc", []byte("null"), xattrs, importOpts, 1) assert.Equal(t, base.ErrEmptyDocument, err) assert.True(t, importedDoc == nil, "Expected no imported doc") } func assertXattrSyncMetaRevGeneration(t *testing.T, dataStore base.DataStore, key string, expectedRevGeneration int) { _, xattrs, _, err := dataStore.GetWithXattrs(base.TestCtx(t), key, []string{base.SyncXattrName}) - assert.NoError(t, err, "Error Getting Xattr") + require.NoError(t, err, "Error Getting Xattr") require.Contains(t, xattrs, base.SyncXattrName) - var xattr map[string]any - require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &xattr)) - revision, ok := xattr["rev"] - assert.True(t, ok) - generation, _ := ParseRevID(base.TestCtx(t), revision.(string)) - log.Printf("assertXattrSyncMetaRevGeneration generation: %d rev: %s", generation, revision) + var syncData SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) + require.True(t, syncData.CurrentRev != "") + generation, _ := ParseRevID(base.TestCtx(t), syncData.CurrentRev) + log.Printf("assertXattrSyncMetaRevGeneration generation: %d rev: %s", generation, syncData.CurrentRev) assert.True(t, generation == expectedRevGeneration) } @@ -639,6 +696,12 @@ func TestImportStampClusterUUID(t *testing.T) { _, cas, err := collection.dataStore.GetRaw(key) require.NoError(t, err) + xattrs, _, err := collection.dataStore.GetXattrs(ctx, key, []string{base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + docXattr, ok := xattrs[base.VirtualXattrRevSeqNo] + require.True(t, ok) + revSeqNo := RetrieveDocRevSeqNo(t, docXattr) + base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeyMigrate, base.KeyImport) body := Body{} @@ -646,13 +709,13 @@ func TestImportStampClusterUUID(t *testing.T) { require.NoError(t, err) existingDoc := &sgbucket.BucketDocument{Body: bodyBytes, Cas: cas} - importedDoc, err := collection.importDoc(ctx, key, body, nil, false, existingDoc, ImportOnDemand) + importedDoc, err := collection.importDoc(ctx, key, body, nil, false, revSeqNo, existingDoc, ImportOnDemand) require.NoError(t, err) if assert.NotNil(t, importedDoc) { require.Len(t, importedDoc.ClusterUUID, 32) } - xattrs, _, err := collection.dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName}) + xattrs, _, err = collection.dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName}) require.NoError(t, err) require.Contains(t, xattrs, base.SyncXattrName) var xattr map[string]any @@ -877,8 +940,8 @@ func TestMetadataOnlyUpdate(t *testing.T) { previousRev := syncData.CurrentRev // verify mou contents - require.Equal(t, base.CasToString(writeCas), mou.PreviousCAS) - require.Equal(t, base.CasToString(importCas), mou.CAS) + require.Equal(t, base.CasToString(writeCas), mou.PreviousHexCAS) + require.Equal(t, base.CasToString(importCas), mou.HexCAS) // 3. Update the previous SDK write via SGW, ensure mou isn't updated again updatedBody := Body{"_rev": previousRev, "foo": "baz"} @@ -974,12 +1037,12 @@ func TestImportConflictWithTombstone(t *testing.T) { // Create rev 2 through SGW body["foo"] = "abc" - _, _, err = collection.PutExistingRevWithBody(ctx, docID, body, []string{"2-abc", rev1ID}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, docID, body, []string{"2-abc", rev1ID}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // Create conflicting rev 2 through SGW body["foo"] = "def" - _, _, err = collection.PutExistingRevWithBody(ctx, docID, body, []string{"2-def", rev1ID}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, docID, body, []string{"2-def", rev1ID}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err) docRev, err := collection.GetRev(ctx, docID, "", false, nil) diff --git a/db/query.go b/db/query.go index 28c9685cd2..6980ee156b 100644 --- a/db/query.go +++ b/db/query.go @@ -154,12 +154,12 @@ var QuerySequences = SGQuery{ } type QueryChannelsRow struct { - Id string `json:"id,omitempty"` - Rev string `json:"rev,omitempty"` - Sequence uint64 `json:"seq,omitempty"` - Flags uint8 `json:"flags,omitempty"` - RemovalRev string `json:"rRev,omitempty"` - RemovalDel bool `json:"rDel,omitempty"` + Id string `json:"id,omitempty"` + Rev channels.RevAndVersion `json:"rev,omitempty"` + Sequence uint64 `json:"seq,omitempty"` + Flags uint8 `json:"flags,omitempty"` + RemovalRev *channels.RevAndVersion `json:"rRev,omitempty"` + RemovalDel bool `json:"rDel,omitempty"` } var QueryPrincipals = SGQuery{ @@ -688,17 +688,17 @@ func (context *DatabaseContext) QueryAllRoles(ctx context.Context, startKey stri type AllDocsViewQueryRow struct { Key string Value struct { - RevID string `json:"r"` - Sequence uint64 `json:"s"` - Channels []string `json:"c"` + RevID channels.RevAndVersion `json:"r"` + Sequence uint64 `json:"s"` + Channels []string `json:"c"` } } type AllDocsIndexQueryRow struct { Id string - RevID string `json:"r"` - Sequence uint64 `json:"s"` - Channels channels.ChannelMap `json:"c"` + RevID channels.RevAndVersion `json:"r"` + Sequence uint64 `json:"s"` + Channels channels.ChannelMap `json:"c"` } // AllDocs returns all non-deleted documents in the bucket between startKey and endKey diff --git a/db/query_test.go b/db/query_test.go index 66f7304c34..19b1a2478f 100644 --- a/db/query_test.go +++ b/db/query_test.go @@ -372,7 +372,7 @@ func TestQueryChannelsActiveOnlyWithLimit(t *testing.T) { // Create 10 added documents for i := 1; i <= 10; i++ { id := "created" + strconv.Itoa(i) - doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false) + doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "1-a", revId) docIdFlagMap[doc.ID] = uint8(0x0) @@ -385,12 +385,12 @@ func TestQueryChannelsActiveOnlyWithLimit(t *testing.T) { // Create 10 deleted documents for i := 1; i <= 10; i++ { id := "deleted" + strconv.Itoa(i) - _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false) + _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "1-a", revId) body[BodyDeleted] = true - doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false) + doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "2-a", revId, "Couldn't create tombstone revision") @@ -402,22 +402,22 @@ func TestQueryChannelsActiveOnlyWithLimit(t *testing.T) { for i := 1; i <= 10; i++ { body["sound"] = "meow" id := "branched" + strconv.Itoa(i) - _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false) + _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document revision 1-a") require.Equal(t, "1-a", revId) body["sound"] = "bark" - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-b") require.Equal(t, "2-b", revId) body["sound"] = "bleat" - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-a") require.Equal(t, "2-a", revId) body[BodyDeleted] = true - doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"3-a", "2-a"}, false) + doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "3-a", revId, "Couldn't create tombstone revision") @@ -429,27 +429,27 @@ func TestQueryChannelsActiveOnlyWithLimit(t *testing.T) { for i := 1; i <= 10; i++ { body["sound"] = "meow" id := "branched|deleted" + strconv.Itoa(i) - _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false) + _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document revision 1-a") require.Equal(t, "1-a", revId) body["sound"] = "bark" - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-b") require.Equal(t, "2-b", revId) body["sound"] = "bleat" - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-a") require.Equal(t, "2-a", revId) body[BodyDeleted] = true - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"3-a", "2-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "3-a", revId, "Couldn't create tombstone revision") body[BodyDeleted] = true - doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"3-b", "2-b"}, false) + doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"3-b", "2-b"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document") require.Equal(t, "3-b", revId, "Couldn't create tombstone revision") @@ -461,17 +461,17 @@ func TestQueryChannelsActiveOnlyWithLimit(t *testing.T) { for i := 1; i <= 10; i++ { body["sound"] = "meow" id := "branched|conflict" + strconv.Itoa(i) - _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false) + _, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create document revision 1-a") require.Equal(t, "1-a", revId) body["sound"] = "bark" - _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false) + _, revId, err = collection.PutExistingRevWithBody(ctx, id, body, []string{"2-b", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-b") require.Equal(t, "2-b", revId) body["sound"] = "bleat" - doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false) + doc, revId, err := collection.PutExistingRevWithBody(ctx, id, body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV) require.NoError(t, err, "Couldn't create revision 2-a") require.Equal(t, "2-a", revId) diff --git a/db/revision.go b/db/revision.go index ba9acec124..4064cfe2b6 100644 --- a/db/revision.go +++ b/db/revision.go @@ -229,10 +229,10 @@ const nonJSONPrefix = byte(1) // Looks up the raw JSON data of a revision that's been archived to a separate doc. // If the revision isn't found (e.g. has been deleted by compaction) returns 404 error. -func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid string, revid string) ([]byte, error) { - data, _, err := c.dataStore.GetRaw(oldRevisionKey(docid, revid)) +func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid string, rev string) ([]byte, error) { + data, _, err := c.dataStore.GetRaw(oldRevisionKey(docid, rev)) if base.IsDocNotFoundError(err) { - base.DebugfCtx(ctx, base.KeyCRUD, "No old revision %q / %q", base.UD(docid), revid) + base.DebugfCtx(ctx, base.KeyCRUD, "No old revision %q / %q", base.UD(docid), rev) err = ErrMissing } if data != nil { @@ -240,7 +240,7 @@ func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid strin if len(data) > 0 && data[0] == nonJSONPrefix { data = data[1:] } - base.DebugfCtx(ctx, base.KeyCRUD, "Got old revision %q / %q --> %d bytes", base.UD(docid), revid, len(data)) + base.DebugfCtx(ctx, base.KeyCRUD, "Got old revision %q / %q --> %d bytes", base.UD(docid), rev, len(data)) } return data, err } @@ -254,12 +254,13 @@ func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid strin // - new revision stored (as duplicate), with expiry rev_max_age_seconds // delta=true && shared_bucket_access=false // - old revision stored, with expiry rev_max_age_seconds -func (db *DatabaseCollectionWithUser) backupRevisionJSON(ctx context.Context, docId, newRevId, oldRevId string, newBody []byte, oldBody []byte, newAtts AttachmentsMeta) { +func (db *DatabaseCollectionWithUser) backupRevisionJSON(ctx context.Context, docId, oldRev string, oldBody []byte) { // Without delta sync, store the old rev for in-flight replication purposes if !db.deltaSyncEnabled() || db.deltaSyncRevMaxAgeSeconds() == 0 { if len(oldBody) > 0 { - _ = db.setOldRevisionJSON(ctx, docId, oldRevId, oldBody, db.oldRevExpirySeconds()) + oldRevHash := base.Crc32cHashString([]byte(oldRev)) + _ = db.setOldRevisionJSON(ctx, docId, oldRevHash, oldBody, db.oldRevExpirySeconds()) } return } @@ -268,33 +269,20 @@ func (db *DatabaseCollectionWithUser) backupRevisionJSON(ctx context.Context, do // Special handling for Xattrs so that SG still has revisions that were updated by an SDK write if db.UseXattrs() { - // Backup the current revision - var newBodyWithAtts = newBody - if len(newAtts) > 0 { - var err error - newBodyWithAtts, err = base.InjectJSONProperties(newBody, base.KVPair{ - Key: BodyAttachments, - Val: newAtts, - }) - if err != nil { - base.WarnfCtx(ctx, "Unable to marshal new revision body during backupRevisionJSON: doc=%q rev=%q err=%v ", base.UD(docId), newRevId, err) - return - } - } - _ = db.setOldRevisionJSON(ctx, docId, newRevId, newBodyWithAtts, db.deltaSyncRevMaxAgeSeconds()) - // Refresh the expiry on the previous revision backup - _ = db.refreshPreviousRevisionBackup(ctx, docId, oldRevId, oldBody, db.deltaSyncRevMaxAgeSeconds()) + oldRevHash := base.Crc32cHashString([]byte(oldRev)) + _ = db.refreshPreviousRevisionBackup(ctx, docId, oldRevHash, oldBody, db.deltaSyncRevMaxAgeSeconds()) return } // Non-xattr only need to store the previous revision, as all writes come through SG if len(oldBody) > 0 { - _ = db.setOldRevisionJSON(ctx, docId, oldRevId, oldBody, db.deltaSyncRevMaxAgeSeconds()) + oldRevHash := base.Crc32cHashString([]byte(oldRev)) + _ = db.setOldRevisionJSON(ctx, docId, oldRevHash, oldBody, db.deltaSyncRevMaxAgeSeconds()) } } -func (db *DatabaseCollectionWithUser) setOldRevisionJSON(ctx context.Context, docid string, revid string, body []byte, expiry uint32) error { +func (db *DatabaseCollectionWithUser) setOldRevisionJSON(ctx context.Context, docid string, rev string, body []byte, expiry uint32) error { // Setting the binary flag isn't sufficient to make N1QL ignore the doc - the binary flag is only used by the SDKs. // To ensure it's not available via N1QL, need to prefix the raw bytes with non-JSON data. @@ -302,11 +290,11 @@ func (db *DatabaseCollectionWithUser) setOldRevisionJSON(ctx context.Context, do nonJSONBytes := make([]byte, 1, len(body)+1) nonJSONBytes[0] = nonJSONPrefix nonJSONBytes = append(nonJSONBytes, body...) - err := db.dataStore.SetRaw(oldRevisionKey(docid, revid), expiry, nil, nonJSONBytes) + err := db.dataStore.SetRaw(oldRevisionKey(docid, rev), expiry, nil, nonJSONBytes) if err == nil { - base.DebugfCtx(ctx, base.KeyCRUD, "Backed up revision body %q/%q (%d bytes, ttl:%d)", base.UD(docid), revid, len(body), expiry) + base.DebugfCtx(ctx, base.KeyCRUD, "Backed up revision body %q/%q (%d bytes, ttl:%d)", base.UD(docid), rev, len(body), expiry) } else { - base.WarnfCtx(ctx, "setOldRevisionJSON failed: doc=%q rev=%q err=%v", base.UD(docid), revid, err) + base.WarnfCtx(ctx, "setOldRevisionJSON failed: doc=%q rev=%q err=%v", base.UD(docid), rev, err) } return err } @@ -330,8 +318,8 @@ func (c *DatabaseCollection) PurgeOldRevisionJSON(ctx context.Context, docid str // ////// UTILITY FUNCTIONS: -func oldRevisionKey(docid string, revid string) string { - return fmt.Sprintf("%s%s:%d:%s", base.RevPrefix, docid, len(revid), revid) +func oldRevisionKey(docid string, rev string) string { + return fmt.Sprintf("%s%s:%d:%s", base.RevPrefix, docid, len(rev), rev) } // Version of FixJSONNumbers (see base/util.go) that operates on a Body diff --git a/db/revision_cache_bypass.go b/db/revision_cache_bypass.go index 355a3e7850..036266960a 100644 --- a/db/revision_cache_bypass.go +++ b/db/revision_cache_bypass.go @@ -30,9 +30,8 @@ func NewBypassRevisionCache(backingStores map[uint32]RevisionCacheBackingStore, } } -// Get fetches the revision for the given docID and revID immediately from the bucket. -func (rc *BypassRevisionCache) Get(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { - +// GetWithRev fetches the revision for the given docID and revID immediately from the bucket. +func (rc *BypassRevisionCache) GetWithRev(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { doc, err := rc.backingStores[collectionID].GetDocument(ctx, docID, DocUnmarshalSync) if err != nil { return DocumentRevision{}, err @@ -41,10 +40,42 @@ func (rc *BypassRevisionCache) Get(ctx context.Context, docID, revID string, col docRev = DocumentRevision{ RevID: revID, } - docRev.BodyBytes, docRev.History, docRev.Channels, docRev.Removed, docRev.Attachments, docRev.Deleted, docRev.Expiry, err = revCacheLoaderForDocument(ctx, rc.backingStores[collectionID], doc, revID) + var hlv *HybridLogicalVector + docRev.BodyBytes, docRev.History, docRev.Channels, docRev.Removed, docRev.Attachments, docRev.Deleted, docRev.Expiry, hlv, err = revCacheLoaderForDocument(ctx, rc.backingStores[collectionID], doc, revID) if err != nil { return DocumentRevision{}, err } + if hlv != nil { + docRev.CV = hlv.ExtractCurrentVersionFromHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() + } + + rc.bypassStat.Add(1) + + return docRev, nil +} + +// GetWithCV fetches the Current Version for the given docID and CV immediately from the bucket. +func (rc *BypassRevisionCache) GetWithCV(ctx context.Context, docID string, cv *Version, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { + + docRev = DocumentRevision{ + CV: cv, + } + + doc, err := rc.backingStores[collectionID].GetDocument(ctx, docID, DocUnmarshalSync) + if err != nil { + return DocumentRevision{}, err + } + + var hlv *HybridLogicalVector + docRev.BodyBytes, docRev.History, docRev.Channels, docRev.Removed, docRev.Attachments, docRev.Deleted, docRev.Expiry, docRev.RevID, hlv, err = revCacheLoaderForDocumentCV(ctx, rc.backingStores[collectionID], doc, *cv) + if err != nil { + return DocumentRevision{}, err + } + if hlv != nil { + docRev.CV = hlv.ExtractCurrentVersionFromHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() + } rc.bypassStat.Add(1) @@ -63,10 +94,15 @@ func (rc *BypassRevisionCache) GetActive(ctx context.Context, docID string, coll RevID: doc.CurrentRev, } - docRev.BodyBytes, docRev.History, docRev.Channels, docRev.Removed, docRev.Attachments, docRev.Deleted, docRev.Expiry, err = revCacheLoaderForDocument(ctx, rc.backingStores[collectionID], doc, doc.SyncData.CurrentRev) + var hlv *HybridLogicalVector + docRev.BodyBytes, docRev.History, docRev.Channels, docRev.Removed, docRev.Attachments, docRev.Deleted, docRev.Expiry, hlv, err = revCacheLoaderForDocument(ctx, rc.backingStores[collectionID], doc, doc.SyncData.CurrentRev) if err != nil { return DocumentRevision{}, err } + if hlv != nil { + docRev.CV = hlv.ExtractCurrentVersionFromHLV() + docRev.hlvHistory = hlv.ToHistoryForHLV() + } rc.bypassStat.Add(1) @@ -88,11 +124,20 @@ func (rc *BypassRevisionCache) Upsert(ctx context.Context, docRev DocumentRevisi // no-op } -func (rc *BypassRevisionCache) Remove(docID, revID string, collectionID uint32) { - // nop +func (rc *BypassRevisionCache) RemoveWithRev(docID, revID string, collectionID uint32) { + // no-op +} + +func (rc *BypassRevisionCache) RemoveWithCV(docID string, cv *Version, collectionID uint32) { + // no-op } // UpdateDelta is a no-op for a BypassRevisionCache func (rc *BypassRevisionCache) UpdateDelta(ctx context.Context, docID, revID string, collectionID uint32, toDelta RevisionDelta) { // no-op } + +// UpdateDeltaCV is a no-op for a BypassRevisionCache +func (rc *BypassRevisionCache) UpdateDeltaCV(ctx context.Context, docID string, cv *Version, collectionID uint32, toDelta RevisionDelta) { + // no-op +} diff --git a/db/revision_cache_interface.go b/db/revision_cache_interface.go index 240103f995..6a71b2ed8e 100644 --- a/db/revision_cache_interface.go +++ b/db/revision_cache_interface.go @@ -27,9 +27,15 @@ const ( // RevisionCache is an interface that can be used to fetch a DocumentRevision for a Doc ID and Rev ID pair. type RevisionCache interface { - // Get returns the given revision, and stores if not already cached. + + // GetWithRev returns the given revision, and stores if not already cached. + // When includeDelta=true, the returned DocumentRevision will include delta - requires additional locking during retrieval. + GetWithRev(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (DocumentRevision, error) + + // GetWithCV returns the given revision by CV, and stores if not already cached. + // When includeBody=true, the returned DocumentRevision will include a mutable shallow copy of the marshaled body. // When includeDelta=true, the returned DocumentRevision will include delta - requires additional locking during retrieval. - Get(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (DocumentRevision, error) + GetWithCV(ctx context.Context, docID string, cv *Version, collectionID uint32, includeDelta bool) (DocumentRevision, error) // GetActive returns the current revision for the given doc ID, and stores if not already cached. GetActive(ctx context.Context, docID string, collectionID uint32) (docRev DocumentRevision, err error) @@ -43,11 +49,17 @@ type RevisionCache interface { // Upsert will remove existing value and re-create new one Upsert(ctx context.Context, docRev DocumentRevision, collectionID uint32) - // Remove eliminates a revision in the cache. - Remove(docID, revID string, collectionID uint32) + // RemoveWithRev evicts a revision from the cache using its revID. + RemoveWithRev(docID, revID string, collectionID uint32) + + // RemoveWithCV evicts a revision from the cache using its current version. + RemoveWithCV(docID string, cv *Version, collectionID uint32) // UpdateDelta stores the given toDelta value in the given rev if cached UpdateDelta(ctx context.Context, docID, revID string, collectionID uint32, toDelta RevisionDelta) + + // UpdateDeltaCV stores the given toDelta value in the given rev if cached but will look up in cache by cv + UpdateDeltaCV(ctx context.Context, docID string, cv *Version, collectionID uint32, toDelta RevisionDelta) } const ( @@ -108,6 +120,7 @@ func DefaultRevisionCacheOptions() *RevisionCacheOptions { type RevisionCacheBackingStore interface { GetDocument(ctx context.Context, docid string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) getRevision(ctx context.Context, doc *Document, revid string) ([]byte, AttachmentsMeta, error) + getCurrentVersion(ctx context.Context, doc *Document, cv Version) ([]byte, AttachmentsMeta, error) } // collectionRevisionCache is a view of a revision cache for a collection. @@ -125,8 +138,13 @@ func newCollectionRevisionCache(revCache *RevisionCache, collectionID uint32) co } // Get is for per collection access to Get method -func (c *collectionRevisionCache) Get(ctx context.Context, docID, revID string, includeDelta bool) (DocumentRevision, error) { - return (*c.revCache).Get(ctx, docID, revID, c.collectionID, includeDelta) +func (c *collectionRevisionCache) GetWithRev(ctx context.Context, docID, revID string, includeDelta bool) (DocumentRevision, error) { + return (*c.revCache).GetWithRev(ctx, docID, revID, c.collectionID, includeDelta) +} + +// Get is for per collection access to Get method +func (c *collectionRevisionCache) GetWithCV(ctx context.Context, docID string, cv *Version, includeDelta bool) (DocumentRevision, error) { + return (*c.revCache).GetWithCV(ctx, docID, cv, c.collectionID, includeDelta) } // GetActive is for per collection access to GetActive method @@ -149,9 +167,14 @@ func (c *collectionRevisionCache) Upsert(ctx context.Context, docRev DocumentRev (*c.revCache).Upsert(ctx, docRev, c.collectionID) } -// Remove is for per collection access to Remove method -func (c *collectionRevisionCache) Remove(docID, revID string) { - (*c.revCache).Remove(docID, revID, c.collectionID) +// RemoveWithRev is for per collection access to Remove method +func (c *collectionRevisionCache) RemoveWithRev(docID, revID string) { + (*c.revCache).RemoveWithRev(docID, revID, c.collectionID) +} + +// RemoveWithCV is for per collection access to Remove method +func (c *collectionRevisionCache) RemoveWithCV(docID string, cv *Version) { + (*c.revCache).RemoveWithCV(docID, cv, c.collectionID) } // UpdateDelta is for per collection access to UpdateDelta method @@ -159,6 +182,11 @@ func (c *collectionRevisionCache) UpdateDelta(ctx context.Context, docID, revID (*c.revCache).UpdateDelta(ctx, docID, revID, c.collectionID, toDelta) } +// UpdateDeltaCV is for per collection access to UpdateDeltaCV method +func (c *collectionRevisionCache) UpdateDeltaCV(ctx context.Context, docID string, cv *Version, toDelta RevisionDelta) { + (*c.revCache).UpdateDeltaCV(ctx, docID, cv, c.collectionID, toDelta) +} + // DocumentRevision stored and returned by the rev cache type DocumentRevision struct { DocID string @@ -173,6 +201,8 @@ type DocumentRevision struct { Deleted bool Removed bool // True if the revision is a removal. MemoryBytes int64 // storage of the doc rev bytes measurement, includes size of delta when present too + CV *Version + hlvHistory string } // MutableBody returns a deep copy of the given document revision as a plain body (without any special properties) @@ -249,7 +279,7 @@ func (rev *DocumentRevision) Inject1xBodyProperties(ctx context.Context, db *Dat // Mutable1xBody returns a copy of the given document revision as a 1.x style body (with special properties) // Callers are free to modify this body without affecting the document revision. -func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseCollectionWithUser, requestedHistory Revisions, attachmentsSince []string, showExp bool) (b Body, err error) { +func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseCollectionWithUser, requestedHistory Revisions, attachmentsSince []string, showExp bool, showCV bool) (b Body, err error) { b, err = rev.Body() if err != nil { return nil, err @@ -270,6 +300,10 @@ func (rev *DocumentRevision) Mutable1xBody(ctx context.Context, db *DatabaseColl b[BodyExpiry] = rev.Expiry.Format(time.RFC3339) } + if showCV && rev.CV != nil { + b["_cv"] = rev.CV.String() + } + if rev.Deleted { b[BodyDeleted] = true } @@ -318,13 +352,22 @@ type IDAndRev struct { CollectionID uint32 } +type IDandCV struct { + DocID string + Version uint64 + Source string + CollectionID uint32 +} + // RevisionDelta stores data about a delta between a revision and ToRevID. type RevisionDelta struct { ToRevID string // Target revID for the delta + ToCV string // Target CV for the delta DeltaBytes []byte // The actual delta AttachmentStorageMeta []AttachmentStorageMeta // Storage metadata of all attachments present on ToRevID ToChannels base.Set // Full list of channels for the to revision RevisionHistory []string // Revision history from parent of ToRevID to source revID, in descending order + HlvHistory string // HLV History in CBL format ToDeleted bool // Flag if ToRevID is a tombstone totalDeltaBytes int64 // totalDeltaBytes is the total bytes for channels, revisions and body on the delta itself } @@ -332,10 +375,12 @@ type RevisionDelta struct { func newRevCacheDelta(deltaBytes []byte, fromRevID string, toRevision DocumentRevision, deleted bool, toRevAttStorageMeta []AttachmentStorageMeta) RevisionDelta { revDelta := RevisionDelta{ ToRevID: toRevision.RevID, + ToCV: toRevision.CV.String(), DeltaBytes: deltaBytes, AttachmentStorageMeta: toRevAttStorageMeta, ToChannels: toRevision.Channels, RevisionHistory: toRevision.History.parseAncestorRevisions(fromRevID), + HlvHistory: toRevision.hlvHistory, ToDeleted: deleted, } revDelta.CalculateDeltaBytes() @@ -344,40 +389,101 @@ func newRevCacheDelta(deltaBytes []byte, fromRevID string, toRevision DocumentRe // This is the RevisionCacheLoaderFunc callback for the context's RevisionCache. // Its job is to load a revision from the bucket when there's a cache miss. -func revCacheLoader(ctx context.Context, backingStore RevisionCacheBackingStore, id IDAndRev) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, err error) { +func revCacheLoader(ctx context.Context, backingStore RevisionCacheBackingStore, id IDAndRev) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, hlv *HybridLogicalVector, err error) { var doc *Document if doc, err = backingStore.GetDocument(ctx, id.DocID, DocUnmarshalSync); doc == nil { - return bodyBytes, history, channels, removed, attachments, deleted, expiry, err + return bodyBytes, history, channels, removed, attachments, deleted, expiry, hlv, err } - return revCacheLoaderForDocument(ctx, backingStore, doc, id.RevID) } +// revCacheLoaderForCv will load a document from the bucket using the CV, compare the fetched doc and the CV specified in the function, +// and will still return revid for purpose of populating the Rev ID lookup map on the cache +func revCacheLoaderForCv(ctx context.Context, backingStore RevisionCacheBackingStore, id IDandCV) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, revid string, hlv *HybridLogicalVector, err error) { + cv := Version{ + Value: id.Version, + SourceID: id.Source, + } + var doc *Document + if doc, err = backingStore.GetDocument(ctx, id.DocID, DocUnmarshalSync); doc == nil { + return bodyBytes, history, channels, removed, attachments, deleted, expiry, revid, hlv, err + } + + return revCacheLoaderForDocumentCV(ctx, backingStore, doc, cv) +} + // Common revCacheLoader functionality used either during a cache miss (from revCacheLoader), or directly when retrieving current rev from cache -func revCacheLoaderForDocument(ctx context.Context, backingStore RevisionCacheBackingStore, doc *Document, revid string) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, err error) { + +func revCacheLoaderForDocument(ctx context.Context, backingStore RevisionCacheBackingStore, doc *Document, revid string) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, hlv *HybridLogicalVector, err error) { if bodyBytes, attachments, err = backingStore.getRevision(ctx, doc, revid); err != nil { // If we can't find the revision (either as active or conflicted body from the document, or as old revision body backup), check whether // the revision was a channel removal. If so, we want to store as removal in the revision cache removalBodyBytes, removalHistory, activeChannels, isRemoval, isDelete, isRemovalErr := doc.IsChannelRemoval(ctx, revid) if isRemovalErr != nil { - return bodyBytes, history, channels, isRemoval, nil, isDelete, nil, isRemovalErr + return bodyBytes, history, channels, isRemoval, nil, isDelete, nil, hlv, isRemovalErr } if isRemoval { - return removalBodyBytes, removalHistory, activeChannels, isRemoval, nil, isDelete, nil, nil + return removalBodyBytes, removalHistory, activeChannels, isRemoval, nil, isDelete, nil, hlv, nil } else { // If this wasn't a removal, return the original error from getRevision - return bodyBytes, history, channels, removed, nil, isDelete, nil, err + return bodyBytes, history, channels, removed, nil, isDelete, nil, hlv, err } } deleted = doc.History[revid].Deleted validatedHistory, getHistoryErr := doc.History.getHistory(revid) if getHistoryErr != nil { - return bodyBytes, history, channels, removed, nil, deleted, nil, getHistoryErr + return bodyBytes, history, channels, removed, nil, deleted, nil, hlv, getHistoryErr } history = encodeRevisions(ctx, doc.ID, validatedHistory) channels = doc.History[revid].Channels + if doc.HLV != nil { + hlv = doc.HLV + } + + return bodyBytes, history, channels, removed, attachments, deleted, doc.Expiry, hlv, err +} + +// revCacheLoaderForDocumentCV used either during cache miss (from revCacheLoaderForCv), or used directly when getting current active CV from cache +// nolint:staticcheck +func revCacheLoaderForDocumentCV(ctx context.Context, backingStore RevisionCacheBackingStore, doc *Document, cv Version) (bodyBytes []byte, history Revisions, channels base.Set, removed bool, attachments AttachmentsMeta, deleted bool, expiry *time.Time, revid string, hlv *HybridLogicalVector, err error) { + if bodyBytes, attachments, err = backingStore.getCurrentVersion(ctx, doc, cv); err != nil { + // TODO: CBG-3814 - pending support of channel removal for CV + base.ErrorfCtx(ctx, "pending CBG-3814 support of channel removal for CV: %v", err) + } + + deleted = doc.Deleted + channels = doc.SyncData.getCurrentChannels() + revid = doc.CurrentRev + hlv = doc.HLV + + return bodyBytes, history, channels, removed, attachments, deleted, doc.Expiry, revid, hlv, err +} + +func (c *DatabaseCollection) getCurrentVersion(ctx context.Context, doc *Document, cv Version) (bodyBytes []byte, attachments AttachmentsMeta, err error) { + if err = doc.HasCurrentVersion(ctx, cv); err != nil { + bodyBytes, err = c.getOldRevisionJSON(ctx, doc.ID, base.Crc32cHashString([]byte(cv.String()))) + if err != nil || bodyBytes == nil { + return nil, nil, err + } + } else { + bodyBytes, err = doc.BodyBytes(ctx) + if err != nil { + base.WarnfCtx(ctx, "Marshal error when retrieving active current version body: %v", err) + return nil, nil, err + } + } - return bodyBytes, history, channels, removed, attachments, deleted, doc.Expiry, err + attachments = doc.Attachments + + // handle backup revision inline attachments, or pre-2.5 meta + if inlineAtts, cleanBodyBytes, _, err := extractInlineAttachments(bodyBytes); err != nil { + return nil, nil, err + } else if len(inlineAtts) > 0 { + // we found some inline attachments, so merge them with attachments, and update the bodies + attachments = mergeAttachments(inlineAtts, attachments) + bodyBytes = cleanBodyBytes + } + return bodyBytes, attachments, err } diff --git a/db/revision_cache_lru.go b/db/revision_cache_lru.go index 235f65833b..2e9eda7d44 100644 --- a/db/revision_cache_lru.go +++ b/db/revision_cache_lru.go @@ -52,8 +52,12 @@ func (sc *ShardedLRURevisionCache) getShard(docID string) *LRURevisionCache { return sc.caches[sgbucket.VBHash(docID, sc.numShards)] } -func (sc *ShardedLRURevisionCache) Get(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { - return sc.getShard(docID).Get(ctx, docID, revID, collectionID, includeDelta) +func (sc *ShardedLRURevisionCache) GetWithRev(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { + return sc.getShard(docID).GetWithRev(ctx, docID, revID, collectionID, includeDelta) +} + +func (sc *ShardedLRURevisionCache) GetWithCV(ctx context.Context, docID string, cv *Version, collectionID uint32, includeDelta bool) (docRev DocumentRevision, err error) { + return sc.getShard(docID).GetWithCV(ctx, docID, cv, collectionID, includeDelta) } func (sc *ShardedLRURevisionCache) Peek(ctx context.Context, docID, revID string, collectionID uint32) (docRev DocumentRevision, found bool) { @@ -64,6 +68,10 @@ func (sc *ShardedLRURevisionCache) UpdateDelta(ctx context.Context, docID, revID sc.getShard(docID).UpdateDelta(ctx, docID, revID, collectionID, toDelta) } +func (sc *ShardedLRURevisionCache) UpdateDeltaCV(ctx context.Context, docID string, cv *Version, collectionID uint32, toDelta RevisionDelta) { + sc.getShard(docID).UpdateDeltaCV(ctx, docID, cv, collectionID, toDelta) +} + func (sc *ShardedLRURevisionCache) GetActive(ctx context.Context, docID string, collectionID uint32) (docRev DocumentRevision, err error) { return sc.getShard(docID).GetActive(ctx, docID, collectionID) } @@ -76,14 +84,19 @@ func (sc *ShardedLRURevisionCache) Upsert(ctx context.Context, docRev DocumentRe sc.getShard(docRev.DocID).Upsert(ctx, docRev, collectionID) } -func (sc *ShardedLRURevisionCache) Remove(docID, revID string, collectionID uint32) { - sc.getShard(docID).Remove(docID, revID, collectionID) +func (sc *ShardedLRURevisionCache) RemoveWithRev(docID, revID string, collectionID uint32) { + sc.getShard(docID).RemoveWithRev(docID, revID, collectionID) +} + +func (sc *ShardedLRURevisionCache) RemoveWithCV(docID string, cv *Version, collectionID uint32) { + sc.getShard(docID).RemoveWithCV(docID, cv, collectionID) } // An LRU cache of document revision bodies, together with their channel access. type LRURevisionCache struct { backingStores map[uint32]RevisionCacheBackingStore cache map[IDAndRev]*list.Element + hlvCache map[IDandCV]*list.Element lruList *list.List cacheHits *base.SgwIntStat cacheMisses *base.SgwIntStat @@ -97,18 +110,22 @@ type LRURevisionCache struct { // The cache payload data. Stored as the Value of a list Element. type revCacheValue struct { - err error - history Revisions - channels base.Set - expiry *time.Time - attachments AttachmentsMeta - delta *RevisionDelta - key IDAndRev - bodyBytes []byte - lock sync.RWMutex - deleted bool - removed bool - itemBytes int64 + err error + history Revisions + channels base.Set + expiry *time.Time + attachments AttachmentsMeta + delta *RevisionDelta + id string + cv Version + hlvHistory string + revID string + bodyBytes []byte + lock sync.RWMutex + deleted bool + removed bool + itemBytes int64 + collectionID uint32 } // Creates a revision cache with the given capacity and an optional loader function. @@ -116,6 +133,7 @@ func NewLRURevisionCache(revCacheOptions *RevisionCacheOptions, backingStores ma return &LRURevisionCache{ cache: map[IDAndRev]*list.Element{}, + hlvCache: map[IDandCV]*list.Element{}, lruList: list.New(), capacity: revCacheOptions.MaxItemCount, backingStores: backingStores, @@ -131,14 +149,18 @@ func NewLRURevisionCache(revCacheOptions *RevisionCacheOptions, backingStores ma // Returns the body of the revision, its history, and the set of channels it's in. // If the cache has a loaderFunction, it will be called if the revision isn't in the cache; // any error returned by the loaderFunction will be returned from Get. -func (rc *LRURevisionCache) Get(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (DocumentRevision, error) { - return rc.getFromCache(ctx, docID, revID, collectionID, true, includeDelta) +func (rc *LRURevisionCache) GetWithRev(ctx context.Context, docID, revID string, collectionID uint32, includeDelta bool) (DocumentRevision, error) { + return rc.getFromCacheByRev(ctx, docID, revID, collectionID, true, includeDelta) +} + +func (rc *LRURevisionCache) GetWithCV(ctx context.Context, docID string, cv *Version, collectionID uint32, includeDelta bool) (DocumentRevision, error) { + return rc.getFromCacheByCV(ctx, docID, cv, collectionID, true, includeDelta) } // Looks up a revision from the cache only. Will not fall back to loader function if not // present in the cache. func (rc *LRURevisionCache) Peek(ctx context.Context, docID, revID string, collectionID uint32) (docRev DocumentRevision, found bool) { - docRev, err := rc.getFromCache(ctx, docID, revID, collectionID, false, RevCacheOmitDelta) + docRev, err := rc.getFromCacheByRev(ctx, docID, revID, collectionID, false, RevCacheOmitDelta) if err != nil { return DocumentRevision{}, false } @@ -159,7 +181,14 @@ func (rc *LRURevisionCache) UpdateDelta(ctx context.Context, docID, revID string } } -func (rc *LRURevisionCache) getFromCache(ctx context.Context, docID, revID string, collectionID uint32, loadOnCacheMiss, includeDelta bool) (DocumentRevision, error) { +func (rc *LRURevisionCache) UpdateDeltaCV(ctx context.Context, docID string, cv *Version, collectionID uint32, toDelta RevisionDelta) { + value := rc.getValueByCV(docID, cv, collectionID, false) + if value != nil { + value.updateDelta(toDelta) + } +} + +func (rc *LRURevisionCache) getFromCacheByRev(ctx context.Context, docID, revID string, collectionID uint32, loadOnCacheMiss, includeDelta bool) (DocumentRevision, error) { value := rc.getValue(docID, revID, collectionID, loadOnCacheMiss) if value == nil { return DocumentRevision{}, nil @@ -173,11 +202,33 @@ func (rc *LRURevisionCache) getFromCache(ctx context.Context, docID, revID strin rc.updateRevCacheMemoryUsage(value.getItemBytes()) // check for memory based eviction rc.revCacheMemoryBasedEviction() + rc.addToHLVMapPostLoad(docID, docRev.RevID, docRev.CV) } if err != nil { rc.removeValue(value) // don't keep failed loads in the cache } + + return docRev, err +} + +func (rc *LRURevisionCache) getFromCacheByCV(ctx context.Context, docID string, cv *Version, collectionID uint32, loadCacheOnMiss bool, includeDelta bool) (DocumentRevision, error) { + value := rc.getValueByCV(docID, cv, collectionID, loadCacheOnMiss) + if value == nil { + return DocumentRevision{}, nil + } + + docRev, cacheHit, err := value.load(ctx, rc.backingStores[collectionID], includeDelta) + rc.statsRecorderFunc(cacheHit) + + if err != nil { + rc.removeValue(value) // don't keep failed loads in the cache + } + + if !cacheHit && err == nil { + rc.addToRevMapPostLoad(docID, docRev.RevID, docRev.CV, collectionID) + } + return docRev, err } @@ -213,7 +264,11 @@ func (rc *LRURevisionCache) GetActive(ctx context.Context, docID string, collect if err != nil { rc.removeValue(value) // don't keep failed loads in the cache + } else { + // add successfully fetched value to CV lookup map too + rc.addToHLVMapPostLoad(docID, docRev.RevID, docRev.CV) } + return docRev, err } @@ -232,33 +287,54 @@ func (rc *LRURevisionCache) Put(ctx context.Context, docRev DocumentRevision, co // TODO: CBG-1948 panic("Missing history for RevisionCache.Put") } - value := rc.getValue(docRev.DocID, docRev.RevID, collectionID, true) + + // doc should always have a cv present in a PUT operation on the cache (update HLV is called before hand in doc update process) + // thus we can call getValueByCV directly the update the rev lookup post this + value := rc.getValueByCV(docRev.DocID, docRev.CV, collectionID, true) // increment incoming bytes docRev.CalculateBytes() rc.updateRevCacheMemoryUsage(docRev.MemoryBytes) value.store(docRev) + + // add new doc version to the rev id lookup map + rc.addToRevMapPostLoad(docRev.DocID, docRev.RevID, docRev.CV, collectionID) + // check for rev cache memory based eviction rc.revCacheMemoryBasedEviction() } // Upsert a revision in the cache. func (rc *LRURevisionCache) Upsert(ctx context.Context, docRev DocumentRevision, collectionID uint32) { - key := IDAndRev{DocID: docRev.DocID, RevID: docRev.RevID, CollectionID: collectionID} + var value *revCacheValue + // similar to PUT operation we should have the CV defined by this point (updateHLV is called before calling this) + key := IDandCV{DocID: docRev.DocID, Source: docRev.CV.SourceID, Version: docRev.CV.Value, CollectionID: collectionID} + legacyKey := IDAndRev{DocID: docRev.DocID, RevID: docRev.RevID, CollectionID: collectionID} rc.lock.Lock() + newItem := true - // If element exists remove from lrulist - if elem := rc.cache[key]; elem != nil { - revItem := elem.Value.(*revCacheValue) + // lookup for element in hlv lookup map, if not found for some reason try rev lookup map + var existingElem *list.Element + var found bool + existingElem, found = rc.hlvCache[key] + if !found { + existingElem, found = rc.cache[legacyKey] + } + if found { + revItem := existingElem.Value.(*revCacheValue) // decrement item bytes by the removed item rc.updateRevCacheMemoryUsage(-revItem.getItemBytes()) - rc.lruList.Remove(elem) + rc.lruList.Remove(existingElem) newItem = false } // Add new value and overwrite existing cache key, pushing to front to maintain order - value := &revCacheValue{key: key} - rc.cache[key] = rc.lruList.PushFront(value) + // also ensure we add to rev id lookup map too + value = &revCacheValue{id: docRev.DocID, cv: *docRev.CV, collectionID: collectionID} + elem := rc.lruList.PushFront(value) + rc.hlvCache[key] = elem + rc.cache[legacyKey] = elem + // only increment if we are inserting new item to cache if newItem { rc.cacheNumItems.Add(1) @@ -277,6 +353,7 @@ func (rc *LRURevisionCache) Upsert(ctx context.Context, docRev DocumentRevision, docRev.CalculateBytes() // add new item bytes to overall count rc.updateRevCacheMemoryUsage(docRev.MemoryBytes) + value.store(docRev) // check we aren't over memory capacity, if so perform eviction @@ -303,13 +380,13 @@ func (rc *LRURevisionCache) getValue(docID, revID string, collectionID uint32, c rc.lruList.MoveToFront(elem) value = elem.Value.(*revCacheValue) } else if create { - value = &revCacheValue{key: key} + value = &revCacheValue{id: docID, revID: revID, collectionID: collectionID} rc.cache[key] = rc.lruList.PushFront(value) rc.cacheNumItems.Add(1) // evict if over number capacity var numItemsRemoved int - for len(rc.cache) > int(rc.capacity) { + for rc.lruList.Len() > int(rc.capacity) { rc.purgeOldest_() numItemsRemoved++ } @@ -321,8 +398,126 @@ func (rc *LRURevisionCache) getValue(docID, revID string, collectionID uint32, c return } +// getValueByCV gets a value from rev cache by CV, if not found and create is true, will add the value to cache and both lookup maps + +func (rc *LRURevisionCache) getValueByCV(docID string, cv *Version, collectionID uint32, create bool) (value *revCacheValue) { + if docID == "" || cv == nil { + return nil + } + + key := IDandCV{DocID: docID, Source: cv.SourceID, Version: cv.Value, CollectionID: collectionID} + rc.lock.Lock() + if elem := rc.hlvCache[key]; elem != nil { + rc.lruList.MoveToFront(elem) + value = elem.Value.(*revCacheValue) + } else if create { + value = &revCacheValue{id: docID, cv: *cv, collectionID: collectionID} + newElem := rc.lruList.PushFront(value) + rc.hlvCache[key] = newElem + rc.cacheNumItems.Add(1) + + // evict if over number capacity + var numItemsRemoved int + for rc.lruList.Len() > int(rc.capacity) { + rc.purgeOldest_() + numItemsRemoved++ + } + + if numItemsRemoved > 0 { + rc.cacheNumItems.Add(int64(-numItemsRemoved)) + } + } + rc.lock.Unlock() + return +} + +// addToRevMapPostLoad will generate and entry in the Rev lookup map for a new document entering the cache +func (rc *LRURevisionCache) addToRevMapPostLoad(docID, revID string, cv *Version, collectionID uint32) { + legacyKey := IDAndRev{DocID: docID, RevID: revID, CollectionID: collectionID} + key := IDandCV{DocID: docID, Source: cv.SourceID, Version: cv.Value, CollectionID: collectionID} + + rc.lock.Lock() + defer rc.lock.Unlock() + // check for existing value in rev cache map (due to concurrent fetch by rev ID) + cvElem, cvFound := rc.hlvCache[key] + revElem, revFound := rc.cache[legacyKey] + if !cvFound { + // its possible the element has been evicted if we don't find the element above (high churn on rev cache) + // need to return doc revision to caller still but no need repopulate the cache + return + } + // Check if another goroutine has already updated the rev map + if revFound { + if cvElem == revElem { + // already match, return + return + } + // if CV map and rev map are targeting different list elements, update to have both use the cv map element + rc.cache[legacyKey] = cvElem + rc.lruList.Remove(revElem) + } else { + // if not found we need to add the element to the rev lookup (for PUT code path) + rc.cache[legacyKey] = cvElem + } +} + +// addToHLVMapPostLoad will generate and entry in the CV lookup map for a new document entering the cache +func (rc *LRURevisionCache) addToHLVMapPostLoad(docID, revID string, cv *Version) { + legacyKey := IDAndRev{DocID: docID, RevID: revID} + key := IDandCV{DocID: docID, Source: cv.SourceID, Version: cv.Value} + + rc.lock.Lock() + defer rc.lock.Unlock() + // check for existing value in rev cache map (due to concurrent fetch by rev ID) + cvElem, cvFound := rc.hlvCache[key] + revElem, revFound := rc.cache[legacyKey] + if !revFound { + // its possible the element has been evicted if we don't find the element above (high churn on rev cache) + // need to return doc revision to caller still but no need repopulate the cache + return + } + // Check if another goroutine has already updated the cv map + if cvFound { + if cvElem == revElem { + // already match, return + return + } + // if CV map and rev map are targeting different list elements, update to have both use the cv map element + rc.cache[legacyKey] = cvElem + rc.lruList.Remove(revElem) + } +} + // Remove removes a value from the revision cache, if present. -func (rc *LRURevisionCache) Remove(docID, revID string, collectionID uint32) { +func (rc *LRURevisionCache) RemoveWithRev(docID, revID string, collectionID uint32) { + rc.removeFromCacheByRev(docID, revID, collectionID) +} + +// RemoveWithCV removes a value from rev cache by CV reference if present +func (rc *LRURevisionCache) RemoveWithCV(docID string, cv *Version, collectionID uint32) { + rc.removeFromCacheByCV(docID, cv, collectionID) +} + +// removeFromCacheByCV removes an entry from rev cache by CV +func (rc *LRURevisionCache) removeFromCacheByCV(docID string, cv *Version, collectionID uint32) { + key := IDandCV{DocID: docID, Source: cv.SourceID, Version: cv.Value, CollectionID: collectionID} + rc.lock.Lock() + defer rc.lock.Unlock() + element, ok := rc.hlvCache[key] + if !ok { + return + } + // grab the revid key from the value to enable us to remove the reference from the rev lookup map too + elem := element.Value.(*revCacheValue) + legacyKey := IDAndRev{DocID: docID, RevID: elem.revID} + rc.lruList.Remove(element) + delete(rc.hlvCache, key) + // remove from rev lookup map too + delete(rc.cache, legacyKey) +} + +// removeFromCacheByRev removes an entry from rev cache by revID +func (rc *LRURevisionCache) removeFromCacheByRev(docID, revID string, collectionID uint32) { key := IDAndRev{DocID: docID, RevID: revID, CollectionID: collectionID} rc.lock.Lock() defer rc.lock.Unlock() @@ -330,28 +525,49 @@ func (rc *LRURevisionCache) Remove(docID, revID string, collectionID uint32) { if !ok { return } + // grab the cv key from the value to enable us to remove the reference from the rev lookup map too + elem := element.Value.(*revCacheValue) + hlvKey := IDandCV{DocID: docID, Source: elem.cv.SourceID, Version: elem.cv.Value} rc.lruList.Remove(element) // decrement the overall memory bytes count revItem := element.Value.(*revCacheValue) rc.updateRevCacheMemoryUsage(-revItem.getItemBytes()) delete(rc.cache, key) rc.cacheNumItems.Add(-1) + // remove from CV lookup map too + delete(rc.hlvCache, hlvKey) } // removeValue removes a value from the revision cache, if present and the value matches the the value. If there's an item in the revision cache with a matching docID and revID but the document is different, this item will not be removed from the rev cache. func (rc *LRURevisionCache) removeValue(value *revCacheValue) { rc.lock.Lock() - if element := rc.cache[value.key]; element != nil && element.Value == value { + defer rc.lock.Unlock() + revKey := IDAndRev{DocID: value.id, RevID: value.revID} + var itemRemoved bool + if element := rc.cache[revKey]; element != nil && element.Value == value { + rc.lruList.Remove(element) + delete(rc.cache, revKey) + itemRemoved = true + } + // need to also check hlv lookup cache map + hlvKey := IDandCV{DocID: value.id, Source: value.cv.SourceID, Version: value.cv.Value} + if element := rc.hlvCache[hlvKey]; element != nil && element.Value == value { rc.lruList.Remove(element) - delete(rc.cache, value.key) + delete(rc.hlvCache, hlvKey) + itemRemoved = true + } + + if itemRemoved { rc.cacheNumItems.Add(-1) } - rc.lock.Unlock() } func (rc *LRURevisionCache) purgeOldest_() { value := rc.lruList.Remove(rc.lruList.Back()).(*revCacheValue) - delete(rc.cache, value.key) + revKey := IDAndRev{DocID: value.id, RevID: value.revID, CollectionID: value.collectionID} + hlvKey := IDandCV{DocID: value.id, Source: value.cv.SourceID, Version: value.cv.Value, CollectionID: value.collectionID} + delete(rc.cache, revKey) + delete(rc.hlvCache, hlvKey) // decrement memory overall size rc.updateRevCacheMemoryUsage(-value.getItemBytes()) } @@ -364,6 +580,7 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache // Reading the delta from the revCacheValue requires holding the read lock, so it's managed outside asDocumentRevision, // to reduce locking when includeDelta=false var delta *RevisionDelta + var revid string // Attempt to read cached value. value.lock.RLock() @@ -385,7 +602,24 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache cacheHit = true } else { cacheHit = false - value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, value.err = revCacheLoader(ctx, backingStore, value.key) + hlv := &HybridLogicalVector{} + if value.revID == "" { + hlvKey := IDandCV{DocID: value.id, Source: value.cv.SourceID, Version: value.cv.Value} + value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, revid, hlv, value.err = revCacheLoaderForCv(ctx, backingStore, hlvKey) + // based off the current value load we need to populate the revid key with what has been fetched from the bucket (for use of populating the opposite lookup map) + value.revID = revid + if hlv != nil { + value.hlvHistory = hlv.ToHistoryForHLV() + } + } else { + revKey := IDAndRev{DocID: value.id, RevID: value.revID} + value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, hlv, value.err = revCacheLoader(ctx, backingStore, revKey) + // based off the revision load we need to populate the hlv key with what has been fetched from the bucket (for use of populating the opposite lookup map) + if hlv != nil { + value.cv = *hlv.ExtractCurrentVersionFromHLV() + value.hlvHistory = hlv.ToHistoryForHLV() + } + } } if includeDelta { @@ -408,8 +642,8 @@ func (value *revCacheValue) load(ctx context.Context, backingStore RevisionCache func (value *revCacheValue) asDocumentRevision(delta *RevisionDelta) (DocumentRevision, error) { docRev := DocumentRevision{ - DocID: value.key.DocID, - RevID: value.key.RevID, + DocID: value.id, + RevID: value.revID, BodyBytes: value.bodyBytes, History: value.history, Channels: value.channels, @@ -417,6 +651,8 @@ func (value *revCacheValue) asDocumentRevision(delta *RevisionDelta) (DocumentRe Attachments: value.attachments.ShallowCopy(), // Avoid caller mutating the stored attachments Deleted: value.deleted, Removed: value.removed, + hlvHistory: value.hlvHistory, + CV: &value.cv, } docRev.Delta = delta @@ -427,6 +663,7 @@ func (value *revCacheValue) asDocumentRevision(delta *RevisionDelta) (DocumentRe // the provided document. func (value *revCacheValue) loadForDoc(ctx context.Context, backingStore RevisionCacheBackingStore, doc *Document) (docRev DocumentRevision, cacheHit bool, err error) { + var revid string value.lock.RLock() if value.bodyBytes != nil || value.err != nil { value.lock.RUnlock() @@ -442,7 +679,17 @@ func (value *revCacheValue) loadForDoc(ctx context.Context, backingStore Revisio cacheHit = true } else { cacheHit = false - value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, value.err = revCacheLoaderForDocument(ctx, backingStore, doc, value.key.RevID) + hlv := &HybridLogicalVector{} + if value.revID == "" { + value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, revid, hlv, value.err = revCacheLoaderForDocumentCV(ctx, backingStore, doc, value.cv) + value.revID = revid + } else { + value.bodyBytes, value.history, value.channels, value.removed, value.attachments, value.deleted, value.expiry, hlv, value.err = revCacheLoaderForDocument(ctx, backingStore, doc, value.revID) + } + if hlv != nil { + value.cv = *hlv.ExtractCurrentVersionFromHLV() + value.hlvHistory = hlv.ToHistoryForHLV() + } } docRev, err = value.asDocumentRevision(nil) // if not cache hit, we loaded from bucket. Calculate doc rev size and assign to rev cache value @@ -458,7 +705,7 @@ func (value *revCacheValue) loadForDoc(ctx context.Context, backingStore Revisio func (value *revCacheValue) store(docRev DocumentRevision) { value.lock.Lock() if value.bodyBytes == nil { - // value already has doc id/rev id in key + value.revID = docRev.RevID value.bodyBytes = docRev.BodyBytes value.history = docRev.History value.channels = docRev.Channels @@ -467,6 +714,7 @@ func (value *revCacheValue) store(docRev DocumentRevision) { value.deleted = docRev.Deleted value.err = nil value.itemBytes = docRev.MemoryBytes + value.hlvHistory = docRev.hlvHistory } value.lock.Unlock() } diff --git a/db/revision_cache_test.go b/db/revision_cache_test.go index 5be5cfd749..18305247d5 100644 --- a/db/revision_cache_test.go +++ b/db/revision_cache_test.go @@ -50,6 +50,16 @@ func (t *testBackingStore) GetDocument(ctx context.Context, docid string, unmars Channels: base.SetOf("*"), }, } + + doc.HLV = &HybridLogicalVector{ + SourceID: "test", + Version: 123, + } + _, _, err = doc.updateChannels(ctx, base.SetOf("*")) + if err != nil { + return nil, err + } + return doc, nil } @@ -66,6 +76,22 @@ func (t *testBackingStore) getRevision(ctx context.Context, doc *Document, revid return bodyBytes, nil, err } +func (t *testBackingStore) getCurrentVersion(ctx context.Context, doc *Document, cv Version) ([]byte, AttachmentsMeta, error) { + t.getRevisionCounter.Add(1) + + b := Body{ + "testing": true, + BodyId: doc.ID, + BodyRev: doc.CurrentRev, + "current_version": &Version{Value: doc.HLV.Version, SourceID: doc.HLV.SourceID}, + } + if err := doc.HasCurrentVersion(ctx, cv); err != nil { + return nil, nil, err + } + bodyBytes, err := base.JSONMarshal(b) + return bodyBytes, nil, err +} + type noopBackingStore struct{} func (*noopBackingStore) GetDocument(ctx context.Context, docid string, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { @@ -76,6 +102,10 @@ func (*noopBackingStore) getRevision(ctx context.Context, doc *Document, revid s return nil, nil, nil } +func (*noopBackingStore) getCurrentVersion(ctx context.Context, doc *Document, cv Version) ([]byte, AttachmentsMeta, error) { + return nil, nil, nil +} + // testCollectionID is a test collection ID to use for a key in the backing store map to point to a tests backing store. // This should only be used in tests that have no database context being created. const testCollectionID = 0 @@ -102,7 +132,8 @@ func TestLRURevisionCacheEviction(t *testing.T) { // Fill up the rev cache with the first 10 docs for docID := 0; docID < 10; docID++ { id := strconv.Itoa(docID) - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: id, RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + vrs := uint64(docID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: id, RevID: "1-abc", CV: &Version{Value: vrs, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) } assert.Equal(t, int64(10), cacheNumItems.Value()) assert.Equal(t, int64(20), memoryBytesCounted.Value()) @@ -110,7 +141,7 @@ func TestLRURevisionCacheEviction(t *testing.T) { // Get them back out for i := 0; i < 10; i++ { docID := strconv.Itoa(i) - docRev, err := cache.Get(ctx, docID, "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err := cache.GetWithRev(ctx, docID, "1-abc", testCollectionID, RevCacheOmitDelta) assert.NoError(t, err) assert.NotNil(t, docRev.BodyBytes, "nil body for %s", docID) assert.Equal(t, docID, docRev.DocID) @@ -123,7 +154,8 @@ func TestLRURevisionCacheEviction(t *testing.T) { // Add 3 more docs to the now full revcache for i := 10; i < 13; i++ { docID := strconv.Itoa(i) - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: docID, RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + vrs := uint64(i) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: docID, RevID: "1-abc", CV: &Version{Value: vrs, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) } assert.Equal(t, int64(10), cacheNumItems.Value()) assert.Equal(t, int64(20), memoryBytesCounted.Value()) @@ -144,7 +176,78 @@ func TestLRURevisionCacheEviction(t *testing.T) { // and check we can Get up to and including the last 3 we put in for i := 0; i < 10; i++ { id := strconv.Itoa(i + 3) - docRev, err := cache.Get(ctx, id, "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err := cache.GetWithRev(ctx, id, "1-abc", testCollectionID, RevCacheOmitDelta) + assert.NoError(t, err) + assert.NotNil(t, docRev.BodyBytes, "nil body for %s", id) + assert.Equal(t, id, docRev.DocID) + assert.Equal(t, int64(0), cacheMissCounter.Value()) + assert.Equal(t, prevCacheHitCount+int64(i)+1, cacheHitCounter.Value()) + } +} + +// TestLRURevisionCacheEvictionMixedRevAndCV: +// - Add 10 docs to the cache +// - Assert that the cache list and relevant lookup maps have correct lengths +// - Add 3 more docs +// - Assert that lookup maps and the cache list still only have 10 elements in +// - Perform a Get with CV specified on all 10 elements in the cache and assert we get a hit for each element and no misses, +// testing the eviction worked correct +// - Then do the same but for rev lookup +func TestLRURevisionCacheEvictionMixedRevAndCV(t *testing.T) { + + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + backingStoreMap := CreateTestSingleBackingStoreMap(&noopBackingStore{}, testCollectionID) + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + ctx := base.TestCtx(t) + + // Fill up the rev cache with the first 10 docs + for docID := 0; docID < 10; docID++ { + id := strconv.Itoa(docID) + vrs := uint64(docID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: id, RevID: "1-abc", CV: &Version{Value: vrs, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) + } + + // assert that the list has 10 elements along with both lookup maps + assert.Equal(t, 10, len(cache.hlvCache)) + assert.Equal(t, 10, len(cache.cache)) + assert.Equal(t, 10, cache.lruList.Len()) + + // Add 3 more docs to the now full rev cache to trigger eviction + for docID := 10; docID < 13; docID++ { + id := strconv.Itoa(docID) + vrs := uint64(docID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{}`), DocID: id, RevID: "1-abc", CV: &Version{Value: vrs, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) + } + // assert the cache and associated lookup maps only have 10 items in them (i.e.e is eviction working?) + assert.Equal(t, 10, len(cache.hlvCache)) + assert.Equal(t, 10, len(cache.cache)) + assert.Equal(t, 10, cache.lruList.Len()) + + // assert we can get a hit on all 10 elements in the cache by CV lookup + prevCacheHitCount := cacheHitCounter.Value() + for i := 0; i < 10; i++ { + id := strconv.Itoa(i + 3) + vrs := uint64(i + 3) + cv := Version{Value: vrs, SourceID: "test"} + docRev, err := cache.GetWithCV(ctx, id, &cv, testCollectionID, RevCacheOmitDelta) + + assert.NoError(t, err) + assert.NotNil(t, docRev.BodyBytes, "nil body for %s", id) + assert.Equal(t, id, docRev.DocID) + assert.Equal(t, int64(0), cacheMissCounter.Value()) + assert.Equal(t, prevCacheHitCount+int64(i)+1, cacheHitCounter.Value()) + } + + // now do same but for rev lookup + prevCacheHitCount = cacheHitCounter.Value() + for i := 0; i < 10; i++ { + id := strconv.Itoa(i + 3) + docRev, err := cache.GetWithRev(ctx, id, "1-abc", testCollectionID, RevCacheOmitDelta) assert.NoError(t, err) assert.NotNil(t, docRev.BodyBytes, "nil body for %s", id) assert.Equal(t, id, docRev.DocID) @@ -196,7 +299,7 @@ func TestLRURevisionCacheEvictionMemoryBased(t *testing.T) { assert.Equal(t, expValue, currMem) // remove doc "1" to give headroom for memory based eviction - db.revisionCache.Remove("1", rev, collection.GetCollectionID()) + db.revisionCache.RemoveWithRev("1", rev, collection.GetCollectionID()) docRev, ok = db.revisionCache.Peek(ctx, "1", rev, collection.GetCollectionID()) assert.False(t, ok) assert.Nil(t, docRev.BodyBytes) @@ -238,7 +341,7 @@ func TestBackingStoreMemoryCalculation(t *testing.T) { cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) ctx := base.TestCtx(t) - docRev, err := cache.Get(ctx, "doc1", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err := cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, RevCacheOmitDelta) require.NoError(t, err) assert.Equal(t, "doc1", docRev.DocID) assert.NotNil(t, docRev.History) @@ -260,7 +363,7 @@ func TestBackingStoreMemoryCalculation(t *testing.T) { assert.Equal(t, newMemStat, memoryBytesCounted.Value()) // test fail load event doesn't increment memory stat - docRev, err = cache.Get(ctx, "doc2", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(ctx, "doc2", "1-abc", testCollectionID, RevCacheOmitDelta) assertHTTPError(t, err, 404) assert.Nil(t, docRev.BodyBytes) assert.Equal(t, newMemStat, memoryBytesCounted.Value()) @@ -270,7 +373,7 @@ func TestBackingStoreMemoryCalculation(t *testing.T) { memStatBeforeThirdLoad := memoryBytesCounted.Value() // test another load from bucket but doing so should trigger memory based eviction - docRev, err = cache.Get(ctx, "doc3", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(ctx, "doc3", "1-abc", testCollectionID, RevCacheOmitDelta) require.NoError(t, err) assert.Equal(t, "doc3", docRev.DocID) assert.NotNil(t, docRev.History) @@ -295,7 +398,7 @@ func TestBackingStore(t *testing.T) { cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) // Get Rev for the first time - miss cache, but fetch the doc and revision to store - docRev, err := cache.Get(base.TestCtx(t), "Jens", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err := cache.GetWithRev(base.TestCtx(t), "Jens", "1-abc", testCollectionID, RevCacheOmitDelta) assert.NoError(t, err) assert.Equal(t, "Jens", docRev.DocID) assert.NotNil(t, docRev.History) @@ -306,7 +409,7 @@ func TestBackingStore(t *testing.T) { assert.Equal(t, int64(1), getRevisionCounter.Value()) // Doc doesn't exist, so miss the cache, and fail when getting the doc - docRev, err = cache.Get(base.TestCtx(t), "Peter", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(base.TestCtx(t), "Peter", "1-abc", testCollectionID, RevCacheOmitDelta) assertHTTPError(t, err, 404) assert.Nil(t, docRev.BodyBytes) assert.Equal(t, int64(0), cacheHitCounter.Value()) @@ -315,7 +418,7 @@ func TestBackingStore(t *testing.T) { assert.Equal(t, int64(1), getRevisionCounter.Value()) // Rev is already resident, but still issue GetDocument to check for later revisions - docRev, err = cache.Get(base.TestCtx(t), "Jens", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(base.TestCtx(t), "Jens", "1-abc", testCollectionID, RevCacheOmitDelta) assert.NoError(t, err) assert.Equal(t, "Jens", docRev.DocID) assert.NotNil(t, docRev.History) @@ -326,7 +429,67 @@ func TestBackingStore(t *testing.T) { assert.Equal(t, int64(1), getRevisionCounter.Value()) // Rev still doesn't exist, make sure it wasn't cached - docRev, err = cache.Get(base.TestCtx(t), "Peter", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(base.TestCtx(t), "Peter", "1-abc", testCollectionID, RevCacheOmitDelta) + assertHTTPError(t, err, 404) + assert.Nil(t, docRev.BodyBytes) + assert.Equal(t, int64(1), cacheHitCounter.Value()) + assert.Equal(t, int64(3), cacheMissCounter.Value()) + assert.Equal(t, int64(3), getDocumentCounter.Value()) + assert.Equal(t, int64(1), getRevisionCounter.Value()) +} + +// TestBackingStoreCV: +// - Perform a Get on a doc by cv that is not currently in the rev cache, assert we get cache miss +// - Perform a Get again on the same doc and assert we get cache hit +// - Perform a Get on doc that doesn't exist, so misses cache and will fail on retrieving doc from bucket +// - Try a Get again on the same doc and assert it wasn't loaded into the cache as it doesn't exist +func TestBackingStoreCV(t *testing.T) { + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted, getDocumentCounter, getRevisionCounter := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + + backingStoreMap := CreateTestSingleBackingStoreMap(&testBackingStore{[]string{"not_found"}, &getDocumentCounter, &getRevisionCounter}, testCollectionID) + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + // Get Rev for the first time - miss cache, but fetch the doc and revision to store + cv := Version{SourceID: "test", Value: 123} + docRev, err := cache.GetWithCV(base.TestCtx(t), "doc1", &cv, testCollectionID, RevCacheOmitDelta) + assert.NoError(t, err) + assert.Equal(t, "doc1", docRev.DocID) + assert.NotNil(t, docRev.Channels) + assert.Equal(t, "test", docRev.CV.SourceID) + assert.Equal(t, uint64(123), docRev.CV.Value) + assert.Equal(t, int64(0), cacheHitCounter.Value()) + assert.Equal(t, int64(1), cacheMissCounter.Value()) + assert.Equal(t, int64(1), getDocumentCounter.Value()) + assert.Equal(t, int64(1), getRevisionCounter.Value()) + + // Perform a get on the same doc as above, check that we get cache hit + docRev, err = cache.GetWithCV(base.TestCtx(t), "doc1", &cv, testCollectionID, RevCacheOmitDelta) + assert.NoError(t, err) + assert.Equal(t, "doc1", docRev.DocID) + assert.Equal(t, "test", docRev.CV.SourceID) + assert.Equal(t, uint64(123), docRev.CV.Value) + assert.Equal(t, int64(1), cacheHitCounter.Value()) + assert.Equal(t, int64(1), cacheMissCounter.Value()) + assert.Equal(t, int64(1), getDocumentCounter.Value()) + assert.Equal(t, int64(1), getRevisionCounter.Value()) + + // Doc doesn't exist, so miss the cache, and fail when getting the doc + cv = Version{SourceID: "test11", Value: 100} + docRev, err = cache.GetWithCV(base.TestCtx(t), "not_found", &cv, testCollectionID, RevCacheOmitDelta) + + assertHTTPError(t, err, 404) + assert.Nil(t, docRev.BodyBytes) + assert.Equal(t, int64(1), cacheHitCounter.Value()) + assert.Equal(t, int64(2), cacheMissCounter.Value()) + assert.Equal(t, int64(2), getDocumentCounter.Value()) + assert.Equal(t, int64(1), getRevisionCounter.Value()) + + // Rev still doesn't exist, make sure it wasn't cached + docRev, err = cache.GetWithCV(base.TestCtx(t), "not_found", &cv, testCollectionID, RevCacheOmitDelta) assertHTTPError(t, err, 404) assert.Nil(t, docRev.BodyBytes) assert.Equal(t, int64(1), cacheHitCounter.Value()) @@ -385,7 +548,7 @@ func TestRevisionCacheInternalProperties(t *testing.T) { } func TestBypassRevisionCache(t *testing.T) { - + t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)") base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll) db, ctx := setupTestDB(t) @@ -416,15 +579,15 @@ func TestBypassRevisionCache(t *testing.T) { assert.False(t, ok) // Get non-existing doc - _, err = rc.Get(ctx, "invalid", rev1, testCollectionID, RevCacheOmitDelta) + _, err = rc.GetWithRev(base.TestCtx(t), "invalid", rev1, testCollectionID, RevCacheOmitDelta) assert.True(t, base.IsDocNotFoundError(err)) // Get non-existing revision - _, err = rc.Get(ctx, key, "3-abc", testCollectionID, RevCacheOmitDelta) + _, err = rc.GetWithRev(base.TestCtx(t), key, "3-abc", testCollectionID, RevCacheOmitDelta) assertHTTPError(t, err, 404) // Get specific revision - doc, err := rc.Get(ctx, key, rev1, testCollectionID, RevCacheOmitDelta) + doc, err := rc.GetWithRev(base.TestCtx(t), key, rev1, testCollectionID, RevCacheOmitDelta) assert.NoError(t, err) require.NotNil(t, doc) assert.Equal(t, `{"value":1234}`, string(doc.BodyBytes)) @@ -511,7 +674,7 @@ func TestPutExistingRevRevisionCacheAttachmentProperty(t *testing.T) { "value": 1235, BodyAttachments: map[string]interface{}{"myatt": map[string]interface{}{"content_type": "text/plain", "data": "SGVsbG8gV29ybGQh"}}, } - _, _, err = collection.PutExistingRevWithBody(ctx, docKey, rev2body, []string{rev2id, rev1id}, false) + _, _, err = collection.PutExistingRevWithBody(ctx, docKey, rev2body, []string{rev2id, rev1id}, false, ExistingVersionWithUpdateToHLV) assert.NoError(t, err, "Unexpected error calling collection.PutExistingRev") // Get the raw document directly from the bucket, validate _attachments property isn't found @@ -522,7 +685,7 @@ func TestPutExistingRevRevisionCacheAttachmentProperty(t *testing.T) { assert.False(t, ok, "_attachments property still present in document body retrieved from bucket: %#v", bucketBody) // Get the raw document directly from the revcache, validate _attachments property isn't found - docRevision, err := collection.revisionCache.Get(ctx, docKey, rev2id, RevCacheOmitDelta) + docRevision, err := collection.revisionCache.GetWithRev(base.TestCtx(t), docKey, rev2id, RevCacheOmitDelta) assert.NoError(t, err, "Unexpected error calling collection.revisionCache.Get") assert.NotContains(t, docRevision.BodyBytes, BodyAttachments, "_attachments property still present in document body retrieved from rev cache: %#v", bucketBody) _, ok = docRevision.Attachments["myatt"] @@ -554,12 +717,12 @@ func TestRevisionImmutableDelta(t *testing.T) { secondDelta := []byte("modified delta") // Trigger load into cache - _, err := cache.Get(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + _, err := cache.GetWithRev(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) assert.NoError(t, err, "Error adding to cache") cache.UpdateDelta(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevisionDelta{ToRevID: "rev2", DeltaBytes: firstDelta}) // Retrieve from cache - retrievedRev, err := cache.Get(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + retrievedRev, err := cache.GetWithRev(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) assert.NoError(t, err, "Error retrieving from cache") assert.Equal(t, "rev2", retrievedRev.Delta.ToRevID) assert.Equal(t, firstDelta, retrievedRev.Delta.DeltaBytes) @@ -570,7 +733,7 @@ func TestRevisionImmutableDelta(t *testing.T) { assert.Equal(t, firstDelta, retrievedRev.Delta.DeltaBytes) // Retrieve again, validate delta is correct - updatedRev, err := cache.Get(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + updatedRev, err := cache.GetWithRev(base.TestCtx(t), "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) assert.NoError(t, err, "Error retrieving from cache") assert.Equal(t, "rev3", updatedRev.Delta.ToRevID) assert.Equal(t, secondDelta, updatedRev.Delta.DeltaBytes) @@ -595,7 +758,7 @@ func TestUpdateDeltaRevCacheMemoryStat(t *testing.T) { ctx := base.TestCtx(t) // Trigger load into cache - docRev, err := cache.Get(ctx, "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + docRev, err := cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) assert.NoError(t, err, "Error adding to cache") revCacheMem := memoryBytesCounted.Value() @@ -632,18 +795,18 @@ func TestImmediateRevCacheMemoryBasedEviction(t *testing.T) { ctx := base.TestCtx(t) cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc1", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc1", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) assert.Equal(t, int64(0), memoryBytesCounted.Value()) assert.Equal(t, int64(0), cacheNumItems.Value()) - cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc2", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc2", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) assert.Equal(t, int64(0), memoryBytesCounted.Value()) assert.Equal(t, int64(0), cacheNumItems.Value()) // assert we can still fetch this upsert doc - docRev, err := cache.Get(ctx, "doc2", "1-abc", testCollectionID, false) + docRev, err := cache.GetWithRev(ctx, "doc2", "1-abc", testCollectionID, false) require.NoError(t, err) assert.Equal(t, "doc2", docRev.DocID) assert.Equal(t, int64(102), docRev.MemoryBytes) @@ -651,7 +814,7 @@ func TestImmediateRevCacheMemoryBasedEviction(t *testing.T) { assert.Equal(t, int64(0), memoryBytesCounted.Value()) assert.Equal(t, int64(0), cacheNumItems.Value()) - docRev, err = cache.Get(ctx, "doc1", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err = cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, RevCacheOmitDelta) require.NoError(t, err) assert.NotNil(t, docRev.BodyBytes) @@ -771,20 +934,20 @@ func TestImmediateRevCacheItemBasedEviction(t *testing.T) { ctx := base.TestCtx(t) cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) // load up item to hit max capacity - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc1", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc1", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) // eviction starts from here in test - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "newDoc", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "newDoc", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) assert.Equal(t, int64(15), memoryBytesCounted.Value()) assert.Equal(t, int64(1), cacheNumItems.Value()) - cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc2", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"some":"test"}`), DocID: "doc2", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) assert.Equal(t, int64(15), memoryBytesCounted.Value()) assert.Equal(t, int64(1), cacheNumItems.Value()) - docRev, err := cache.Get(ctx, "doc3", "1-abc", testCollectionID, RevCacheOmitDelta) + docRev, err := cache.GetWithRev(ctx, "doc3", "1-abc", testCollectionID, RevCacheOmitDelta) require.NoError(t, err) assert.NotNil(t, docRev.BodyBytes) @@ -842,7 +1005,7 @@ func TestBasicOperationsOnCacheWithMemoryStat(t *testing.T) { assert.Equal(t, int64(docSize), cacheStats.RevisionCacheTotalMemory.Value()) // Test Get with item in the cache - docRev, err := db.revisionCache.Get(ctx, "doc1", revID, collctionID, RevCacheOmitDelta) + docRev, err := db.revisionCache.GetWithRev(ctx, "doc1", revID, collctionID, RevCacheOmitDelta) require.NoError(t, err) assert.NotNil(t, docRev.BodyBytes) assert.Equal(t, int64(docSize), cacheStats.RevisionCacheTotalMemory.Value()) @@ -850,9 +1013,9 @@ func TestBasicOperationsOnCacheWithMemoryStat(t *testing.T) { // Test Get operation with load from bucket, need to first create and remove from rev cache prevMemStat := cacheStats.RevisionCacheTotalMemory.Value() - revIDDoc2 := createThenRemoveFromRevCache(t, ctx, "doc2", db, collection) + revDoc2 := createThenRemoveFromRevCache(t, ctx, "doc2", db, collection) // load from doc from bucket - docRev, err = db.revisionCache.Get(ctx, "doc2", docRev.RevID, collctionID, RevCacheOmitDelta) + docRev, err = db.revisionCache.GetWithRev(ctx, "doc2", docRev.RevID, collctionID, RevCacheOmitDelta) require.NoError(t, err) assert.NotNil(t, docRev.BodyBytes) assert.Equal(t, "doc2", docRev.DocID) @@ -867,7 +1030,7 @@ func TestBasicOperationsOnCacheWithMemoryStat(t *testing.T) { // Test Get active with item to be loaded from bucket, need to first create and remove from rev cache prevMemStat = cacheStats.RevisionCacheTotalMemory.Value() - revIDDoc3 := createThenRemoveFromRevCache(t, ctx, "doc3", db, collection) + revDoc3 := createThenRemoveFromRevCache(t, ctx, "doc3", db, collection) docRev, err = db.revisionCache.GetActive(ctx, "doc3", collctionID) require.NoError(t, err) assert.Equal(t, "doc3", docRev.DocID) @@ -881,7 +1044,7 @@ func TestBasicOperationsOnCacheWithMemoryStat(t *testing.T) { assert.Equal(t, prevMemStat, cacheStats.RevisionCacheTotalMemory.Value()) // Test Peek in cache, assert stat unchanged - docRev, ok = db.revisionCache.Peek(ctx, "doc3", revIDDoc3, collctionID) + docRev, ok = db.revisionCache.Peek(ctx, "doc3", revDoc3.RevTreeID, collctionID) require.True(t, ok) assert.Equal(t, "doc3", docRev.DocID) assert.Equal(t, prevMemStat, cacheStats.RevisionCacheTotalMemory.Value()) @@ -890,45 +1053,38 @@ func TestBasicOperationsOnCacheWithMemoryStat(t *testing.T) { docRev.CalculateBytes() doc3Size := docRev.MemoryBytes expMem := cacheStats.RevisionCacheTotalMemory.Value() - doc3Size - newDocRev := DocumentRevision{ - DocID: "doc3", - RevID: revIDDoc3, - BodyBytes: []byte(`"some": "body"`), - } + newDocRev := documentRevisionForCacheTestUpdate("doc3", `"some": "body"`, revDoc3) + expMem = expMem + 14 // size for above doc rev db.revisionCache.Upsert(ctx, newDocRev, collctionID) assert.Equal(t, expMem, cacheStats.RevisionCacheTotalMemory.Value()) // Test Upsert with item not in cache, assert stat is as expected - newDocRev = DocumentRevision{ - DocID: "doc5", - RevID: "1-abc", - BodyBytes: []byte(`"some": "body"`), - } + newDocRev = documentRevisionForCacheTest("doc5", `"some": "body"`) expMem = cacheStats.RevisionCacheTotalMemory.Value() + 14 // size for above doc rev db.revisionCache.Upsert(ctx, newDocRev, collctionID) assert.Equal(t, expMem, cacheStats.RevisionCacheTotalMemory.Value()) // Test Remove with something in cache, assert stat decrements by expected value - db.revisionCache.Remove("doc5", "1-abc", collctionID) + db.revisionCache.RemoveWithRev("doc5", "1-abc", collctionID) expMem -= 14 assert.Equal(t, expMem, cacheStats.RevisionCacheTotalMemory.Value()) // Test Remove with item not in cache, assert stat is unchanged prevMemStat = cacheStats.RevisionCacheTotalMemory.Value() - db.revisionCache.Remove("doc6", "1-abc", collctionID) + db.revisionCache.RemoveWithRev("doc6", "1-abc", collctionID) assert.Equal(t, prevMemStat, cacheStats.RevisionCacheTotalMemory.Value()) // Test Update Delta, assert stat increases as expected revDelta := newRevCacheDelta([]byte(`"rev":"delta"`), "1-abc", newDocRev, false, nil) expMem = prevMemStat + revDelta.totalDeltaBytes - db.revisionCache.UpdateDelta(ctx, "doc3", revIDDoc3, collctionID, revDelta) + db.revisionCache.UpdateDelta(ctx, "doc3", revDoc3.RevTreeID, collctionID, revDelta) assert.Equal(t, expMem, cacheStats.RevisionCacheTotalMemory.Value()) // Empty cache and see memory stat is 0 - db.revisionCache.Remove("doc3", revIDDoc3, collctionID) - db.revisionCache.Remove("doc2", revIDDoc2, collctionID) - db.revisionCache.Remove("doc1", revIDDoc1, collctionID) + db.revisionCache.RemoveWithRev("doc3", revDoc3.RevTreeID, collctionID) + db.revisionCache.RemoveWithRev("doc2", revDoc2.RevTreeID, collctionID) + db.revisionCache.RemoveWithRev("doc1", revIDDoc1, collctionID) // TODO: pending CBG-4135 assert rev cache had 0 items in it assert.Equal(t, int64(0), cacheStats.RevisionCacheTotalMemory.Value()) @@ -944,8 +1100,9 @@ func TestSingleLoad(t *testing.T) { } cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) - cache.Put(base.TestCtx(t), DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc123", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) - _, err := cache.Get(base.TestCtx(t), "doc123", "1-abc", testCollectionID, false) + cache.Put(base.TestCtx(t), DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc123", RevID: "1-abc", CV: &Version{Value: 123, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) + _, err := cache.GetWithRev(base.TestCtx(t), "doc123", "1-abc", testCollectionID, false) + assert.NoError(t, err) } @@ -959,14 +1116,14 @@ func TestConcurrentLoad(t *testing.T) { } cache := NewLRURevisionCache(cacheOptions, backingStoreMap, &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) - cache.Put(base.TestCtx(t), DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc1", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(base.TestCtx(t), DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc1", RevID: "1-abc", CV: &Version{Value: 1234, SourceID: "test"}, History: Revisions{"start": 1}}, testCollectionID) // Trigger load into cache var wg sync.WaitGroup wg.Add(20) for i := 0; i < 20; i++ { go func() { - _, err := cache.Get(base.TestCtx(t), "doc1", "1-abc", testCollectionID, false) + _, err := cache.GetWithRev(base.TestCtx(t), "doc1", "1-abc", testCollectionID, false) assert.NoError(t, err) wg.Done() }() @@ -984,14 +1141,14 @@ func TestRevisionCacheRemove(t *testing.T) { rev1id, _, err := collection.Put(ctx, "doc", Body{"val": 123}) assert.NoError(t, err) - docRev, err := collection.revisionCache.Get(ctx, "doc", rev1id, true) + docRev, err := collection.revisionCache.GetWithRev(base.TestCtx(t), "doc", rev1id, true) assert.NoError(t, err) assert.Equal(t, rev1id, docRev.RevID) assert.Equal(t, int64(0), db.DbStats.Cache().RevisionCacheMisses.Value()) - collection.revisionCache.Remove("doc", rev1id) + collection.revisionCache.RemoveWithRev("doc", rev1id) - docRev, err = collection.revisionCache.Get(ctx, "doc", rev1id, true) + docRev, err = collection.revisionCache.GetWithRev(base.TestCtx(t), "doc", rev1id, true) assert.NoError(t, err) assert.Equal(t, rev1id, docRev.RevID) assert.Equal(t, int64(1), db.DbStats.Cache().RevisionCacheMisses.Value()) @@ -1068,6 +1225,8 @@ func TestRevCacheHitMultiCollection(t *testing.T) { // - This in turn evicts the second doc // - Perform Get on that second doc to trigger load from the bucket, assert doc is as expected func TestRevCacheHitMultiCollectionLoadFromBucket(t *testing.T) { + + t.Skip("Pending CBG-4164") base.TestRequiresCollections(t) tb := base.GetTestBucket(t) @@ -1135,25 +1294,25 @@ func TestRevCacheCapacityStat(t *testing.T) { assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Create a new doc + asert num items increments - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc1", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(ctx, documentRevisionForCacheTest("doc1", `{"test":"1234"}`), testCollectionID) assert.Equal(t, int64(1), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // test not found doc, assert that the stat isn't incremented - _, err := cache.Get(ctx, "badDoc", "1-abc", testCollectionID, false) + _, err := cache.GetWithRev(ctx, "badDoc", "1-abc", testCollectionID, false) require.Error(t, err) assert.Equal(t, int64(1), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Get on a doc that doesn't exist in cache, assert num items increments - docRev, err := cache.Get(ctx, "doc2", "1-abc", testCollectionID, false) + docRev, err := cache.GetWithRev(ctx, "doc2", "1-abc", testCollectionID, false) require.NoError(t, err) assert.Equal(t, "doc2", docRev.DocID) assert.Equal(t, int64(2), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Get on item in cache, assert num items remains the same - docRev, err = cache.Get(ctx, "doc1", "1-abc", testCollectionID, false) + docRev, err = cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, false) require.NoError(t, err) assert.Equal(t, "doc1", docRev.DocID) assert.Equal(t, int64(2), cacheNumItems.Value()) @@ -1174,12 +1333,12 @@ func TestRevCacheCapacityStat(t *testing.T) { assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Upsert a doc resident in cache, assert stat is unchanged - cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"test":"12345"}`), DocID: "doc1", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Upsert(ctx, documentRevisionForCacheTest("doc1", `{"test":"12345"}`), testCollectionID) assert.Equal(t, int64(3), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Upsert new doc, assert the num items stat increments - cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"test":"1234}`), DocID: "doc4", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Upsert(ctx, documentRevisionForCacheTest("doc4", `{"test":"1234}`), testCollectionID) assert.Equal(t, int64(4), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) @@ -1197,26 +1356,113 @@ func TestRevCacheCapacityStat(t *testing.T) { assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Eviction situation and assert stat doesn't go over the capacity set - cache.Put(ctx, DocumentRevision{BodyBytes: []byte(`{"test":"1234"}`), DocID: "doc5", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Put(ctx, documentRevisionForCacheTest("doc5", `{"test":"1234"}`), testCollectionID) assert.Equal(t, int64(4), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // test case of eviction for upsert - cache.Upsert(ctx, DocumentRevision{BodyBytes: []byte(`{"test":"12345"}`), DocID: "doc6", RevID: "1-abc", History: Revisions{"start": 1}}, testCollectionID) + cache.Upsert(ctx, documentRevisionForCacheTest("doc6", `{"test":"12345"}`), testCollectionID) assert.Equal(t, int64(4), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) // Empty cache - cache.Remove("doc1", "1-abc", testCollectionID) - cache.Remove("doc4", "1-abc", testCollectionID) - cache.Remove("doc5", "1-abc", testCollectionID) - cache.Remove("doc6", "1-abc", testCollectionID) + cache.RemoveWithRev("doc1", "1-abc", testCollectionID) + cache.RemoveWithRev("doc4", "1-abc", testCollectionID) + cache.RemoveWithRev("doc5", "1-abc", testCollectionID) + cache.RemoveWithRev("doc6", "1-abc", testCollectionID) // Assert num items goes back to 0 assert.Equal(t, int64(0), cacheNumItems.Value()) assert.Equal(t, int64(len(cache.cache)), cacheNumItems.Value()) } +// documentRevisionForCacheTest creates a document revision with the specified body and key, and a hardcoded revID, cv and history: +// +// RevID: 1-abc +// CV: Version{SourceID: "test", Value: 123} +// History: Revisions{"start": 1}} +func documentRevisionForCacheTest(key string, body string) DocumentRevision { + cv := Version{SourceID: "test", Value: 123} + return DocumentRevision{ + BodyBytes: []byte(body), + DocID: key, + RevID: "1-abc", + History: Revisions{"start": 1}, + CV: &cv, + } +} + +// documentRevisionForCacheTestUpsert creates a document revision with the specified body and key and Version +// +// History: Revisions{"start": 1}} +func documentRevisionForCacheTestUpdate(key string, body string, version DocVersion) DocumentRevision { + return DocumentRevision{ + BodyBytes: []byte(body), + DocID: key, + RevID: version.RevTreeID, + History: Revisions{"start": 1}, + CV: &version.CV, + } +} + +// TestRevCacheOperationsCV: +// - Create doc revision, put the revision into the cache +// - Perform a get on that doc by cv and assert that it has correctly been handled +// - Updated doc revision and upsert the cache +// - Get the updated doc by cv and assert iot has been correctly handled +// - Peek the doc by cv and assert it has been found +// - Peek the rev id cache for the same doc and assert that doc also has been updated in that lookup cache +// - Remove the doc by cv, and asser that the doc is gone +func TestRevCacheOperationsCV(t *testing.T) { + + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted, getDocumentCounter, getRevisionCounter := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, CreateTestSingleBackingStoreMap(&testBackingStore{[]string{"test_doc"}, &getDocumentCounter, &getRevisionCounter}, testCollectionID), &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + cv := Version{SourceID: "test", Value: 123} + documentRevision := DocumentRevision{ + DocID: "doc1", + RevID: "1-abc", + BodyBytes: []byte(`{"test":"1234"}`), + Channels: base.SetOf("chan1"), + History: Revisions{"start": 1}, + CV: &cv, + } + cache.Put(base.TestCtx(t), documentRevision, testCollectionID) + + docRev, err := cache.GetWithCV(base.TestCtx(t), "doc1", &cv, testCollectionID, RevCacheOmitDelta) + require.NoError(t, err) + assert.Equal(t, "doc1", docRev.DocID) + assert.Equal(t, base.SetOf("chan1"), docRev.Channels) + assert.Equal(t, "test", docRev.CV.SourceID) + assert.Equal(t, uint64(123), docRev.CV.Value) + assert.Equal(t, int64(1), cacheHitCounter.Value()) + assert.Equal(t, int64(0), cacheMissCounter.Value()) + + documentRevision.BodyBytes = []byte(`{"test":"12345"}`) + + cache.Upsert(base.TestCtx(t), documentRevision, testCollectionID) + + docRev, err = cache.GetWithCV(base.TestCtx(t), "doc1", &cv, testCollectionID, RevCacheOmitDelta) + require.NoError(t, err) + assert.Equal(t, "doc1", docRev.DocID) + assert.Equal(t, base.SetOf("chan1"), docRev.Channels) + assert.Equal(t, "test", docRev.CV.SourceID) + assert.Equal(t, uint64(123), docRev.CV.Value) + assert.Equal(t, []byte(`{"test":"12345"}`), docRev.BodyBytes) + assert.Equal(t, int64(2), cacheHitCounter.Value()) + assert.Equal(t, int64(0), cacheMissCounter.Value()) + + // remove the doc rev from the cache and assert that the document is no longer present in cache + cache.RemoveWithCV("doc1", &cv, testCollectionID) + assert.Equal(t, 0, len(cache.cache)) + assert.Equal(t, 0, len(cache.hlvCache)) + assert.Equal(t, 0, cache.lruList.Len()) +} + func BenchmarkRevisionCacheRead(b *testing.B) { base.SetUpBenchmarkLogging(b, base.LevelDebug, base.KeyAll) @@ -1232,7 +1478,7 @@ func BenchmarkRevisionCacheRead(b *testing.B) { // trigger load into cache for i := 0; i < 5000; i++ { - _, _ = cache.Get(ctx, fmt.Sprintf("doc%d", i), "1-abc", testCollectionID, RevCacheOmitDelta) + _, _ = cache.GetWithRev(ctx, fmt.Sprintf("doc%d", i), "1-abc", testCollectionID, RevCacheOmitDelta) } b.ResetTimer() @@ -1240,19 +1486,24 @@ func BenchmarkRevisionCacheRead(b *testing.B) { // GET the document until test run has completed for pb.Next() { docId := fmt.Sprintf("doc%d", rand.Intn(5000)) - _, _ = cache.Get(ctx, docId, "1-abc", testCollectionID, RevCacheOmitDelta) + _, _ = cache.GetWithRev(ctx, docId, "1-abc", testCollectionID, RevCacheOmitDelta) } }) } // createThenRemoveFromRevCache will create a doc and then immediately remove it from the rev cache -func createThenRemoveFromRevCache(t *testing.T, ctx context.Context, docID string, db *Database, collection *DatabaseCollectionWithUser) string { - revIDDoc, _, err := collection.Put(ctx, docID, Body{"test": "doc"}) +func createThenRemoveFromRevCache(t *testing.T, ctx context.Context, docID string, db *Database, collection *DatabaseCollectionWithUser) DocVersion { + revIDDoc, doc, err := collection.Put(ctx, docID, Body{"test": "doc"}) require.NoError(t, err) - db.revisionCache.Remove(docID, revIDDoc, collection.GetCollectionID()) - - return revIDDoc + db.revisionCache.RemoveWithRev(docID, revIDDoc, collection.GetCollectionID()) + docVersion := DocVersion{ + RevTreeID: doc.CurrentRev, + } + if doc.HLV != nil { + docVersion.CV = *doc.HLV.ExtractCurrentVersionFromHLV() + } + return docVersion } // createDocAndReturnSizeAndRev creates a rev and measures its size based on rev cache measurements @@ -1277,3 +1528,155 @@ func createDocAndReturnSizeAndRev(t *testing.T, ctx context.Context, docID strin return expectedSize, rev } + +// TestLoaderMismatchInCV: +// - Get doc that is not in cache by CV to trigger a load from bucket +// - Ensure the CV passed into the GET operation won't match the doc in the bucket +// - Assert we get error and the value is not loaded into the cache +func TestLoaderMismatchInCV(t *testing.T) { + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted, getDocumentCounter, getRevisionCounter := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, CreateTestSingleBackingStoreMap(&testBackingStore{[]string{"test_doc"}, &getDocumentCounter, &getRevisionCounter}, testCollectionID), &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + // create cv with incorrect version to the one stored in backing store + cv := Version{SourceID: "test", Value: 1234} + + _, err := cache.GetWithCV(base.TestCtx(t), "doc1", &cv, testCollectionID, RevCacheOmitDelta) + require.Error(t, err) + require.Error(t, err, base.ErrNotFound) + assert.Equal(t, int64(0), cacheHitCounter.Value()) + assert.Equal(t, int64(1), cacheMissCounter.Value()) + assert.Equal(t, 0, cache.lruList.Len()) + assert.Equal(t, 0, len(cache.hlvCache)) + assert.Equal(t, 0, len(cache.cache)) +} + +// TestConcurrentLoadByCVAndRevOnCache: +// - Create cache +// - Now perform two concurrent Gets, one by CV and one by revid on a document that doesn't exist in the cache +// - This will trigger two concurrent loads from bucket in the CV code path and revid code path +// - In doing so we will have two processes trying to update lookup maps at the same time and a race condition will appear +// - In doing so will cause us to potentially have two of the same elements the cache, one with nothing referencing it +// - Assert after both gets are processed, that the cache only has one element in it and that both lookup maps have only one +// element +// - Grab the single element in the list and assert that both maps point to that element in the cache list +func TestConcurrentLoadByCVAndRevOnCache(t *testing.T) { + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted, getDocumentCounter, getRevisionCounter := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, CreateTestSingleBackingStoreMap(&testBackingStore{[]string{"test_doc"}, &getDocumentCounter, &getRevisionCounter}, testCollectionID), &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + ctx := base.TestCtx(t) + + wg := sync.WaitGroup{} + wg.Add(2) + + cv := Version{SourceID: "test", Value: 123} + go func() { + _, err := cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + require.NoError(t, err) + wg.Done() + }() + + go func() { + _, err := cache.GetWithCV(ctx, "doc1", &cv, testCollectionID, RevCacheIncludeDelta) + require.NoError(t, err) + wg.Done() + }() + + wg.Wait() + + revElement := cache.cache[IDAndRev{RevID: "1-abc", DocID: "doc1"}] + cvElement := cache.hlvCache[IDandCV{DocID: "doc1", Source: "test", Version: 123}] + assert.Equal(t, 1, cache.lruList.Len()) + assert.Equal(t, 1, len(cache.cache)) + assert.Equal(t, 1, len(cache.hlvCache)) + // grab the single elem in the cache list + cacheElem := cache.lruList.Front() + // assert that both maps point to the same element in cache list + assert.Equal(t, cacheElem, cvElement) + assert.Equal(t, cacheElem, revElement) +} + +// TestGetActive: +// - Create db, create a doc on the db +// - Call GetActive pn the rev cache and assert that the rev and cv are correct +func TestGetActive(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close(ctx) + collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) + + rev1id, doc, err := collection.Put(ctx, "doc", Body{"val": 123}) + require.NoError(t, err) + + expectedCV := Version{ + SourceID: db.EncodedSourceID, + Value: doc.Cas, + } + + // remove the entry form the rev cache to force the cache to not have the active version in it + collection.revisionCache.RemoveWithCV("doc", &expectedCV) + + // call get active to get the active version from the bucket + docRev, err := collection.revisionCache.GetActive(base.TestCtx(t), "doc") + assert.NoError(t, err) + assert.Equal(t, rev1id, docRev.RevID) + assert.Equal(t, expectedCV, *docRev.CV) +} + +// TestConcurrentPutAndGetOnRevCache: +// - Perform a Get with rev on the cache for a doc not in the cache +// - Concurrently perform a PUT on the cache with doc revision the same as the GET +// - Assert we get consistent cache with only 1 entry in lookup maps and the cache itself +func TestConcurrentPutAndGetOnRevCache(t *testing.T) { + cacheHitCounter, cacheMissCounter, cacheNumItems, memoryBytesCounted, getDocumentCounter, getRevisionCounter := base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{}, base.SgwIntStat{} + cacheOptions := &RevisionCacheOptions{ + MaxItemCount: 10, + MaxBytes: 0, + } + cache := NewLRURevisionCache(cacheOptions, CreateTestSingleBackingStoreMap(&testBackingStore{[]string{"test_doc"}, &getDocumentCounter, &getRevisionCounter}, testCollectionID), &cacheHitCounter, &cacheMissCounter, &cacheNumItems, &memoryBytesCounted) + + ctx := base.TestCtx(t) + + wg := sync.WaitGroup{} + wg.Add(2) + + cv := Version{SourceID: "test", Value: 123} + docRev := DocumentRevision{ + DocID: "doc1", + RevID: "1-abc", + BodyBytes: []byte(`{"test":"1234"}`), + Channels: base.SetOf("chan1"), + History: Revisions{"start": 1}, + CV: &cv, + } + + go func() { + _, err := cache.GetWithRev(ctx, "doc1", "1-abc", testCollectionID, RevCacheIncludeDelta) + require.NoError(t, err) + wg.Done() + }() + + go func() { + cache.Put(ctx, docRev, testCollectionID) + wg.Done() + }() + + wg.Wait() + + revElement := cache.cache[IDAndRev{RevID: "1-abc", DocID: "doc1"}] + cvElement := cache.hlvCache[IDandCV{DocID: "doc1", Source: "test", Version: 123}] + + assert.Equal(t, 1, cache.lruList.Len()) + assert.Equal(t, 1, len(cache.cache)) + assert.Equal(t, 1, len(cache.hlvCache)) + cacheElem := cache.lruList.Front() + // assert that both maps point to the same element in cache list + assert.Equal(t, cacheElem, cvElement) + assert.Equal(t, cacheElem, revElement) +} diff --git a/db/revision_test.go b/db/revision_test.go index 20898958d5..6882482352 100644 --- a/db/revision_test.go +++ b/db/revision_test.go @@ -112,7 +112,7 @@ func TestBackupOldRevision(t *testing.T) { docID := t.Name() - rev1ID, _, err := collection.Put(ctx, docID, Body{"test": true}) + rev1ID, docRev1, err := collection.Put(ctx, docID, Body{"test": true}) require.NoError(t, err) // make sure we didn't accidentally store an empty old revision @@ -121,7 +121,8 @@ func TestBackupOldRevision(t *testing.T) { assert.Equal(t, "404 missing", err.Error()) // check for current rev backup in xattr+delta case (to support deltas by sdk imports) - _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, rev1ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) if deltasEnabled && xattrsEnabled { require.NoError(t, err) } else { @@ -131,15 +132,17 @@ func TestBackupOldRevision(t *testing.T) { // create rev 2 and check backups for both revs rev2ID := "2-abc" - _, _, err = collection.PutExistingRevWithBody(ctx, docID, Body{"test": true, "updated": true}, []string{rev2ID, rev1ID}, true) + docRev2, _, err := collection.PutExistingRevWithBody(ctx, docID, Body{"test": true, "updated": true}, []string{rev2ID, rev1ID}, true, ExistingVersionWithUpdateToHLV) require.NoError(t, err) // now in all cases we'll have rev 1 backed up (for at least 5 minutes) - _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, rev1ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, base.Crc32cHashString([]byte(docRev1.HLV.GetCurrentVersionString()))) require.NoError(t, err) // check for current rev backup in xattr+delta case (to support deltas by sdk imports) - _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, rev2ID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + _, err = collection.getOldRevisionJSON(base.TestCtx(t), docID, base.Crc32cHashString([]byte(docRev2.HLV.GetCurrentVersionString()))) if deltasEnabled && xattrsEnabled { require.NoError(t, err) } else { diff --git a/db/util_testing.go b/db/util_testing.go index 0b6e9d05f1..456e5c0641 100644 --- a/db/util_testing.go +++ b/db/util_testing.go @@ -14,6 +14,7 @@ import ( "context" "errors" "fmt" + "strconv" "sync/atomic" "testing" "time" @@ -178,7 +179,17 @@ func purgeWithDCPFeed(ctx context.Context, dataStore sgbucket.DataStore, tbp *ba key := string(event.Key) if base.TestUseXattrs() { - purgeErr = dataStore.DeleteWithXattrs(ctx, key, []string{base.SyncXattrName}) + systemXattrNames, decodeErr := sgbucket.DecodeXattrNames(event.Value, true) + if decodeErr != nil { + purgeErrors = purgeErrors.Append(decodeErr) + tbp.Logf(ctx, "Error decoding DCP event xattrs for key %s. %v", key, decodeErr) + return false + } + if len(systemXattrNames) > 0 { + purgeErr = dataStore.DeleteWithXattrs(ctx, key, systemXattrNames) + } else { + purgeErr = dataStore.Delete(key) + } } else { purgeErr = dataStore.Delete(key) } @@ -698,3 +709,97 @@ func WriteDirect(t *testing.T, collection *DatabaseCollection, channelArray []st require.NoError(t, err) } } + +func createTestDocument(docID string, revID string, body Body, deleted bool, expiry uint32) (newDoc *Document) { + newDoc = &Document{ + ID: docID, + Deleted: deleted, + DocExpiry: expiry, + RevID: revID, + _body: body, + } + return newDoc +} + +// requireCurrentVersion fetches the document by key, and validates that cv matches. +func (c *DatabaseCollection) RequireCurrentVersion(t *testing.T, key string, source string, version uint64) { + ctx := base.TestCtx(t) + doc, err := c.GetDocument(ctx, key, DocUnmarshalSync) + require.NoError(t, err) + if doc.HLV == nil { + require.Equal(t, "", source) + require.Equal(t, "", version) + return + } + + require.Equal(t, doc.HLV.SourceID, source) + require.Equal(t, doc.HLV.Version, version) +} + +// GetDocumentCurrentVersion fetches the document by key and returns the current version +func (c *DatabaseCollection) GetDocumentCurrentVersion(t testing.TB, key string) (source string, version uint64) { + ctx := base.TestCtx(t) + doc, err := c.GetDocument(ctx, key, DocUnmarshalSync) + require.NoError(t, err) + if doc.HLV == nil { + return "", 0 + } + return doc.HLV.SourceID, doc.HLV.Version +} + +// retrieveDocRevSeNo will take the $document xattr and return the revSeqNo defined in that xattr +func RetrieveDocRevSeqNo(t *testing.T, docxattr []byte) uint64 { + // virtual xattr not implemented for rosmar CBG-4233 + if base.UnitTestUrlIsWalrus() { + return 0 + } + require.NotNil(t, docxattr) + var retrievedDocumentRevNo string + require.NoError(t, base.JSONUnmarshal(docxattr, &retrievedDocumentRevNo)) + + revNo, err := strconv.ParseUint(retrievedDocumentRevNo, 10, 64) + require.NoError(t, err) + return revNo +} + +// MoveAttachmentXattrFromGlobalToSync is a test only function that will move any defined attachment metadata in global xattr to sync data xattr +func MoveAttachmentXattrFromGlobalToSync(t *testing.T, ctx context.Context, docID string, cas uint64, value, syncXattr []byte, attachments AttachmentsMeta, macroExpand bool, dataStore base.DataStore) { + var docSync SyncData + err := base.JSONUnmarshal(syncXattr, &docSync) + require.NoError(t, err) + docSync.Attachments = attachments + + opts := &sgbucket.MutateInOptions{} + // this should be true for cases we want to move the attachment metadata without causing a new import feed event + if macroExpand { + spec := macroExpandSpec(base.SyncXattrName) + opts.MacroExpansion = spec + } else { + opts = nil + docSync.Cas = "" + } + + newSync, err := base.JSONMarshal(docSync) + require.NoError(t, err) + + _, err = dataStore.WriteWithXattrs(ctx, docID, 0, cas, value, map[string][]byte{base.SyncXattrName: newSync}, []string{base.GlobalXattrName}, opts) + require.NoError(t, err) +} + +func RequireBackgroundManagerState(t *testing.T, ctx context.Context, mgr *BackgroundManager, expState BackgroundProcessState) { + require.EventuallyWithT(t, func(c *assert.CollectT) { + var status BackgroundManagerStatus + rawStatus, err := mgr.GetStatus(ctx) + assert.NoError(c, err) + assert.NoError(c, base.JSONUnmarshal(rawStatus, &status)) + assert.Equal(c, expState, status.State) + }, time.Second*10, time.Millisecond*100) +} + +// AssertSyncInfoMetaVersion will assert that meta version is equal to current product version +func AssertSyncInfoMetaVersion(t *testing.T, ds base.DataStore) { + var syncInfo base.SyncInfo + _, err := ds.Get(base.SGSyncInfo, &syncInfo) + require.NoError(t, err) + assert.Equal(t, "4.0.0", syncInfo.MetaDataVersion) +} diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go new file mode 100644 index 0000000000..9832a44526 --- /dev/null +++ b/db/utilities_hlv_testing.go @@ -0,0 +1,219 @@ +/* +Copyright 2017-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package db + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/stretchr/testify/require" +) + +// DocVersion represents a specific version of a document in an revID/HLV agnostic manner. +type DocVersion struct { + RevTreeID string + CV Version +} + +func (v *DocVersion) String() string { + return fmt.Sprintf("RevTreeID: %s", v.RevTreeID) +} + +func (v DocVersion) Equal(o DocVersion) bool { + if v.RevTreeID != o.RevTreeID { + return false + } + return true +} + +func (v DocVersion) GetRev(useHLV bool) string { + if useHLV { + if v.CV.SourceID == "" { + return "" + } + return v.CV.String() + } else { + return v.RevTreeID + } +} + +// Digest returns the digest for the current version +func (v DocVersion) Digest() string { + return strings.Split(v.RevTreeID, "-")[1] +} + +// HLVAgent performs HLV updates directly (not via SG) for simulating/testing interaction with non-SG HLV agents +type HLVAgent struct { + t *testing.T + datastore base.DataStore + Source string // All writes by the HLVHelper are done as this source + xattrName string // xattr name to store the HLV +} + +var defaultHelperBody = map[string]interface{}{"version": 1} + +func NewHLVAgent(t *testing.T, datastore base.DataStore, source string, xattrName string) *HLVAgent { + return &HLVAgent{ + t: t, + datastore: datastore, + Source: EncodeSource(source), // all writes by the HLVHelper are done as this source + xattrName: xattrName, + } +} + +// InsertWithHLV inserts a new document into the bucket with a populated HLV (matching a write from +// a different, non-SGW HLV-aware peer) +func (h *HLVAgent) InsertWithHLV(ctx context.Context, key string) (casOut uint64) { + hlv := &HybridLogicalVector{} + err := hlv.AddVersion(CreateVersion(h.Source, expandMacroCASValueUint64)) + require.NoError(h.t, err) + hlv.CurrentVersionCAS = expandMacroCASValueUint64 + + vvDataBytes := base.MustJSONMarshal(h.t, hlv) + mutateInOpts := &sgbucket.MutateInOptions{ + MacroExpansion: hlv.computeMacroExpansions(), + } + + docBody := base.MustJSONMarshal(h.t, defaultHelperBody) + xattrData := map[string][]byte{ + h.xattrName: vvDataBytes, + } + + cas, err := h.datastore.WriteWithXattrs(ctx, key, 0, 0, docBody, xattrData, nil, mutateInOpts) + require.NoError(h.t, err) + return cas +} + +// UpdateWithHLV will update and existing doc in bucket mocking write from another hlv aware peer +func (h *HLVAgent) UpdateWithHLV(ctx context.Context, key string, inputCas uint64, hlv *HybridLogicalVector) (casOut uint64) { + err := hlv.AddVersion(CreateVersion(h.Source, expandMacroCASValueUint64)) + require.NoError(h.t, err) + hlv.CurrentVersionCAS = expandMacroCASValueUint64 + + vvXattr, err := hlv.MarshalJSON() + require.NoError(h.t, err) + mutateInOpts := &sgbucket.MutateInOptions{ + MacroExpansion: hlv.computeMacroExpansions(), + } + + docBody := base.MustJSONMarshal(h.t, defaultHelperBody) + xattrData := map[string][]byte{ + h.xattrName: vvXattr, + } + cas, err := h.datastore.WriteWithXattrs(ctx, key, 0, inputCas, docBody, xattrData, nil, mutateInOpts) + require.NoError(h.t, err) + return cas +} + +// EncodeTestVersion converts a simplified string version of the form 1@abc to a hex-encoded version and base64 encoded +// source, like 169a05acd705ffc0@YWJj. Allows use of simplified versions in tests for readability, ease of use. +func EncodeTestVersion(versionString string) (encodedString string) { + timestampString, source, found := strings.Cut(versionString, "@") + if !found { + return versionString + } + if len(timestampString) > 0 && timestampString[0] == ' ' { + timestampString = timestampString[1:] + } + timestampUint, err := strconv.ParseUint(timestampString, 10, 64) + if err != nil { + return "" + } + hexTimestamp := strconv.FormatUint(timestampUint, 16) + base64Source := EncodeSource(source) + return hexTimestamp + "@" + base64Source +} + +// GetHelperBody returns the body contents of a document written by HLVAgent. +func (h *HLVAgent) GetHelperBody() string { + return string(base.MustJSONMarshal(h.t, defaultHelperBody)) +} + +// SourceID returns the encoded source ID for the HLVAgent +func (h *HLVAgent) SourceID() string { + return h.Source +} + +// encodeTestHistory converts a simplified version history of the form "1@abc,2@def;3@ghi" to use hex-encoded versions and +// base64 encoded sources +func EncodeTestHistory(historyString string) (encodedString string) { + // possible versionSets are pv;mv + // possible versionSets are pv;mv + versionSets := strings.Split(historyString, ";") + if len(versionSets) == 0 { + return "" + } + for index, versionSet := range versionSets { + // versionSet delimiter + if index > 0 { + encodedString += ";" + } + versions := strings.Split(versionSet, ",") + for index, version := range versions { + // version delimiter + if index > 0 { + encodedString += "," + } + encodedString += EncodeTestVersion(version) + } + } + return encodedString +} + +// ParseTestHistory takes a string test history in the form 1@abc,2@def;3@ghi,4@jkl and formats this +// as pv and mv maps keyed by encoded source, with encoded values +func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HLVVersions) { + versionSets := strings.Split(historyString, ";") + + pv = make(HLVVersions) + mv = make(HLVVersions) + + var pvString, mvString string + switch len(versionSets) { + case 1: + pvString = versionSets[0] + case 2: + mvString = versionSets[0] + pvString = versionSets[1] + default: + return pv, mv + } + + // pv + for _, versionStr := range strings.Split(pvString, ",") { + version, err := ParseVersion(versionStr) + require.NoError(t, err) + pv[EncodeSource(version.SourceID)] = version.Value + } + + // mv + if mvString != "" { + for _, versionStr := range strings.Split(mvString, ",") { + version, err := ParseVersion(versionStr) + require.NoError(t, err) + mv[EncodeSource(version.SourceID)] = version.Value + } + } + return pv, mv +} + +// Requires that the CV for the provided HLV matches the expected CV (sent in simplified test format) +func RequireCVEqual(t *testing.T, hlv *HybridLogicalVector, expectedCV string) { + testVersion, err := ParseVersion(expectedCV) + require.NoError(t, err) + require.Equal(t, EncodeSource(testVersion.SourceID), hlv.SourceID) + require.Equal(t, testVersion.Value, hlv.Version) +} diff --git a/docs/api/admin.yaml b/docs/api/admin.yaml index 7136a22fdb..955d327f2e 100644 --- a/docs/api/admin.yaml +++ b/docs/api/admin.yaml @@ -124,6 +124,8 @@ paths: $ref: ./paths/admin/_all_dbs.yaml '/{db}/_compact': $ref: './paths/admin/db-_compact.yaml' + '/{db}/_attachment_migration': + $ref: './paths/admin/db-_attachment_migration.yaml' '/{db}/': $ref: './paths/admin/db-.yaml' '/{keyspace}/': diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index 26a6c2d490..0144a906b0 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -385,6 +385,13 @@ show_exp: schema: type: boolean description: Whether to show the expiry property (`_exp`) in the response. +show_cv: + name: show_cv + in: query + required: false + schema: + type: boolean + description: Output the current version of the version vector in the response as property `_cv`. startkey: name: startkey in: query diff --git a/docs/api/components/responses.yaml b/docs/api/components/responses.yaml index 54c730f9e0..5ac82de2a5 100644 --- a/docs/api/components/responses.yaml +++ b/docs/api/components/responses.yaml @@ -138,6 +138,8 @@ all-docs: properties: rev: type: string + cv: + type: string uniqueItems: true total_rows: type: number diff --git a/docs/api/components/schemas.yaml b/docs/api/components/schemas.yaml index 5088275e4d..ac82f2c8ae 100644 --- a/docs/api/components/schemas.yaml +++ b/docs/api/components/schemas.yaml @@ -2099,6 +2099,41 @@ Resync-status: - docs_changed - docs_processed title: Resync-status +Attachment-Migration-status: + description: The status of a attachment migration operation + type: object + properties: + status: + description: The status of the current attachment migration operation. + type: string + enum: + - running + - completed + - stopping + - stopped + - error + start_time: + description: The ISO-8601 date and time the attachment migration operation was started. + type: string + last_error: + description: The last error that occurred in the attachment migration operation (if any). + type: string + migration_id: + description: The UUID given to the attachment migration operation. + type: string + docs_changed: + description: The amount of documents that have had attachment metadata migrated as a result of attachment migration operation. + type: integer + docs_processed: + description: The amount of docs that have been processed through the attachment migration operation. + type: integer + required: + - status + - start_time + - last_error + - docs_changed + - docs_processed + title: Attachment-Migration-status Compact-status: description: The status returned from a compaction. type: object diff --git a/docs/api/paths/admin/db-_attachment_migration.yaml b/docs/api/paths/admin/db-_attachment_migration.yaml new file mode 100644 index 0000000000..8d9ab0300c --- /dev/null +++ b/docs/api/paths/admin/db-_attachment_migration.yaml @@ -0,0 +1,73 @@ +# Copyright 2024-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. +parameters: + - $ref: ../../components/parameters.yaml#/db +post: + summary: Manage a attachment migration operation + description: |- + This allows a new attachment migration operation to be done on the database, or to stop an existing running attachment migration operation. + + Attachment Migration is a single node process and can only one node can be running it at one point. + + Required Sync Gateway RBAC roles: + + * Sync Gateway Architect + parameters: + - name: action + in: query + description: Defines whether the an attachment migration operation is being started or stopped. + schema: + type: string + default: start + enum: + - start + - stop + - name: reset + in: query + description: |- + This forces a fresh attachment migration start instead of trying to resume the previous failed migration operation. + schema: + type: boolean + responses: + '200': + description: Started or stopped compact operation successfully + '400': + $ref: ../../components/responses.yaml#/request-problem + '404': + $ref: ../../components/responses.yaml#/Not-found + '503': + description: Cannot start attachment migration due to another migration operation still running. + content: + application/json: + schema: + $ref: ../../components/schemas.yaml#/HTTP-Error + tags: + - Database Management + operationId: post_db-_attachment_migration +get: + summary: Get the status of the most recent attachment migration operation + description: |- + This will retrieve the current status of the most recent attachment migration operation. + + Required Sync Gateway RBAC roles: + + * Sync Gateway Architect + responses: + '200': + description: Attachment migration status retrieved successfully + content: + application/json: + schema: + $ref: ../../components/schemas.yaml#/Attachment-Migration-status + '400': + $ref: ../../components/responses.yaml#/request-problem + '404': + $ref: ../../components/responses.yaml#/Not-found + tags: + - Database Management + operationId: get_db-_attachment_migration diff --git a/docs/api/paths/admin/keyspace-_all_docs.yaml b/docs/api/paths/admin/keyspace-_all_docs.yaml index b2b60afae9..1f50b8bff1 100644 --- a/docs/api/paths/admin/keyspace-_all_docs.yaml +++ b/docs/api/paths/admin/keyspace-_all_docs.yaml @@ -26,6 +26,7 @@ get: - $ref: ../../components/parameters.yaml#/startkey - $ref: ../../components/parameters.yaml#/endkey - $ref: ../../components/parameters.yaml#/limit-result-rows + - $ref: ../../components/parameters.yaml#/show_cv responses: '200': $ref: ../../components/responses.yaml#/all-docs diff --git a/docs/api/paths/admin/keyspace-_bulk_get.yaml b/docs/api/paths/admin/keyspace-_bulk_get.yaml index ad53207e90..f313f9c799 100644 --- a/docs/api/paths/admin/keyspace-_bulk_get.yaml +++ b/docs/api/paths/admin/keyspace-_bulk_get.yaml @@ -45,6 +45,7 @@ post: description: If this header includes `gzip` then the the HTTP response will be compressed. This takes priority over `X-Accept-Part-Encoding`. Only part compression will be done if `X-Accept-Part-Encoding=gzip` and the `User-Agent` is below 1.2 due to clients not being able to handle full compression. schema: type: string + - $ref: ../../components/parameters.yaml#/show_cv requestBody: content: application/json: diff --git a/docs/api/paths/admin/keyspace-docid.yaml b/docs/api/paths/admin/keyspace-docid.yaml index e8c0d26438..1b0e877e43 100644 --- a/docs/api/paths/admin/keyspace-docid.yaml +++ b/docs/api/paths/admin/keyspace-docid.yaml @@ -21,6 +21,7 @@ get: - $ref: ../../components/parameters.yaml#/rev - $ref: ../../components/parameters.yaml#/open_revs - $ref: ../../components/parameters.yaml#/show_exp + - $ref: ../../components/parameters.yaml#/show_cv - $ref: ../../components/parameters.yaml#/revs_from - $ref: ../../components/parameters.yaml#/atts_since - $ref: ../../components/parameters.yaml#/revs_limit @@ -52,6 +53,7 @@ get: - Bob _id: AliceSettings _rev: 1-64d4a1f179db5c1848fe52967b47c166 + _cv: 1@src '400': $ref: ../../components/responses.yaml#/invalid-doc-id '404': diff --git a/docs/api/paths/public/keyspace-_all_docs.yaml b/docs/api/paths/public/keyspace-_all_docs.yaml index 9cefe219b6..bc49f0c393 100644 --- a/docs/api/paths/public/keyspace-_all_docs.yaml +++ b/docs/api/paths/public/keyspace-_all_docs.yaml @@ -20,6 +20,7 @@ get: - $ref: ../../components/parameters.yaml#/startkey - $ref: ../../components/parameters.yaml#/endkey - $ref: ../../components/parameters.yaml#/limit-result-rows + - $ref: ../../components/parameters.yaml#/show_cv responses: '200': $ref: ../../components/responses.yaml#/all-docs diff --git a/docs/api/paths/public/keyspace-_bulk_get.yaml b/docs/api/paths/public/keyspace-_bulk_get.yaml index ff2e977daf..920859566a 100644 --- a/docs/api/paths/public/keyspace-_bulk_get.yaml +++ b/docs/api/paths/public/keyspace-_bulk_get.yaml @@ -40,6 +40,7 @@ post: description: If this header includes `gzip` then the the HTTP response will be compressed. This takes priority over `X-Accept-Part-Encoding`. Only part compression will be done if `X-Accept-Part-Encoding=gzip` and the `User-Agent` is below 1.2 due to clients not being able to handle full compression. schema: type: string + - $ref: ../../components/parameters.yaml#/show_cv requestBody: content: application/json: diff --git a/docs/api/paths/public/keyspace-docid.yaml b/docs/api/paths/public/keyspace-docid.yaml index 6ad314512f..8eb9a9ee0d 100644 --- a/docs/api/paths/public/keyspace-docid.yaml +++ b/docs/api/paths/public/keyspace-docid.yaml @@ -15,6 +15,7 @@ get: - $ref: ../../components/parameters.yaml#/rev - $ref: ../../components/parameters.yaml#/open_revs - $ref: ../../components/parameters.yaml#/show_exp + - $ref: ../../components/parameters.yaml#/show_cv - $ref: ../../components/parameters.yaml#/revs_from - $ref: ../../components/parameters.yaml#/atts_since - $ref: ../../components/parameters.yaml#/revs_limit @@ -39,6 +40,9 @@ get: _rev: description: The revision ID of the document. type: string + _cv: + description: The current version of version vector of the document. + type: string additionalProperties: true example: FailedLoginAttempts: 5 @@ -46,6 +50,7 @@ get: - Bob _id: AliceSettings _rev: 1-64d4a1f179db5c1848fe52967b47c166 + _cv: 1@src '400': $ref: ../../components/responses.yaml#/invalid-doc-id '404': diff --git a/go.mod b/go.mod index 1f47a44a3f..541f0bb4f9 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,17 @@ require ( dario.cat/mergo v1.0.0 github.com/KimMachineGun/automemlimit v0.6.1 github.com/coreos/go-oidc/v3 v3.11.0 - github.com/couchbase/cbgt v1.3.9 + github.com/couchbase/cbgt v1.4.2-0.20241112001929-b9fdd9b009b1 github.com/couchbase/clog v0.1.0 github.com/couchbase/go-blip v0.0.0-20241014144256-13a798c348fd github.com/couchbase/gocb/v2 v2.9.1 - github.com/couchbase/gocbcore/v10 v10.5.1 + github.com/couchbase/gocbcore/v10 v10.5.2 github.com/couchbase/gomemcached v0.2.1 github.com/couchbase/goutils v0.1.2 github.com/couchbase/sg-bucket v0.0.0-20241018143914-45ef51a0c1be github.com/couchbaselabs/go-fleecedelta v0.0.0-20220909152808-6d09efa7a338 github.com/couchbaselabs/gocbconnstr v1.0.5 - github.com/couchbaselabs/rosmar v0.0.0-20240610211258-c856107e8e78 + github.com/couchbaselabs/rosmar v0.0.0-20240924211003-933f0fd5bba0 github.com/elastic/gosigar v0.14.3 github.com/felixge/fgprof v0.9.5 github.com/go-jose/go-jose/v4 v4.0.4 @@ -42,22 +42,20 @@ require ( ) require ( - github.com/aws/aws-sdk-go v1.44.299 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/containerd/cgroups/v3 v3.0.1 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/couchbase/blance v0.1.5 // indirect - github.com/couchbase/cbauth v0.1.11 // indirect + github.com/couchbase/blance v0.1.6 // indirect + github.com/couchbase/cbauth v0.1.12 // indirect github.com/couchbase/go-couchbase v0.1.1 // indirect github.com/couchbase/gocbcoreps v0.1.3 // indirect github.com/couchbase/goprotostellar v1.0.2 // indirect - github.com/couchbase/tools-common/cloud v1.0.0 // indirect - github.com/couchbase/tools-common/fs v1.0.0 // indirect - github.com/couchbase/tools-common/testing v1.0.0 // indirect - github.com/couchbase/tools-common/types v1.0.0 // indirect - github.com/couchbase/tools-common/utils v1.0.0 // indirect + github.com/couchbase/tools-common/cloud/v5 v5.0.3 // indirect + github.com/couchbase/tools-common/fs v1.0.2 // indirect + github.com/couchbase/tools-common/testing v1.0.1 // indirect + github.com/couchbase/tools-common/types v1.1.4 // indirect github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect @@ -70,7 +68,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/mattn/go-sqlite3 v1.14.23 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -97,7 +95,7 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index c44acf08a0..1377143b38 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= -github.com/aws/aws-sdk-go v1.44.299 h1:HVD9lU4CAFHGxleMJp95FV/sRhtg7P4miHD1v88JAQk= -github.com/aws/aws-sdk-go v1.44.299/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -36,12 +34,12 @@ github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/ github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/blance v0.1.5 h1:kNSAwhb8FXSJpicJ8R8Kk7+0V1+MyTcY1MOHIDbU79w= -github.com/couchbase/blance v0.1.5/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= -github.com/couchbase/cbauth v0.1.11 h1:LLyGiVnsKxyHp9wbOQk87oF9eDUSh1in2vh/l6vaezg= -github.com/couchbase/cbauth v0.1.11/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= -github.com/couchbase/cbgt v1.3.9 h1:MAT3FwD1ctekxuFe0yau0H1BCTvgLXvh1ipbZ3nZhBE= -github.com/couchbase/cbgt v1.3.9/go.mod h1:MImhtmvk0qjJit5HbmA34tnYThZoNtvgjL7jJH/kCAE= +github.com/couchbase/blance v0.1.6 h1:zyNew/SN2AheIoJxQ2LqqA1u3bMp03eGCer6hSDMUDs= +github.com/couchbase/blance v0.1.6/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= +github.com/couchbase/cbauth v0.1.12 h1:JOAWjjp2BdubvrrggvN4yQo3oEc2ndXcRN1ONCklUOM= +github.com/couchbase/cbauth v0.1.12/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= +github.com/couchbase/cbgt v1.4.2-0.20241112001929-b9fdd9b009b1 h1:w8lHraA/oMGQbc0yhu/rsLQbI1pKL2sIQNHRZBmBpJc= +github.com/couchbase/cbgt v1.4.2-0.20241112001929-b9fdd9b009b1/go.mod h1:lEYqydKcZbXMd9a4+eEvexODYkQ36ku1KxncsjN+Pqw= github.com/couchbase/clog v0.1.0 h1:4Kh/YHkhRjMCbdQuvRVsm39XZh4FtL1d8fAwJsHrEPY= github.com/couchbase/clog v0.1.0/go.mod h1:7tzUpEOsE+fgU81yfcjy5N1H6XtbVC8SgOz/3mCjmd4= github.com/couchbase/go-blip v0.0.0-20241014144256-13a798c348fd h1:ERQXaXuX1eix3NUqrxQ5VY0hqHH90vcfrWdbEWKzlEY= @@ -50,8 +48,8 @@ github.com/couchbase/go-couchbase v0.1.1 h1:ClFXELcKj/ojyoTYbsY34QUrrYCBi/1G749s github.com/couchbase/go-couchbase v0.1.1/go.mod h1:+/bddYDxXsf9qt0xpDUtRR47A2GjaXmGGAqQ/k3GJ8A= github.com/couchbase/gocb/v2 v2.9.1 h1:yB2ZhRLk782Y9sZlATaUwglZe9+2QpvFmItJXTX4stQ= github.com/couchbase/gocb/v2 v2.9.1/go.mod h1:TMAeK34yUdcASdV4mGcYuwtkAWckRBYN5uvMCEgPfXo= -github.com/couchbase/gocbcore/v10 v10.5.1 h1:bwlV/zv/fSQLuO14M9k49K7yWgcWfjSgMyfRGhW1AyU= -github.com/couchbase/gocbcore/v10 v10.5.1/go.mod h1:rulbgUK70EuyRUiLQ0LhQAfSI/Rl+jWws8tTbHzvB6M= +github.com/couchbase/gocbcore/v10 v10.5.2 h1:DHK042E1RfhPBR3b14CITl5XHRsLjH3hpERuwUc5UIg= +github.com/couchbase/gocbcore/v10 v10.5.2/go.mod h1:rulbgUK70EuyRUiLQ0LhQAfSI/Rl+jWws8tTbHzvB6M= github.com/couchbase/gocbcoreps v0.1.3 h1:fILaKGCjxFIeCgAUG8FGmRDSpdrRggohOMKEgO9CUpg= github.com/couchbase/gocbcoreps v0.1.3/go.mod h1:hBFpDNPnRno6HH5cRXExhqXYRmTsFJlFHQx7vztcXPk= github.com/couchbase/gomemcached v0.2.1 h1:lDONROGbklo8pOt4Sr4eV436PVEaKDr3o9gUlhv9I2U= @@ -62,16 +60,14 @@ github.com/couchbase/goutils v0.1.2 h1:gWr8B6XNWPIhfalHNog3qQKfGiYyh4K4VhO3P2o9B github.com/couchbase/goutils v0.1.2/go.mod h1:h89Ek/tiOxxqjz30nPPlwZdQbdB8BwgnuBxeoUe/ViE= github.com/couchbase/sg-bucket v0.0.0-20241018143914-45ef51a0c1be h1:QM2afa9Xhbhy1ywVEVCRV0vEQvHIPplDkc6NsNug78Y= github.com/couchbase/sg-bucket v0.0.0-20241018143914-45ef51a0c1be/go.mod h1:Tw3QSBP+nkDjw1cpHwMFP4pBORs0UOP+KbF2hXBVwqM= -github.com/couchbase/tools-common/cloud v1.0.0 h1:SQZIccXoedbrThehc/r9BJbpi/JhwJ8X00PDjZ2gEBE= -github.com/couchbase/tools-common/cloud v1.0.0/go.mod h1:6KVlRpbcnDWrvickUJ+xpqCWx1vgYYlEli/zL4xmZAg= -github.com/couchbase/tools-common/fs v1.0.0 h1:HFA4xCF/r3BtZShFJUxzVvGuXtDkqGnaPzYJP3Kp1mw= -github.com/couchbase/tools-common/fs v1.0.0/go.mod h1:se8Dr4gDPfy2A8qYnsv3TX1lyBn0Nn9+4Y9xNaFpubU= -github.com/couchbase/tools-common/testing v1.0.0 h1:FHa/rwTunvb9+j/4+DT0RSaXg/fWW6XAfj8jyGu5e5Y= -github.com/couchbase/tools-common/testing v1.0.0/go.mod h1:x1TTvkYyXSle7ZpTkyvzEhKCxthvTEaOsgCJcpKgyto= -github.com/couchbase/tools-common/types v1.0.0 h1:C9MjHmTPcZyPo2Yp9Dt86WeZH+2XQgydorCC9jb+/dQ= -github.com/couchbase/tools-common/types v1.0.0/go.mod h1:r700V2xUuJqBGNG2aWbQYn5S0sJdqO3TLIa2AIQVaGU= -github.com/couchbase/tools-common/utils v1.0.0 h1:6mWXqWWj7aM0Kp2LWpSKEu9pLAYm7il3gWdqpvxnJV4= -github.com/couchbase/tools-common/utils v1.0.0/go.mod h1:i6cN5Z5hB9vQRLxe2j1v6Nu8bv+pKl9BFXjbQUHSah8= +github.com/couchbase/tools-common/cloud/v5 v5.0.3 h1:+mAZtjEGWX+Vt74HWMKhykmOuul6KBKPC40gmwSDaJ8= +github.com/couchbase/tools-common/cloud/v5 v5.0.3/go.mod h1:goFa2Uy5qZDUAs5KfXHVJ4jxbubxMvG7g812Y/CYrlA= +github.com/couchbase/tools-common/fs v1.0.2 h1:rmHHed8HCbIriTHVVTpDvWyUAvG0Xfq/hD4Altet2w0= +github.com/couchbase/tools-common/fs v1.0.2/go.mod h1:+aQlBU/0OpWmvJ7EQNhZM51oysy7zoL96ltXleZusDM= +github.com/couchbase/tools-common/testing v1.0.1 h1:GVc5OjMN5gj79cnjMTocouwXBSW6VeiRl86pVPaogPU= +github.com/couchbase/tools-common/testing v1.0.1/go.mod h1:HeOA1IU1H+u83li+Qe6G8f7dnlVPrKJuhsF9I5r83S8= +github.com/couchbase/tools-common/types v1.1.4 h1:YAZn9VOkkmn05YC24/TEm7eXa/j8k/R4tqy6folkSWo= +github.com/couchbase/tools-common/types v1.1.4/go.mod h1:089L74+qhIDvDLEZzWk7PoQKAxij9j7KwUnw2aMYUv4= github.com/couchbaselabs/go-fleecedelta v0.0.0-20220909152808-6d09efa7a338 h1:xMeDnMiapTiq8n8J83Mo2tPjQNIU7GssSsbQsP1CLOY= github.com/couchbaselabs/go-fleecedelta v0.0.0-20220909152808-6d09efa7a338/go.mod h1:0f+dmhfcTKK+4quAe6rwqQUVVWtHX/eztNB8cmBUniQ= github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4= @@ -80,8 +76,8 @@ github.com/couchbaselabs/gocbconnstr v1.0.5 h1:e0JokB5qbcz7rfnxEhNRTKz8q1svoRvDo github.com/couchbaselabs/gocbconnstr v1.0.5/go.mod h1:KV3fnIKMi8/AzX0O9zOrO9rofEqrRF1d2rG7qqjxC7o= github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 h1:lhGOw8rNG6RAadmmaJAF3PJ7MNt7rFuWG7BHCYMgnGE= github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28/go.mod h1:o7T431UOfFVHDNvMBUmUxpHnhivwv7BziUao/nMl81E= -github.com/couchbaselabs/rosmar v0.0.0-20240610211258-c856107e8e78 h1:pdMO4naNb0W68OisY0Y7LEE6xOXlrlZow5IWmwow2Wc= -github.com/couchbaselabs/rosmar v0.0.0-20240610211258-c856107e8e78/go.mod h1:BZgg7zjF7c8e7BR5/JBuSXZ+PLIHgyrNKwE0eLFeglw= +github.com/couchbaselabs/rosmar v0.0.0-20240924211003-933f0fd5bba0 h1:CQil6oxiHYhJBITdKTlxEUOetPdcgN6bk8wOZd4maDM= +github.com/couchbaselabs/rosmar v0.0.0-20240924211003-933f0fd5bba0/go.mod h1:Abf5EPwi/7j5caDy2OPmo+L36I02H7sp9dkgek5t4bM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -136,9 +132,6 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -162,8 +155,8 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -239,7 +232,6 @@ github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqTosly github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -264,7 +256,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -276,7 +267,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -285,9 +275,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -298,7 +285,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -308,30 +294,20 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -341,7 +317,6 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/jenkins-integration-build.sh b/jenkins-integration-build.sh index 5c74d617f2..d26807132d 100755 --- a/jenkins-integration-build.sh +++ b/jenkins-integration-build.sh @@ -25,7 +25,7 @@ if [ "${1:-}" == "-m" ]; then RUN_COUNT="1" # CBS server settings COUCHBASE_SERVER_PROTOCOL="couchbase" - COUCHBASE_SERVER_VERSION="enterprise-7.2.2" + COUCHBASE_SERVER_VERSION="ghcr.io/cb-vanilla/server:7.6.5-5535" SG_TEST_BUCKET_POOL_SIZE="3" SG_TEST_BUCKET_POOL_DEBUG="true" GSI="true" diff --git a/rest/access_test.go b/rest/access_test.go index 86b870fe93..6c41e1f177 100644 --- a/rest/access_test.go +++ b/rest/access_test.go @@ -25,6 +25,12 @@ import ( "github.com/stretchr/testify/require" ) +type allDocsResponse struct { + TotalRows int `json:"total_rows"` + Offset int `json:"offset"` + Rows []allDocsRow `json:"rows"` +} + func TestPublicChanGuestAccess(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{ @@ -70,17 +76,6 @@ func TestStarAccess(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyChanges) - type allDocsRow struct { - ID string `json:"id"` - Key string `json:"key"` - Value struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } `json:"value"` - Doc db.Body `json:"doc,omitempty"` - Error string `json:"error"` - } var allDocsResult struct { TotalRows int `json:"total_rows"` Offset int `json:"offset"` @@ -385,11 +380,11 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // User has no permissions to access rev - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusOK) // Guest has no permissions to access rev - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusOK) // Attachments should be forbidden as well @@ -397,7 +392,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // Attachment revs should be forbidden as well - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusNotFound) // Attachments should be forbidden for guests as well @@ -405,7 +400,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusForbidden) // Attachment revs should be forbidden for guests as well - resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodGet, "/{{.keyspace}}/doc/attach?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusNotFound) // Document does not exist should cause 403 @@ -422,7 +417,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // PUT with rev - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "NoPerms", "password") assertRespStatus(resp, http.StatusForbidden) // PUT with incorrect rev @@ -434,7 +429,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // PUT with rev as Guest - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{}`, nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{}`, nil, "", "") assertRespStatus(resp, http.StatusForbidden) // PUT with incorrect rev as Guest @@ -466,7 +461,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assert.NotContains(t, user.GetChannels(s, c).ToArray(), "chan2") // Successful PUT which will grant access grants - resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevID, `{"channels": "chan"}`, nil, "Perms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodPut, "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{"channels": "chan"}`, nil, "Perms", "password") AssertStatus(t, resp, http.StatusCreated) // Make sure channel access grant was successful @@ -483,7 +478,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // Attempt to delete document rev with no permissions - resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "NoPerms", "password") + resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "NoPerms", "password") assertRespStatus(resp, http.StatusConflict) // Attempt to delete document with wrong rev @@ -499,7 +494,7 @@ func TestForceAPIForbiddenErrors(t *testing.T) { assertRespStatus(resp, http.StatusConflict) // Attempt to delete document rev with no write perms as guest - resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevID, "", nil, "", "") + resp = rt.SendUserRequestWithHeaders(http.MethodDelete, "/{{.keyspace}}/doc?rev="+version.RevTreeID, "", nil, "", "") assertRespStatus(resp, http.StatusConflict) // Attempt to delete document with wrong rev as guest @@ -552,23 +547,6 @@ func TestAllDocsAccessControl(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{SyncFn: channels.DocChannelsSyncFunction}) defer rt.Close() - type allDocsRow struct { - ID string `json:"id"` - Key string `json:"key"` - Value struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } `json:"value"` - Doc db.Body `json:"doc,omitempty"` - Error string `json:"error"` - } - type allDocsResponse struct { - TotalRows int `json:"total_rows"` - Offset int `json:"offset"` - Rows []allDocsRow `json:"rows"` - } - // Create some docs: a := auth.NewAuthenticator(rt.MetadataStore(), nil, rt.GetDatabase().AuthenticatorOptions(rt.Context())) a.Collections = rt.GetDatabase().CollectionNames @@ -708,13 +686,13 @@ func TestAllDocsAccessControl(t *testing.T) { assert.Equal(t, []string{"Cinemax"}, allDocsResult.Rows[0].Value.Channels) assert.Equal(t, "doc1", allDocsResult.Rows[1].Key) assert.Equal(t, "forbidden", allDocsResult.Rows[1].Error) - assert.Equal(t, "", allDocsResult.Rows[1].Value.Rev) + assert.Nil(t, allDocsResult.Rows[1].Value) assert.Equal(t, "doc3", allDocsResult.Rows[2].ID) assert.Equal(t, []string{"Cinemax"}, allDocsResult.Rows[2].Value.Channels) assert.Equal(t, "1-20912648f85f2bbabefb0993ddd37b41", allDocsResult.Rows[2].Value.Rev) assert.Equal(t, "b0gus", allDocsResult.Rows[3].Key) assert.Equal(t, "not_found", allDocsResult.Rows[3].Error) - assert.Equal(t, "", allDocsResult.Rows[3].Value.Rev) + assert.Nil(t, allDocsResult.Rows[3].Value) // Check GET to _all_docs with keys parameter: response = rt.SendUserRequest(http.MethodGet, "/{{.keyspace}}/_all_docs?channels=true&keys=%5B%22doc4%22%2C%22doc1%22%2C%22doc3%22%2C%22b0gus%22%5D", "", "alice") @@ -1123,7 +1101,7 @@ func TestRoleChannelGrantInheritance(t *testing.T) { RequireStatus(t, response, 200) // Revoke access to chan2 (dynamic) - response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1") + response = rt.SendUserRequest("PUT", "/{{.keyspace}}/grant1?rev="+grant1Version.RevTreeID, `{"type":"setaccess", "owner":"none", "channel":"chan2"}`, "user1") RequireStatus(t, response, 201) // Verify user cannot access doc in revoked channel, but can successfully access remaining documents @@ -1178,3 +1156,43 @@ func TestPublicChannel(t *testing.T) { response = rt.SendUserRequest("GET", "/{{.keyspace}}/privateDoc", "", "user1") RequireStatus(t, response, 403) } + +func TestAllDocsCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + const docID = "foo" + docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + + testCases := []struct { + name string + url string + output string + }{ + { + name: "no query string", + url: "/{{.keyspace}}/_all_docs", + output: fmt.Sprintf(`{ + "total_rows": 1, + "update_seq": 1, + "rows": [{"key": "%s", "id": "%s", "value": {"rev": "%s"}}] + }`, docID, docID, docVersion.RevTreeID), + }, + { + name: "cvs=true", + url: "/{{.keyspace}}/_all_docs?show_cv=true", + output: fmt.Sprintf(`{ + "total_rows": 1, + "update_seq": 1, + "rows": [{"key": "%s", "id": "%s", "value": {"rev": "%s", "cv": "%s"}}] + }`, docID, docID, docVersion.RevTreeID, docVersion.CV.String()), + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequest(http.MethodGet, testCase.url, "") + RequireStatus(t, response, http.StatusOK) + require.JSONEq(t, testCase.output, response.Body.String()) + }) + } +} diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index d28ac74ef5..fb8ff2d0b0 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -2346,7 +2346,7 @@ func TestRawTombstone(t *testing.T) { resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``) assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`) - assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevID+`"`) + assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+version.RevTreeID+`"`) assert.Contains(t, string(resp.BodyBytes()), `"foo":"bar"`) assert.NotContains(t, string(resp.BodyBytes()), `"_deleted":true`) @@ -2356,7 +2356,7 @@ func TestRawTombstone(t *testing.T) { resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docID, ``) assert.Equal(t, "application/json", resp.Header().Get("Content-Type")) assert.NotContains(t, string(resp.BodyBytes()), `"_id":"`+docID+`"`) - assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevID+`"`) + assert.NotContains(t, string(resp.BodyBytes()), `"_rev":"`+deletedVersion.RevTreeID+`"`) assert.NotContains(t, string(resp.BodyBytes()), `"foo":"bar"`) assert.Contains(t, string(resp.BodyBytes()), `"_deleted":true`) } @@ -2498,7 +2498,7 @@ func TestHandlePutDbConfigWithBackticksCollections(t *testing.T) { reqBodyWithBackticks := `{ "server": "walrus:", "bucket": "backticks", - "enable_shared_bucket_access":false, + "enable_shared_bucket_access":true, "scopes": { "scope1": { "collections" : { @@ -4086,9 +4086,9 @@ func TestPutIDRevMatchBody(t *testing.T) { docRev := test.rev docBody := test.docBody if test.docID == "" { - docID = "doc" // Used for the rev tests to branch off of - docBody = strings.ReplaceAll(docBody, "[REV]", version.RevID) // FIX for HLV? - docRev = strings.ReplaceAll(docRev, "[REV]", version.RevID) + docID = "doc" // Used for the rev tests to branch off of + docBody = strings.ReplaceAll(docBody, "[REV]", version.RevTreeID) // FIX for HLV? + docRev = strings.ReplaceAll(docRev, "[REV]", version.RevTreeID) } resp := rt.SendAdminRequest("PUT", fmt.Sprintf("/{{.keyspace}}/%s?rev=%s", docID, docRev), docBody) diff --git a/rest/api.go b/rest/api.go index 3cd66f9f7f..e770a27e81 100644 --- a/rest/api.go +++ b/rest/api.go @@ -130,6 +130,52 @@ func (h *handler) handleGetCompact() error { return nil } +func (h *handler) handleAttachmentMigration() error { + action := h.getQuery("action") + if action == "" { + action = string(db.BackgroundProcessActionStart) + } + reset := h.getBoolQuery("reset") + + if action != string(db.BackgroundProcessActionStart) && action != string(db.BackgroundProcessActionStop) { + return base.HTTPErrorf(http.StatusBadRequest, "Unknown parameter for 'action'. Must be start or stop") + } + + if action == string(db.BackgroundProcessActionStart) { + err := h.db.AttachmentMigrationManager.Start(h.ctx(), map[string]interface{}{ + "reset": reset, + }) + if err != nil { + return err + } + status, err := h.db.AttachmentMigrationManager.GetStatus(h.ctx()) + if err != nil { + return err + } + h.writeRawJSON(status) + } else if action == string(db.BackgroundProcessActionStop) { + err := h.db.AttachmentMigrationManager.Stop() + if err != nil { + return err + } + status, err := h.db.AttachmentMigrationManager.GetStatus(h.ctx()) + if err != nil { + return err + } + h.writeRawJSON(status) + } + return nil +} + +func (h *handler) handleGetAttachmentMigration() error { + status, err := h.db.AttachmentMigrationManager.GetStatus(h.ctx()) + if err != nil { + return err + } + h.writeRawJSON(status) + return nil +} + func (h *handler) handleCompact() error { action := h.getQuery("action") if action == "" { diff --git a/rest/api_test.go b/rest/api_test.go index f4ecfe9212..cc420c5dfe 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -221,9 +221,9 @@ func TestDocLifecycle(t *testing.T) { defer rt.Close() version := rt.CreateTestDoc("doc") - assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevID) + assert.Equal(t, "1-45ca73d819d5b1c9b8eea95290e79004", version.RevTreeID) - response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevID, "") + response := rt.SendAdminRequest("DELETE", "/{{.keyspace}}/doc?rev="+version.RevTreeID, "") RequireStatus(t, response, 200) } @@ -1665,10 +1665,9 @@ func TestWriteTombstonedDocUsingXattrs(t *testing.T) { xattrs, _, err := rt.GetSingleDataStore().GetXattrs(rt.Context(), "-21SK00U-ujxUO9fU2HezxL", []string{base.SyncXattrName}) require.NoError(t, err) require.Contains(t, xattrs, base.SyncXattrName) - var retrievedXattr map[string]any - require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &retrievedXattr)) - assert.NoError(t, err, "Unexpected Error") - assert.Equal(t, "2-466a1fab90a810dc0a63565b70680e4e", retrievedXattr["rev"]) + var retrievedSyncData db.SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &retrievedSyncData)) + assert.Equal(t, "2-466a1fab90a810dc0a63565b70680e4e", retrievedSyncData.CurrentRev) } @@ -2718,7 +2717,7 @@ func TestNullDocHandlingForMutable1xBody(t *testing.T) { documentRev := db.DocumentRevision{DocID: "doc1", BodyBytes: []byte("null")} - body, err := documentRev.Mutable1xBody(ctx, collection, nil, nil, false) + body, err := documentRev.Mutable1xBody(ctx, collection, nil, nil, false, false) require.Error(t, err) require.Nil(t, body) assert.Contains(t, err.Error(), "null doc body for doc") @@ -2729,6 +2728,221 @@ func TestNullDocHandlingForMutable1xBody(t *testing.T) { assert.Contains(t, err.Error(), "b is not a JSON object") } +// TestDatabaseXattrConfigHandlingForDBConfigUpdate: +// - Create database with xattrs enabled +// - Test updating the config to disable the use of xattrs in this database through replacing + upserting the config +// - Assert error code is returned and response contains error string +func TestDatabaseXattrConfigHandlingForDBConfigUpdate(t *testing.T) { + base.LongRunningTest(t) + const ( + dbName = "db1" + errResp = "sync gateway requires enable_shared_bucket_access=true" + ) + + testCases := []struct { + name string + upsertConfig bool + }{ + { + name: "POST update", + upsertConfig: true, + }, + { + name: "PUT update", + upsertConfig: false, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + rt := NewRestTester(t, &RestTesterConfig{ + PersistentConfig: true, + }) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + + resp := rt.CreateDatabase(dbName, dbConfig) + RequireStatus(t, resp, http.StatusCreated) + assert.NoError(t, rt.WaitForDBOnline()) + + dbConfig.EnableXattrs = base.BoolPtr(false) + + if testCase.upsertConfig { + resp = rt.UpsertDbConfig(dbName, dbConfig) + RequireStatus(t, resp, http.StatusInternalServerError) + assert.Contains(t, resp.Body.String(), errResp) + } else { + resp = rt.ReplaceDbConfig(dbName, dbConfig) + RequireStatus(t, resp, http.StatusInternalServerError) + assert.Contains(t, resp.Body.String(), errResp) + } + }) + } +} + +// TestCreateDBWithXattrsDisbaled: +// - Test that you cannot create a database with xattrs disabled +// - Assert error code is returned and response contains error string +func TestCreateDBWithXattrsDisbaled(t *testing.T) { + rt := NewRestTester(t, &RestTesterConfig{ + PersistentConfig: true, + }) + defer rt.Close() + const ( + dbName = "db1" + errResp = "sync gateway requires enable_shared_bucket_access=true" + ) + + dbConfig := rt.NewDbConfig() + dbConfig.EnableXattrs = base.BoolPtr(false) + + resp := rt.CreateDatabase(dbName, dbConfig) + RequireStatus(t, resp, http.StatusInternalServerError) + assert.Contains(t, resp.Body.String(), errResp) +} + +// TestPvDeltaReadAndWrite: +// - Write a doc from another hlv aware peer to the bucket +// - Force import of this doc, then update this doc via rest tester source +// - Assert that the document hlv is as expected +// - Update the doc from a new hlv aware peer and force the import of this new write +// - Assert that the new hlv is as expected, testing that the hlv went through transformation to the persisted delta +// version and back to the in memory version as expected +func TestPvDeltaReadAndWrite(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + testSource := rt.GetDatabase().EncodedSourceID + + const docID = "doc1" + otherSource := "otherSource" + hlvHelper := db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") + existingHLVKey := docID + cas := hlvHelper.InsertWithHLV(ctx, existingHLVKey) + casV1 := cas + encodedSourceV1 := db.EncodeSource(otherSource) + + // force import of this write + version1, _ := rt.GetDoc(docID) + + // update the above doc, this should push CV to PV and adds a new CV + version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"new": "update!"}) + newDoc, _, err := collection.GetDocWithXattrs(ctx, existingHLVKey, db.DocUnmarshalAll) + require.NoError(t, err) + casV2 := newDoc.Cas + encodedSourceV2 := testSource + + // assert that we have a prev CV drop to pv and a new CV pair, assert pv values are as expected after delta conversions + assert.Equal(t, testSource, newDoc.HLV.SourceID) + assert.Equal(t, version2.CV.Value, newDoc.HLV.Version) + assert.Len(t, newDoc.HLV.PreviousVersions, 1) + assert.Equal(t, casV1, newDoc.HLV.PreviousVersions[encodedSourceV1]) + + otherSource = "diffSource" + hlvHelper = db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") + cas = hlvHelper.UpdateWithHLV(ctx, existingHLVKey, newDoc.Cas, newDoc.HLV) + encodedSourceV3 := db.EncodeSource(otherSource) + casV3 := cas + + // import and get raw doc + _, _ = rt.GetDoc(docID) + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, docID, db.DocUnmarshalAll) + require.NoError(t, err) + + // assert that we have two entries in previous versions, and they are correctly converted from deltas back to full value + assert.Equal(t, encodedSourceV3, bucketDoc.HLV.SourceID) + assert.Equal(t, casV3, bucketDoc.HLV.Version) + assert.Len(t, bucketDoc.HLV.PreviousVersions, 2) + assert.Equal(t, casV1, bucketDoc.HLV.PreviousVersions[encodedSourceV1]) + assert.Equal(t, casV2, bucketDoc.HLV.PreviousVersions[encodedSourceV2]) +} + +// TestPutDocUpdateVersionVector: +// - Put a doc and assert that the versions and the source for the hlv is correctly updated +// - Update that doc and assert HLV has also been updated +// - Delete the doc and assert that the HLV has been updated in deletion event +func TestPutDocUpdateVersionVector(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + + bucketUUID := rt.GetDatabase().EncodedSourceID + + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc1", `{"key": "value"}`) + RequireStatus(t, resp, http.StatusCreated) + + collection, _ := rt.GetSingleTestDatabaseCollection() + syncData, err := collection.GetDocSyncData(base.TestCtx(t), "doc1") + assert.NoError(t, err) + + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // Put a new revision of this doc and assert that the version vector SourceID and Version is updated + resp = rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc1?rev="+syncData.CurrentRev, `{"key1": "value1"}`) + RequireStatus(t, resp, http.StatusCreated) + + syncData, err = collection.GetDocSyncData(base.TestCtx(t), "doc1") + assert.NoError(t, err) + + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // Delete doc and assert that the version vector SourceID and Version is updated + resp = rt.SendAdminRequest(http.MethodDelete, "/{{.keyspace}}/doc1?rev="+syncData.CurrentRev, "") + RequireStatus(t, resp, http.StatusOK) + + syncData, err = collection.GetDocSyncData(base.TestCtx(t), "doc1") + assert.NoError(t, err) + + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) +} + +// TestHLVOnPutWithImportRejection: +// - Put a doc successfully and assert the HLV is updated correctly +// - Put a doc that will be rejected by the custom import filter +// - Assert that the HLV values on the sync data are still correctly updated/preserved +func TestHLVOnPutWithImportRejection(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport) + importFilter := `function (doc) { return doc.type == "mobile"}` + rtConfig := RestTesterConfig{ + DatabaseConfig: &DatabaseConfig{DbConfig: DbConfig{ + AutoImport: false, + ImportFilter: &importFilter, + }}, + } + rt := NewRestTester(t, &rtConfig) + defer rt.Close() + + bucketUUID := rt.GetDatabase().EncodedSourceID + + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc1", `{"type": "mobile"}`) + RequireStatus(t, resp, http.StatusCreated) + + collection, _ := rt.GetSingleTestDatabaseCollection() + syncData, err := collection.GetDocSyncData(base.TestCtx(t), "doc1") + assert.NoError(t, err) + + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // Put a doc that will be rejected by the import filter on the attempt to perform on demand import for write + resp = rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc2", `{"type": "not-mobile"}`) + RequireStatus(t, resp, http.StatusCreated) + + // assert that the hlv is correctly updated and in tact after the import was cancelled on the doc + syncData, err = collection.GetDocSyncData(base.TestCtx(t), "doc2") + assert.NoError(t, err) + + assert.Equal(t, bucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) +} + func TestTombstoneCompactionAPI(t *testing.T) { rt := NewRestTester(t, nil) defer rt.Close() diff --git a/rest/api_test_helpers.go b/rest/api_test_helpers.go index 7c7d01ca66..bdbf6552ab 100644 --- a/rest/api_test_helpers.go +++ b/rest/api_test_helpers.go @@ -50,13 +50,13 @@ func (rt *RestTester) PutNewEditsFalse(docID string, newVersion DocVersion, pare require.NoError(rt.TB(), marshalErr) requestBody := body.ShallowCopy() - newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB()), newVersion.RevID) + newRevGeneration, newRevDigest := db.ParseRevID(base.TestCtx(rt.TB()), newVersion.RevTreeID) revisions := make(map[string]interface{}) revisions["start"] = newRevGeneration ids := []string{newRevDigest} - if parentVersion.RevID != "" { - _, parentDigest := db.ParseRevID(base.TestCtx(rt.TB()), parentVersion.RevID) + if parentVersion.RevTreeID != "" { + _, parentDigest := db.ParseRevID(base.TestCtx(rt.TB()), parentVersion.RevTreeID) ids = append(ids, parentDigest) } revisions["ids"] = ids diff --git a/rest/attachment_test.go b/rest/attachment_test.go index 96f15bc61d..138317b268 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -43,25 +43,25 @@ func TestDocEtag(t *testing.T) { version := DocVersionFromPutResponse(t, response) // Validate Etag returned on doc creation - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) response = rt.SendRequest("GET", "/{{.keyspace}}/doc", "") RequireStatus(t, response, 200) // Validate Etag returned when retrieving doc - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) // Validate Etag returned when updating doc - response = rt.SendRequest("PUT", "/{{.keyspace}}/doc?rev="+version.RevID, `{"prop":false}`) + response = rt.SendRequest("PUT", "/{{.keyspace}}/doc?rev="+version.RevTreeID, `{"prop":false}`) version = DocVersionFromPutResponse(t, response) - assert.Equal(t, strconv.Quote(version.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(version.RevTreeID), response.Header().Get("Etag")) // Test Attachments attachmentBody := "this is the body of attachment" attachmentContentType := "content/type" // attach to existing document with correct rev (should succeed), manual request to change etag - resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", "doc", "attach1", version.RevID) + resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", "doc", "attach1", version.RevTreeID) response = rt.SendAdminRequestWithHeaders(http.MethodPut, resource, attachmentBody, attachmentHeaders()) RequireStatus(t, response, http.StatusCreated) var body db.Body @@ -71,7 +71,7 @@ func TestDocEtag(t *testing.T) { RequireDocVersionNotEqual(t, version, afterAttachmentVersion) // validate Etag returned from adding an attachment - assert.Equal(t, strconv.Quote(afterAttachmentVersion.RevID), response.Header().Get("Etag")) + assert.Equal(t, strconv.Quote(afterAttachmentVersion.RevTreeID), response.Header().Get("Etag")) // retrieve attachment response = rt.SendRequest("GET", "/{{.keyspace}}/doc/attach1", "") @@ -115,7 +115,7 @@ func TestDocAttachment(t *testing.T) { assert.Equal(t, attachmentContentType, response.Header().Get("Content-Type")) // attempt to delete an attachment that is not on the document - response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach2?rev="+version.RevID, "") + response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach2?rev="+version.RevTreeID, "") RequireStatus(t, response, 404) // attempt to delete attachment from non existing doc @@ -127,7 +127,7 @@ func TestDocAttachment(t *testing.T) { RequireStatus(t, response, 409) // delete the attachment calling the delete attachment endpoint - response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach1?rev="+version.RevID, "") + response = rt.SendRequest("DELETE", "/{{.keyspace}}/doc/attach1?rev="+version.RevTreeID, "") RequireStatus(t, response, 200) // attempt to access deleted attachment (should return error) @@ -221,7 +221,7 @@ func TestDocAttachmentOnRemovedRev(t *testing.T) { } // attach to existing document with correct rev (should fail) - response := rt.SendUserRequestWithHeaders("PUT", "/{{.keyspace}}/doc/attach1?rev="+version.RevID, attachmentBody, reqHeaders, "user1", "letmein") + response := rt.SendUserRequestWithHeaders("PUT", "/{{.keyspace}}/doc/attach1?rev="+version.RevTreeID, attachmentBody, reqHeaders, "user1", "letmein") RequireStatus(t, response, 404) } @@ -429,7 +429,7 @@ func TestAttachmentsNoCrossTalk(t *testing.T) { "Accept": "application/json", } - response := rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevID, doc1Version.RevID), "", reqHeaders) + response := rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevTreeID, doc1Version.RevTreeID), "", reqHeaders) assert.Equal(t, 200, response.Code) // validate attachment has data property body := db.Body{} @@ -440,7 +440,7 @@ func TestAttachmentsNoCrossTalk(t *testing.T) { data := attach1["data"] assert.True(t, data != nil) - response = rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevID, afterAttachmentVersion.RevID), "", reqHeaders) + response = rt.SendAdminRequestWithHeaders("GET", fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s&revs=true&attachments=true&atts_since=[\"%s\"]", afterAttachmentVersion.RevTreeID, afterAttachmentVersion.RevTreeID), "", reqHeaders) assert.Equal(t, 200, response.Code) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) log.Printf("response body revid1 = %s", body) @@ -593,7 +593,7 @@ func TestBulkGetBadAttachmentReproIssue2528(t *testing.T) { version, _ := rt.GetDoc(doc1ID) // Do a bulk_get to get the doc -- this was causing a panic prior to the fix for #2528 - bulkGetDocs := fmt.Sprintf(`{"docs": [{"id": "%v", "rev": "%v"}, {"id": "%v", "rev": "%v"}]}`, doc1ID, version.RevID, doc2ID, doc2Version.RevID) + bulkGetDocs := fmt.Sprintf(`{"docs": [{"id": "%v", "rev": "%v"}, {"id": "%v", "rev": "%v"}]}`, doc1ID, version.RevTreeID, doc2ID, doc2Version.RevTreeID) bulkGetResponse := rt.SendAdminRequest("POST", "/{{.keyspace}}/_bulk_get?revs=true&attachments=true&revs_limit=2", bulkGetDocs) if bulkGetResponse.Code != 200 { panic(fmt.Sprintf("Got unexpected response: %v", bulkGetResponse)) @@ -684,6 +684,7 @@ func TestBulkGetBadAttachmentReproIssue2528(t *testing.T) { } func TestConflictWithInvalidAttachment(t *testing.T) { + t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)") rt := NewRestTester(t, nil) defer rt.Close() @@ -698,15 +699,15 @@ func TestConflictWithInvalidAttachment(t *testing.T) { // Set attachment attachmentBody := "aGVsbG8gd29ybGQ=" // hello.txt - response := rt.SendAdminRequestWithHeaders("PUT", "/{{.keyspace}}/doc1/attach1?rev="+version.RevID, attachmentBody, reqHeaders) + response := rt.SendAdminRequestWithHeaders("PUT", "/{{.keyspace}}/doc1/attach1?rev="+version.RevTreeID, attachmentBody, reqHeaders) RequireStatus(t, response, http.StatusCreated) - docrevId2 := DocVersionFromPutResponse(t, response).RevID + docrevId2 := DocVersionFromPutResponse(t, response).RevTreeID // Update Doc rev3Input := `{"_attachments":{"attach1":{"content-type": "content/type", "digest":"sha1-b7fDq/pHG8Nf5F3fe0K2nu0xcw0=", "length": 16, "revpos": 2, "stub": true}}, "_id": "doc1", "_rev": "` + docrevId2 + `", "prop":true}` response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1", rev3Input) RequireStatus(t, response, http.StatusCreated) - docrevId3 := DocVersionFromPutResponse(t, response).RevID + docrevId3 := DocVersionFromPutResponse(t, response).RevTreeID // Get Existing Doc & Update rev rev4Input := `{"_attachments":{"attach1":{"content-type": "content/type", "digest":"sha1-b7fDq/pHG8Nf5F3fe0K2nu0xcw0=", "length": 16, "revpos": 2, "stub": true}}, "_id": "doc1", "_rev": "` + docrevId3 + `", "prop":true}` @@ -796,7 +797,7 @@ func TestConflictingBranchAttachments(t *testing.T) { response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev2a) RequireStatus(t, response, http.StatusCreated) docVersion2a := DocVersionFromPutResponse(t, response) - assert.Equal(t, "2-twoa", docVersion2a.RevID) + assert.Equal(t, "2-twoa", docVersion2a.RevTreeID) // Put attachment on doc1 rev 2 rev3Attachment := `aGVsbG8gd29ybGQ=` // hello.txt @@ -815,8 +816,8 @@ func TestConflictingBranchAttachments(t *testing.T) { docVersion4a := rt.UpdateDoc("doc1", docVersion3a, rev4aBody) // Ensure the two attachments are different - response1 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]&rev="+docVersion4.RevID, "") - response2 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?rev="+docVersion4a.RevID, "") + response1 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]&rev="+docVersion4.RevTreeID, "") + response2 := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?rev="+docVersion4a.RevTreeID, "") var body1 db.Body var body2 db.Body @@ -865,14 +866,14 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { `}` _ = rt.UpdateDoc("doc1", docVersion5, rev6Body) - response := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") log.Printf("Rev6 GET: %s", response.Body.Bytes()) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) _, attachmentsPresent := body["_attachments"] assert.True(t, attachmentsPresent) // Create conflicting rev 6 that doesn't have attachments - reqBodyRev6a := `{"_rev": "6-a", "_revisions": {"ids": ["a", "` + docVersion5.RevID + `"], "start": 6}}` + reqBodyRev6a := `{"_rev": "6-a", "_revisions": {"ids": ["a", "` + docVersion5.RevTreeID + `"], "start": 6}}` response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev6a) RequireStatus(t, response, http.StatusCreated) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) @@ -880,7 +881,7 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { assert.Equal(t, "6-a", docRevId2a) var rev6Response db.Body - response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &rev6Response)) _, attachmentsPresent = rev6Response["_attachments"] assert.False(t, attachmentsPresent) @@ -891,7 +892,7 @@ func TestAttachmentsWithTombstonedConflict(t *testing.T) { // Retrieve current winning rev with attachments var rev7Response db.Body - response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevID+"\"]", "") + response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?atts_since=[\""+version.RevTreeID+"\"]", "") log.Printf("Rev6 GET: %s", response.Body.Bytes()) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &rev7Response)) _, attachmentsPresent = rev7Response["_attachments"] @@ -2115,7 +2116,7 @@ func TestAttachmentRemovalWithConflicts(t *testing.T) { var doc1 docResp // Get losing rev and ensure attachment is still there and has not been deleted - resp := rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+losingVersion3.RevID, "", map[string]string{"Accept": "application/json"}) + resp := rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+losingVersion3.RevTreeID, "", map[string]string{"Accept": "application/json"}) RequireStatus(t, resp, http.StatusOK) err := base.JSONUnmarshal(resp.BodyBytes(), &doc1) @@ -2132,7 +2133,7 @@ func TestAttachmentRemovalWithConflicts(t *testing.T) { var doc2 docResp // Get winning rev and ensure attachment is indeed removed from this rev - resp = rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+finalVersion4.RevID, "", map[string]string{"Accept": "application/json"}) + resp = rt.SendAdminRequestWithHeaders("GET", "/{{.keyspace}}/doc?attachments=true&rev="+finalVersion4.RevTreeID, "", map[string]string{"Accept": "application/json"}) RequireStatus(t, resp, http.StatusOK) err = base.JSONUnmarshal(resp.BodyBytes(), &doc2) @@ -2253,6 +2254,77 @@ func TestAttachmentDeleteOnExpiry(t *testing.T) { } } + +// TestUpdateViaBlipMigrateAttachment: +// - Tests document update through blip to a doc with attachment metadata defined in sync data +// - Assert that the c doc update this way will migrate the attachment metadata from sync data to global sync data +func TestUpdateViaBlipMigrateAttachment(t *testing.T) { + rtConfig := &RestTesterConfig{ + GuestEnabled: true, + } + + btcRunner := NewBlipTesterClientRunner(t) + const ( + doc1ID = "doc1" + ) + btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { + rt := NewRestTester(t, rtConfig) + defer rt.Close() + + opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols} + btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) + defer btc.Close() + ds := rt.GetSingleDataStore() + ctx := base.TestCtx(t) + + initialVersion := btc.rt.PutDocWithAttachment(doc1ID, "{}", "hello.txt", "aGVsbG8gd29ybGQ=") + btc.rt.WaitForPendingChanges() + btcRunner.StartOneshotPull(btc.id) + btcRunner.WaitForVersion(btc.id, doc1ID, initialVersion) + + value, xattrs, cas, err := ds.GetWithXattrs(ctx, doc1ID, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok := xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok := xattrs[base.GlobalXattrName] + require.True(t, ok) + + var attachs db.GlobalSyncData + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + // move attachment metadata from global xattr to sync xattr + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, doc1ID, cas, value, syncXattr, attachs.GlobalAttachments, true, ds) + + // push revision from client + doc1Version, err := btcRunner.PushRev(btc.id, doc1ID, initialVersion, []byte(`{"new": "val", "_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`)) + require.NoError(t, err) + assert.NoError(t, rt.WaitForVersion(doc1ID, doc1Version)) + + // assert the pushed rev updates the doc in bucket and migrates attachment metadata in process + xattrs, _, err = ds.GetXattrs(ctx, doc1ID, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + + // empty global sync, + attachs = db.GlobalSyncData{} + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + var syncData db.SyncData + err = base.JSONUnmarshal(syncXattr, &syncData) + require.NoError(t, err) + + // assert that the attachment metadata has been moved + assert.NotNil(t, attachs.GlobalAttachments) + assert.Nil(t, syncData.Attachments) + att := attachs.GlobalAttachments["hello.txt"].(map[string]interface{}) + assert.Equal(t, float64(11), att["length"]) + }) +} + func TestUpdateExistingAttachment(t *testing.T) { rtConfig := &RestTesterConfig{ GuestEnabled: true, @@ -2272,8 +2344,8 @@ func TestUpdateExistingAttachment(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() - doc1Version := rt.PutDoc(doc1ID, `{}`) - doc2Version := rt.PutDoc(doc2ID, `{}`) + doc1Version := btc.PutDoc(doc1ID, `{}`) + doc2Version := btc.PutDoc(doc2ID, `{}`) rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) @@ -2329,7 +2401,7 @@ func TestPushUnknownAttachmentAsStub(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() // Add doc1 and doc2 - doc1Version := btc.rt.PutDoc(doc1ID, `{}`) + doc1Version := btc.PutDoc(doc1ID, `{}`) btc.rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) @@ -2377,7 +2449,8 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() // Push an initial rev with attachment data - initialVersion := btc.rt.PutDoc(docID, `{"_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) + initialVersion := btc.rt.PutDocWithAttachment(docID, "{}", "hello.txt", "aGVsbG8gd29ybGQ=") + btc.rt.WaitForPendingChanges() // Replicate data to client and ensure doc arrives btc.rt.WaitForPendingChanges() @@ -2387,7 +2460,7 @@ func TestMinRevPosWorkToAvoidUnnecessaryProveAttachment(t *testing.T) { // Push a revision with a bunch of history simulating doc updated on mobile device // Note this references revpos 1 and therefore SGW has it - Shouldn't need proveAttachment proveAttachmentBefore := btc.pushReplication.replicationStats.ProveAttachment.Value() - revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.RevID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) + revid, err := btcRunner.PushRevWithHistory(btc.id, docID, initialVersion.GetRev(btc.UseHLV()), []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`), 25, 5) assert.NoError(t, err) proveAttachmentAfter := btc.pushReplication.replicationStats.ProveAttachment.Value() assert.Equal(t, proveAttachmentBefore, proveAttachmentAfter) @@ -2406,7 +2479,6 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) defer rt.Close() @@ -2416,7 +2488,9 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { defer btc.Close() // Create rev 1 with the hello.txt attachment const docID = "doc" - version := btc.rt.PutDoc(docID, `{"val": "val", "_attachments": {"hello.txt": {"data": "aGVsbG8gd29ybGQ="}}}`) + + version := btc.rt.PutDocWithAttachment(docID, `{"val": "val"}`, "hello.txt", "aGVsbG8gd29ybGQ=") + btc.rt.WaitForPendingChanges() // Pull rev and attachment down to client btc.rt.WaitForPendingChanges() @@ -2430,7 +2504,7 @@ func TestAttachmentWithErroneousRevPos(t *testing.T) { btcRunner.AttachmentsLock(btc.id).Unlock() // Put doc with an erroneous revpos 1 but with a different digest, referring to the above attachment - _, err := btcRunner.PushRevWithHistory(btc.id, docID, version.RevID, []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) + _, err := btcRunner.PushRevWithHistory(btc.id, docID, version.GetRev(btc.UseHLV()), []byte(`{"_attachments": {"hello.txt": {"revpos":1,"stub":true,"length": 19,"digest":"sha1-l+N7VpXGnoxMm8xfvtWPbz2YvDc="}}}`), 1, 0) require.NoError(t, err) // Ensure message and attachment is pushed up @@ -2597,8 +2671,9 @@ func TestCBLRevposHandling(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, &opts) defer btc.Close() - doc1Version := btc.rt.PutDoc(doc1ID, `{}`) - doc2Version := btc.rt.PutDoc(doc2ID, `{}`) + startingBody := db.Body{"foo": "bar"} + doc1Version := btc.rt.PutDocDirectly(doc1ID, startingBody) + doc2Version := btc.rt.PutDocDirectly(doc2ID, startingBody) btc.rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) @@ -2672,6 +2747,22 @@ func CreateDocWithLegacyAttachment(t *testing.T, rt *RestTester, docID string, r require.Len(t, attachments, 1) } +// CreateDocWithLegacyAttachmentNoMigration create a doc with legacy attachment defined (v1) and will not attempt to migrate that attachment to v2 +func CreateDocWithLegacyAttachmentNoMigration(t *testing.T, rt *RestTester, docID string, rawDoc []byte, attKey string, attBody []byte) { + // Write attachment directly to the datastore. + dataStore := rt.GetSingleDataStore() + _, err := dataStore.Add(attKey, 0, attBody) + require.NoError(t, err) + + body := db.Body{} + err = body.Unmarshal(rawDoc) + require.NoError(t, err, "Error unmarshalling body") + + // Write raw document to the datastore. + _, err = dataStore.Add(docID, 0, rawDoc) + require.NoError(t, err) +} + func retrieveAttachmentMeta(t *testing.T, rt *RestTester, docID string) (attMeta map[string]interface{}) { body := rt.GetDocBody(docID) attachments, ok := body["_attachments"].(map[string]interface{}) @@ -2730,7 +2821,7 @@ func (rt *RestTester) storeAttachment(docID string, version DocVersion, attName, // storeAttachmentWithHeaders adds an attachment to a document version and returns the new version using rev= syntax. func (rt *RestTester) storeAttachmentWithHeaders(docID string, version DocVersion, attName, attBody string, reqHeaders map[string]string) DocVersion { - resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", docID, attName, version.RevID) + resource := fmt.Sprintf("/{{.keyspace}}/%s/%s?rev=%s", docID, attName, version.RevTreeID) response := rt.SendAdminRequestWithHeaders(http.MethodPut, resource, attBody, reqHeaders) RequireStatus(rt.TB(), response, http.StatusCreated) var body db.Body @@ -2742,7 +2833,7 @@ func (rt *RestTester) storeAttachmentWithHeaders(docID string, version DocVersio // storeAttachmentWithIfMatch adds an attachment to a document version and returns the new version, using If-Match. func (rt *RestTester) storeAttachmentWithIfMatch(docID string, version DocVersion, attName, attBody string) DocVersion { reqHeaders := attachmentHeaders() - reqHeaders["If-Match"] = `"` + version.RevID + `"` + reqHeaders["If-Match"] = `"` + version.RevTreeID + `"` resource := fmt.Sprintf("/{{.keyspace}}/%s/%s", docID, attName) response := rt.SendRequestWithHeaders(http.MethodPut, resource, attBody, reqHeaders) RequireStatus(rt.TB(), response, http.StatusCreated) @@ -2751,3 +2842,161 @@ func (rt *RestTester) storeAttachmentWithIfMatch(docID string, version DocVersio require.True(rt.TB(), body["ok"].(bool)) return DocVersionFromPutResponse(rt.TB(), response) } + +// TestLegacyAttachmentMigrationToGlobalXattrOnImport: +// - Create legacy attachment and perform a read to migrate the attachment to xattr +// - Assert that this migrated attachment is moved to global xattr not sync data xattr +// - Add new doc with legacy attachment but do not attempt to migrate after write +// - Trigger on demand import for write and assert that the attachment is moved ot global xattr +func TestLegacyAttachmentMigrationToGlobalXattrOnImport(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + docID := "foo16" + attBody := []byte(`hi`) + digest := db.Sha1DigestKey(attBody) + attKey := db.MakeAttachmentKey(db.AttVersion1, docID, digest) + rawDoc := rawDocWithAttachmentAndSyncMeta() + + // Create a document with legacy attachment. + CreateDocWithLegacyAttachment(t, rt, docID, rawDoc, attKey, attBody) + + // get global xattr and assert the attachment is there + xattrs, _, err := collection.GetCollectionDatastore().GetXattrs(ctx, docID, []string{base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + var globalXattr db.GlobalSyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalXattr)) + hi := globalXattr.GlobalAttachments["hi.txt"].(map[string]interface{}) + + assert.Len(t, globalXattr.GlobalAttachments, 1) + assert.Equal(t, float64(2), hi["length"]) + + // Create a document with legacy attachment but do not attempt to migrate + docID = "baa16" + CreateDocWithLegacyAttachmentNoMigration(t, rt, docID, rawDoc, attKey, attBody) + + // Trigger on demand import for write + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/baa16", `{}`) + RequireStatus(t, resp, http.StatusConflict) + + // get xattrs of new doc we had the conflict update for, assert that the attachment metadata has been moved to global xattr + xattrs, _, err = collection.GetCollectionDatastore().GetXattrs(ctx, docID, []string{base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + globalXattr = db.GlobalSyncData{} + require.NoError(t, base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalXattr)) + newatt := globalXattr.GlobalAttachments["hi.txt"].(map[string]interface{}) + + assert.Len(t, globalXattr.GlobalAttachments, 1) + assert.Equal(t, float64(2), newatt["length"]) +} + +// TestAttachmentMigrationToGlobalXattrOnUpdate: +// - Create doc with attachment defined +// - Set doc in bucket to move attachment from global xattr to old location in sync data +// - Update this doc through sync gateway +// - Assert that the attachment metadata in moved from sync data to global xattr on update +func TestAttachmentMigrationToGlobalXattrOnUpdate(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + docID := "baa" + + body := `{"test":"doc","_attachments":{"camera.txt":{"data":"Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}` + vrs := rt.PutDoc(docID, body) + + // get xattrs, remove the global xattr and move attachments back to sync data in the bucket + xattrs, cas, err := collection.GetCollectionDatastore().GetXattrs(ctx, docID, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + require.Contains(t, xattrs, base.SyncXattrName) + + var bucketSyncData db.SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &bucketSyncData)) + var globalXattr db.GlobalSyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalXattr)) + + bucketSyncData.Attachments = globalXattr.GlobalAttachments + syncBytes := base.MustJSONMarshal(t, bucketSyncData) + xattrBytes := map[string][]byte{ + base.SyncXattrName: syncBytes, + } + // add new update sync data but also remove global xattr from doc + _, err = collection.GetCollectionDatastore().WriteWithXattrs(ctx, docID, 0, cas, []byte(`{"test":"doc"}`), xattrBytes, []string{base.GlobalXattrName}, nil) + require.NoError(t, err) + + // update doc + body = `{"some":"update","_attachments":{"camera.txt":{"data":"Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}` + _ = rt.UpdateDoc(docID, vrs, body) + + // assert that the attachments moved to global xattr after doc update + xattrs, _, err = collection.GetCollectionDatastore().GetXattrs(ctx, docID, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + require.Contains(t, xattrs, base.GlobalXattrName) + require.Contains(t, xattrs, base.SyncXattrName) + + bucketSyncData = db.SyncData{} + globalXattr = db.GlobalSyncData{} + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &bucketSyncData)) + require.NoError(t, base.JSONUnmarshal(xattrs[base.GlobalXattrName], &globalXattr)) + + assert.Nil(t, bucketSyncData.Attachments) + assert.NotNil(t, globalXattr.GlobalAttachments) + attMeta := globalXattr.GlobalAttachments["camera.txt"].(map[string]interface{}) + assert.Equal(t, float64(20), attMeta["length"]) +} + +func TestBlipPushRevWithAttachment(t *testing.T) { + btcRunner := NewBlipTesterClientRunner(t) + + btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { + // Setup + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + const username = "bernard" + + opts := &BlipTesterClientOpts{Username: username, SupportedBLIPProtocols: SupportedBLIPProtocols} + btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) + docID := "doc1" + attachmentName := "attachment1" + attachmentData := "attachmentContents" + + contentType := "text/plain" + + length, digest, err := btcRunner.saveAttachment(btc.id, contentType, base64.StdEncoding.EncodeToString([]byte(attachmentData))) + require.NoError(t, err) + + blipBody := db.Body{ + "key": "val", + "_attachments": db.Body{ + "attachment1": db.Body{ + "digest": digest, + "stub": true, + "length": length, + }, + }, + } + version1, err := btcRunner.PushRev(btc.id, docID, EmptyDocVersion(), base.MustJSONMarshal(t, blipBody)) + require.NoError(t, err) + body := rt.GetDocBody(docID) + require.Equal(t, db.Body{ + "key": "val", + "_attachments": map[string]any{ + "attachment1": map[string]any{ + "digest": digest, + "stub": true, + "revpos": float64(1), + "length": float64(length), + }, + }, + "_id": docID, + "_rev": version1.RevTreeID, + }, body) + response := rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s/%s", docID, attachmentName), "") + RequireStatus(t, response, http.StatusOK) + require.Equal(t, attachmentData, string(response.BodyBytes())) + }) +} diff --git a/rest/attachmentmigrationtest/attachment_migration_api_test.go b/rest/attachmentmigrationtest/attachment_migration_api_test.go new file mode 100644 index 0000000000..ede191b03f --- /dev/null +++ b/rest/attachmentmigrationtest/attachment_migration_api_test.go @@ -0,0 +1,255 @@ +/* +Copyright 2024-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package attachmentmigrationtest + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAttachmentMigrationAPI(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + + rt := rest.NewRestTester(t, &rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: false, // turn off import feed to stop the feed migrating attachments + }}, + }) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + // Perform GET as automatic migration kicks in upon db start + resp := rt.SendAdminRequest("GET", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + + var migrationStatus db.AttachmentMigrationManagerResponse + err := base.JSONUnmarshal(resp.BodyBytes(), &migrationStatus) + require.NoError(t, err) + require.Equal(t, db.BackgroundProcessStateRunning, migrationStatus.State) + assert.Equal(t, int64(0), migrationStatus.DocsChanged) + assert.Equal(t, int64(0), migrationStatus.DocsProcessed) + assert.Empty(t, migrationStatus.LastErrorMessage) + + // Wait for run on startup to complete + _ = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + + // add some docs for migration + addDocsForMigrationProcess(t, ctx, collection) + + // kick off migration + resp = rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + + // attempt to kick off again, should error + resp = rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusServiceUnavailable) + + // Wait for run to complete + _ = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + + // Perform GET after migration has been ran, ensure it starts in valid 'stopped' state + resp = rt.SendAdminRequest("GET", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + + migrationStatus = db.AttachmentMigrationManagerResponse{} + err = base.JSONUnmarshal(resp.BodyBytes(), &migrationStatus) + require.NoError(t, err) + require.Equal(t, db.BackgroundProcessStateCompleted, migrationStatus.State) + assert.Equal(t, int64(5), migrationStatus.DocsChanged) + assert.Equal(t, int64(10), migrationStatus.DocsProcessed) + assert.Empty(t, migrationStatus.LastErrorMessage) +} + +func TestAttachmentMigrationAbort(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + + rt := rest.NewRestTester(t, &rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: false, // turn off import feed to stop the feed migrating attachments + }}, + }) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + // Wait for run on startup to complete + _ = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + + // add some docs to arrive over dcp + for i := 0; i < 20; i++ { + key := fmt.Sprintf("%s_%d", t.Name(), i) + docBody := db.Body{ + "value": 1234, + } + _, _, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + } + + // start migration + resp := rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + + // stop the migration job + resp = rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?action=stop", "") + rest.RequireStatus(t, resp, http.StatusOK) + + status := rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateStopped) + assert.Equal(t, int64(0), status.DocsChanged) +} + +func TestAttachmentMigrationReset(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + + rt := rest.NewRestTester(t, &rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: false, // turn off import feed to stop the feed migrating attachments + }}, + }) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + // Wait for run on startup to complete + _ = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + + // add some docs for migration + addDocsForMigrationProcess(t, ctx, collection) + + // start migration + resp := rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + status := rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateRunning) + migrationID := status.MigrationID + + // Stop migration + resp = rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?action=stop", "") + rest.RequireStatus(t, resp, http.StatusOK) + status = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateStopped) + + // make sure status is stopped + resp = rt.SendAdminRequest("GET", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + var migrationStatus db.AttachmentManagerResponse + err := base.JSONUnmarshal(resp.BodyBytes(), &migrationStatus) + assert.NoError(t, err) + assert.Equal(t, db.BackgroundProcessStateStopped, migrationStatus.State) + + // reset migration run + resp = rt.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?reset=true", "") + rest.RequireStatus(t, resp, http.StatusOK) + status = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateRunning) + assert.NotEqual(t, migrationID, status.MigrationID) + + // wait to complete + status = rt.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + // assert all 10 docs are processed again + assert.Equal(t, int64(10), status.DocsProcessed) +} + +func TestAttachmentMigrationMultiNode(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + tb := base.GetTestBucket(t) + noCloseTB := tb.NoCloseClone() + + rt1 := rest.NewRestTester(t, &rest.RestTesterConfig{ + CustomTestBucket: noCloseTB, + }) + rt2 := rest.NewRestTester(t, &rest.RestTesterConfig{ + CustomTestBucket: tb, + }) + defer rt2.Close() + defer rt1.Close() + collection, ctx := rt1.GetSingleTestDatabaseCollectionWithUser() + + // Wait for startup run to complete, assert completed status is on both nodes + _ = rt1.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + _ = rt2.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + + // add some docs for migration + addDocsForMigrationProcess(t, ctx, collection) + + // kick off migration on node 1 + resp := rt1.SendAdminRequest("POST", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + status := rt1.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateRunning) + migrationID := status.MigrationID + + // stop migration + resp = rt1.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?action=stop", "") + rest.RequireStatus(t, resp, http.StatusOK) + _ = rt1.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateStopped) + + // assert that node 2 also has stopped status + var rt2MigrationStatus db.AttachmentMigrationManagerResponse + resp = rt2.SendAdminRequest("GET", "/{{.db}}/_attachment_migration", "") + rest.RequireStatus(t, resp, http.StatusOK) + err := base.JSONUnmarshal(resp.BodyBytes(), &rt2MigrationStatus) + assert.NoError(t, err) + assert.Equal(t, db.BackgroundProcessStateStopped, rt2MigrationStatus.State) + + // kick off migration run again on node 2. Should resume and have same migration id + resp = rt2.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?action=start", "") + rest.RequireStatus(t, resp, http.StatusOK) + _ = rt2.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateRunning) + + // assert starting on another node when already running should error + resp = rt1.SendAdminRequest("POST", "/{{.db}}/_attachment_migration?action=start", "") + rest.RequireStatus(t, resp, http.StatusServiceUnavailable) + + // Wait for run to be marked as complete on both nodes + status = rt1.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) + assert.Equal(t, migrationID, status.MigrationID) + _ = rt2.WaitForAttachmentMigrationStatus(t, db.BackgroundProcessStateCompleted) +} + +func addDocsForMigrationProcess(t *testing.T, ctx context.Context, collection *db.DatabaseCollectionWithUser) { + for i := 0; i < 10; i++ { + docBody := db.Body{ + "value": 1234, + db.BodyAttachments: map[string]interface{}{"myatt": map[string]interface{}{"content_type": "text/plain", "data": "SGVsbG8gV29ybGQh"}}, + } + key := fmt.Sprintf("%s_%d", t.Name(), i) + _, doc, err := collection.Put(ctx, key, docBody) + require.NoError(t, err) + assert.NotNil(t, doc.SyncData.Attachments) + } + + // Move some subset of the documents attachment metadata from global sync to sync data + for j := 0; j < 5; j++ { + key := fmt.Sprintf("%s_%d", t.Name(), j) + value, xattrs, cas, err := collection.GetCollectionDatastore().GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok := xattrs[base.SyncXattrName] + assert.True(t, ok) + globalXattr, ok := xattrs[base.GlobalXattrName] + assert.True(t, ok) + + var attachs db.GlobalSyncData + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, true, collection.GetCollectionDatastore()) + } +} diff --git a/rest/attachmentmigrationtest/attachment_migration_test.go b/rest/attachmentmigrationtest/attachment_migration_test.go new file mode 100644 index 0000000000..bf352d0932 --- /dev/null +++ b/rest/attachmentmigrationtest/attachment_migration_test.go @@ -0,0 +1,422 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package attachmentmigrationtest + +import ( + "fmt" + "net/http" + "testing" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMigrationJobStartOnDbStart: +// - Create a db +// - Grab attachment migration manager and assert it has run upon db startup +// - Assert job has written syncInfo metaVersion as expected to the bucket +func TestMigrationJobStartOnDbStart(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + rt := rest.NewRestTesterPersistentConfig(t) + defer rt.Close() + ctx := rt.Context() + + ds := rt.GetSingleDataStore() + dbCtx := rt.GetDatabase() + + mgr := dbCtx.AttachmentMigrationManager + + // wait for migration job to finish + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateCompleted) + + // assert that sync info with metadata version written to the collection + db.AssertSyncInfoMetaVersion(t, ds) +} + +// TestChangeDbCollectionsRestartMigrationJob: +// - Add docs before job starts, this will test that the dcp checkpoint are correctly reset upon db update later in test +// - Create db with collection one +// - Assert the attachment migration job is running +// - Update db config to include a new collection +// - Assert job runs/completes +// - As the job should've purged dcp collections upon new collection being added to db we expect some added docs +// to be processed twice in the job, so we can assert that the job has processed more docs than we added +// - Assert sync info: metaVersion is written to BOTH collections in the db config +func TestChangeDbCollectionsRestartMigrationJob(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + base.TestRequiresCollections(t) + base.RequireNumTestDataStores(t, 2) + base.LongRunningTest(t) + base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll) + tb := base.GetTestBucket(t) + rtConfig := &rest.RestTesterConfig{ + CustomTestBucket: tb, + PersistentConfig: true, + } + + rt := rest.NewRestTesterMultipleCollections(t, rtConfig, 2) + defer rt.Close() + ctx := rt.Context() + _ = rt.Bucket() + + const ( + dbName = "db1" + totalDocsAdded = 8000 + ) + + ds0, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + ds1, err := tb.GetNamedDataStore(1) + require.NoError(t, err) + opts := &sgbucket.MutateInOptions{} + + // add some docs (with xattr so they won't be ignored in the background job) to both collections + // we want to add large number of docs to stop the migration job from finishing before we can assert on state + bodyBytes := []byte(`{"some": "body"}`) + for i := 0; i < 4000; i++ { + key := fmt.Sprintf("%s_%d", t.Name(), i) + xattrsInput := map[string][]byte{ + "_xattr": []byte(`{"some":"xattr"}`), + } + _, writeErr := ds0.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + + _, writeErr = ds1.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + } + + scopesConfigC1Only := rest.GetCollectionsConfig(t, tb, 2) + dataStoreNames := rest.GetDataStoreNamesFromScopesConfig(scopesConfigC1Only) + scope := dataStoreNames[0].ScopeName() + collection1 := dataStoreNames[0].CollectionName() + collection2 := dataStoreNames[1].CollectionName() + delete(scopesConfigC1Only[scope].Collections, collection2) + + scopesConfigBothCollection := rest.GetCollectionsConfig(t, tb, 2) + + // Create a db1 with one collection initially + dbConfig := rt.NewDbConfig() + // ensure import is off to stop the docs we add from being imported by sync gateway, this could cause extra overhead + // on the migration job (more doc writes going to bucket). We want to avoid for purpose of this test + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfigC1Only + + resp := rt.CreateDatabase(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx := rt.GetDatabase() + mgr := dbCtx.AttachmentMigrationManager + scNames := base.ScopeAndCollectionNames{base.ScopeAndCollectionName{Scope: scope, Collection: collection1}} + assert.ElementsMatch(t, scNames, dbCtx.RequireAttachmentMigration) + // wait for migration job to start + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateRunning) + + // update db config to include second collection + dbConfig = rt.NewDbConfig() + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfigBothCollection + resp = rt.UpsertDbConfig(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + // wait for attachment migration job to start and finish + dbCtx = rt.GetDatabase() + mgr = dbCtx.AttachmentMigrationManager + scNames = append(scNames, base.ScopeAndCollectionName{Scope: scope, Collection: collection2}) + assert.ElementsMatch(t, scNames, dbCtx.RequireAttachmentMigration) + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateRunning) + + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateCompleted) + + var mgrStatus db.AttachmentMigrationManagerResponse + stat, err := mgr.GetStatus(ctx) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(stat, &mgrStatus)) + // assert that number of docs precessed is greater than the total docs added, this will be because when updating + // the db config to include a new collection this should force reset of DCP checkpoints and start DCP feed from 0 again + assert.Greater(t, mgrStatus.DocsProcessed, int64(totalDocsAdded)) + + // assert that sync info with metadata version written to both collections + db.AssertSyncInfoMetaVersion(t, ds0) + db.AssertSyncInfoMetaVersion(t, ds1) +} + +// TestMigrationNewCollectionToDbNoRestart: +// - Create db with one collection +// - Wait for attachment migration job to finish on that single collection +// - Assert syncInfo: metaVersion is present in collection +// - Update db config to include new collection +// - Assert that the attachment migration task is restarted but only on the one (new) collection +// - We can do this though asserting the new run only process amount of docs added in second collection +// after update to db config + assert on collections requiring migration +// - Assert that syncInfo: metaVersion is written for new collection (and is still present in original collection) +func TestMigrationNewCollectionToDbNoRestart(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + base.TestRequiresCollections(t) + base.RequireNumTestDataStores(t, 2) + base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll) + tb := base.GetTestBucket(t) + rtConfig := &rest.RestTesterConfig{ + CustomTestBucket: tb, + PersistentConfig: true, + } + + rt := rest.NewRestTesterMultipleCollections(t, rtConfig, 2) + defer rt.Close() + ctx := rt.Context() + _ = rt.Bucket() + + const ( + dbName = "db1" + totalDocsAddedCollOne = 10 + totalDocsAddedCollTwo = 10 + ) + + ds0, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + ds1, err := tb.GetNamedDataStore(1) + require.NoError(t, err) + opts := &sgbucket.MutateInOptions{} + + // add some docs (with xattr so they won't be ignored in the background job) to both collections + bodyBytes := []byte(`{"some": "body"}`) + for i := 0; i < 10; i++ { + key := fmt.Sprintf("%s_%d", t.Name(), i) + xattrsInput := map[string][]byte{ + "_xattr": []byte(`{"some":"xattr"}`), + } + _, writeErr := ds0.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + + _, writeErr = ds1.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + } + + scopesConfigC1Only := rest.GetCollectionsConfig(t, tb, 2) + dataStoreNames := rest.GetDataStoreNamesFromScopesConfig(scopesConfigC1Only) + scope := dataStoreNames[0].ScopeName() + collection2 := dataStoreNames[1].CollectionName() + delete(scopesConfigC1Only[scope].Collections, collection2) + + // Create a db1 with one collection initially + dbConfig := rt.NewDbConfig() + // ensure import is off to stop the docs we add from being imported by sync gateway, this could cause extra overhead + // on the migration job (more doc writes going to bucket). We want to avoid for purpose of this test + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfigC1Only + resp := rt.CreateDatabase(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx := rt.GetDatabase() + mgr := dbCtx.AttachmentMigrationManager + assert.Len(t, dbCtx.RequireAttachmentMigration, 1) + // wait for migration job to finish on single collection + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateCompleted) + + var mgrStatus db.AttachmentMigrationManagerResponse + stat, err := mgr.GetStatus(ctx) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(stat, &mgrStatus)) + // assert that number of docs precessed is equal to docs in collection 1 + assert.Equal(t, int64(totalDocsAddedCollOne), mgrStatus.DocsProcessed) + + // assert sync info meta version exists for this collection + db.AssertSyncInfoMetaVersion(t, ds0) + + // create db with second collection, background job should only run on new collection added given + // existent of sync info meta version on collection 1 + scopesConfigBothCollection := rest.GetCollectionsConfig(t, tb, 2) + dbConfig = rt.NewDbConfig() + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfigBothCollection + resp = rt.UpsertDbConfig(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx = rt.GetDatabase() + mgr = dbCtx.AttachmentMigrationManager + assert.Len(t, dbCtx.RequireAttachmentMigration, 1) + // wait for migration job to finish on the new collection + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateCompleted) + + mgrStatus = db.AttachmentMigrationManagerResponse{} + stat, err = mgr.GetStatus(ctx) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(stat, &mgrStatus)) + // assert that number of docs precessed is equal to docs in collection 2 (not the total number of docs added across + // the collections, as we'd expect if the process had reset) + assert.Equal(t, int64(totalDocsAddedCollTwo), mgrStatus.DocsProcessed) + + // assert that sync info with metadata version written to both collections + db.AssertSyncInfoMetaVersion(t, ds0) + db.AssertSyncInfoMetaVersion(t, ds1) +} + +// TestMigrationNoReRunStartStopDb: +// - Create db +// - Wait for attachment migration task to finish +// - Update db config to trigger reload of db +// - Assert that the migration job is not re-run (docs processed is the same as before + collections +// requiring migration is empty) +func TestMigrationNoReRunStartStopDb(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + base.TestRequiresCollections(t) + base.RequireNumTestDataStores(t, 2) + base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll) + tb := base.GetTestBucket(t) + rtConfig := &rest.RestTesterConfig{ + CustomTestBucket: tb, + PersistentConfig: true, + } + + rt := rest.NewRestTesterMultipleCollections(t, rtConfig, 2) + defer rt.Close() + ctx := rt.Context() + _ = rt.Bucket() + + const ( + dbName = "db1" + totalDocsAdded = 20 + ) + + ds0, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + ds1, err := tb.GetNamedDataStore(1) + require.NoError(t, err) + opts := &sgbucket.MutateInOptions{} + + // add some docs (with xattr so they won't be ignored in the background job) to both collections + bodyBytes := []byte(`{"some": "body"}`) + for i := 0; i < 10; i++ { + key := fmt.Sprintf("%s_%d", t.Name(), i) + xattrsInput := map[string][]byte{ + "_xattr": []byte(`{"some":"xattr"}`), + } + _, writeErr := ds0.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + + _, writeErr = ds1.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + } + + scopesConfigBothCollection := rest.GetCollectionsConfig(t, tb, 2) + dbConfig := rt.NewDbConfig() + // ensure import is off to stop the docs we add from being imported by sync gateway, this could cause extra overhead + // on the migration job (more doc writes going to bucket). We want to avoid for purpose of this test + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfigBothCollection + resp := rt.CreateDatabase(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx := rt.GetDatabase() + assert.Len(t, dbCtx.RequireAttachmentMigration, 2) + mgr := dbCtx.AttachmentMigrationManager + // wait for migration job to finish on both collections + db.RequireBackgroundManagerState(t, ctx, mgr, db.BackgroundProcessStateCompleted) + + var mgrStatus db.AttachmentMigrationManagerResponse + stat, err := mgr.GetStatus(ctx) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(stat, &mgrStatus)) + // assert that number of docs precessed is equal to docs in collection 1 + assert.Equal(t, int64(totalDocsAdded), mgrStatus.DocsProcessed) + + // assert that sync info with metadata version written to both collections + db.AssertSyncInfoMetaVersion(t, ds0) + db.AssertSyncInfoMetaVersion(t, ds1) + + // reload db config with a config change + dbConfig = rt.NewDbConfig() + dbConfig.AutoImport = true + dbConfig.Scopes = scopesConfigBothCollection + resp = rt.UpsertDbConfig(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx = rt.GetDatabase() + mgr = dbCtx.AttachmentMigrationManager + // assert that the job remains in completed state (not restarted) + mgrStatus = db.AttachmentMigrationManagerResponse{} + stat, err = mgr.GetStatus(ctx) + require.NoError(t, err) + require.NoError(t, base.JSONUnmarshal(stat, &mgrStatus)) + assert.Equal(t, db.BackgroundProcessStateCompleted, mgrStatus.State) + assert.Equal(t, int64(totalDocsAdded), mgrStatus.DocsProcessed) + assert.Len(t, dbCtx.RequireAttachmentMigration, 0) +} + +// TestStartMigrationAlreadyRunningProcess: +// - Create db +// - Wait for migration job to start +// - Attempt to start job again on manager, assert we get error +func TestStartMigrationAlreadyRunningProcess(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("rosmar does not support DCP client, pending CBG-4249") + } + base.TestRequiresCollections(t) + base.RequireNumTestDataStores(t, 1) + base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll) + tb := base.GetTestBucket(t) + rtConfig := &rest.RestTesterConfig{ + CustomTestBucket: tb, + PersistentConfig: true, + } + + rt := rest.NewRestTester(t, rtConfig) + defer rt.Close() + ctx := rt.Context() + _ = rt.Bucket() + + const ( + dbName = "db1" + ) + + ds0, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + opts := &sgbucket.MutateInOptions{} + + // add some docs (with xattr so they won't be ignored in the background job) to both collections + // we want to add large number of docs to stop the migration job from finishing before we can try start the job + // again (whilst already running) + bodyBytes := []byte(`{"some": "body"}`) + for i := 0; i < 2000; i++ { + key := fmt.Sprintf("%s_%d", t.Name(), i) + xattrsInput := map[string][]byte{ + "_xattr": []byte(`{"some":"xattr"}`), + } + _, writeErr := ds0.WriteWithXattrs(ctx, key, 0, 0, bodyBytes, xattrsInput, nil, opts) + require.NoError(t, writeErr) + } + + scopesConfig := rest.GetCollectionsConfig(t, tb, 1) + dbConfig := rt.NewDbConfig() + // ensure import is off to stop the docs we add from being imported by sync gateway, this could cause extra overhead + // on the migration job (more doc writes going to bucket). We want to avoid for purpose of this test + dbConfig.AutoImport = false + dbConfig.Scopes = scopesConfig + resp := rt.CreateDatabase(dbName, dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + dbCtx := rt.GetDatabase() + nodeMgr := dbCtx.AttachmentMigrationManager + // wait for migration job to start + db.RequireBackgroundManagerState(t, ctx, nodeMgr, db.BackgroundProcessStateRunning) + + err = nodeMgr.Start(ctx, nil) + assert.Error(t, err) + assert.ErrorContains(t, err, "Process already running") +} diff --git a/rest/attachmentmigrationtest/main_test.go b/rest/attachmentmigrationtest/main_test.go new file mode 100644 index 0000000000..347a1ae26a --- /dev/null +++ b/rest/attachmentmigrationtest/main_test.go @@ -0,0 +1,25 @@ +/* +Copyright 2024-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package attachmentmigrationtest + +import ( + "context" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" +) + +func TestMain(m *testing.M) { + ctx := context.Background() // start of test process + tbpOptions := base.TestBucketPoolOptions{MemWatermarkThresholdMB: 8192} + db.TestBucketPoolWithIndexes(ctx, m, tbpOptions) +} diff --git a/rest/audit_test.go b/rest/audit_test.go index c9f311cc04..01c273e93b 100644 --- a/rest/audit_test.go +++ b/rest/audit_test.go @@ -770,21 +770,21 @@ func TestAuditDocumentRead(t *testing.T) { method: http.MethodGet, path: "/{{.keyspace}}/doc1", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { name: "get doc with rev", method: http.MethodGet, - path: "/{{.keyspace}}/doc1?rev=" + docVersion.RevID, + path: "/{{.keyspace}}/doc1?rev=" + docVersion.RevTreeID, docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { name: "get doc with openrevs", method: http.MethodGet, path: "/{{.keyspace}}/doc1?open_revs=all", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { name: "_bulk_get", @@ -792,11 +792,11 @@ func TestAuditDocumentRead(t *testing.T) { path: "/{{.keyspace}}/_bulk_get", requestBody: string(base.MustJSONMarshal(t, db.Body{ "docs": []db.Body{ - {"id": "doc1", "rev": docVersion.RevID}, + {"id": "doc1", "rev": docVersion.RevTreeID}, }, })), docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { // this doesn't actually provide the document body, no audit events @@ -804,7 +804,7 @@ func TestAuditDocumentRead(t *testing.T) { method: http.MethodPost, path: "/{{.keyspace}}/_revs_diff", requestBody: string(base.MustJSONMarshal(t, db.Body{ - "doc1": []string{docVersion.RevID}, + "doc1": []string{docVersion.RevTreeID}, })), docID: "doc1", docReadVersions: nil, @@ -822,14 +822,14 @@ func TestAuditDocumentRead(t *testing.T) { method: http.MethodGet, path: "/{{.keyspace}}/_all_docs?include_docs=true", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { name: "all_docs with include_docs=true&channels=true", method: http.MethodGet, path: "/{{.keyspace}}/_all_docs?include_docs=true&channels=true", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, docMetadataReadCount: 1, }, { @@ -867,14 +867,14 @@ func TestAuditDocumentRead(t *testing.T) { method: http.MethodGet, path: "/{{.keyspace}}/_changes?since=0&include_docs=true", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, { name: "raw", method: http.MethodGet, path: "/{{.keyspace}}/_raw/doc1", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, docMetadataReadCount: 1, }, { @@ -901,7 +901,7 @@ func TestAuditDocumentRead(t *testing.T) { method: http.MethodGet, path: "/{{.keyspace}}/doc1?replicator2=true", docID: "doc1", - docReadVersions: []string{docVersion.RevID}, + docReadVersions: []string{docVersion.RevTreeID}, }, ) } @@ -912,7 +912,7 @@ func TestAuditDocumentRead(t *testing.T) { RequireStatus(t, resp, http.StatusOK) }) requireDocumentReadEvents(rt, output, testCase.docID, testCase.docReadVersions) - requireDocumentMetadataReadEvents(rt, output, testCase.docID, docVersion.RevID, testCase.docMetadataReadCount) + requireDocumentMetadataReadEvents(rt, output, testCase.docID, docVersion.RevTreeID, testCase.docMetadataReadCount) }) } } @@ -937,7 +937,7 @@ func TestAuditAttachmentEvents(t *testing.T) { return rt.CreateTestDoc(docID) }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevID, "content"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevTreeID, "content"), http.StatusCreated) }, attachmentCreateCount: 1, }, @@ -947,7 +947,7 @@ func TestAuditAttachmentEvents(t *testing.T) { return rt.CreateTestDoc(docID) }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevID, `{"_attachments":{"attachment1":{"data": "YQ=="}}}`), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevTreeID, `{"_attachments":{"attachment1":{"data": "YQ=="}}}`), http.StatusCreated) }, attachmentCreateCount: 1, }, @@ -955,12 +955,12 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "get attachment with rev", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevID, ""), http.StatusOK) + RequireStatus(t, rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevTreeID, ""), http.StatusOK) }, attachmentReadCount: 1, }, @@ -968,14 +968,14 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "bulk_get attachment with rev", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { body := string(base.MustJSONMarshal(t, db.Body{ "docs": []db.Body{ - {"id": docID, "rev": docVersion.RevID}, + {"id": docID, "rev": docVersion.RevTreeID}, }, })) RequireStatus(t, rt.SendAdminRequest(http.MethodPost, "/{{.keyspace}}/_bulk_get?attachments=true", body), http.StatusOK) @@ -996,12 +996,12 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "update attachment", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevID, "content-update"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevTreeID, "content-update"), http.StatusCreated) }, attachmentUpdateCount: 1, }, @@ -1009,12 +1009,12 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "update inline attachment", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevID, `{"_attachments":{"attachment1":{"data": "YQ=="}}}`), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevTreeID, `{"_attachments":{"attachment1":{"data": "YQ=="}}}`), http.StatusCreated) }, attachmentUpdateCount: 1, }, @@ -1022,12 +1022,12 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "delete attachment", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodDelete, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevID, ""), http.StatusOK) + RequireStatus(t, rt.SendAdminRequest(http.MethodDelete, "/{{.keyspace}}/"+docID+"/attachment1?rev="+docVersion.RevTreeID, ""), http.StatusOK) }, attachmentDeleteCount: 1, }, @@ -1035,12 +1035,12 @@ func TestAuditAttachmentEvents(t *testing.T) { name: "delete inline attachment", setupCode: func(t testing.TB, docID string) DocVersion { initialDocVersion := rt.CreateTestDoc(docID) - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevID, "contentdoc2"), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"/attachment1?rev="+initialDocVersion.RevTreeID, "contentdoc2"), http.StatusCreated) docVersion, _ := rt.GetDoc(docID) return docVersion }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevID, `{"foo": "bar", "_attachments":{}}`), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevTreeID, `{"foo": "bar", "_attachments":{}}`), http.StatusCreated) }, attachmentDeleteCount: 1, }, @@ -1055,7 +1055,7 @@ func TestAuditAttachmentEvents(t *testing.T) { }) postAttachmentVersion, _ := rt.GetDoc(docID) - requireAttachmentEvents(rt, base.AuditIDAttachmentDelete, output, docID, postAttachmentVersion.RevID, attachmentName, testCase.attachmentDeleteCount) + requireAttachmentEvents(rt, base.AuditIDAttachmentDelete, output, docID, postAttachmentVersion.RevTreeID, attachmentName, testCase.attachmentDeleteCount) }) } } @@ -1091,7 +1091,7 @@ func TestAuditDocumentCreateUpdateEvents(t *testing.T) { return rt.CreateTestDoc(docID) }, auditableCode: func(t testing.TB, docID string, docVersion DocVersion) { - RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevID, `{"foo": "bar"}`), http.StatusCreated) + RequireStatus(t, rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+docVersion.RevTreeID, `{"foo": "bar"}`), http.StatusCreated) }, documentUpdateCount: 1, }, @@ -1130,14 +1130,15 @@ func TestAuditDocumentCreateUpdateEvents(t *testing.T) { testCase.auditableCode(t, docID, docVersion) }) postAttachmentVersion, _ := rt.GetDoc(docID) - requireDocumentEvents(rt, base.AuditIDDocumentCreate, output, docID, postAttachmentVersion.RevID, testCase.documentCreateCount) - requireDocumentEvents(rt, base.AuditIDDocumentUpdate, output, docID, postAttachmentVersion.RevID, testCase.documentUpdateCount) + requireDocumentEvents(rt, base.AuditIDDocumentCreate, output, docID, postAttachmentVersion.RevTreeID, testCase.documentCreateCount) + requireDocumentEvents(rt, base.AuditIDDocumentUpdate, output, docID, postAttachmentVersion.RevTreeID, testCase.documentUpdateCount) }) } } func TestAuditChangesFeedStart(t *testing.T) { btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // CBG-4166 btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := createAuditLoggingRestTester(t) @@ -1529,10 +1530,10 @@ func TestAuditBlipCRUD(t *testing.T) { }) postAttachmentVersion, _ := rt.GetDoc(docID) - requireAttachmentEvents(rt, base.AuditIDAttachmentCreate, output, docID, postAttachmentVersion.RevID, testCase.attachmentName, testCase.attachmentCreateCount) - requireAttachmentEvents(rt, base.AuditIDAttachmentRead, output, docID, postAttachmentVersion.RevID, testCase.attachmentName, testCase.attachmentReadCount) - requireAttachmentEvents(rt, base.AuditIDAttachmentUpdate, output, docID, postAttachmentVersion.RevID, testCase.attachmentName, testCase.attachmentUpdateCount) - requireAttachmentEvents(rt, base.AuditIDAttachmentDelete, output, docID, postAttachmentVersion.RevID, testCase.attachmentName, testCase.attachmentDeleteCount) + requireAttachmentEvents(rt, base.AuditIDAttachmentCreate, output, docID, postAttachmentVersion.RevTreeID, testCase.attachmentName, testCase.attachmentCreateCount) + requireAttachmentEvents(rt, base.AuditIDAttachmentRead, output, docID, postAttachmentVersion.RevTreeID, testCase.attachmentName, testCase.attachmentReadCount) + requireAttachmentEvents(rt, base.AuditIDAttachmentUpdate, output, docID, postAttachmentVersion.RevTreeID, testCase.attachmentName, testCase.attachmentUpdateCount) + requireAttachmentEvents(rt, base.AuditIDAttachmentDelete, output, docID, postAttachmentVersion.RevTreeID, testCase.attachmentName, testCase.attachmentDeleteCount) }) } }) diff --git a/rest/blip_api_attachment_test.go b/rest/blip_api_attachment_test.go index 5861c46334..3184e635a9 100644 --- a/rest/blip_api_attachment_test.go +++ b/rest/blip_api_attachment_test.go @@ -45,8 +45,7 @@ func TestBlipPushPullV2AttachmentV2Client(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - // given this test is for v2 protocol, skip version vector test - btcRunner.SkipVersionVectorInitialization = true + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Doesn't require HLV - attachment v2 protocol test const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { @@ -119,6 +118,7 @@ func TestBlipPushPullV2AttachmentV3Client(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Doesn't require HLV - attachment v2 protocol test const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { @@ -190,7 +190,7 @@ func TestBlipProveAttachmentV2(t *testing.T) { ) btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipVersionVectorInitialization = true // v2 protocol test + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Doesn't require HLV - attachment v2 protocol test btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -247,7 +247,7 @@ func TestBlipProveAttachmentV2Push(t *testing.T) { ) btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipVersionVectorInitialization = true // v2 protocol test + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Doesn't require HLV - attachment v2 protocol test btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -287,13 +287,13 @@ func TestBlipPushPullNewAttachmentCommonAncestor(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) - const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { + docID := t.Name() rt := NewRestTester(t, &rtConfig) defer rt.Close() - opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols} + opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols, SourceID: "abc"} btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() @@ -301,50 +301,59 @@ func TestBlipPushPullNewAttachmentCommonAncestor(t *testing.T) { // CBL creates revisions 1-abc,2-abc on the client, with an attachment associated with rev 2. bodyText := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` - err := btcRunner.StoreRevOnClient(btc.id, docID, "2-abc", []byte(bodyText)) + + rev := "2-abc" + if btc.UseHLV() { + rev = db.EncodeTestVersion("2@abc") + } + err := btcRunner.StoreRevOnClient(btc.id, docID, rev, []byte(bodyText)) require.NoError(t, err) bodyText = `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` revId, err := btcRunner.PushRevWithHistory(btc.id, docID, "", []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "2-abc", revId) + assert.Equal(t, rev, revId) // Wait for the documents to be replicated at SG btc.pushReplication.WaitForMessage(2) - resp := btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) + collection, ctx := rt.GetSingleTestDatabaseCollection() + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + + attachmentRevPos, _ := db.ParseRevID(ctx, doc.CurrentRev) // CBL updates the doc w/ two more revisions, 3-abc, 4-abc, // these are sent to SG as 4-abc, history:[4-abc,3-abc,2-abc], the attachment has revpos=2 bodyText = `{"greetings":[{"hi":"bob"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` revId, err = btcRunner.PushRevWithHistory(btc.id, docID, revId, []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "4-abc", revId) + expectedRev := "4-abc" + if btc.UseHLV() { + expectedRev = db.EncodeTestVersion("4@abc") + } + assert.Equal(t, expectedRev, revId) // Wait for the document to be replicated at SG btc.pushReplication.WaitForMessage(4) - resp = btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) - - var respBody db.Body - assert.NoError(t, base.JSONUnmarshal(resp.Body.Bytes(), &respBody)) + doc, err = collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) - assert.Equal(t, docID, respBody[db.BodyId]) - assert.Equal(t, "4-abc", respBody[db.BodyRev]) - greetings := respBody["greetings"].([]interface{}) + btc.RequireRev(t, expectedRev, doc) + body := doc.Body(ctx) + greetings := body["greetings"].([]interface{}) assert.Len(t, greetings, 1) assert.Equal(t, map[string]interface{}{"hi": "bob"}, greetings[0]) - attachments, ok := respBody[db.BodyAttachments].(map[string]interface{}) - require.True(t, ok) - assert.Len(t, attachments, 1) - hello, ok := attachments["hello.txt"].(map[string]interface{}) + assert.Len(t, doc.Attachments, 1) + hello, ok := doc.Attachments["hello.txt"].(map[string]interface{}) require.True(t, ok) assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) assert.Equal(t, float64(11), hello["length"]) - assert.Equal(t, float64(2), hello["revpos"]) + + // revpos should mach the generation of the original revision + assert.Equal(t, float64(attachmentRevPos), hello["revpos"]) assert.True(t, hello["stub"].(bool)) // Check the number of sendProveAttachment/sendGetAttachment calls. @@ -359,14 +368,14 @@ func TestBlipPushPullNewAttachmentNoCommonAncestor(t *testing.T) { GuestEnabled: true, } - const docID = "doc1" btcRunner := NewBlipTesterClientRunner(t) + const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) defer rt.Close() - opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols} + opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols, SourceID: "abc"} btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() btcRunner.StartPull(btc.id) @@ -375,37 +384,48 @@ func TestBlipPushPullNewAttachmentNoCommonAncestor(t *testing.T) { // rev tree pruning on the CBL side, so 1-abc no longer exists. // CBL replicates, sends to client as 4-abc history:[4-abc, 3-abc, 2-abc], attachment has revpos=2 bodyText := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` - err := btcRunner.StoreRevOnClient(btc.id, docID, "2-abc", []byte(bodyText)) + + rev := "2-abc" + if btc.UseHLV() { + rev = db.EncodeTestVersion("2@abc") + } + err := btcRunner.StoreRevOnClient(btc.id, docID, rev, []byte(bodyText)) require.NoError(t, err) bodyText = `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":2,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` - revId, err := btcRunner.PushRevWithHistory(btc.id, docID, "2-abc", []byte(bodyText), 2, 0) + currentRev, err := btcRunner.PushRevWithHistory(btc.id, docID, rev, []byte(bodyText), 2, 0) require.NoError(t, err) - assert.Equal(t, "4-abc", revId) + expectedRev := "4-abc" + if btc.UseHLV() { + expectedRev = db.EncodeTestVersion("4@abc") + } + assert.Equal(t, expectedRev, currentRev) // Wait for the document to be replicated at SG btc.pushReplication.WaitForMessage(2) - resp := btc.rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+revId, "") - assert.Equal(t, http.StatusOK, resp.Code) + collection, ctx := rt.GetSingleTestDatabaseCollection() + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) - var respBody db.Body - assert.NoError(t, base.JSONUnmarshal(resp.Body.Bytes(), &respBody)) + btc.RequireRev(t, expectedRev, doc) - assert.Equal(t, docID, respBody[db.BodyId]) - assert.Equal(t, "4-abc", respBody[db.BodyRev]) - greetings := respBody["greetings"].([]interface{}) + body := doc.Body(ctx) + greetings := body["greetings"].([]interface{}) assert.Len(t, greetings, 1) assert.Equal(t, map[string]interface{}{"hi": "alice"}, greetings[0]) - attachments, ok := respBody[db.BodyAttachments].(map[string]interface{}) - require.True(t, ok) - assert.Len(t, attachments, 1) - hello, ok := attachments["hello.txt"].(map[string]interface{}) + assert.Len(t, doc.Attachments, 1) + hello, ok := doc.Attachments["hello.txt"].(map[string]interface{}) require.True(t, ok) assert.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", hello["digest"]) assert.Equal(t, float64(11), hello["length"]) - assert.Equal(t, float64(4), hello["revpos"]) + + // revpos should match the generation of the current revision, since it's new to SGW with that revision. + // The actual revTreeID will differ when running this test as HLV client (multiple updates to HLV on client + // don't result in multiple revTree revisions) + expectedRevPos, _ := db.ParseRevID(ctx, doc.CurrentRev) + assert.Equal(t, float64(expectedRevPos), hello["revpos"]) assert.True(t, hello["stub"].(bool)) // Check the number of sendProveAttachment/sendGetAttachment calls. @@ -519,6 +539,7 @@ func TestPutAttachmentViaBlipGetViaBlip(t *testing.T) { // TestBlipAttachNameChange tests CBL handling - attachments with changed names are sent as stubs, and not new attachments func TestBlipAttachNameChange(t *testing.T) { base.SetUpTestLogging(t, base.LevelInfo, base.KeySync, base.KeySyncMsg, base.KeyWebSocket, base.KeyWebSocketFrame, base.KeyHTTP, base.KeyCRUD) + rtConfig := &RestTesterConfig{ GuestEnabled: true, } @@ -529,6 +550,7 @@ func TestBlipAttachNameChange(t *testing.T) { rt := NewRestTester(t, rtConfig) defer rt.Close() + docID := "doc" opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols} client1 := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer client1.Close() @@ -538,20 +560,20 @@ func TestBlipAttachNameChange(t *testing.T) { digest := db.Sha1DigestKey(attachmentA) // Push initial attachment data - version, err := btcRunner.PushRev(client1.id, "doc", EmptyDocVersion(), []byte(`{"key":"val","_attachments":{"attachment": {"data":"`+attachmentAData+`"}}}`)) + version, err := btcRunner.PushRev(client1.id, docID, EmptyDocVersion(), []byte(`{"key":"val","_attachments":{"attachment": {"data":"`+attachmentAData+`"}}}`)) require.NoError(t, err) // Confirm attachment is in the bucket - attachmentAKey := db.MakeAttachmentKey(2, "doc", digest) + attachmentAKey := db.MakeAttachmentKey(2, docID, digest) bucketAttachmentA, _, err := client1.rt.GetSingleDataStore().GetRaw(attachmentAKey) require.NoError(t, err) require.EqualValues(t, bucketAttachmentA, attachmentA) // Simulate changing only the attachment name over CBL // Use revpos 2 to simulate revpos bug in CBL 2.8 - 3.0.0 - version, err = btcRunner.PushRev(client1.id, "doc", version, []byte(`{"key":"val","_attachments":{"attach":{"revpos":2,"content_type":"","length":11,"stub":true,"digest":"`+digest+`"}}}`)) + version, err = btcRunner.PushRev(client1.id, docID, version, []byte(`{"key":"val","_attachments":{"attach":{"revpos":2,"content_type":"","length":11,"stub":true,"digest":"`+digest+`"}}}`)) require.NoError(t, err) - err = client1.rt.WaitForVersion("doc", version) + err = client1.rt.WaitForVersion(docID, version) require.NoError(t, err) // Check if attachment is still in bucket @@ -559,7 +581,7 @@ func TestBlipAttachNameChange(t *testing.T) { assert.NoError(t, err) assert.Equal(t, bucketAttachmentA, attachmentA) - resp := client1.rt.SendAdminRequest("GET", "/{{.keyspace}}/doc/attach", "") + resp := client1.rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"/attach", "") RequireStatus(t, resp, http.StatusOK) assert.Equal(t, attachmentA, resp.BodyBytes()) }) @@ -573,6 +595,7 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires legacy attachment upgrade to HLV (CBG-3806) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -592,10 +615,10 @@ func TestBlipLegacyAttachNameChange(t *testing.T) { CreateDocWithLegacyAttachment(t, client1.rt, docID, rawDoc, attKey, attBody) // Get the document and grab the revID. - docVersion, _ := client1.rt.GetDoc(docID) + docVersion := client1.GetDocVersion(docID) // Store the document and attachment on the test client - err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.RevID, rawDoc) + err := btcRunner.StoreRevOnClient(client1.id, docID, docVersion.GetRev(client1.UseHLV()), rawDoc) require.NoError(t, err) btcRunner.AttachmentsLock(client1.id).Lock() @@ -630,6 +653,7 @@ func TestBlipLegacyAttachDocUpdate(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // Requires legacy attachment upgrade to HLV (CBG-3806) btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) @@ -652,7 +676,7 @@ func TestBlipLegacyAttachDocUpdate(t *testing.T) { version, _ := client1.rt.GetDoc(docID) // Store the document and attachment on the test client - err := btcRunner.StoreRevOnClient(client1.id, docID, version.RevID, rawDoc) + err := btcRunner.StoreRevOnClient(client1.id, docID, version.RevTreeID, rawDoc) require.NoError(t, err) btcRunner.AttachmentsLock(client1.id).Lock() btcRunner.Attachments(client1.id)[digest] = attBody diff --git a/rest/blip_api_collections_test.go b/rest/blip_api_collections_test.go index 8dceac486a..5d26a824bd 100644 --- a/rest/blip_api_collections_test.go +++ b/rest/blip_api_collections_test.go @@ -258,7 +258,7 @@ func TestCollectionsReplication(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() - version := btc.rt.PutDoc(docID, "{}") + version := btc.rt.PutDocDirectly(docID, db.Body{}) btc.rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) @@ -284,10 +284,9 @@ func TestBlipReplicationMultipleCollections(t *testing.T) { docName := "doc1" body := `{"foo":"bar"}` versions := make([]DocVersion, 0, len(btc.rt.GetKeyspaces())) - for _, keyspace := range btc.rt.GetKeyspaces() { - resp := btc.rt.SendAdminRequest(http.MethodPut, "/"+keyspace+"/"+docName, `{"foo":"bar"}`) - RequireStatus(t, resp, http.StatusCreated) - versions = append(versions, DocVersionFromPutResponse(t, resp)) + for _, collection := range btc.rt.GetDbCollections() { + docVersion := rt.PutDocDirectlyInCollection(collection, docName, db.Body{"foo": "bar"}) + versions = append(versions, docVersion) } btc.rt.WaitForPendingChanges() @@ -326,7 +325,7 @@ func TestBlipReplicationMultipleCollectionsMismatchedDocSizes(t *testing.T) { collectionDocIDs := make(map[string][]string) collectionVersions := make(map[string][]DocVersion) require.Len(t, btc.rt.GetKeyspaces(), 2) - for i, keyspace := range btc.rt.GetKeyspaces() { + for i, collection := range btc.rt.GetDbCollections() { // intentionally create collections with different size replications to ensure one collection finishing won't cancel another one docCount := 10 if i == 0 { @@ -335,10 +334,8 @@ func TestBlipReplicationMultipleCollectionsMismatchedDocSizes(t *testing.T) { blipName := btc.rt.getCollectionsForBLIP()[i] for j := 0; j < docCount; j++ { docName := fmt.Sprintf("doc%d", j) - resp := btc.rt.SendAdminRequest(http.MethodPut, "/"+keyspace+"/"+docName, body) - RequireStatus(t, resp, http.StatusCreated) + version := rt.PutDocDirectlyInCollection(collection, docName, db.Body{"foo": "bar"}) - version := DocVersionFromPutResponse(t, resp) collectionVersions[blipName] = append(collectionVersions[blipName], version) collectionDocIDs[blipName] = append(collectionDocIDs[blipName], docName) } diff --git a/rest/blip_api_crud_test.go b/rest/blip_api_crud_test.go index 3143bba4b8..363d7f6406 100644 --- a/rest/blip_api_crud_test.go +++ b/rest/blip_api_crud_test.go @@ -17,6 +17,7 @@ import ( "log" "net/http" "net/url" + "reflect" "strconv" "strings" "sync" @@ -651,21 +652,21 @@ func TestProposedChangesIncludeConflictingRev(t *testing.T) { // Write existing docs to server directly (not via blip) rt := bt.restTester resp := rt.PutDoc("conflictingInsert", `{"version":1}`) - conflictingInsertRev := resp.RevID + conflictingInsertRev := resp.RevTreeID resp = rt.PutDoc("matchingInsert", `{"version":1}`) - matchingInsertRev := resp.RevID + matchingInsertRev := resp.RevTreeID resp = rt.PutDoc("conflictingUpdate", `{"version":1}`) - conflictingUpdateRev1 := resp.RevID - conflictingUpdateRev2 := rt.UpdateDocRev("conflictingUpdate", resp.RevID, `{"version":2}`) + conflictingUpdateRev1 := resp.RevTreeID + conflictingUpdateRev2 := rt.UpdateDocRev("conflictingUpdate", resp.RevTreeID, `{"version":2}`) resp = rt.PutDoc("matchingUpdate", `{"version":1}`) - matchingUpdateRev1 := resp.RevID - matchingUpdateRev2 := rt.UpdateDocRev("matchingUpdate", resp.RevID, `{"version":2}`) + matchingUpdateRev1 := resp.RevTreeID + matchingUpdateRev2 := rt.UpdateDocRev("matchingUpdate", resp.RevTreeID, `{"version":2}`) resp = rt.PutDoc("newUpdate", `{"version":1}`) - newUpdateRev1 := resp.RevID + newUpdateRev1 := resp.RevTreeID type proposeChangesCase struct { key string @@ -1719,6 +1720,109 @@ func TestPutRevConflictsMode(t *testing.T) { } +// TestPutRevV4: +// - Create blip tester to run with V4 protocol +// - Use send rev with CV defined in rev field and history field with PV/MV defined +// - Retrieve the doc from bucket and assert that the HLV is set to what has been sent over the blip tester +func TestPutRevV4(t *testing.T) { + base.SetUpTestLogging(t, base.LevelInfo, base.KeyHTTP, base.KeySync, base.KeySyncMsg) + + // Create blip tester with v4 protocol + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + connectingUsername: "user1", + connectingPassword: "1234", + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + require.NoError(t, err, "Unexpected error creating BlipTester") + defer bt.Close() + collection, _ := bt.restTester.GetSingleTestDatabaseCollection() + + docID := t.Name() + + // 1. Send rev with history + history := "1@def, 2@abc" + sent, _, resp, err := bt.SendRev(docID, db.EncodeTestVersion("3@efg"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(history)}) + assert.True(t, sent) + require.NoError(t, err) + assert.Equal(t, "", resp.Properties["Error-Code"]) + + // Validate against the bucket doc's HLV + doc, _, err := collection.GetDocWithXattrs(base.TestCtx(t), docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + pv, _ := db.ParseTestHistory(t, history) + db.RequireCVEqual(t, doc.HLV, "3@efg") + assert.Equal(t, doc.Cas, doc.HLV.CurrentVersionCAS) + assert.True(t, reflect.DeepEqual(pv, doc.HLV.PreviousVersions)) + + // 2. Update the document with a non-conflicting revision, where only cv is updated + sent, _, resp, err = bt.SendRev(docID, db.EncodeTestVersion("4@efg"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(history)}) + assert.True(t, sent) + require.NoError(t, err) + assert.Equal(t, "", resp.Properties["Error-Code"]) + + // Validate against the bucket doc's HLV + doc, _, err = collection.GetDocWithXattrs(base.TestCtx(t), docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + db.RequireCVEqual(t, doc.HLV, "4@efg") + assert.Equal(t, doc.Cas, doc.HLV.CurrentVersionCAS) + assert.True(t, reflect.DeepEqual(pv, doc.HLV.PreviousVersions)) + + // 3. Update the document again with a non-conflicting revision from a different source (previous cv moved to pv) + updatedHistory := "1@def, 2@abc, 4@efg" + sent, _, resp, err = bt.SendRev(docID, db.EncodeTestVersion("1@jkl"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(updatedHistory)}) + assert.True(t, sent) + require.NoError(t, err) + assert.Equal(t, "", resp.Properties["Error-Code"]) + + // Validate against the bucket doc's HLV + doc, _, err = collection.GetDocWithXattrs(base.TestCtx(t), docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + pv, _ = db.ParseTestHistory(t, updatedHistory) + db.RequireCVEqual(t, doc.HLV, "1@jkl") + assert.Equal(t, doc.Cas, doc.HLV.CurrentVersionCAS) + assert.True(t, reflect.DeepEqual(pv, doc.HLV.PreviousVersions)) + + // 4. Update the document again with a non-conflicting revision from a different source, and additional sources in history (previous cv moved to pv, and pv expanded) + updatedHistory = "1@def, 2@abc, 4@efg, 1@jkl, 1@mmm" + sent, _, resp, err = bt.SendRev(docID, db.EncodeTestVersion("1@nnn"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(updatedHistory)}) + assert.True(t, sent) + require.NoError(t, err) + assert.Equal(t, "", resp.Properties["Error-Code"]) + + // Validate against the bucket doc's HLV + doc, _, err = collection.GetDocWithXattrs(base.TestCtx(t), docID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + pv, _ = db.ParseTestHistory(t, updatedHistory) + db.RequireCVEqual(t, doc.HLV, "1@nnn") + assert.Equal(t, doc.Cas, doc.HLV.CurrentVersionCAS) + assert.True(t, reflect.DeepEqual(pv, doc.HLV.PreviousVersions)) + + // 5. Attempt to update the document again with a conflicting revision from a different source (previous cv not in pv), expect conflict + sent, _, resp, err = bt.SendRev(docID, db.EncodeTestVersion("1@pqr"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(updatedHistory)}) + assert.True(t, sent) + require.Error(t, err) + assert.Equal(t, "409", resp.Properties["Error-Code"]) + + // 6. Test sending rev with merge versions included in history (note new key) + newDocID := t.Name() + "_2" + mvHistory := "3@def, 3@abc; 1@def, 2@abc" + sent, _, resp, err = bt.SendRev(newDocID, db.EncodeTestVersion("3@efg"), []byte(`{"key": "val"}`), blip.Properties{"history": db.EncodeTestHistory(mvHistory)}) + assert.True(t, sent) + require.NoError(t, err) + assert.Equal(t, "", resp.Properties["Error-Code"]) + + // assert on bucket doc + doc, _, err = collection.GetDocWithXattrs(base.TestCtx(t), newDocID, db.DocUnmarshalNoHistory) + require.NoError(t, err) + + pv, mv := db.ParseTestHistory(t, mvHistory) + db.RequireCVEqual(t, doc.HLV, "3@efg") + assert.Equal(t, doc.Cas, doc.HLV.CurrentVersionCAS) + assert.True(t, reflect.DeepEqual(pv, doc.HLV.PreviousVersions)) + assert.True(t, reflect.DeepEqual(mv, doc.HLV.MergeVersions)) +} + // Repro attempt for SG #3281 // // - Set up a user w/ access to channel A @@ -1732,7 +1836,7 @@ func TestPutRevConflictsMode(t *testing.T) { // Actual: // - Same as Expected (this test is unable to repro SG #3281, but is being left in as a regression test) func TestGetRemovedDoc(t *testing.T) { - + t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)") base.SetUpTestLogging(t, base.LevelInfo, base.KeyHTTP, base.KeySync, base.KeySyncMsg) rt := NewRestTester(t, &RestTesterConfig{SyncFn: channels.DocChannelsSyncFunction}) @@ -1911,6 +2015,8 @@ func TestSendReplacementRevision(t *testing.T) { } btcRunner := NewBlipTesterClientRunner(t) + + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // requires cv in PUT rest response btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1921,17 +2027,19 @@ func TestSendReplacementRevision(t *testing.T) { defer rt.Close() docID := test.name - version1 := rt.PutDoc(docID, fmt.Sprintf(`{"foo":"bar","channels":["%s"]}`, rev1Channel)) + version1 := rt.PutDocDirectly(docID, JsonToMap(t, fmt.Sprintf(`{"foo":"bar","channels":["%s"]}`, rev1Channel))) updatedVersion := make(chan DocVersion) collection, ctx := rt.GetSingleTestDatabaseCollection() // underneath the client's response to changes - we'll update the document so the requested rev is not available by the time SG receives the changes response. changesEntryCallbackFn := func(changeEntryDocID, changeEntryRevID string) { - if changeEntryDocID == docID && changeEntryRevID == version1.RevID { + if changeEntryDocID == docID && changeEntryRevID == version1.RevTreeID { updatedVersion <- rt.UpdateDoc(docID, version1, fmt.Sprintf(`{"foo":"buzz","channels":["%s"]}`, test.replacementRevChannel)) // also purge revision backup and flush cache to ensure request for rev 1-... cannot be fulfilled - err := collection.PurgeOldRevisionJSON(ctx, docID, version1.RevID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + cvHash := base.Crc32cHashString([]byte(version1.CV.String())) + err := collection.PurgeOldRevisionJSON(ctx, docID, cvHash) require.NoError(t, err) rt.GetDatabase().FlushRevisionCacheForTest() } @@ -1959,15 +2067,15 @@ func TestSendReplacementRevision(t *testing.T) { _ = btcRunner.SingleCollection(btc.id).WaitForVersion(docID, version2) // rev message with a replacedRev property referring to the originally requested rev - msg2, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version2.RevID) + msg2, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version2.RevTreeID) require.True(t, ok) assert.Equal(t, db.MessageRev, msg2.Profile()) - assert.Equal(t, version2.RevID, msg2.Properties[db.RevMessageRev]) - assert.Equal(t, version1.RevID, msg2.Properties[db.RevMessageReplacedRev]) + assert.Equal(t, version2.RevTreeID, msg2.Properties[db.RevMessageRev]) + assert.Equal(t, version1.RevTreeID, msg2.Properties[db.RevMessageReplacedRev]) // the blip test framework records a message entry for the originally requested rev as well, but it should point to the message sent for rev 2 // this is an artifact of the test framework to make assertions for tests not explicitly testing replacement revs easier - msg1, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version1.RevID) + msg1, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version1.RevTreeID) require.True(t, ok) assert.Equal(t, msg1, msg2) @@ -1979,11 +2087,11 @@ func TestSendReplacementRevision(t *testing.T) { assert.Nil(t, data) // no message for rev 2 - _, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version2.RevID) + _, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version2.RevTreeID) require.False(t, ok) // norev message for the requested rev - msg, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version1.RevID) + msg, ok := btcRunner.SingleCollection(btc.id).GetBlipRevMessage(docID, version1.RevTreeID) require.True(t, ok) assert.Equal(t, db.MessageNoRev, msg.Profile()) @@ -2024,19 +2132,76 @@ func TestBlipPullRevMessageHistory(t *testing.T) { const docID = "doc1" // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version1 := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version1 := rt.PutDocDirectly(docID, db.Body{"hello": "world!"}) data := btcRunner.WaitForVersion(client.id, docID, version1) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) + assert.Equal(t, `{"hello":"world!"}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version2 := rt.UpdateDoc(docID, version1, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 12345678901234567890}]}`) + version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"hello": "alice"}) data = btcRunner.WaitForVersion(client.id, docID, version2) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":12345678901234567890}]}`, string(data)) + assert.Equal(t, `{"hello":"alice"}`, string(data)) msg := client.pullReplication.WaitForMessage(5) - assert.Equal(t, version1.RevID, msg.Properties[db.RevMessageHistory]) // CBG-3268 update to use version + client.AssertOnBlipHistory(t, msg, version1) + }) +} + +// TestPullReplicationUpdateOnOtherHLVAwarePeer: +// - Main purpose is to test if history is correctly populated on HLV aware replication +// - Making use of HLV agent to mock a doc from a HLV aware peer coming over replicator +// - Update this same doc through sync gateway then assert that the history is populated with the old current version +func TestPullReplicationUpdateOnOtherHLVAwarePeer(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) + rtConfig := RestTesterConfig{ + GuestEnabled: true, + } + btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[RevtreeSubtestName] = true // V4 replication only test + + btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { + rt := NewRestTester(t, &rtConfig) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + + opts := &BlipTesterClientOpts{SupportedBLIPProtocols: SupportedBLIPProtocols} + client := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) + defer client.Close() + + btcRunner.StartPull(client.id) + + const docID = "doc1" + otherSource := "otherSource" + hlvHelper := db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") + existingHLVKey := "doc1" + cas := hlvHelper.InsertWithHLV(ctx, existingHLVKey) + + // force import of this write + _, _ = rt.GetDoc(docID) + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, docID, db.DocUnmarshalAll) + require.NoError(t, err) + + // create doc version of the above doc write + version1 := DocVersion{ + RevTreeID: bucketDoc.CurrentRev, + CV: db.Version{ + SourceID: hlvHelper.Source, + Value: cas, + }, + } + + _ = btcRunner.WaitForVersion(client.id, docID, version1) + + // update the above doc + version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"hello": "world!"}) + + data := btcRunner.WaitForVersion(client.id, docID, version2) + assert.Equal(t, `{"hello":"world!"}`, string(data)) + + // assert that history in blip properties is correct + msg := client.pullReplication.WaitForMessage(5) + client.AssertOnBlipHistory(t, msg, version1) }) } @@ -2057,7 +2222,7 @@ func TestActiveOnlyContinuous(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer btc.Close() - version := rt.PutDoc(docID, `{"test":true}`) + version := rt.PutDocDirectly(docID, db.Body{"test": true}) // start an initial pull btcRunner.StartPullSince(btc.id, BlipTesterPullOptions{Continuous: true, Since: "0", ActiveOnly: true}) @@ -2065,7 +2230,7 @@ func TestActiveOnlyContinuous(t *testing.T) { assert.Equal(t, `{"test":true}`, string(rev)) // delete the doc and make sure the client still gets the tombstone replicated - deletedVersion := rt.DeleteDocReturnVersion(docID, version) + deletedVersion := rt.DeleteDocDirectly(docID, version) rev = btcRunner.WaitForVersion(btc.id, docID, deletedVersion) assert.Equal(t, `{}`, string(rev)) @@ -2148,7 +2313,7 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { defer btc.Close() const docID = "doc" - version := rt.PutDoc(docID, `{"channels": ["A", "B"]}`) + version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"A", "B"}}) changes, err := rt.WaitForChanges(1, "/{{.keyspace}}/_changes?since=0&revocations=true", "user", true) require.NoError(t, err) @@ -2159,7 +2324,7 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDoc(docID, version, `{"channels": ["B"]}`) + version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{"B"}}) changes, err = rt.WaitForChanges(1, fmt.Sprintf("/{{.keyspace}}/_changes?since=%s&revocations=true", changes.Last_Seq), "user", true) require.NoError(t, err) @@ -2170,9 +2335,9 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDoc(docID, version, `{"channels": []}`) + version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{}}) const docMarker = "docmarker" - docMarkerVersion := rt.PutDoc(docMarker, `{"channels": ["!"]}`) + docMarkerVersion := rt.PutDocDirectly(docMarker, db.Body{"channels": []string{"!"}}) changes, err = rt.WaitForChanges(2, fmt.Sprintf("/{{.keyspace}}/_changes?since=%s&revocations=true", changes.Last_Seq), "user", true) require.NoError(t, err) @@ -2254,7 +2419,7 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi const ( docID = "doc" ) - version := rt.PutDoc(docID, `{"channels": ["A", "B"]}`) + version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"A", "B"}}) changes, err := rt.WaitForChanges(1, "/{{.keyspace}}/_changes?since=0&revocations=true", "user", true) require.NoError(t, err) @@ -2265,8 +2430,9 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDoc(docID, version, `{"channels": ["C"]}`) + version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{"C"}}) rt.WaitForPendingChanges() + // At this point changes should send revocation, as document isn't in any of the user's channels changes, err = rt.WaitForChanges(1, "/{{.keyspace}}/_changes?filter=sync_gateway/bychannel&channels=A&since=0&revocations=true", "user", true) require.NoError(t, err) @@ -2279,7 +2445,7 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi _ = rt.UpdateDoc(docID, version, `{"channels": ["B"]}`) markerID := "docmarker" - markerVersion := rt.PutDoc(markerID, `{"channels": ["A"]}`) + markerVersion := rt.PutDocDirectly(markerID, db.Body{"channels": []string{"A"}}) rt.WaitForPendingChanges() // Revocation should not be sent over blip, as document is now in user's channels - only marker document should be received @@ -2741,7 +2907,7 @@ func TestSendRevisionNoRevHandling(t *testing.T) { recievedNoRevs <- msg } - version := rt.PutDoc(docName, `{"foo":"bar"}`) + version := rt.PutDocDirectly(docName, db.Body{"foo": "bar"}) // Make the LeakyBucket return an error leakyDataStore.SetGetRawCallback(func(key string) error { @@ -2803,7 +2969,7 @@ func TestUnsubChanges(t *testing.T) { // Sub changes btcRunner.StartPull(btc.id) - doc1Version := rt.PutDoc(doc1ID, `{"key":"val1"}`) + doc1Version := rt.PutDocDirectly(doc1ID, db.Body{"key": "val1"}) _ = btcRunner.WaitForVersion(btc.id, doc1ID, doc1Version) activeReplStat := rt.GetDatabase().DbStats.CBLReplicationPull().NumPullReplActiveContinuous @@ -2817,7 +2983,7 @@ func TestUnsubChanges(t *testing.T) { base.RequireWaitForStat(t, activeReplStat.Value, 0) // Confirm no more changes are being sent - doc2Version := rt.PutDoc(doc2ID, `{"key":"val1"}`) + doc2Version := rt.PutDocDirectly(doc2ID, db.Body{"key": "val1"}) err = rt.WaitForConditionWithOptions(func() bool { _, found := btcRunner.GetVersion(btc.id, "doc2", doc2Version) return found @@ -2992,7 +3158,7 @@ func TestBlipRefreshUser(t *testing.T) { // add chan1 explicitly rt.CreateUser(username, []string{"chan1"}) - version := rt.PutDoc(docID, `{"channels":["chan1"]}`) + version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"chan1"}}) // Start a regular one-shot pull btcRunner.StartPullSince(btc.id, BlipTesterPullOptions{Continuous: true, Since: "0"}) @@ -3038,6 +3204,7 @@ func TestOnDemandImportBlipFailure(t *testing.T) { } base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeyCache, base.KeyChanges) btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // CBG-4166 btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { syncFn := `function(doc) { if (doc.invalid) { @@ -3143,7 +3310,7 @@ func TestOnDemandImportBlipFailure(t *testing.T) { btcRunner.WaitForDoc(btc2.id, markerDoc) // Validate that the latest client message for the requested doc/rev was a norev - msg, ok := btcRunner.SingleCollection(btc2.id).GetBlipRevMessage(docID, revID.RevID) + msg, ok := btcRunner.SingleCollection(btc2.id).GetBlipRevMessage(docID, revID.RevTreeID) require.True(t, ok) require.Equal(t, db.MessageNoRev, msg.Profile()) @@ -3185,3 +3352,25 @@ func TestBlipDatabaseClose(t *testing.T) { }, time.Second*10, time.Millisecond*100) }) } + +func TestPutRevBlip(t *testing.T) { + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{GuestEnabled: true, blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}}) + require.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + + _, _, _, err = bt.SendRev( + "foo", + "2@stZPWD8vS/O3nsx9yb2Brw", + []byte(`{"key": "val"}`), + blip.Properties{}, + ) + require.NoError(t, err) + + _, _, _, err = bt.SendRev( + "foo", + "fa1@stZPWD8vS/O3nsx9yb2Brw", + []byte(`{"key": "val2"}`), + blip.Properties{}, + ) + require.NoError(t, err) +} diff --git a/rest/blip_api_delta_sync_test.go b/rest/blip_api_delta_sync_test.go index 8600b61b07..f2bce3309d 100644 --- a/rest/blip_api_delta_sync_test.go +++ b/rest/blip_api_delta_sync_test.go @@ -23,10 +23,8 @@ import ( ) // TestBlipDeltaSyncPushAttachment tests updating a doc that has an attachment with a delta that doesn't modify the attachment. - func TestBlipDeltaSyncPushAttachment(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) - if !base.IsEnterpriseEdition() { t.Skip("Delta test requires EE") } @@ -42,6 +40,7 @@ func TestBlipDeltaSyncPushAttachment(t *testing.T) { const docID = "pushAttachmentDoc" btcRunner := NewBlipTesterClientRunner(t) + btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, rtConfig) defer rt.Close() @@ -126,7 +125,8 @@ func TestBlipDeltaSyncPushPullNewAttachment(t *testing.T) { // Create doc1 rev 1-77d9041e49931ceef58a1eef5fd032e8 on SG with an attachment bodyText := `{"greetings":[{"hi": "alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` - version := rt.PutDoc(docID, bodyText) + // put doc directly needs to be here + version := rt.PutDocDirectly(docID, JsonToMap(t, bodyText)) data := btcRunner.WaitForVersion(btc.id, docID, version) bodyTextExpected := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":1,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` @@ -195,15 +195,15 @@ func TestBlipDeltaSyncNewAttachmentPull(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDoc(doc1ID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version := rt.PutDocDirectly(doc1ID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) data := btcRunner.WaitForVersion(client.id, doc1ID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-10000d5ec533b29b117e60274b1e3653 on SG with the first attachment - version = rt.UpdateDoc(doc1ID, version, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}], "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`) + version2 := rt.UpdateDocDirectly(doc1ID, version, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}], "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`)) - data = btcRunner.WaitForVersion(client.id, doc1ID, version) + data = btcRunner.WaitForVersion(client.id, doc1ID, version2) var dataMap map[string]interface{} assert.NoError(t, base.JSONUnmarshal(data, &dataMap)) atts, ok := dataMap[db.BodyAttachments].(map[string]interface{}) @@ -222,11 +222,12 @@ func TestBlipDeltaSyncNewAttachmentPull(t *testing.T) { // Check EE is delta, and CE is full-body replication // msg, ok = client.pullReplication.WaitForMessage(5) - msg = btcRunner.WaitForBlipRevMessage(client.id, doc1ID, version) - - if base.IsEnterpriseEdition() { + msg = btcRunner.WaitForBlipRevMessage(client.id, doc1ID, version2) + // Delta sync only works for Version vectors, CBG-3748 (backwards compatibility for revID) + sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV() + if sgCanUseDeltas { // Check the request was sent with the correct deltaSrc property - assert.Equal(t, "1-0335a345b6ffed05707ccc4cbc1b67f4", msg.Properties[db.RevMessageDeltaSrc]) + client.AssertDeltaSrcProperty(t, msg, version) // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) @@ -238,11 +239,14 @@ func TestBlipDeltaSyncNewAttachmentPull(t *testing.T) { msgBody, err := msg.Body() assert.NoError(t, err) assert.NotEqual(t, `{"_attachments":[{"hello.txt":{"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=","length":11,"revpos":2,"stub":true}}]}`, string(msgBody)) - assert.Contains(t, string(msgBody), `"_attachments":{"hello.txt":{"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=","length":11,"revpos":2,"stub":true}}`) + assert.Contains(t, string(msgBody), `"stub":true`) + assert.Contains(t, string(msgBody), `"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="`) + assert.Contains(t, string(msgBody), `"revpos":2`) + assert.Contains(t, string(msgBody), `"length":11`) assert.Contains(t, string(msgBody), `"greetings":[{"hello":"world!"},{"hi":"alice"}]`) } - respBody := rt.GetDocVersion(doc1ID, version) + respBody := rt.GetDocVersion(doc1ID, version2) assert.Equal(t, doc1ID, respBody[db.BodyId]) greetings := respBody["greetings"].([]interface{}) assert.Len(t, greetings, 2) @@ -293,26 +297,28 @@ func TestBlipDeltaSyncPull(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version = rt.UpdateDoc(docID, version, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 12345678901234567890}]}`) + version2 := rt.UpdateDocDirectly(docID, version, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`)) - data = btcRunner.WaitForVersion(client.id, docID, version) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":12345678901234567890}]}`, string(data)) - msg := btcRunner.WaitForBlipRevMessage(client.id, docID, version) + data = btcRunner.WaitForVersion(client.id, docID, version2) + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(data)) + msg := btcRunner.WaitForBlipRevMessage(client.id, docID, version2) // Check EE is delta, and CE is full-body replication - if base.IsEnterpriseEdition() { + // Delta sync only works for Version vectors, CBG-3748 (backwards compatibility for revID) + sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV() + if sgCanUseDeltas { // Check the request was sent with the correct deltaSrc property - assert.Equal(t, "1-0335a345b6ffed05707ccc4cbc1b67f4", msg.Properties[db.RevMessageDeltaSrc]) + client.AssertDeltaSrcProperty(t, msg, version) // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) - assert.Equal(t, `{"greetings":{"2-":[{"howdy":12345678901234567890}]}}`, string(msgBody)) + assert.Equal(t, `{"greetings":{"2-":[{"howdy":1234567890123}]}}`, string(msgBody)) assert.Equal(t, deltaSentCount+1, rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value()) } else { // Check the request was NOT sent with a deltaSrc property @@ -320,8 +326,8 @@ func TestBlipDeltaSyncPull(t *testing.T) { // Check the request body was NOT the delta msgBody, err := msg.Body() assert.NoError(t, err) - assert.NotEqual(t, `{"greetings":{"2-":[{"howdy":12345678901234567890}]}}`, string(msgBody)) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":12345678901234567890}]}`, string(msgBody)) + assert.NotEqual(t, `{"greetings":{"2-":[{"howdy":1234567890123}]}}`, string(msgBody)) + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(msgBody)) var afterDeltaSyncCount int64 if rt.GetDatabase().DbStats.DeltaSync() != nil { @@ -351,6 +357,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { GuestEnabled: true, } btcRunner := NewBlipTesterClientRunner(t) + btcRunner.SkipSubtest[RevtreeSubtestName] = true // delta sync not implemented for rev tree replication, CBG-3748 btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { rt := NewRestTester(t, &rtConfig) @@ -358,7 +365,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { docID := "doc1" // create doc1 rev 1 - docVersion1 := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + docVersion1 := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) deltaSentCount := rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value() @@ -367,7 +374,11 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { defer client.Close() // reject deltas built ontop of rev 1 - client.rejectDeltasForSrcRev = docVersion1.RevID + if client.UseHLV() { + client.rejectDeltasForSrcRev = docVersion1.CV.String() + } else { + client.rejectDeltasForSrcRev = docVersion1.RevTreeID + } client.ClientDeltas = true btcRunner.StartPull(client.id) @@ -375,19 +386,20 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2 - docVersion2 := rt.UpdateDoc(docID, docVersion1, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 12345678901234567890}]}`) + docVersion2 := rt.UpdateDocDirectly(docID, docVersion1, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`)) data = btcRunner.WaitForVersion(client.id, docID, docVersion2) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":12345678901234567890}]}`, string(data)) + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(data)) msg := client.pullReplication.WaitForMessage(5) // Check the request was initially sent with the correct deltaSrc property - assert.Equal(t, docVersion1.RevID, msg.Properties[db.RevMessageDeltaSrc]) + client.AssertDeltaSrcProperty(t, msg, docVersion1) + // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) - assert.Equal(t, `{"greetings":{"2-":[{"howdy":12345678901234567890}]}}`, string(msgBody)) + assert.Equal(t, `{"greetings":{"2-":[{"howdy":1234567890123}]}}`, string(msgBody)) assert.Equal(t, deltaSentCount+1, rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value()) msg = btcRunner.WaitForBlipRevMessage(client.id, docID, docVersion2) @@ -397,8 +409,8 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { // Check the request body was NOT the delta msgBody, err = msg.Body() assert.NoError(t, err) - assert.NotEqual(t, `{"greetings":{"2-":[{"howdy":12345678901234567890}]}}`, string(msgBody)) - assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":12345678901234567890}]}`, string(msgBody)) + assert.NotEqual(t, `{"greetings":{"2-":[{"howdy":1234567890123}]}}`, string(msgBody)) + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(msgBody)) }) } @@ -419,7 +431,7 @@ func TestBlipDeltaSyncPullRemoved(t *testing.T) { SyncFn: channels.DocChannelsSyncFunction, } btcRunner := NewBlipTesterClientRunner(t) - btcRunner.SkipVersionVectorInitialization = true // v2 protocol test + btcRunner.SkipSubtest[VersionVectorSubtestName] = true // test requires v2 subprotocol const docID = "doc1" btcRunner.Run(func(t *testing.T, SupportedBLIPProtocols []string) { @@ -438,14 +450,14 @@ func TestBlipDeltaSyncPullRemoved(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-1513b53e2738671e634d9dd111f48de0 - version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // create doc1 rev 2-ff91e11bc1fd12bbb4815a06571859a9 - version = rt.UpdateDoc(docID, version, `{"channels": ["private"], "greetings": [{"hello": "world!"}, {"hi": "bob"}]}`) + version = rt.UpdateDocDirectly(docID, version, JsonToMap(t, `{"channels": ["private"], "greetings": [{"hello": "world!"}, {"hi": "bob"}]}`)) data = btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"_removed":true}`, string(data)) @@ -513,13 +525,13 @@ func TestBlipDeltaSyncPullTombstoned(t *testing.T) { const docID = "doc1" // create doc1 rev 1-e89945d756a1d444fa212bffbbb31941 - version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // tombstone doc1 at rev 2-2db70833630b396ef98a3ec75b3e90fc - version = rt.DeleteDocReturnVersion(docID, version) + version = rt.DeleteDocDirectly(docID, version) data = btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{}`, string(data)) @@ -541,8 +553,9 @@ func TestBlipDeltaSyncPullTombstoned(t *testing.T) { deltasRequestedEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasRequested.Value() deltasSentEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value() } - - if sgUseDeltas { + // delta sync not implemented for rev tree replication, CBG-3748 + sgCanUseDelta := base.IsEnterpriseEdition() && client.UseHLV() + if sgCanUseDelta { assert.Equal(t, deltaCacheHitsStart, deltaCacheHitsEnd) assert.Equal(t, deltaCacheMissesStart+1, deltaCacheMissesEnd) assert.Equal(t, deltasRequestedStart+1, deltasRequestedEnd) @@ -614,7 +627,7 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) { btcRunner.StartPull(client1.id) // create doc1 rev 1-e89945d756a1d444fa212bffbbb31941 - version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) data := btcRunner.WaitForVersion(client1.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) @@ -627,7 +640,7 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) { assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // tombstone doc1 at rev 2-2db70833630b396ef98a3ec75b3e90fc - version = rt.DeleteDocReturnVersion(docID, version) + version = rt.DeleteDocDirectly(docID, version) data = btcRunner.WaitForVersion(client1.id, docID, version) assert.Equal(t, `{}`, string(data)) @@ -681,7 +694,9 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) { deltasSentEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value() } - if sgUseDeltas { + // delta sync not implemented for rev tree replication, CBG-3748 + sgCanUseDelta := base.IsEnterpriseEdition() && client1.UseHLV() + if sgCanUseDelta { assert.Equal(t, deltaCacheHitsStart+1, deltaCacheHitsEnd) assert.Equal(t, deltaCacheMissesStart+1, deltaCacheMissesEnd) assert.Equal(t, deltasRequestedStart+2, deltasRequestedEnd) @@ -727,10 +742,11 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { defer client.Close() client.ClientDeltas = true + sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV() btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version1 := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version1 := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version1) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) @@ -745,7 +761,7 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version2 := rt.UpdateDoc(docID, version1, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": "bob"}]}`) + version2 := rt.UpdateDocDirectly(docID, version1, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": "bob"}]}`)) data = btcRunner.WaitForVersion(client.id, docID, version2) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":"bob"}]}`, string(data)) @@ -753,11 +769,20 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { // Check EE is delta // Check the request was sent with the correct deltaSrc property - assert.Equal(t, "1-0335a345b6ffed05707ccc4cbc1b67f4", msg.Properties[db.RevMessageDeltaSrc]) + // delta sync not implemented for rev tree replication, CBG-3748 + if sgCanUseDeltas { + client.AssertDeltaSrcProperty(t, msg, version1) + } else { + assert.Equal(t, "", msg.Properties[db.RevMessageDeltaSrc]) + } // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) - assert.Equal(t, `{"greetings":{"2-":[{"howdy":"bob"}]}}`, string(msgBody)) + if sgCanUseDeltas { + assert.Equal(t, `{"greetings":{"2-":[{"howdy":"bob"}]}}`, string(msgBody)) + } else { + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":"bob"}]}`, string(msgBody)) + } deltaCacheHits := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheHit.Value() deltaCacheMisses := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheMiss.Value() @@ -768,17 +793,31 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { msg2 := btcRunner.WaitForBlipRevMessage(client2.id, docID, version2) // Check the request was sent with the correct deltaSrc property - assert.Equal(t, "1-0335a345b6ffed05707ccc4cbc1b67f4", msg2.Properties[db.RevMessageDeltaSrc]) + if sgCanUseDeltas { + client2.AssertDeltaSrcProperty(t, msg2, version1) + } else { + assert.Equal(t, "", msg2.Properties[db.RevMessageDeltaSrc]) + } // Check the request body was the actual delta msgBody2, err := msg2.Body() assert.NoError(t, err) - assert.Equal(t, `{"greetings":{"2-":[{"howdy":"bob"}]}}`, string(msgBody2)) + if sgCanUseDeltas { + assert.Equal(t, `{"greetings":{"2-":[{"howdy":"bob"}]}}`, string(msgBody2)) + } else { + assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":"bob"}]}`, string(msgBody2)) + } updatedDeltaCacheHits := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheHit.Value() updatedDeltaCacheMisses := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheMiss.Value() - assert.Equal(t, deltaCacheHits+1, updatedDeltaCacheHits) - assert.Equal(t, deltaCacheMisses, updatedDeltaCacheMisses) + // delta sync not implemented for rev tree replication, CBG-3748 + if sgCanUseDeltas { + assert.Equal(t, deltaCacheHits+1, updatedDeltaCacheHits) + assert.Equal(t, deltaCacheMisses, updatedDeltaCacheMisses) + } else { + assert.Equal(t, deltaCacheHits, updatedDeltaCacheHits) + assert.Equal(t, deltaCacheMisses, updatedDeltaCacheMisses) + } }) } @@ -809,11 +848,12 @@ func TestBlipDeltaSyncPush(t *testing.T) { client := btcRunner.NewBlipTesterClientOptsWithRT(rt, opts) defer client.Close() client.ClientDeltas = true + sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV() btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) @@ -824,9 +864,9 @@ func TestBlipDeltaSyncPush(t *testing.T) { // Check EE is delta, and CE is full-body replication msg := client.waitForReplicationMessage(collection, 2) - if base.IsEnterpriseEdition() { + if base.IsEnterpriseEdition() && sgCanUseDeltas { // Check the request was sent with the correct deltaSrc property - assert.Equal(t, "1-0335a345b6ffed05707ccc4cbc1b67f4", msg.Properties[db.RevMessageDeltaSrc]) + client.AssertDeltaSrcProperty(t, msg, version) // Check the request body was the actual delta msgBody, err := msg.Body() assert.NoError(t, err) @@ -834,7 +874,7 @@ func TestBlipDeltaSyncPush(t *testing.T) { collection, ctx := rt.GetSingleTestDatabaseCollection() // Validate that generation of a delta didn't mutate the revision body in the revision cache - docRev, cacheErr := collection.GetRevisionCacheForTest().Get(ctx, "doc1", "1-0335a345b6ffed05707ccc4cbc1b67f4", db.RevCacheOmitDelta) + docRev, cacheErr := collection.GetRevisionCacheForTest().GetWithRev(ctx, "doc1", "1-0335a345b6ffed05707ccc4cbc1b67f4", db.RevCacheOmitDelta) assert.NoError(t, cacheErr) assert.NotContains(t, docRev.BodyBytes, "bob") } else { @@ -856,7 +896,7 @@ func TestBlipDeltaSyncPush(t *testing.T) { assert.Equal(t, map[string]interface{}{"howdy": "bob"}, greetings[2]) // tombstone doc1 (gets rev 3-f3be6c85e0362153005dae6f08fc68bb) - deletedVersion := rt.DeleteDocReturnVersion(docID, newRev) + deletedVersion := rt.DeleteDocDirectly(docID, newRev) data = btcRunner.WaitForVersion(client.id, docID, deletedVersion) assert.Equal(t, `{}`, string(data)) @@ -869,7 +909,7 @@ func TestBlipDeltaSyncPush(t *testing.T) { _, err = btcRunner.PushRev(client.id, docID, deletedVersion, []byte(`{"undelete":true}`)) - if base.IsEnterpriseEdition() { + if base.IsEnterpriseEdition() && sgCanUseDeltas { // Now make the client push up a delta that has the parent of the tombstone. // This is not a valid scenario, and is actively prevented on the CBL side. assert.Error(t, err) @@ -919,7 +959,7 @@ func TestBlipNonDeltaSyncPush(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) + version := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) diff --git a/rest/blip_legacy_revid_test.go b/rest/blip_legacy_revid_test.go new file mode 100644 index 0000000000..6c99a96859 --- /dev/null +++ b/rest/blip_legacy_revid_test.go @@ -0,0 +1,843 @@ +/* +Copyright 2024-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package rest + +import ( + "encoding/json" + "log" + "strings" + "sync" + "testing" + "time" + + "github.com/couchbase/go-blip" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLegacyProposeChanges: +// - Build propose changes request of docs that are all new to SGW in legacy format +// - Assert that the response is as expected (empty response) +func TestLegacyProposeChanges(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + + proposeChangesRequest := bt.newRequest() + proposeChangesRequest.SetProfile("proposeChanges") + proposeChangesRequest.SetCompressed(true) + + changesBody := ` +[["foo", "1-abc"], +["foo2", "1-abc"]] +` + proposeChangesRequest.SetBody([]byte(changesBody)) + sent := bt.sender.Send(proposeChangesRequest) + assert.True(t, sent) + proposeChangesResponse := proposeChangesRequest.Response() + body, err := proposeChangesResponse.Body() + require.NoError(t, err) + + var changeList [][]interface{} + err = base.JSONUnmarshal(body, &changeList) + require.NoError(t, err) + + assert.Len(t, changeList, 0) +} + +// TestProposeChangesHandlingWithExistingRevs: +// - Build up propose changes request for conflicting and non conflicting docs with legacy revs +// - Assert that the response sent from SGW is as expected +func TestProposeChangesHandlingWithExistingRevs(t *testing.T) { + base.SetUpTestLogging(t, base.LevelInfo, base.KeyHTTP, base.KeySync, base.KeySyncMsg) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + + resp := rt.PutDoc("conflictingInsert", `{"version":1}`) + conflictingInsertRev := resp.RevTreeID + + resp = rt.PutDoc("conflictingUpdate", `{"version":1}`) + conflictingUpdateRev1 := resp.RevTreeID + conflictingUpdateRev2 := rt.UpdateDocRev("conflictingUpdate", resp.RevTreeID, `{"version":2}`) + + resp = rt.PutDoc("newUpdate", `{"version":1}`) + newUpdateRev1 := resp.RevTreeID + + resp = rt.PutDoc("existingDoc", `{"version":1}`) + existingDocRev := resp.RevTreeID + + type proposeChangesCase struct { + key string + revID string + parentRevID string + expectedValue interface{} + } + + proposeChangesCases := []proposeChangesCase{ + proposeChangesCase{ + key: "conflictingInsert", + revID: "1-abc", + parentRevID: "", + expectedValue: map[string]interface{}{"status": float64(db.ProposedRev_Conflict), "rev": conflictingInsertRev}, + }, + proposeChangesCase{ + key: "newInsert", + revID: "1-abc", + parentRevID: "", + expectedValue: float64(db.ProposedRev_OK), + }, + proposeChangesCase{ + key: "conflictingUpdate", + revID: "2-abc", + parentRevID: conflictingUpdateRev1, + expectedValue: map[string]interface{}{"status": float64(db.ProposedRev_Conflict), "rev": conflictingUpdateRev2}, + }, + proposeChangesCase{ + key: "newUpdate", + revID: "2-abc", + parentRevID: newUpdateRev1, + expectedValue: float64(db.ProposedRev_OK), + }, + proposeChangesCase{ + key: "existingDoc", + revID: existingDocRev, + parentRevID: "", + expectedValue: float64(db.ProposedRev_Exists), + }, + } + + proposeChangesRequest := bt.newRequest() + proposeChangesRequest.SetProfile("proposeChanges") + proposeChangesRequest.SetCompressed(true) + proposeChangesRequest.Properties[db.ProposeChangesConflictsIncludeRev] = "true" + + proposedChanges := make([][]interface{}, 0) + for _, c := range proposeChangesCases { + changeEntry := []interface{}{ + c.key, + c.revID, + } + if c.parentRevID != "" { + changeEntry = append(changeEntry, c.parentRevID) + } + proposedChanges = append(proposedChanges, changeEntry) + } + proposeChangesBody, marshalErr := json.Marshal(proposedChanges) + require.NoError(t, marshalErr) + + proposeChangesRequest.SetBody(proposeChangesBody) + sent := bt.sender.Send(proposeChangesRequest) + assert.True(t, sent) + proposeChangesResponse := proposeChangesRequest.Response() + bodyReader, err := proposeChangesResponse.BodyReader() + require.NoError(t, err) + + var changeList []interface{} + decoder := base.JSONDecoder(bodyReader) + decodeErr := decoder.Decode(&changeList) + require.NoError(t, decodeErr) + + for i, entry := range changeList { + assert.Equal(t, proposeChangesCases[i].expectedValue, entry) + } +} + +// TestProcessLegacyRev: +// - Create doc on SGW +// - Push new revision of this doc form client in legacy rev mode +// - Assert that the new doc is created and given a new source version pair +// - Send a new rev that SGW hasn;t yet seen unsolicited and assert that the doc is added correctly and given a source version pair +func TestProcessLegacyRev(t *testing.T) { + base.SetUpTestLogging(t, base.LevelInfo, base.KeyHTTP, base.KeySync, base.KeySyncMsg) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, _ := rt.GetSingleTestDatabaseCollection() + + // add doc to SGW + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + // Send another rev of same doc + history := []string{rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", "2-bcd", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + assert.NoError(t, err) + require.NoError(t, rt.WaitForVersion("doc1", DocVersion{RevTreeID: "2-bcd"})) + + // assert we can fetch this doc rev + resp := rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1?rev=2-bcd", "") + RequireStatus(t, resp, 200) + + // assert this legacy doc has been given source version pair + docSource, docVrs := collection.GetDocumentCurrentVersion(t, "doc1") + assert.Equal(t, docVersion.CV.SourceID, docSource) + assert.NotEqual(t, docVersion.CV.Value, docVrs) + + // try new rev to process + _, _, _, err = bt.SendRev( + "foo", + "1-abc", + []byte(`{"key": "val"}`), + blip.Properties{}, + ) + assert.NoError(t, err) + + require.NoError(t, rt.WaitForVersion("foo", DocVersion{RevTreeID: "1-abc"})) + // assert we can fetch this doc rev + resp = rt.SendAdminRequest("GET", "/{{.keyspace}}/foo?rev=1-abc", "") + RequireStatus(t, resp, 200) + + // assert this legacy doc has been given source version pair + docSource, docVrs = collection.GetDocumentCurrentVersion(t, "doc1") + assert.NotEqual(t, "", docSource) + assert.NotEqual(t, uint64(0), docVrs) +} + +// TestProcessRevWithLegacyHistory: +// - 1. CBL sends rev=1010@CBL1, history=1-abc when SGW has current rev 1-abc (document underwent an update before being pushed to SGW) +// - 2. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 1-abc (document underwent multiple p2p updates before being pushed to SGW) +// - 3. CBL sends rev=1010@CBL1, history=1000@CBL2,2-abc,1-abc when SGW has current rev 1-abc (document underwent multiple legacy and p2p updates before being pushed to SGW) +// - 4. CBL sends rev=1010@CBL1, history=1-abc when SGW does not have the doc (document underwent multiple legacy and p2p updates before being pushed to SGW) +// - Assert that the bucket doc resulting on each operation is as expected +func TestProcessRevWithLegacyHistory(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + ds := rt.GetSingleDataStore() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + const ( + docID = "doc1" + docID2 = "doc2" + docID3 = "doc3" + docID4 = "doc4" + ) + + // 1. CBL sends rev=1010@CBL1, history=1-abc when SGW has current rev 1-abc (document underwent an update before being pushed to SGW) + docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, docID, []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + // Have CBL send an update to that doc, with history in revTreeID format + history := []string{rev1ID} + sent, _, _, err := bt.SendRevWithHistory(docID, "1000@CBL1", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, docID, db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, "1000@CBL1", bucketDoc.HLV.GetCurrentVersionString()) + assert.NotNil(t, bucketDoc.History[rev1ID]) + + // 2. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 1-abc (document underwent multiple p2p updates before being pushed to SGW) + docVersion = rt.PutDocDirectly(docID2, db.Body{"test": "doc"}) + rev1ID = docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, docID2, []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + // Have CBL send an update to that doc, with history in HLV + revTreeID format + history = []string{"1000@CBL2", rev1ID} + sent, _, _, err = bt.SendRevWithHistory(docID2, "1001@CBL1", history, []byte(`{"some": "update"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err = collection.GetDocWithXattrs(ctx, docID2, db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, "1001@CBL1", bucketDoc.HLV.GetCurrentVersionString()) + assert.Equal(t, uint64(4096), bucketDoc.HLV.PreviousVersions["CBL2"]) + assert.NotNil(t, bucketDoc.History[rev1ID]) + + // 3. CBL sends rev=1010@CBL1, history=1000@CBL2,2-abc,1-abc when SGW has current rev 1-abc (document underwent multiple legacy and p2p updates before being pushed to SGW) + docVersion = rt.PutDocDirectly(docID3, db.Body{"test": "doc"}) + rev1ID = docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, docID3, []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + history = []string{"1000@CBL2", "2-abc", rev1ID} + sent, _, _, err = bt.SendRevWithHistory(docID3, "1010@CBL1", history, []byte(`{"some": "update"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err = collection.GetDocWithXattrs(ctx, docID3, db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, "1010@CBL1", bucketDoc.HLV.GetCurrentVersionString()) + assert.Equal(t, uint64(4096), bucketDoc.HLV.PreviousVersions["CBL2"]) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.NotNil(t, bucketDoc.History["2-abc"]) + + // 4. CBL sends rev=1010@CBL1, history=1-abc when SGW does not have the doc (document underwent multiple legacy and p2p updates before being pushed to SGW) + history = []string{"1000@CBL2", "1-abc"} + sent, _, _, err = bt.SendRevWithHistory(docID4, "1010@CBL1", history, []byte(`{"some": "update"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err = collection.GetDocWithXattrs(ctx, docID4, db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, "1010@CBL1", bucketDoc.HLV.GetCurrentVersionString()) + assert.Equal(t, uint64(4096), bucketDoc.HLV.PreviousVersions["CBL2"]) + assert.NotNil(t, bucketDoc.History["1-abc"]) +} + +// TestProcessRevWithLegacyHistoryConflict: +// - 1. conflicting changes with legacy rev on both sides of communication (no upgrade of doc at all) +// - 2. conflicting changes with legacy rev on client side and HLV on SGW side +// - 3. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 2-abc (document underwent multiple p2p updates before being pushed to SGW) +// - 4. CBL sends rev=1010@CBL1, history=2-abc and SGW has 1000@CBL2, 2-abc +func TestProcessRevWithLegacyHistoryConflict(t *testing.T) { + base.SetUpTestLogging(t, base.LevelTrace, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyCRUD, base.KeyChanges, base.KeyImport) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + ds := rt.GetSingleDataStore() + const ( + docID = "doc1" + docID2 = "doc2" + docID3 = "doc3" + docID4 = "doc4" + ) + + // 1. conflicting changes with legacy rev on both sides of communication (no upgrade of doc at all) + docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly(docID, docVersion, db.Body{"some": "update"}) + rev2ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly(docID, docVersion, db.Body{"some": "update2"}) + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(base.TestCtx(t), docID, []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + history := []string{rev2ID, rev1ID} + sent, _, _, err := bt.SendRevWithHistory(docID, "3-abc", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // 2. same as above but not having the rev be legacy on SGW side (don't remove the hlv) + docVersion = rt.PutDocDirectly(docID2, db.Body{"test": "doc"}) + rev1ID = docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly(docID2, docVersion, db.Body{"some": "update"}) + rev2ID = docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly(docID2, docVersion, db.Body{"some": "update2"}) + + history = []string{rev2ID, rev1ID} + sent, _, _, err = bt.SendRevWithHistory(docID2, "3-abc", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // 3. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 2-abc (document underwent multiple p2p updates before being pushed to SGW) + docVersion = rt.PutDocDirectly(docID3, db.Body{"test": "doc"}) + rev1ID = docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly(docID3, docVersion, db.Body{"some": "update"}) + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(base.TestCtx(t), docID3, []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + history = []string{"1000@CBL2", rev1ID} + sent, _, _, err = bt.SendRevWithHistory(docID3, "1010@CBL1", history, []byte(`{"some": "update"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // 4. CBL sends rev=1010@CBL1, history=2-abc and SGW has 1000@CBL2, 2-abc + docVersion = rt.PutDocDirectly(docID4, db.Body{"test": "doc"}) + + docVersion = rt.UpdateDocDirectly(docID4, docVersion, db.Body{"some": "update"}) + version := docVersion.CV.Value + pushedRev := db.Version{ + Value: version + 1000, + SourceID: "CBL1", + } + + history = []string{"2-abc"} + sent, _, _, err = bt.SendRevWithHistory(docID4, pushedRev.String(), history, []byte(`{"some": "update"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") +} + +// TestChangesResponseLegacyRev: +// - Create doc +// - Update doc through SGW, creating a new revision +// - Send subChanges request and have custom changes handler to force a revID change being constructed +// - Have custom rev handler to assert the subsequent rev message is as expected with cv as rev + full rev +// tree in history. No hlv in history is expected here. +func TestChangesResponseLegacyRev(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion2 := rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + // wait for pending change to avoid flakes where changes feed didn't pick up this change + rt.WaitForPendingChanges() + receivedChangesRequestWg := sync.WaitGroup{} + revsFinishedWg := sync.WaitGroup{} + + bt.blipContext.HandlerForProfile["rev"] = func(request *blip.Message) { + defer revsFinishedWg.Done() + log.Printf("received rev request") + + // assert the rev property contains cv + rev := request.Properties["rev"] + assert.Equal(t, docVersion2.CV.String(), rev) + + // assert that history contain current revID and previous revID + history := request.Properties["history"] + historyList := strings.Split(history, ",") + assert.Len(t, historyList, 2) + assert.Equal(t, docVersion2.RevTreeID, historyList[0]) + assert.Equal(t, docVersion.RevTreeID, historyList[1]) + } + + bt.blipContext.HandlerForProfile["changes"] = func(request *blip.Message) { + + log.Printf("got changes message: %+v", request) + body, err := request.Body() + log.Printf("changes body: %v, err: %v", string(body), err) + + knownRevs := []interface{}{} + + if string(body) != "null" { + var changesReqs [][]interface{} + err = base.JSONUnmarshal(body, &changesReqs) + require.NoError(t, err) + + knownRevs = make([]interface{}, len(changesReqs)) + + for i, changesReq := range changesReqs { + docID := changesReq[1].(string) + revID := changesReq[2].(string) + log.Printf("change: %s %s", docID, revID) + + // fill known rev with revision 1 of doc1, this will replicate a situation where client has legacy rev of + // a document that SGW had a newer version of + knownRevs[i] = []string{rev1ID} + } + } + + if !request.NoReply() { + response := request.Response() + emptyResponseValBytes, err := base.JSONMarshal(knownRevs) + require.NoError(t, err) + response.SetBody(emptyResponseValBytes) + } + receivedChangesRequestWg.Done() + } + + subChangesRequest := bt.newRequest() + subChangesRequest.SetProfile("subChanges") + subChangesRequest.Properties["continuous"] = "false" + sent := bt.sender.Send(subChangesRequest) + assert.True(t, sent) + // changes will be called again with empty changes so hence the wait group of 2 + receivedChangesRequestWg.Add(2) + + // expect 1 rev message + revsFinishedWg.Add(1) + + subChangesResponse := subChangesRequest.Response() + assert.Equal(t, subChangesRequest.SerialNumber(), subChangesResponse.SerialNumber()) + + timeoutErr := WaitWithTimeout(&receivedChangesRequestWg, time.Second*10) + require.NoError(t, timeoutErr, "Timed out waiting") + + timeoutErr = WaitWithTimeout(&revsFinishedWg, time.Second*10) + require.NoError(t, timeoutErr, "Timed out waiting") + +} + +// TestChangesResponseWithHLVInHistory: +// - Create doc +// - Update doc with hlv agent to mock update from a another peer +// - Send subChanges request and have custom changes handler to force a revID change being constructed +// - Have custom rev handler to asser the subsequent rev message is as expected with cv as rev and pv + full rev +// tree in history +func TestChangesResponseWithHLVInHistory(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + newDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + + agent := db.NewHLVAgent(t, rt.GetSingleDataStore(), "newSource", base.VvXattrName) + _ = agent.UpdateWithHLV(ctx, "doc1", newDoc.Cas, newDoc.HLV) + + // force import + newDoc, err = collection.GetDocument(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + // wait for pending change to avoid flakes where changes feed didn't pick up this change + rt.WaitForPendingChanges() + + receivedChangesRequestWg := sync.WaitGroup{} + revsFinishedWg := sync.WaitGroup{} + + bt.blipContext.HandlerForProfile["rev"] = func(request *blip.Message) { + defer revsFinishedWg.Done() + log.Printf("received rev request") + + // assert the rev property contains cv + rev := request.Properties["rev"] + assert.Equal(t, newDoc.HLV.GetCurrentVersionString(), rev) + + // assert that history contain current revID and previous revID + pv of HLV + history := request.Properties["history"] + historyList := strings.Split(history, ",") + assert.Len(t, historyList, 3) + assert.Equal(t, newDoc.CurrentRev, historyList[1]) + assert.Equal(t, docVersion.RevTreeID, historyList[2]) + assert.Equal(t, docVersion.CV.String(), historyList[0]) + } + + bt.blipContext.HandlerForProfile["changes"] = func(request *blip.Message) { + + log.Printf("got changes message: %+v", request) + body, err := request.Body() + log.Printf("changes body: %v, err: %v", string(body), err) + + knownRevs := []interface{}{} + + if string(body) != "null" { + var changesReqs [][]interface{} + err = base.JSONUnmarshal(body, &changesReqs) + require.NoError(t, err) + + knownRevs = make([]interface{}, len(changesReqs)) + + for i, changesReq := range changesReqs { + docID := changesReq[1].(string) + revID := changesReq[2].(string) + log.Printf("change: %s %s", docID, revID) + + // fill known rev with revision 1 of doc1, this will replicate a situation where client has legacy rev of + // a document that SGW had a newer version of + knownRevs[i] = []string{rev1ID} + } + } + + if !request.NoReply() { + response := request.Response() + emptyResponseValBytes, err := base.JSONMarshal(knownRevs) + require.NoError(t, err) + response.SetBody(emptyResponseValBytes) + } + receivedChangesRequestWg.Done() + } + + subChangesRequest := bt.newRequest() + subChangesRequest.SetProfile("subChanges") + subChangesRequest.Properties["continuous"] = "false" + sent := bt.sender.Send(subChangesRequest) + assert.True(t, sent) + // changes will be called again with empty changes so hence the wait group of 2 + receivedChangesRequestWg.Add(2) + + // expect 1 rev message + revsFinishedWg.Add(1) + + subChangesResponse := subChangesRequest.Response() + assert.Equal(t, subChangesRequest.SerialNumber(), subChangesResponse.SerialNumber()) + + timeoutErr := WaitWithTimeout(&receivedChangesRequestWg, time.Second*10) + require.NoError(t, timeoutErr, "Timed out waiting") + + timeoutErr = WaitWithTimeout(&revsFinishedWg, time.Second*10) + require.NoError(t, timeoutErr, "Timed out waiting") +} + +// test case 2 of non conflict plan +func TestCBLHasPreUpgradeMutationThatHasNotBeenReplicated(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + ds := rt.GetSingleDataStore() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, "doc1", []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + history := []string{rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", "2-abc", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + // assert a cv was assigned + assert.NotEqual(t, "", bucketDoc.HLV.GetCurrentVersionString()) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.Equal(t, "2-abc", bucketDoc.CurrentRev) +} + +// test case 3 of non conflict plan +func TestCBLHasOfPreUpgradeMutationThatSGWAlreadyKnows(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + ds := rt.GetSingleDataStore() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + rev2ID := docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, "doc1", []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + history := []string{rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", rev2ID, history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, rev2ID, bucketDoc.CurrentRev) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.NotNil(t, bucketDoc.History[rev2ID]) +} + +// test case 6 of non conflict plan +func TestPushOfPostUpgradeMutationThatHasCommonAncestorToSGWVersion(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + ds := rt.GetSingleDataStore() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + rev2ID := docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, "doc1", []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + // send 100@CBL1 + sent, _, _, err := bt.SendRevWithHistory("doc1", "100@CBL1", nil, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.NoError(t, err) + + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + assert.NotEqual(t, rev2ID, bucketDoc.CurrentRev) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.NotNil(t, bucketDoc.History[rev2ID]) + assert.Equal(t, "100@CBL1", bucketDoc.HLV.GetCurrentVersionString()) +} + +// case 1 of conflivt test plan +func TestPushDocConflictBetweenPreUpgradeCBLMutationAndPreUpgradeSGWMutation(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + ds := rt.GetSingleDataStore() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + rev2ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update1"}) + rev3ID := docVersion.RevTreeID + + // remove hlv here to simulate a legacy rev + require.NoError(t, ds.RemoveXattrs(ctx, "doc1", []string{base.VvXattrName}, docVersion.CV.Value)) + rt.GetDatabase().FlushRevisionCacheForTest() + + // send rev 3-def + history := []string{rev2ID, rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", "3-def", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, rev3ID, bucketDoc.CurrentRev) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.NotNil(t, bucketDoc.History[rev2ID]) +} + +// case 3 of oconflict test plan +func TestPushDocConflictBetweenPreUpgradeCBLMutationAndPostUpgradeSGWMutation(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + rev2ID := docVersion.RevTreeID + + docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update1"}) + rev3ID := docVersion.RevTreeID + + // send rev 3-def + history := []string{rev2ID, rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", "3-def", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, rev3ID, bucketDoc.CurrentRev) + assert.NotNil(t, bucketDoc.History[rev1ID]) + assert.NotNil(t, bucketDoc.History[rev2ID]) +} + +// test case 6 of conlfuct plan +func TestConflictBetweenPostUpgradeCBLMutationAndPostUpgradeSGWMutation(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyHTTP, base.KeySync, base.KeySyncMsg, base.KeyChanges) + + bt, err := NewBlipTesterFromSpec(t, BlipTesterSpec{ + noConflictsMode: true, + GuestEnabled: true, + blipProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + }) + assert.NoError(t, err, "Error creating BlipTester") + defer bt.Close() + rt := bt.restTester + collection, ctx := rt.GetSingleTestDatabaseCollection() + + docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + rev1ID := docVersion.RevTreeID + + history := []string{rev1ID} + sent, _, _, err := bt.SendRevWithHistory("doc1", "100@CBL1", history, []byte(`{"key": "val"}`), blip.Properties{}) + assert.True(t, sent) + require.ErrorContains(t, err, "Document revision conflict") + + // assert that the bucket doc is as expected + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) + require.NoError(t, err) + assert.Equal(t, rev1ID, bucketDoc.CurrentRev) + assert.Equal(t, docVersion.CV.String(), bucketDoc.HLV.GetCurrentVersionString()) +} diff --git a/rest/bulk_api.go b/rest/bulk_api.go index c3f2b532dc..73265a35f0 100644 --- a/rest/bulk_api.go +++ b/rest/bulk_api.go @@ -25,6 +25,25 @@ import ( "github.com/couchbase/sync_gateway/db" ) +// allDocsRowValue is a struct that represents possible values returned in a document from /ks/_all_docs endpoint +type allDocsRowValue struct { + Rev string `json:"rev"` + CV string `json:"cv,omitempty"` + Channels []string `json:"channels,omitempty"` + Access map[string]base.Set `json:"access,omitempty"` // for admins only +} + +// allDocsRow is a struct that represents a linefrom /ks/_all_docs endpoint +type allDocsRow struct { + Key string `json:"key"` + ID string `json:"id,omitempty"` + Value *allDocsRowValue `json:"value,omitempty"` + Doc json.RawMessage `json:"doc,omitempty"` + UpdateSeq uint64 `json:"update_seq,omitempty"` + Error string `json:"error,omitempty"` + Status int `json:"status,omitempty"` +} + // HTTP handler for _all_docs func (h *handler) handleAllDocs() error { // http://wiki.apache.org/couchdb/HTTP_Bulk_Document_API @@ -32,6 +51,7 @@ func (h *handler) handleAllDocs() error { includeChannels := h.getBoolQuery("channels") includeAccess := h.getBoolQuery("access") && h.user == nil includeRevs := h.getBoolQuery("revs") + includeCVs := h.getBoolQuery("show_cv") includeSeqs := h.getBoolQuery("update_seq") // Get the doc IDs if this is a POST request: @@ -99,21 +119,6 @@ func (h *handler) handleAllDocs() error { return result } - type allDocsRowValue struct { - Rev string `json:"rev"` - Channels []string `json:"channels,omitempty"` - Access map[string]base.Set `json:"access,omitempty"` // for admins only - } - type allDocsRow struct { - Key string `json:"key"` - ID string `json:"id,omitempty"` - Value *allDocsRowValue `json:"value,omitempty"` - Doc json.RawMessage `json:"doc,omitempty"` - UpdateSeq uint64 `json:"update_seq,omitempty"` - Error string `json:"error,omitempty"` - Status int `json:"status,omitempty"` - } - // Subroutine that creates a response row for a document: totalRows := 0 createRow := func(doc db.IDRevAndSequence, channels []string) *allDocsRow { @@ -142,7 +147,7 @@ func (h *handler) handleAllDocs() error { row.Status = http.StatusForbidden return row } - // handle the case where the incoming doc.RevID == "" + // handle the case where the incoming doc.RevTreeID == "" // and Get1xRevAndChannels returns the current revision doc.RevID = currentRevID } @@ -169,6 +174,9 @@ func (h *handler) handleAllDocs() error { if includeChannels { row.Value.Channels = channels } + if includeCVs { + row.Value.CV = doc.CV + } return row } @@ -219,7 +227,8 @@ func (h *handler) handleAllDocs() error { if explicitDocIDs != nil { count := uint64(0) for _, docID := range explicitDocIDs { - _, _ = writeDoc(db.IDRevAndSequence{DocID: docID, RevID: "", Sequence: 0}, nil) + // no revtreeid or cv if explicitDocIDs are specified + _, _ = writeDoc(db.IDRevAndSequence{DocID: docID, RevID: "", Sequence: 0, CV: ""}, nil) count++ if options.Limit > 0 && count == options.Limit { break @@ -363,6 +372,7 @@ func (h *handler) handleBulkGet() error { includeAttachments := h.getBoolQuery("attachments") showExp := h.getBoolQuery("show_exp") + showCV := h.getBoolQuery("show_cv") showRevs := h.getBoolQuery("revs") globalRevsLimit := int(h.getIntQuery("revs_limit", math.MaxInt32)) @@ -439,7 +449,12 @@ func (h *handler) handleBulkGet() error { } if err == nil { - body, err = h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, docRevsLimit, revsFrom, attsSince, showExp) + body, err = h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: docRevsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attsSince, + ShowExp: showExp, + ShowCV: showCV}) } if err != nil { @@ -542,7 +557,7 @@ func (h *handler) handleBulkDocs() error { err = base.HTTPErrorf(http.StatusBadRequest, "Bad _revisions") } else { revid = revisions[0] - _, _, err = h.collection.PutExistingRevWithBody(h.ctx(), docid, doc, revisions, false) + _, _, err = h.collection.PutExistingRevWithBody(h.ctx(), docid, doc, revisions, false, db.ExistingVersionWithUpdateToHLV) } } diff --git a/rest/changes_test.go b/rest/changes_test.go index 20525f7b71..1d5b164c64 100644 --- a/rest/changes_test.go +++ b/rest/changes_test.go @@ -229,7 +229,7 @@ func TestWebhookWinningRevChangedEvent(t *testing.T) { // push winning branch wg.Add(2) - res := rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"buzz","_revisions":{"start":3,"ids":["buzz","bar","`+version1.RevID+`"]}}`) + res := rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"buzz","_revisions":{"start":3,"ids":["buzz","bar","`+version1.RevTreeID+`"]}}`) RequireStatus(t, res, http.StatusCreated) winningVersion := DocVersionFromPutResponse(t, res) @@ -252,7 +252,7 @@ func TestWebhookWinningRevChangedEvent(t *testing.T) { // push a separate winning branch wg.Add(2) - res = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"quux","_revisions":{"start":4,"ids":["quux", "buzz","bar","`+version1.RevID+`"]}}`) + res = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", `{"foo":"quux","_revisions":{"start":4,"ids":["quux", "buzz","bar","`+version1.RevTreeID+`"]}}`) RequireStatus(t, res, http.StatusCreated) newWinningVersion := DocVersionFromPutResponse(t, res) @@ -333,7 +333,7 @@ func TestJumpInSequencesAtAllocatorSkippedSequenceFill(t *testing.T) { changes, err := rt.WaitForChanges(2, "/{{.keyspace}}/_changes", "", true) require.NoError(t, err) changes.RequireDocIDs(t, []string{"doc1", "doc"}) - changes.RequireRevID(t, []string{docVrs.RevID, doc1Vrs.RevID}) + changes.RequireRevID(t, []string{docVrs.RevTreeID, doc1Vrs.RevTreeID}) } // TestJumpInSequencesAtAllocatorRangeInPending: @@ -404,5 +404,67 @@ func TestJumpInSequencesAtAllocatorRangeInPending(t *testing.T) { changes, err := rt.WaitForChanges(2, "/{{.keyspace}}/_changes", "", true) require.NoError(t, err) changes.RequireDocIDs(t, []string{"doc1", "doc"}) - changes.RequireRevID(t, []string{docVrs.RevID, doc1Vrs.RevID}) + changes.RequireRevID(t, []string{docVrs.RevTreeID, doc1Vrs.RevTreeID}) +} + +func TestCVPopulationOnChangesViaAPI(t *testing.T) { + t.Skip("Disabled until REST support for version is added") + rtConfig := RestTesterConfig{ + SyncFn: `function(doc) {channel(doc.channels)}`, + } + rt := NewRestTester(t, &rtConfig) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollection() + bucketUUID := rt.GetDatabase().EncodedSourceID + const DocID = "doc1" + + // activate channel cache + _, err := rt.WaitForChanges(0, "/{{.keyspace}}/_changes", "", true) + require.NoError(t, err) + + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+DocID, `{"channels": ["ABC"]}`) + RequireStatus(t, resp, http.StatusCreated) + + require.NoError(t, collection.WaitForPendingChanges(base.TestCtx(t))) + + changes, err := rt.WaitForChanges(1, "/{{.keyspace}}/_changes", "", true) + require.NoError(t, err) + + fetchedDoc, _, err := collection.GetDocWithXattrs(ctx, DocID, db.DocUnmarshalCAS) + require.NoError(t, err) + + assert.Equal(t, "doc1", changes.Results[0].ID) + assert.Equal(t, bucketUUID, changes.Results[0].CurrentVersion.SourceID) + assert.Equal(t, fetchedDoc.Cas, changes.Results[0].CurrentVersion.Value) +} + +func TestCVPopulationOnDocIDChanges(t *testing.T) { + t.Skip("Disabled until REST support for version is added") + rtConfig := RestTesterConfig{ + SyncFn: `function(doc) {channel(doc.channels)}`, + } + rt := NewRestTester(t, &rtConfig) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollection() + bucketUUID := rt.GetDatabase().EncodedSourceID + const DocID = "doc1" + + // activate channel cache + _, err := rt.WaitForChanges(0, "/{{.keyspace}}/_changes", "", true) + require.NoError(t, err) + + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+DocID, `{"channels": ["ABC"]}`) + RequireStatus(t, resp, http.StatusCreated) + + require.NoError(t, collection.WaitForPendingChanges(base.TestCtx(t))) + + changes, err := rt.WaitForChanges(1, fmt.Sprintf(`/{{.keyspace}}/_changes?filter=_doc_ids&doc_ids=%s`, DocID), "", true) + require.NoError(t, err) + + fetchedDoc, _, err := collection.GetDocWithXattrs(ctx, DocID, db.DocUnmarshalCAS) + require.NoError(t, err) + + assert.Equal(t, "doc1", changes.Results[0].ID) + assert.Equal(t, bucketUUID, changes.Results[0].CurrentVersion.SourceID) + assert.Equal(t, fetchedDoc.Cas, changes.Results[0].CurrentVersion.Value) } diff --git a/rest/changestest/changes_api_test.go b/rest/changestest/changes_api_test.go index e80d552fc1..e2a0b19898 100644 --- a/rest/changestest/changes_api_test.go +++ b/rest/changestest/changes_api_test.go @@ -689,10 +689,10 @@ func TestPostChangesAdminChannelGrantRemovalWithLimit(t *testing.T) { cacheWaiter.AddAndWait(4) // Mark the first four PBS docs as removals - _ = rt.PutDoc("pbs-1", fmt.Sprintf(`{"_rev":%q}`, pbs1.RevID)) - _ = rt.PutDoc("pbs-2", fmt.Sprintf(`{"_rev":%q}`, pbs2.RevID)) - _ = rt.PutDoc("pbs-3", fmt.Sprintf(`{"_rev":%q}`, pbs3.RevID)) - _ = rt.PutDoc("pbs-4", fmt.Sprintf(`{"_rev":%q}`, pbs4.RevID)) + _ = rt.PutDoc("pbs-1", fmt.Sprintf(`{"_rev":%q}`, pbs1.RevTreeID)) + _ = rt.PutDoc("pbs-2", fmt.Sprintf(`{"_rev":%q}`, pbs2.RevTreeID)) + _ = rt.PutDoc("pbs-3", fmt.Sprintf(`{"_rev":%q}`, pbs3.RevTreeID)) + _ = rt.PutDoc("pbs-4", fmt.Sprintf(`{"_rev":%q}`, pbs4.RevTreeID)) cacheWaiter.AddAndWait(4) @@ -742,9 +742,9 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { } }`}) defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollection() // Create user with access to channel NBC: - ctx := rt.Context() a := rt.ServerContext().Database(ctx, "db").Authenticator(ctx) alice, err := a.NewUser("alice", "letmein", channels.BaseSetOf(t, "NBC")) assert.NoError(t, err) @@ -767,7 +767,7 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { cacheWaiter.AddAndWait(4) // remove channels/tombstone a couple of docs to ensure they're not backfilled after a dynamic grant - _ = rt.PutDoc("hbo-2", fmt.Sprintf(`{"_rev":%q}`, hbo2.RevID)) + _ = rt.PutDoc("hbo-2", fmt.Sprintf(`{"_rev":%q}`, hbo2.RevTreeID)) rt.DeleteDoc(pbs2ID, pbs2Version) cacheWaiter.AddAndWait(2) @@ -781,9 +781,7 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { changes, err := rt.WaitForChanges(len(expectedResults), "/{{.keyspace}}/_changes", "bernard", false) require.NoError(t, err, "Error retrieving changes results") for index, result := range changes.Results { - var expectedChange db.ChangeEntry - require.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - assert.Equal(t, expectedChange, result) + assertChangeEntryMatches(t, expectedResults[index], result) } // create doc that dynamically grants both users access to PBS and HBO @@ -801,14 +799,16 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { fmt.Sprintf("/{{.keyspace}}/_changes?since=%s", changes.Last_Seq), "bernard", false) require.NoError(t, err, "Error retrieving changes results") for index, result := range changes.Results { - var expectedChange db.ChangeEntry - require.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - assert.Equal(t, expectedChange, result) + assertChangeEntryMatches(t, expectedResults[index], result) } // Write another doc _ = rt.PutDoc("mix-1", `{"channel":["ABC", "PBS", "HBO"]}`) + fetchedDoc, _, err := collection.GetDocWithXattrs(ctx, "mix-1", db.DocUnmarshalSync) + require.NoError(t, err) + mixSource, mixVersion := fetchedDoc.HLV.GetCurrentVersion() + cacheWaiter.AddAndWait(1) // Issue a changes request with a compound since value from the last changes response @@ -816,16 +816,14 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { expectedResults = []string{ `{"seq":"8:2","id":"hbo-1","changes":[{"rev":"1-46f8c67c004681619052ee1a1cc8e104"}]}`, `{"seq":8,"id":"grant-1","changes":[{"rev":"1-c5098bb14d12d647c901850ff6a6292a"}]}`, - `{"seq":9,"id":"mix-1","changes":[{"rev":"1-32f69cdbf1772a8e064f15e928a18f85"}]}`, + fmt.Sprintf(`{"seq":9,"id":"mix-1","changes":[{"rev":"1-32f69cdbf1772a8e064f15e928a18f85"}], "current_version":{"source_id": "%s", "version": "%d"}}`, mixSource, mixVersion), } rt.Run("grant via existing channel", func(t *testing.T) { changes, err = rt.WaitForChanges(len(expectedResults), "/{{.keyspace}}/_changes?since=8:1", "alice", false) require.NoError(t, err, "Error retrieving changes results for alice") for index, result := range changes.Results { - var expectedChange db.ChangeEntry - require.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - assert.Equal(t, expectedChange, result) + assertChangeEntryMatches(t, expectedResults[index], result) } }) @@ -833,13 +831,33 @@ func TestChangesFromCompoundSinceViaDocGrant(t *testing.T) { changes, err = rt.WaitForChanges(len(expectedResults), "/{{.keyspace}}/_changes?since=8:1", "bernard", false) require.NoError(t, err, "Error retrieving changes results for bernard") for index, result := range changes.Results { - var expectedChange db.ChangeEntry - require.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - assert.Equal(t, expectedChange, result) + assertChangeEntryMatches(t, expectedResults[index], result) } }) } +// TODO: enhance to compare source/version when expectedChanges are updated to include +func assertChangeEntryMatches(t *testing.T, expectedChangeEntryString string, result db.ChangeEntry) { + var expectedChange db.ChangeEntry + require.NoError(t, base.JSONUnmarshal([]byte(expectedChangeEntryString), &expectedChange)) + assert.Equal(t, expectedChange.Seq, result.Seq) + assert.Equal(t, expectedChange.ID, result.ID) + assert.Equal(t, expectedChange.Changes, result.Changes) + assert.Equal(t, expectedChange.Deleted, result.Deleted) + assert.Equal(t, expectedChange.Removed, result.Removed) + + if expectedChange.Doc != nil { + // result.Doc is json.RawMessage, and properties may not be in the same order for a direct comparison + var expectedBody db.Body + var resultBody db.Body + assert.NoError(t, expectedBody.Unmarshal(expectedChange.Doc)) + assert.NoError(t, resultBody.Unmarshal(result.Doc)) + db.AssertEqualBodies(t, expectedBody, resultBody) + } else { + assert.Equal(t, expectedChange.Doc, result.Doc) + } +} + // Ensures that changes feed goroutines blocked on a ChangeWaiter are closed when the changes feed is terminated. // Reproduces CBG-1113 and #1329 (even with the fix in PR #1360) // Tests all combinations of HTTP feed types, admin/non-admin, and with and without a manual notify to wake up. @@ -1755,7 +1773,6 @@ func updateTestDoc(rt *rest.RestTester, docid string, revid string, body string) // Validate retrieval of various document body types using include_docs. func TestChangesIncludeDocs(t *testing.T) { - base.SetUpTestLogging(t, base.LevelInfo, base.KeyNone) rtConfig := rest.RestTesterConfig{ @@ -1765,7 +1782,7 @@ func TestChangesIncludeDocs(t *testing.T) { testDB := rt.GetDatabase() testDB.RevsLimit = 3 defer rt.Close() - collection, _ := rt.GetSingleTestDatabaseCollection() + collection, ctx := rt.GetSingleTestDatabaseCollection() rt.CreateUser("user1", []string{"alpha", "beta"}) @@ -1807,9 +1824,15 @@ func TestChangesIncludeDocs(t *testing.T) { assert.NoError(t, err, "Error updating doc") // Generate more revs than revs_limit (3) revid = prunedRevId + var cvs []string for i := 0; i < 5; i++ { - revid, err = updateTestDoc(rt, "doc_pruned", revid, `{"type": "pruned", "channels":["gamma"]}`) - assert.NoError(t, err, "Error updating doc") + body := db.Body{ + "type": "pruned", + "channels": []string{"gamma"}, + } + docVersion := rt.UpdateDocDirectly("doc_pruned", db.DocVersion{RevTreeID: revid}, body) + revid = docVersion.RevTreeID + cvs = append(cvs, docVersion.CV.String()) } // Doc w/ attachment @@ -1866,49 +1889,25 @@ func TestChangesIncludeDocs(t *testing.T) { expectedResults[9] = `{"seq":26,"id":"doc_resolved_conflict","doc":{"_id":"doc_resolved_conflict","_rev":"2-251ba04e5889887152df5e7a350745b4","channels":["alpha"],"type":"resolved_conflict"},"changes":[{"rev":"2-251ba04e5889887152df5e7a350745b4"}]}` for index, result := range changes.Results { - var expectedChange db.ChangeEntry - assert.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - - assert.Equal(t, expectedChange.ID, result.ID) - assert.Equal(t, expectedChange.Seq, result.Seq) - assert.Equal(t, expectedChange.Deleted, result.Deleted) - assert.Equal(t, expectedChange.Changes, result.Changes) - assert.Equal(t, expectedChange.Err, result.Err) - assert.Equal(t, expectedChange.Removed, result.Removed) - - if expectedChange.Doc != nil { - // result.Doc is json.RawMessage, and properties may not be in the same order for a direct comparison - var expectedBody db.Body - var resultBody db.Body - assert.NoError(t, expectedBody.Unmarshal(expectedChange.Doc)) - assert.NoError(t, resultBody.Unmarshal(result.Doc)) - db.AssertEqualBodies(t, expectedBody, resultBody) - } else { - assert.Equal(t, expectedChange.Doc, result.Doc) - } + assertChangeEntryMatches(t, expectedResults[index], result) } // Flush the rev cache, and issue changes again to ensure successful handling for rev cache misses rt.GetDatabase().FlushRevisionCacheForTest() // Also nuke temporary revision backup of doc_pruned. Validates that the body for the pruned revision is generated correctly when no longer resident in the rev cache - data := collection.GetCollectionDatastore() - assert.NoError(t, data.Delete(base.RevPrefix+"doc_pruned:34:2-5afcb73bd3eb50615470e3ba54b80f00")) - + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + cvHash := base.Crc32cHashString([]byte(cvs[0])) + err = collection.PurgeOldRevisionJSON(ctx, "doc_pruned", cvHash) + require.NoError(t, err) postFlushChanges := rt.GetChanges("/{{.keyspace}}/_changes?include_docs=true", "user1") assert.Equal(t, len(expectedResults), len(postFlushChanges.Results)) for index, result := range postFlushChanges.Results { + + assertChangeEntryMatches(t, expectedResults[index], result) var expectedChange db.ChangeEntry assert.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - - assert.Equal(t, expectedChange.ID, result.ID) - assert.Equal(t, expectedChange.Seq, result.Seq) - assert.Equal(t, expectedChange.Deleted, result.Deleted) - assert.Equal(t, expectedChange.Changes, result.Changes) - assert.Equal(t, expectedChange.Err, result.Err) - assert.Equal(t, expectedChange.Removed, result.Removed) - if expectedChange.Doc != nil { // result.Doc is json.RawMessage, and properties may not be in the same order for a direct comparison var expectedBody db.Body @@ -1947,26 +1946,7 @@ func TestChangesIncludeDocs(t *testing.T) { assert.Equal(t, len(expectedResults), len(combinedChanges.Results)) for index, result := range combinedChanges.Results { - var expectedChange db.ChangeEntry - assert.NoError(t, base.JSONUnmarshal([]byte(expectedResults[index]), &expectedChange)) - - assert.Equal(t, expectedChange.ID, result.ID) - assert.Equal(t, expectedChange.Seq, result.Seq) - assert.Equal(t, expectedChange.Deleted, result.Deleted) - assert.Equal(t, expectedChange.Changes, result.Changes) - assert.Equal(t, expectedChange.Err, result.Err) - assert.Equal(t, expectedChange.Removed, result.Removed) - - if expectedChange.Doc != nil { - // result.Doc is json.RawMessage, and properties may not be in the same order for a direct comparison - var expectedBody db.Body - var resultBody db.Body - assert.NoError(t, expectedBody.Unmarshal(expectedChange.Doc)) - assert.NoError(t, resultBody.Unmarshal(result.Doc)) - db.AssertEqualBodies(t, expectedBody, resultBody) - } else { - assert.Equal(t, expectedChange.Doc, result.Doc) - } + assertChangeEntryMatches(t, expectedResults[index], result) } } diff --git a/rest/config_test.go b/rest/config_test.go index 1772b0d93f..75ff917f63 100644 --- a/rest/config_test.go +++ b/rest/config_test.go @@ -2363,9 +2363,8 @@ func TestInvalidJavascriptFunctions(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.Name, func(t *testing.T) { - safeDbName := strings.ToLower(strings.ReplaceAll(testCase.Name, " ", "-")) dbConfig := DbConfig{ - Name: safeDbName, + Name: SafeDatabaseName(t, testCase.Name), } if testCase.SyncFunction != nil { @@ -3139,3 +3138,27 @@ func TestRevCacheMemoryLimitConfig(t *testing.T) { assert.Equal(t, uint32(100), *dbConfig.CacheConfig.RevCacheConfig.MaxItemCount) assert.Equal(t, uint32(0), *dbConfig.CacheConfig.RevCacheConfig.MaxMemoryCountMB) } + +func TestTLSWithoutCerts(t *testing.T) { + base.ResetCBGTCertPools(t) // CBG-4394: removing root certs for the bucket should be done, but it is keyed based on the bucket UUID, and multiple dbs can use the same bucket + rt := NewRestTester(t, &RestTesterConfig{ + PersistentConfig: true, + MutateStartupConfig: func(config *StartupConfig) { + config.Bootstrap.Server = strings.ReplaceAll(config.Bootstrap.Server, "couchbase://", "couchbases://") + config.Bootstrap.ServerTLSSkipVerify = base.BoolPtr(true) + config.Bootstrap.UseTLSServer = base.BoolPtr(true) + }, + }) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + dbConfig.AutoImport = true + rt.CreateDatabase("db", dbConfig) + // ensure import feed works without TLS + err := rt.GetSingleDataStore().Set("doc1", 0, nil, []byte(`{"foo": "bar"}`)) + require.NoError(t, err) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, int64(1), rt.GetDatabase().DbStats.SharedBucketImportStats.ImportCount.Value()) + }, time.Second*10, time.Millisecond*100) + +} diff --git a/rest/doc_api.go b/rest/doc_api.go index 653f420362..074a069153 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -30,6 +30,7 @@ func (h *handler) handleGetDoc() error { revid := h.getQuery("rev") openRevs := h.getQuery("open_revs") showExp := h.getBoolQuery("show_exp") + showCV := h.getBoolQuery("show_cv") if replicator2, _ := h.getOptBoolQuery("replicator2", false); replicator2 { return h.handleGetDocReplicator2(docid, revid) @@ -68,7 +69,12 @@ func (h *handler) handleGetDoc() error { if openRevs == "" { // Single-revision GET: - value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { if err == base.ErrImportCancelledPurged { base.DebugfCtx(h.ctx(), base.KeyImport, fmt.Sprintf("Import cancelled as document %v is purged", base.UD(docid))) @@ -130,7 +136,12 @@ func (h *handler) handleGetDoc() error { if h.requestAccepts("multipart/") { err := h.writeMultipart("mixed", func(writer *multipart.Writer) error { for _, revid := range revids { - revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { revBody = db.Body{"missing": revid} // TODO: More specific error } @@ -152,7 +163,12 @@ func (h *handler) handleGetDoc() error { _, _ = h.response.Write([]byte(`[` + "\n")) separator := []byte(``) for _, revid := range revids { - revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, revsLimit, revsFrom, attachmentsSince, showExp) + revBody, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + MaxHistory: revsLimit, + HistoryFrom: revsFrom, + AttachmentsSince: attachmentsSince, + ShowExp: showExp, + ShowCV: showCV}) if err != nil { revBody = db.Body{"missing": revid} // TODO: More specific error } else { @@ -494,7 +510,7 @@ func (h *handler) handlePutDoc() error { if revisions == nil { return base.HTTPErrorf(http.StatusBadRequest, "Bad _revisions") } - doc, newRev, err = h.collection.PutExistingRevWithBody(h.ctx(), docid, body, revisions, false) + doc, newRev, err = h.collection.PutExistingRevWithBody(h.ctx(), docid, body, revisions, false, db.ExistingVersionWithUpdateToHLV) if err != nil { return err } @@ -571,7 +587,7 @@ func (h *handler) handlePutDocReplicator2(docid string, roundTrip bool) (err err newDoc.UpdateBody(body) } - doc, rev, err := h.collection.PutExistingRev(h.ctx(), newDoc, history, true, false, nil) + doc, rev, err := h.collection.PutExistingRev(h.ctx(), newDoc, history, true, false, nil, db.ExistingVersionWithUpdateToHLV) if err != nil { return err @@ -628,7 +644,7 @@ func (h *handler) handleDeleteDoc() error { return err } } - newRev, err := h.collection.DeleteDoc(h.ctx(), docid, revid) + newRev, _, err := h.collection.DeleteDoc(h.ctx(), docid, revid) if err == nil { h.writeRawJSONStatus(http.StatusOK, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`"}`)) } diff --git a/rest/doc_api_test.go b/rest/doc_api_test.go index 5ab381b547..ecf3264393 100644 --- a/rest/doc_api_test.go +++ b/rest/doc_api_test.go @@ -12,7 +12,10 @@ package rest import ( "fmt" + "io" "log" + "mime" + "mime/multipart" "net/http" "strings" "testing" @@ -173,3 +176,129 @@ func TestGuestReadOnly(t *testing.T) { RequireStatus(t, response, http.StatusForbidden) } + +func TestGetDocWithCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + docID := "doc1" + docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + testCases := []struct { + name string + url string + output string + headers map[string]string + multipart bool + }{ + { + name: "get doc", + url: "/{{.keyspace}}/doc1", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, docID, docVersion.RevTreeID), + }, + { + name: "get doc with rev", + url: fmt.Sprintf("/{{.keyspace}}/doc1?rev=%s", docVersion.RevTreeID), + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, docID, docVersion.RevTreeID), + }, + { + name: "get doc with cv", + url: "/{{.keyspace}}/doc1?show_cv=true", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}`, docID, docVersion.RevTreeID, docVersion.CV), + }, + { + name: "get doc with open_revs=all and cv no multipart", + url: "/{{.keyspace}}/doc1?open_revs=all&show_cv=true", + output: fmt.Sprintf(`[{"ok": {"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}}]`, docID, docVersion.RevTreeID, docVersion.CV), + headers: map[string]string{ + "Accept": "application/json", + }, + }, + + { + name: "get doc with open_revs=all and cv", + url: "/{{.keyspace}}/doc1?open_revs=all&show_cv=true", + output: fmt.Sprintf(`{"_id":"%s","_rev":"%s","_cv":"%s","foo":"bar"}`, docID, docVersion.RevTreeID, docVersion.CV), + multipart: true, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequestWithHeaders("GET", testCase.url, "", testCase.headers) + RequireStatus(t, response, http.StatusOK) + output := response.BodyString() + if testCase.multipart { + multipartOutput := readMultiPartBody(t, response) + require.Len(t, multipartOutput, 1) + output = multipartOutput[0] + } + assert.JSONEq(t, testCase.output, output) + }) + } + +} + +func TestBulkGetWithCV(t *testing.T) { + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + doc1ID := "doc1" + doc2ID := "doc2" + doc1Version := rt.PutDocDirectly(doc1ID, db.Body{"foo": "bar"}) + doc2Version := rt.PutDocDirectly(doc2ID, db.Body{"foo": "baz"}) + testCases := []struct { + name string + url string + input string + output []string + }{ + { + name: "get doc multipart", + url: "/{{.keyspace}}/_bulk_get", + input: fmt.Sprintf(`{"docs":[{"id":"%s"},{"id":"%s"}]}`, doc1ID, doc2ID), + output: []string{ + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar"}`, doc1ID, doc1Version.RevTreeID), + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"baz"}`, doc2ID, doc2Version.RevTreeID), + }, + }, + { + name: "get doc multipart", + url: "/{{.keyspace}}/_bulk_get?show_cv=true", + input: fmt.Sprintf(`{"docs":[{"id":"%s"},{"id":"%s"}]}`, doc1ID, doc2ID), + output: []string{ + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"bar", "_cv": "%s"}`, doc1ID, doc1Version.RevTreeID, doc1Version.CV), + fmt.Sprintf(`{"_id":"%s","_rev":"%s","foo":"baz", "_cv": "%s"}`, doc2ID, doc2Version.RevTreeID, doc2Version.CV), + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := rt.SendAdminRequest(http.MethodPost, testCase.url, testCase.input) + RequireStatus(t, response, http.StatusOK) + bodies := readMultiPartBody(t, response) + require.Len(t, bodies, len(testCase.output)) + for i, body := range bodies { + assert.JSONEq(t, testCase.output[i], body) + } + }) + } + +} + +// readMultiPartBody reads a multipart response body and returns the parts as strings +func readMultiPartBody(t *testing.T, response *TestResponse) []string { + _, params, err := mime.ParseMediaType(response.Header().Get("Content-Type")) + require.NoError(t, err) + mr := multipart.NewReader(response.Body, params["boundary"]) + var output []string + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + bodyBytes, err := io.ReadAll(p) + require.NoError(t, err) + output = append(output, string(bodyBytes)) + } + return output +} diff --git a/rest/importtest/import_test.go b/rest/importtest/import_test.go index b8a9329c62..9d8e417199 100644 --- a/rest/importtest/import_test.go +++ b/rest/importtest/import_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" "github.com/couchbase/sync_gateway/rest" @@ -177,16 +178,18 @@ func TestXattrImportOldDocRevHistory(t *testing.T) { // 1. Create revision with history docID := t.Name() - version := rt.PutDoc(docID, `{"val":-1}`) - revID := version.RevID + version := rt.PutDocDirectly(docID, rest.JsonToMap(t, `{"val":-1}`)) + cv := version.CV.String() collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() for i := 0; i < 10; i++ { - version = rt.UpdateDoc(docID, version, fmt.Sprintf(`{"val":%d}`, i)) + version = rt.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"val":%d}`, i))) // Purge old revision JSON to simulate expiry, and to verify import doesn't attempt multiple retrievals - purgeErr := collection.PurgeOldRevisionJSON(ctx, docID, revID) + // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) + cvHash := base.Crc32cHashString([]byte(cv)) + purgeErr := collection.PurgeOldRevisionJSON(ctx, docID, cvHash) require.NoError(t, purgeErr) - revID = version.RevID + cv = version.CV.String() } // 2. Modify doc via SDK @@ -460,6 +463,8 @@ func TestXattrDoubleDelete(t *testing.T) { } func TestViewQueryTombstoneRetrieval(t *testing.T) { + base.SkipImportTestsIfNotEnabled(t) + if !base.TestsDisableGSI() { t.Skip("views tests are not applicable under GSI") } @@ -1608,7 +1613,7 @@ func TestImportRevisionCopy(t *testing.T) { var rawInsertResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawInsertResponse) assert.NoError(t, err, "Unable to unmarshal raw response") - rev1id := rawInsertResponse.Sync.Rev + rev1id := rawInsertResponse.Sync.Rev.RevTreeID // 3. Update via SDK updatedBody := make(map[string]interface{}) @@ -1669,7 +1674,7 @@ func TestImportRevisionCopyUnavailable(t *testing.T) { var rawInsertResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawInsertResponse) assert.NoError(t, err, "Unable to unmarshal raw response") - rev1id := rawInsertResponse.Sync.Rev + rev1id := rawInsertResponse.Sync.Rev.RevTreeID // 3. Flush the rev cache (simulates attempted retrieval by a different SG node, since testing framework isn't great // at simulating multiple SG instances) @@ -1727,7 +1732,7 @@ func TestImportRevisionCopyDisabled(t *testing.T) { var rawInsertResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawInsertResponse) assert.NoError(t, err, "Unable to unmarshal raw response") - rev1id := rawInsertResponse.Sync.Rev + rev1id := rawInsertResponse.Sync.Rev.RevTreeID // 3. Update via SDK updatedBody := make(map[string]interface{}) @@ -1873,15 +1878,14 @@ func assertDocProperty(t *testing.T, getDocResponse *rest.TestResponse, property func assertXattrSyncMetaRevGeneration(t *testing.T, dataStore base.DataStore, key string, expectedRevGeneration int) { xattrs, _, err := dataStore.GetXattrs(base.TestCtx(t), key, []string{base.SyncXattrName}) - assert.NoError(t, err, "Error Getting Xattr") - xattr := map[string]interface{}{} + require.NoError(t, err, "Error Getting Xattr") require.Contains(t, xattrs, base.SyncXattrName) - require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &xattr)) - revision, ok := xattr["rev"] - assert.True(t, ok) - generation, _ := db.ParseRevID(base.TestCtx(t), revision.(string)) - log.Printf("assertXattrSyncMetaRevGeneration generation: %d rev: %s", generation, revision) - assert.True(t, generation == expectedRevGeneration) + var syncData db.SyncData + require.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) + assert.True(t, syncData.CurrentRev != "") + generation, _ := db.ParseRevID(base.TestCtx(t), syncData.CurrentRev) + log.Printf("assertXattrSyncMetaRevGeneration generation: %d rev: %s", generation, syncData.CurrentRev) + assert.Equal(t, expectedRevGeneration, generation) } func TestDeletedEmptyDocumentImport(t *testing.T) { @@ -1905,13 +1909,12 @@ func TestDeletedEmptyDocumentImport(t *testing.T) { // Get the doc and check deleted revision is getting imported response = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_raw/"+docId, "") assert.Equal(t, http.StatusOK, response.Code) - rawResponse := make(map[string]interface{}) + var rawResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawResponse) require.NoError(t, err, "Unable to unmarshal raw response") - assert.True(t, rawResponse[db.BodyDeleted].(bool)) - syncMeta := rawResponse["_sync"].(map[string]interface{}) - assert.Equal(t, "2-5d3308aae9930225ed7f6614cf115366", syncMeta["rev"]) + assert.True(t, rawResponse.Deleted) + assert.Equal(t, "2-5d3308aae9930225ed7f6614cf115366", rawResponse.Sync.Rev.RevTreeID) } // Check deleted document via SDK is getting imported if it is included in through ImportFilter function. @@ -1945,10 +1948,9 @@ func TestDeletedDocumentImportWithImportFilter(t *testing.T) { endpoint := fmt.Sprintf("/{{.keyspace}}/_raw/%s?redact=false", key) response := rt.SendAdminRequest(http.MethodGet, endpoint, "") assert.Equal(t, http.StatusOK, response.Code) - var respBody db.Body + var respBody rest.RawResponse require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &respBody)) - syncMeta := respBody[base.SyncPropertyName].(map[string]interface{}) - assert.NotEmpty(t, syncMeta["rev"].(string)) + assert.NotEmpty(t, respBody.Sync.Rev.RevTreeID) // Delete the document via SDK err = dataStore.Delete(key) @@ -1958,9 +1960,8 @@ func TestDeletedDocumentImportWithImportFilter(t *testing.T) { response = rt.SendAdminRequest(http.MethodGet, endpoint, "") assert.Equal(t, http.StatusOK, response.Code) require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &respBody)) - assert.True(t, respBody[db.BodyDeleted].(bool)) - syncMeta = respBody[base.SyncPropertyName].(map[string]interface{}) - assert.NotEmpty(t, syncMeta["rev"].(string)) + assert.True(t, respBody.Deleted) + assert.NotEmpty(t, respBody.Sync.Rev.RevTreeID) } // CBG-1995: Test the support for using an underscore prefix in the top-level body of a document @@ -2129,7 +2130,7 @@ func TestImportTouch(t *testing.T) { var rawInsertResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawInsertResponse) require.NoError(t, err, "Unable to unmarshal raw response") - initialRev := rawInsertResponse.Sync.Rev + initialRev := rawInsertResponse.Sync.Rev.RevTreeID // 2. Test import behaviour after SDK touch _, err = dataStore.Touch(key, 1000000) @@ -2141,7 +2142,7 @@ func TestImportTouch(t *testing.T) { var rawUpdateResponse rest.RawResponse err = base.JSONUnmarshal(response.Body.Bytes(), &rawUpdateResponse) require.NoError(t, err, "Unable to unmarshal raw response") - require.Equal(t, initialRev, rawUpdateResponse.Sync.Rev) + require.Equal(t, initialRev, rawUpdateResponse.Sync.Rev.RevTreeID) } func TestImportingPurgedDocument(t *testing.T) { if !base.TestUseXattrs() { @@ -2380,3 +2381,303 @@ func TestImportUpdateExpiry(t *testing.T) { }) } } + +func TestPrevRevNoPopulationImportFeed(t *testing.T) { + base.SkipImportTestsIfNotEnabled(t) + if base.UnitTestUrlIsWalrus() { + t.Skipf("test requires CBS for previous rev no assertion, CBG-4233") + } + + rtConfig := rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: true, + }}, + } + + rt := rest.NewRestTester(t, &rtConfig) + defer rt.Close() + dataStore := rt.GetSingleDataStore() + ctx := base.TestCtx(t) + + if !rt.Bucket().IsSupported(sgbucket.BucketStoreFeatureMultiXattrSubdocOperations) { + t.Skip("Test requires multi-xattr subdoc operations, CBS 7.6 or higher") + } + + // Create doc via the SDK + mobileKey := t.Name() + mobileBody := make(map[string]interface{}) + mobileBody["channels"] = "ABC" + _, err := dataStore.Add(mobileKey, 0, mobileBody) + assert.NoError(t, err, "Error writing SDK doc") + + // Wait for import + base.RequireWaitForStat(t, func() int64 { + return rt.GetDatabase().DbStats.SharedBucketImportStats.ImportCount.Value() + }, 1) + + xattrs, _, err := dataStore.GetXattrs(ctx, mobileKey, []string{base.MouXattrName, base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + + var mou *db.MetadataOnlyUpdate + mouXattr, ok := xattrs[base.MouXattrName] + require.True(t, ok) + docxattr, ok := xattrs[base.VirtualXattrRevSeqNo] + require.True(t, ok) + require.NoError(t, base.JSONUnmarshal(mouXattr, &mou)) + revNo := db.RetrieveDocRevSeqNo(t, docxattr) + // curr rev no should be 2, so prev rev is 1 + assert.Equal(t, revNo-1, mou.PreviousRevSeqNo) + + err = dataStore.Set(mobileKey, 0, nil, []byte(`{"test":"update"}`)) + require.NoError(t, err) + + base.RequireWaitForStat(t, func() int64 { + return rt.GetDatabase().DbStats.SharedBucketImportStats.ImportCount.Value() + }, 2) + + xattrs, _, err = dataStore.GetXattrs(ctx, mobileKey, []string{base.MouXattrName, base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + + mou = nil + mouXattr, ok = xattrs[base.MouXattrName] + require.True(t, ok) + docxattr, ok = xattrs[base.VirtualXattrRevSeqNo] + require.True(t, ok) + require.NoError(t, base.JSONUnmarshal(mouXattr, &mou)) + revNo = db.RetrieveDocRevSeqNo(t, docxattr) + // curr rev no should be 4, so prev rev is 3 + assert.Equal(t, revNo-1, mou.PreviousRevSeqNo) + +} + +// TestMigrationOfAttachmentsOnImport: +// - Create a doc and move the attachment metadata from global xattr to sync data xattr in a way that when the doc +// arrives over import feed it will be determined that it doesn't require import +// - Wait for the doc to arrive over import feed and assert even though the doc is not imported it will still get +// attachment metadata migrated from sync data to global xattr +// - Create a doc and move the attachment metadata from global xattr to sync data xattr in a way that when the doc +// arrives over import feed it will be determined that it does require import +// - Wait for the doc to arrive over the import feed and assert that once doc was imported the attachment metadata +// was migrated from sync data xattr to global xattr +func TestMigrationOfAttachmentsOnImport(t *testing.T) { + base.SkipImportTestsIfNotEnabled(t) + + rtConfig := rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: true, + }}, + } + rt := rest.NewRestTester(t, &rtConfig) + defer rt.Close() + dataStore := rt.GetSingleDataStore() + ctx := base.TestCtx(t) + + // add new doc to test a doc arriving import feed that doesn't need importing still has attachment migration take place + key := "doc1" + body := `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` + rt.PutDoc(key, body) + + // grab defined attachment metadata to move to sync data + value, xattrs, cas, err := dataStore.GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok := xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok := xattrs[base.GlobalXattrName] + require.True(t, ok) + + var attachs db.GlobalSyncData + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, true, dataStore) + + // retry loop to wait for import event to arrive over dcp, as doc won't be 'imported' we can't wait for import stat + var retryXattrs map[string][]byte + err = rt.WaitForCondition(func() bool { + retryXattrs, _, err = dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + _, ok := retryXattrs[base.GlobalXattrName] + return ok + }) + require.NoError(t, err) + + syncXattr, ok = retryXattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = retryXattrs[base.GlobalXattrName] + require.True(t, ok) + + // empty global sync, + attachs = db.GlobalSyncData{} + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + var syncData db.SyncData + err = base.JSONUnmarshal(syncXattr, &syncData) + require.NoError(t, err) + + // assert that the attachment metadata has been moved + assert.NotNil(t, attachs.GlobalAttachments) + assert.Nil(t, syncData.Attachments) + att := attachs.GlobalAttachments["hello.txt"].(map[string]interface{}) + assert.Equal(t, float64(11), att["length"]) + + // assert that no import took place + base.RequireWaitForStat(t, func() int64 { + return rt.GetDatabase().DbStats.SharedBucketImportStats.ImportCount.Value() + }, 0) + + // add new doc to test import of doc over feed moves attachments + key = "doc2" + body = `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` + rt.PutDoc(key, body) + + _, xattrs, cas, err = dataStore.GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + // grab defined attachment metadata to move to sync data + attachs = db.GlobalSyncData{} + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + // change doc body to trigger import on feed + value = []byte(`{"test": "doc"}`) + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, false, dataStore) + + // Wait for import + base.RequireWaitForStat(t, func() int64 { + return rt.GetDatabase().DbStats.SharedBucketImportStats.ImportCount.Value() + }, 1) + + // grab the sync and global xattr from doc2 + xattrs, _, err = dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + syncData = db.SyncData{} + err = base.JSONUnmarshal(syncXattr, &syncData) + require.NoError(t, err) + + // assert that the attachment metadata has been moved + assert.NotNil(t, attachs.GlobalAttachments) + assert.Nil(t, syncData.Attachments) + att = attachs.GlobalAttachments["hello.txt"].(map[string]interface{}) + assert.Equal(t, float64(11), att["length"]) +} + +// TestMigrationOfAttachmentsOnDemandImport: +// - Create a doc and move the attachment metadata from global xattr to sync data xattr +// - Trigger on demand import for get +// - Assert that the attachment metadata is migrated from sync data xattr to global sync xattr +// - Create a new doc and move the attachment metadata from global xattr to sync data xattr +// - Trigger an on demand import for write +// - Assert that the attachment metadata is migrated from sync data xattr to global sync xattr +func TestMigrationOfAttachmentsOnDemandImport(t *testing.T) { + base.SkipImportTestsIfNotEnabled(t) + + rtConfig := rest.RestTesterConfig{ + DatabaseConfig: &rest.DatabaseConfig{DbConfig: rest.DbConfig{ + AutoImport: false, // avoid anything arriving over import feed for this test + }}, + } + rt := rest.NewRestTester(t, &rtConfig) + defer rt.Close() + dataStore := rt.GetSingleDataStore() + ctx := base.TestCtx(t) + + key := "doc1" + body := `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` + rt.PutDoc(key, body) + + _, xattrs, cas, err := dataStore.GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok := xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok := xattrs[base.GlobalXattrName] + require.True(t, ok) + + // grab defined attachment metadata to move to sync data + var attachs db.GlobalSyncData + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + + value := []byte(`{"update": "doc"}`) + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, false, dataStore) + + // on demand import for get + _, _ = rt.GetDoc(key) + + xattrs, _, err = dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + + // empty global sync, + attachs = db.GlobalSyncData{} + + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + var syncData db.SyncData + err = base.JSONUnmarshal(syncXattr, &syncData) + require.NoError(t, err) + + // assert that the attachment metadata has been moved + assert.NotNil(t, attachs.GlobalAttachments) + assert.Nil(t, syncData.Attachments) + att := attachs.GlobalAttachments["hello.txt"].(map[string]interface{}) + assert.Equal(t, float64(11), att["length"]) + + key = "doc2" + body = `{"test": true, "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}` + rt.PutDoc(key, body) + + _, xattrs, cas, err = dataStore.GetWithXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + + // grab defined attachment metadata to move to sync data + attachs = db.GlobalSyncData{} + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + value = []byte(`{"update": "doc"}`) + db.MoveAttachmentXattrFromGlobalToSync(t, ctx, key, cas, value, syncXattr, attachs.GlobalAttachments, false, dataStore) + + // trigger on demand import for write + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc2", `{}`) + rest.RequireStatus(t, resp, http.StatusConflict) + + // assert that the attachments metadata is migrated + xattrs, _, err = dataStore.GetXattrs(ctx, key, []string{base.SyncXattrName, base.GlobalXattrName}) + require.NoError(t, err) + syncXattr, ok = xattrs[base.SyncXattrName] + require.True(t, ok) + globalXattr, ok = xattrs[base.GlobalXattrName] + require.True(t, ok) + + // empty global sync, + attachs = db.GlobalSyncData{} + err = base.JSONUnmarshal(globalXattr, &attachs) + require.NoError(t, err) + syncData = db.SyncData{} + err = base.JSONUnmarshal(syncXattr, &syncData) + require.NoError(t, err) + + // assert that the attachment metadata has been moved + assert.NotNil(t, attachs.GlobalAttachments) + assert.Nil(t, syncData.Attachments) + att = attachs.GlobalAttachments["hello.txt"].(map[string]interface{}) + assert.Equal(t, float64(11), att["length"]) +} diff --git a/rest/importuserxattrtest/revid_import_test.go b/rest/importuserxattrtest/revid_import_test.go index 83f5cc3cef..e16b4a32cb 100644 --- a/rest/importuserxattrtest/revid_import_test.go +++ b/rest/importuserxattrtest/revid_import_test.go @@ -60,7 +60,7 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { assert.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData)) collection, ctx := rt.GetSingleTestDatabaseCollection() - docRev, err := collection.GetRevisionCacheForTest().Get(ctx, docKey, syncData.CurrentRev, false) + docRev, err := collection.GetRevisionCacheForTest().GetWithRev(ctx, docKey, syncData.CurrentRev, false) assert.NoError(t, err) assert.Len(t, docRev.Channels.ToArray(), 0) assert.Equal(t, syncData.CurrentRev, docRev.RevID) @@ -82,7 +82,7 @@ func TestUserXattrAvoidRevisionIDGeneration(t *testing.T) { require.Contains(t, xattrs, base.SyncXattrName) assert.NoError(t, base.JSONUnmarshal(xattrs[base.SyncXattrName], &syncData2)) - docRev2, err := collection.GetRevisionCacheForTest().Get(ctx, docKey, syncData.CurrentRev, false) + docRev2, err := collection.GetRevisionCacheForTest().GetWithRev(ctx, docKey, syncData.CurrentRev, false) assert.NoError(t, err) assert.Equal(t, syncData2.CurrentRev, docRev2.RevID) diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index ef0fd83f06..aec89a9747 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -2833,7 +2833,7 @@ func TestActiveReplicatorPullMergeConflictingAttachments(t *testing.T) { rt1.WaitForReplicationStatus("repl1", db.ReplicationStateStopped) - resp = rt1.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevID, test.localConflictingRevBody) + resp = rt1.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevTreeID, test.localConflictingRevBody) rest.RequireStatus(t, resp, http.StatusCreated) changesResults, err = rt1.WaitForChanges(1, "/{{.keyspace}}/_changes?since="+lastSeq, "", true) @@ -2842,7 +2842,7 @@ func TestActiveReplicatorPullMergeConflictingAttachments(t *testing.T) { assert.Equal(t, docID, changesResults.Results[0].ID) lastSeq = changesResults.Last_Seq.String() - resp = rt2.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevID, test.remoteConflictingRevBody) + resp = rt2.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+docID+"?rev="+version1.RevTreeID, test.remoteConflictingRevBody) rest.RequireStatus(t, resp, http.StatusCreated) resp = rt1.SendAdminRequest(http.MethodPut, "/{{.db}}/_replicationStatus/repl1?action=start", "") @@ -5956,7 +5956,7 @@ func TestActiveReplicatorPullConflictReadWriteIntlProps(t *testing.T) { createVersion := func(generation int, parentRevID string, body db.Body) rest.DocVersion { rev, err := db.CreateRevID(generation, parentRevID, body) require.NoError(t, err, "Error creating revision") - return rest.DocVersion{RevID: rev} + return rest.DocVersion{RevTreeID: rev} } docExpiry := time.Now().Local().Add(time.Hour * time.Duration(4)).Format(time.RFC3339) @@ -6403,7 +6403,7 @@ func TestSGR2TombstoneConflictHandling(t *testing.T) { assert.NoError(t, err) // Create another rev and then delete doc on local - ie tree is longer - version := localActiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevID: "3-abc"}, `{"foo":"bar"}`) + version := localActiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevTreeID: "3-abc"}, `{"foo":"bar"}`) localActiveRT.DeleteDoc(doc2ID, version) // Validate local is CBS tombstone, expect not found error @@ -6430,7 +6430,7 @@ func TestSGR2TombstoneConflictHandling(t *testing.T) { assert.NoError(t, err) // Create another rev and then delete doc on remotePassiveRT (passive) - ie, tree is longer - version := remotePassiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevID: "3-abc"}, `{"foo":"bar"}`) + version := remotePassiveRT.UpdateDoc(doc2ID, rest.DocVersion{RevTreeID: "3-abc"}, `{"foo":"bar"}`) remotePassiveRT.DeleteDoc(doc2ID, version) // Validate local is CBS tombstone, expect not found error @@ -7390,7 +7390,7 @@ func TestReplicatorDoNotSendDeltaWhenSrcIsTombstone(t *testing.T) { // Get revision 2 on passive peer to assert it has been (a) replicated and (b) deleted var rawResponse *rest.TestResponse err = passiveRT.WaitForCondition(func() bool { - rawResponse = passiveRT.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/test?rev="+deletedVersion.RevID, "") + rawResponse = passiveRT.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/test?rev="+deletedVersion.RevTreeID, "") return rawResponse.Code == http.StatusOK }) require.NoError(t, err) @@ -7553,23 +7553,25 @@ func TestReplicatorIgnoreRemovalBodies(t *testing.T) { }) defer activeRT.Close() activeCtx := activeRT.Context() + collection, _ := activeRT.GetSingleTestDatabaseCollection() docID := t.Name() // Create the docs // // Doc rev 1 - version1 := activeRT.PutDoc(docID, `{"key":"12","channels": ["rev1chan"]}`) + version1 := activeRT.PutDocDirectly(docID, rest.JsonToMap(t, `{"key":"12","channels": ["rev1chan"]}`)) require.NoError(t, activeRT.WaitForVersion(docID, version1)) // doc rev 2 - version2 := activeRT.UpdateDoc(docID, version1, `{"key":"12","channels":["rev2+3chan"]}`) + version2 := activeRT.UpdateDocDirectly(docID, version1, rest.JsonToMap(t, `{"key":"12","channels":["rev2+3chan"]}`)) require.NoError(t, activeRT.WaitForVersion(docID, version2)) // Doc rev 3 - version3 := activeRT.UpdateDoc(docID, version2, `{"key":"3","channels":["rev2+3chan"]}`) + version3 := activeRT.UpdateDocDirectly(docID, version2, rest.JsonToMap(t, `{"key":"3","channels":["rev2+3chan"]}`)) require.NoError(t, activeRT.WaitForVersion(docID, version3)) activeRT.GetDatabase().FlushRevisionCacheForTest() - err := activeRT.GetSingleDataStore().Delete(fmt.Sprintf("_sync:rev:%s:%d:%s", t.Name(), len(version2.RevID), version2.RevID)) + cvHash := base.Crc32cHashString([]byte(version2.CV.String())) + err := collection.PurgeOldRevisionJSON(activeCtx, docID, cvHash) require.NoError(t, err) // Set-up replicator // passiveDBURL, err := url.Parse(srv.URL + "/db") @@ -8578,3 +8580,47 @@ func requireBodyEqual(t *testing.T, expected string, doc *db.Document) { require.NoError(t, base.JSONUnmarshal([]byte(expected), &expectedBody)) require.Equal(t, expectedBody, doc.Body(base.TestCtx(t))) } + +// TestReplicatorUpdateHLVOnPut: +// - For purpose of testing the PutExistingRev code path +// - Put a doc on a active rest tester +// - Create replication and wait for the doc to be replicated to passive node +// - Assert on the HLV in the metadata of the replicated document +func TestReplicatorUpdateHLVOnPut(t *testing.T) { + + activeRT, passiveRT, remoteURL, teardown := rest.SetupSGRPeers(t) + defer teardown() + + // Grab the bucket UUIDs for both rest testers + activeBucketUUID := activeRT.GetDatabase().EncodedSourceID + passiveBucketUUID := passiveRT.GetDatabase().EncodedSourceID + + const rep = "replication" + + // Put a doc and assert on the HLV update in the sync data + resp := activeRT.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/doc1", `{"source": "activeRT"}`) + rest.RequireStatus(t, resp, http.StatusCreated) + + activeCollection, activeCtx := activeRT.GetSingleTestDatabaseCollection() + syncData, err := activeCollection.GetDocSyncData(activeCtx, "doc1") + assert.NoError(t, err) + + assert.Equal(t, activeBucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + + // create the replication to push the doc to the passive node and wait for the doc to be replicated + activeRT.CreateReplication(rep, remoteURL, db.ActiveReplicatorTypePush, nil, false, db.ConflictResolverDefault) + + _, err = passiveRT.WaitForChanges(1, "/{{.keyspace}}/_changes", "", true) + require.NoError(t, err) + + // assert on the HLV update on the passive node + passiveCollection, passiveCtx := passiveRT.GetSingleTestDatabaseCollection() + syncData, err = passiveCollection.GetDocSyncData(passiveCtx, "doc1") + assert.NoError(t, err) + + assert.Equal(t, passiveBucketUUID, syncData.HLV.SourceID) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) + assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) +} diff --git a/rest/replicatortest/replicator_test_helper.go b/rest/replicatortest/replicator_test_helper.go index e9468af6ab..ec0b6cf263 100644 --- a/rest/replicatortest/replicator_test_helper.go +++ b/rest/replicatortest/replicator_test_helper.go @@ -71,7 +71,7 @@ func addActiveRT(t *testing.T, dbName string, testBucket *base.TestBucket) (acti // requireDocumentVersion asserts that the given ChangeRev has the expected version for a given entry returned by _changes feed func requireDocumentVersion(t testing.TB, expected rest.DocVersion, doc *db.Document) { - rest.RequireDocVersionEqual(t, expected, rest.DocVersion{RevID: doc.SyncData.CurrentRev}) + rest.RequireDocVersionEqual(t, expected, rest.DocVersion{RevTreeID: doc.SyncData.CurrentRev}) } // requireRevID asserts that the specified document version is written to the diff --git a/rest/revocation_test.go b/rest/revocation_test.go index d0bab3e9fa..0717f2ae4d 100644 --- a/rest/revocation_test.go +++ b/rest/revocation_test.go @@ -1008,10 +1008,10 @@ func TestRevocationResumeAndLowSeqCheck(t *testing.T) { changes = revocationTester.getChanges(changes.Last_Seq, 2) assert.Equal(t, doc1ID, changes.Results[0].ID) - assert.Equal(t, doc1Version.RevID, changes.Results[0].Changes[0]["rev"]) + assert.Equal(t, doc1Version.RevTreeID, changes.Results[0].Changes[0]["rev"]) assert.True(t, changes.Results[0].Revoked) assert.Equal(t, doc2ID, changes.Results[1].ID) - assert.Equal(t, doc2Version.RevID, changes.Results[1].Changes[0]["rev"]) + assert.Equal(t, doc2Version.RevTreeID, changes.Results[1].Changes[0]["rev"]) assert.True(t, changes.Results[1].Revoked) changes = revocationTester.getChanges("20:40", 1) @@ -2241,7 +2241,7 @@ func TestRevocationMessage(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDoc("doc", `{"channels": "A"}`) + version := rt.PutDocDirectly("doc", db.Body{"channels": "A"}) // Start pull rt.WaitForPendingChanges() @@ -2254,10 +2254,10 @@ func TestRevocationMessage(t *testing.T) { revocationTester.removeRole("user", "foo") const doc1ID = "doc1" - version = rt.PutDoc(doc1ID, `{"channels": "!"}`) + version = rt.PutDocDirectly(doc1ID, db.Body{"channels": "!"}) revocationTester.fillToSeq(10) - version = rt.UpdateDoc(doc1ID, version, "{}") + version = rt.UpdateDocDirectly(doc1ID, version, db.Body{}) // Start a pull since 5 to receive revocation and removal rt.WaitForPendingChanges() @@ -2351,7 +2351,9 @@ func TestRevocationNoRev(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDoc(docID, `{"channels": "A"}`) + + version := rt.PutDocDirectly(docID, db.Body{"channels": "A"}) + rt.WaitForPendingChanges() firstOneShotSinceSeq := rt.GetDocumentSequence("doc") // OneShot pull to grab doc @@ -2363,10 +2365,11 @@ func TestRevocationNoRev(t *testing.T) { // Remove role from user revocationTester.removeRole("user", "foo") - _ = rt.UpdateDoc(docID, version, `{"channels": "A", "val": "mutate"}`) + _ = rt.UpdateDocDirectly(docID, version, db.Body{"channels": "A", "val": "mutate"}) - waitMarkerVersion := rt.PutDoc(waitMarkerID, `{"channels": "!"}`) + waitMarkerVersion := rt.PutDocDirectly(waitMarkerID, db.Body{"channels": "!"}) rt.WaitForPendingChanges() + lastSeqStr := strconv.FormatUint(firstOneShotSinceSeq, 10) btcRunner.StartPullSince(btc.id, BlipTesterPullOptions{Continuous: false, Since: lastSeqStr}) @@ -2443,7 +2446,7 @@ func TestRevocationGetSyncDataError(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDoc(docID, `{"channels": "A"}}`) + version := rt.PutDocDirectly(docID, db.Body{"channels": "A"}) // OneShot pull to grab doc rt.WaitForPendingChanges() @@ -2457,9 +2460,10 @@ func TestRevocationGetSyncDataError(t *testing.T) { // Remove role from user revocationTester.removeRole("user", "foo") - _ = rt.UpdateDoc(docID, version, `{"channels": "A", "val": "mutate"}`) + _ = rt.UpdateDocDirectly(docID, version, db.Body{"channels": "A", "val": "mutate"}) - waitMarkerVersion := rt.PutDoc(waitMarkerID, `{"channels": "!"}`) + waitMarkerVersion := rt.PutDocDirectly(waitMarkerID, db.Body{"channels": "!"}) + rt.WaitForPendingChanges() rt.WaitForPendingChanges() lastSeqStr := strconv.FormatUint(firstOneShotSinceSeq, 10) diff --git a/rest/routing.go b/rest/routing.go index 37765078e4..9a22b6c62d 100644 --- a/rest/routing.go +++ b/rest/routing.go @@ -164,6 +164,10 @@ func CreateAdminRouter(sc *ServerContext) *mux.Router { makeHandler(sc, adminPrivs, []Permission{PermUpdateDb}, nil, (*handler).handleCompact)).Methods("POST") dbr.Handle("/_compact", makeHandler(sc, adminPrivs, []Permission{PermUpdateDb}, nil, (*handler).handleGetCompact)).Methods("GET") + dbr.Handle("/_attachment_migration", + makeHandler(sc, adminPrivs, []Permission{PermUpdateDb}, nil, (*handler).handleAttachmentMigration)).Methods("POST") + dbr.Handle("/_attachment_migration", + makeHandler(sc, adminPrivs, []Permission{PermUpdateDb}, nil, (*handler).handleGetAttachmentMigration)).Methods("GET") dbr.Handle("/_session", makeHandler(sc, adminPrivs, []Permission{PermWritePrincipal}, nil, (*handler).createUserSession)).Methods("POST") dbr.Handle("/_session/{sessionid}", diff --git a/rest/server_context.go b/rest/server_context.go index 56887d8efd..4c089f7e39 100644 --- a/rest/server_context.go +++ b/rest/server_context.go @@ -651,6 +651,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config hasDefaultCollection := false collectionsRequiringResync := make([]base.ScopeAndCollectionName, 0) + collectionsRequiringAttachmentMigration := make([]base.ScopeAndCollectionName, 0) if len(config.Scopes) > 0 { if !bucket.IsSupported(sgbucket.BucketStoreFeatureCollections) { return nil, errCollectionsUnsupported @@ -676,13 +677,18 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config } // Verify whether the collection is associated with a different database's metadataID - if so, add to set requiring resync - resyncRequired, err := base.InitSyncInfo(dataStore, config.MetadataID) + resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, dataStore, config.MetadataID) if err != nil { return nil, err } if resyncRequired { collectionsRequiringResync = append(collectionsRequiringResync, scName) } + + if requiresAttachmentMigration { + collectionsRequiringAttachmentMigration = append(collectionsRequiringAttachmentMigration, scName) + } + } } } @@ -692,13 +698,17 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config // No explicitly defined scopes means we'll initialize this as a usable default collection, otherwise it's for metadata only if len(config.Scopes) == 0 { scName := base.DefaultScopeAndCollectionName() - resyncRequired, err := base.InitSyncInfo(ds, config.MetadataID) + resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, ds, config.MetadataID) if err != nil { return nil, err } if resyncRequired { collectionsRequiringResync = append(collectionsRequiringResync, scName) } + + if requiresAttachmentMigration { + collectionsRequiringAttachmentMigration = append(collectionsRequiringAttachmentMigration, base.ScopeAndCollectionName{Scope: base.DefaultScope, Collection: base.DefaultCollection}) + } } if useViews { if err := db.InitializeViews(ctx, ds); err != nil { @@ -815,6 +825,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config dbcontext.ServerContextHasStarted = sc.hasStarted dbcontext.NoX509HTTPClient = sc.NoX509HTTPClient dbcontext.RequireResync = collectionsRequiringResync + dbcontext.RequireAttachmentMigration = collectionsRequiringAttachmentMigration if config.CORS != nil { dbcontext.CORS = config.DbConfig.CORS @@ -1071,6 +1082,11 @@ func dbcOptionsFromConfig(ctx context.Context, sc *ServerContext, config *DbConf } } + // In sync gateway version 4.0+ we do not support the disabling of use of xattrs + if !config.UseXattrs() { + return db.DatabaseContextOptions{}, fmt.Errorf("sync gateway requires enable_shared_bucket_access=true") + } + oldRevExpirySeconds := base.DefaultOldRevExpirySeconds if config.OldRevExpirySeconds != nil { oldRevExpirySeconds = *config.OldRevExpirySeconds diff --git a/rest/serverless_test.go b/rest/serverless_test.go index 3f7b270f69..8e8d54baf7 100644 --- a/rest/serverless_test.go +++ b/rest/serverless_test.go @@ -317,6 +317,7 @@ func TestServerlessSuspendDatabase(t *testing.T) { assert.NotNil(t, sc.dbConfigs["db"]) // Update config in bucket to see if unsuspending check for updates + sc.dbConfigs["db"].EnableXattrs = base.BoolPtr(true) // xattrs must be enabled cas, err := sc.BootstrapContext.UpdateConfig(base.TestCtx(t), tb.GetName(), sc.Config.Bootstrap.ConfigGroupID, "db", func(bucketDbConfig *DatabaseConfig) (updatedConfig *DatabaseConfig, err error) { config := sc.dbConfigs["db"].ToDatabaseConfig() config.cfgCas = bucketDbConfig.cfgCas diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 6675831acd..69ed7d7363 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -853,7 +853,7 @@ func (cr ChangesResults) RequireRevID(t testing.TB, revIDs []string) { // RequireChangeRevVersion asserts that the given ChangeRev has the expected version for a given entry returned by _changes feed func RequireChangeRevVersion(t *testing.T, expected DocVersion, changeRev db.ChangeRev) { - RequireDocVersionEqual(t, expected, DocVersion{RevID: changeRev["rev"]}) + RequireDocVersionEqual(t, expected, DocVersion{RevTreeID: changeRev["rev"]}) } func (rt *RestTester) CreateWaitForChangesRetryWorker(numChangesExpected int, changesURL, username string, useAdminPort bool) (worker base.RetryWorker) { @@ -1137,12 +1137,13 @@ func (rt *RestTester) SetAdminChannels(username string, keyspace string, channel type SimpleSync struct { Channels map[string]interface{} - Rev string + Rev channels.RevAndVersion Sequence uint64 } type RawResponse struct { - Sync SimpleSync `json:"_sync"` + Sync SimpleSync `json:"_sync"` + Deleted bool `json:"_deleted"` } // GetDocumentSequence looks up the sequence for a document using the _raw endpoint. @@ -2432,30 +2433,11 @@ func WaitAndAssertBackgroundManagerExpiredHeartbeat(t testing.TB, bm *db.Backgro return assert.Truef(t, base.IsDocNotFoundError(err), "expected heartbeat doc to expire, but got a different error: %v", err) } -// DocVersion represents a specific version of a document in an revID/HLV agnostic manner. -type DocVersion struct { - RevID string -} - -func (v *DocVersion) String() string { - return fmt.Sprintf("RevID: %s", v.RevID) -} - -func (v DocVersion) Equal(o DocVersion) bool { - if v.RevID != o.RevID { - return false - } - return true -} - -// Digest returns the digest for the current version -func (v DocVersion) Digest() string { - return strings.Split(v.RevID, "-")[1] -} +type DocVersion = db.DocVersion // RequireDocVersionNotNil calls t.Fail if two document version is not specified. func RequireDocVersionNotNil(t *testing.T, version DocVersion) { - require.NotEqual(t, "", version.RevID) + require.NotEqual(t, "", version.RevTreeID) } // RequireDocVersionEqual calls t.Fail if two document versions are not equal. @@ -2470,12 +2452,12 @@ func RequireDocVersionNotEqual(t *testing.T, expected, actual DocVersion) { // EmptyDocVersion reprents an empty document version. func EmptyDocVersion() DocVersion { - return DocVersion{RevID: ""} + return DocVersion{RevTreeID: ""} } // NewDocVersionFromFakeRev returns a new DocVersion from the given fake rev ID, intended for use when we explicit create conflicts. func NewDocVersionFromFakeRev(fakeRev string) DocVersion { - return DocVersion{RevID: fakeRev} + return DocVersion{RevTreeID: fakeRev} } // DocVersionFromPutResponse returns a DocRevisionID from the given response to PUT /{, or fails the given test if a rev ID was not found. @@ -2487,7 +2469,7 @@ func DocVersionFromPutResponse(t testing.TB, response *TestResponse) DocVersion require.NoError(t, json.Unmarshal(response.BodyBytes(), &r)) require.NotNil(t, r.RevID, "expecting non-nil rev ID from response: %s", string(response.BodyBytes())) require.NotEqual(t, "", *r.RevID, "expecting non-empty rev ID from response: %s", string(response.BodyBytes())) - return DocVersion{RevID: *r.RevID} + return DocVersion{RevTreeID: *r.RevID} } func MarshalConfig(t *testing.T, config db.ReplicationConfig) string { @@ -2824,3 +2806,19 @@ func RequireGocbDCPResync(t *testing.T) { t.Skip("This test only works against Couchbase Server since rosmar has no support for DCP resync") } } + +// SafeDatabaseName returns a database name free of any special characters for use in tests. +func SafeDatabaseName(t *testing.T, name string) string { + dbName := strings.ToLower(name) + for _, c := range []string{" ", "<", ">", "/", "="} { + dbName = strings.ReplaceAll(dbName, c, "_") + } + return dbName +} + +func JsonToMap(t *testing.T, jsonStr string) map[string]interface{} { + result := make(map[string]interface{}) + err := json.Unmarshal([]byte(jsonStr), &result) + require.NoError(t, err) + return result +} diff --git a/rest/utilities_testing_attachment.go b/rest/utilities_testing_attachment.go index 4f02b3135b..e3dc9986ea 100644 --- a/rest/utilities_testing_attachment.go +++ b/rest/utilities_testing_attachment.go @@ -12,6 +12,7 @@ import ( "context" "net/http" "testing" + "time" sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" @@ -85,3 +86,17 @@ func CreateLegacyAttachmentDoc(t *testing.T, ctx context.Context, collection *db return attDocID } + +func (rt *RestTester) WaitForAttachmentMigrationStatus(t *testing.T, state db.BackgroundProcessState) db.AttachmentMigrationManagerResponse { + var response db.AttachmentMigrationManagerResponse + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp := rt.SendAdminRequest("GET", "/{{.db}}/_attachment_migration", "") + require.Equal(c, http.StatusOK, resp.Code) + + err := base.JSONUnmarshal(resp.BodyBytes(), &response) + require.NoError(c, err) + assert.Equal(c, state, response.State) + }, time.Second*20, time.Millisecond*100) + + return response +} diff --git a/rest/blip_client_test.go b/rest/utilities_testing_blip_client.go similarity index 74% rename from rest/blip_client_test.go rename to rest/utilities_testing_blip_client.go index 919c40bd3f..69c7d22eba 100644 --- a/rest/blip_client_test.go +++ b/rest/utilities_testing_blip_client.go @@ -31,6 +31,11 @@ import ( "github.com/stretchr/testify/require" ) +const ( + VersionVectorSubtestName = "versionVector" + RevtreeSubtestName = "revTree" +) + type BlipTesterClientOpts struct { ClientDeltas bool // Support deltas on the client side Username string @@ -39,6 +44,7 @@ type BlipTesterClientOpts struct { SupportedBLIPProtocols []string SkipCollectionsInitialization bool + AllowCreationWithoutBlipTesterClientRunner bool // Allow the client to be created outside of a BlipTesterClientRunner.Run() subtest // a deltaSrc rev ID for which to reject a delta rejectDeltasForSrcRev string @@ -50,6 +56,9 @@ type BlipTesterClientOpts struct { // sendReplacementRevs opts into the replacement rev behaviour in the event that we do not find the requested one. sendReplacementRevs bool + + // SourceID is used to define the SourceID for the blip client + SourceID string } // BlipTesterClient is a fully fledged client to emulate CBL behaviour on both push and pull replications through methods on this type. @@ -66,31 +75,111 @@ type BlipTesterClient struct { } type BlipTesterCollectionClient struct { - parent *BlipTesterClient - - collection string - collectionIdx int - - docs map[string]map[string]*BodyMessagePair // Client's local store of documents - Map of docID - // to rev ID to bytes - attachments map[string][]byte // Client's local store of attachments - Map of digest to bytes - lastReplicatedRev map[string]string // Latest known rev pulled or pushed - docsLock sync.RWMutex // lock for docs map - attachmentsLock sync.RWMutex // lock for attachments map - lastReplicatedRevLock sync.RWMutex // lock for lastReplicatedRev map + parent *BlipTesterClient + collection string + collectionIdx int + docs map[string]*BlipTesterDoc // Client's local store of documents, indexed by DocID + attachments map[string][]byte // Client's local store of attachments - Map of digest to bytes + docsLock sync.RWMutex // lock for docs map + attachmentsLock sync.RWMutex // lock for attachments map } // BlipTestClientRunner is for running the blip tester client and its associated methods in test framework type BlipTestClientRunner struct { - clients map[uint32]*BlipTesterClient // map of created BlipTesterClient's - t *testing.T - initialisedInsideRunnerCode bool // flag to check that the BlipTesterClient is being initialised in the correct area (inside the Run() method) - SkipVersionVectorInitialization bool // used to skip the version vector subtest + clients map[uint32]*BlipTesterClient // map of created BlipTesterClient's + t *testing.T + initialisedInsideRunnerCode bool // flag to check that the BlipTesterClient is being initialised in the correct area (inside the Run() method) + SkipSubtest map[string]bool // map of sub tests on the blip tester runner to skip } -type BodyMessagePair struct { - body []byte - message *blip.Message +type BlipTesterDoc struct { + revMode blipTesterRevMode + body []byte + revMessageHistory map[string]*blip.Message // History of rev messages received for this document, indexed by revID + revHistory []string // ordered history of revTreeIDs (newest first), populated when mode = revtree + HLV db.HybridLogicalVector // HLV, populated when mode = HLV +} + +const ( + revModeRevTree blipTesterRevMode = iota + revModeHLV +) + +type blipTesterRevMode uint32 + +func (doc *BlipTesterDoc) isRevKnown(revID string) bool { + if doc.revMode == revModeHLV { + version := VersionFromRevID(revID) + return doc.HLV.IsVersionKnown(version) + } else { + for _, revTreeID := range doc.revHistory { + if revTreeID == revID { + return true + } + } + } + return false +} + +func (doc *BlipTesterDoc) makeRevHistoryForChangesResponse() []string { + if doc.revMode == revModeHLV { + // For HLV, a changes response only needs to send cv, since rev message will always send full HLV + return []string{doc.HLV.GetCurrentVersionString()} + } else { + var revList []string + if len(doc.revHistory) < 20 { + revList = doc.revHistory + } else { + revList = doc.revHistory[0:19] + } + return revList + } +} + +func (doc *BlipTesterDoc) getCurrentRevID() string { + if doc.revMode == revModeHLV { + return doc.HLV.GetCurrentVersionString() + } else { + if len(doc.revHistory) == 0 { + return "" + } + return doc.revHistory[0] + } +} + +func (doc *BlipTesterDoc) addRevision(revID string, body []byte, message *blip.Message) { + doc.revMessageHistory[revID] = message + doc.body = body + if doc.revMode == revModeHLV { + _ = doc.HLV.AddVersion(VersionFromRevID(revID)) + } else { + // prepend revID to revHistory + doc.revHistory = append([]string{revID}, doc.revHistory...) + } +} + +func (btcr *BlipTesterCollectionClient) NewBlipTesterDoc(revID string, body []byte, message *blip.Message) *BlipTesterDoc { + doc := &BlipTesterDoc{ + body: body, + revMessageHistory: map[string]*blip.Message{revID: message}, + } + if btcr.UseHLV() { + doc.revMode = revModeHLV + doc.HLV = *db.NewHybridLogicalVector() + _ = doc.HLV.AddVersion(VersionFromRevID(revID)) + } else { + doc.revMode = revModeRevTree + doc.revHistory = []string{revID} + } + return doc +} + +func VersionFromRevID(revID string) db.Version { + version, err := db.ParseVersion(revID) + if err != nil { + panic(err) + } + return version } // BlipTesterReplicator is a BlipTester which stores a map of messages keyed by Serial Number @@ -107,8 +196,9 @@ type BlipTesterReplicator struct { // NewBlipTesterClientRunner creates a BlipTestClientRunner type func NewBlipTesterClientRunner(t *testing.T) *BlipTestClientRunner { return &BlipTestClientRunner{ - t: t, - clients: make(map[uint32]*BlipTesterClient), + t: t, + clients: make(map[uint32]*BlipTesterClient), + SkipSubtest: make(map[string]bool), } } @@ -150,7 +240,6 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { btr.bt.blipContext.HandlerForProfile[db.MessageChanges] = func(msg *blip.Message) { btr.storeMessage(msg) - btcr := btc.getCollectionClientFromMessage(msg) // Exit early when there's nothing to do @@ -190,33 +279,16 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { // Build up a list of revisions known to the client for each change // The first element of each revision list must be the parent revision of the change - if revs, haveDoc := btcr.docs[docID]; haveDoc { - revList := make([]string, 0, len(revs)) - - // Insert the highest ancestor rev generation at the start of the revList - latest, ok := btcr.getLastReplicatedRev(docID) - if ok { - revList = append(revList, latest) + if doc, haveDoc := btcr.docs[docID]; haveDoc { + if deletedInt&2 == 2 { + continue } - for knownRevID := range revs { - if deletedInt&2 == 2 { - continue - } - - if revID == knownRevID { - knownRevs[i] = nil // Send back null to signal we don't need this change - continue outer - } else if latest == knownRevID { - // We inserted this rev as the first element above, so skip it here - continue - } - - // TODO: Limit known revs to 20 to copy CBL behaviour - revList = append(revList, knownRevID) + if doc.isRevKnown(revID) { + knownRevs[i] = nil + continue outer } - - knownRevs[i] = revList + knownRevs[i] = doc.makeRevHistoryForChangesResponse() } else { knownRevs[i] = []interface{}{} // sending empty array means we've not seen the doc before, but still want it } @@ -257,21 +329,17 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { if msg.Properties[db.RevMessageDeleted] == "1" { btcr.docsLock.Lock() defer btcr.docsLock.Unlock() - if _, ok := btcr.docs[docID]; ok { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID][revID] = bodyMessagePair - if replacedRev != "" { - // store a pointer to the message from the replaced rev for tests waiting for this specific rev - btcr.docs[docID][replacedRev] = bodyMessagePair - } - } else { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} - if replacedRev != "" { - btcr.docs[docID][replacedRev] = bodyMessagePair - } + var doc *BlipTesterDoc + var ok bool + if doc, ok = btcr.docs[docID]; !ok { + doc = btcr.NewBlipTesterDoc(revID, body, msg) + btcr.docs[docID] = doc + } + // Add replacedRev first to maintain ordering + if replacedRev != "" { + doc.addRevision(replacedRev, body, msg) } - btcr.updateLastReplicatedRev(docID, revID) + doc.addRevision(revID, body, msg) if !msg.NoReply() { response := msg.Response() @@ -302,7 +370,12 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { var old db.Body btcr.docsLock.RLock() - oldBytes := btcr.docs[docID][deltaSrc].body + // deltaSrc must be the current rev + doc := btcr.docs[docID] + if doc.getCurrentRevID() != deltaSrc { + panic("current rev doesn't match deltaSrc") + } + oldBytes := doc.body btcr.docsLock.RUnlock() err = old.Unmarshal(oldBytes) require.NoError(btc.TB(), err) @@ -428,20 +501,16 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { btcr.docsLock.Lock() defer btcr.docsLock.Unlock() - if _, ok := btcr.docs[docID]; ok { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID][revID] = bodyMessagePair - if replacedRev != "" { - btcr.docs[docID][replacedRev] = bodyMessagePair - } - } else { - bodyMessagePair := &BodyMessagePair{body: body, message: msg} - btcr.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} - if replacedRev != "" { - btcr.docs[docID][replacedRev] = bodyMessagePair - } + var doc *BlipTesterDoc + var ok bool + if doc, ok = btcr.docs[docID]; !ok { + doc = btcr.NewBlipTesterDoc(revID, body, msg) + btcr.docs[docID] = doc + } + if replacedRev != "" { + doc.addRevision(replacedRev, body, msg) } - btcr.updateLastReplicatedRev(docID, revID) + doc.addRevision(revID, body, msg) if !msg.NoReply() { response := msg.Response() @@ -480,12 +549,10 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { btcr.docsLock.Lock() defer btcr.docsLock.Unlock() - if _, ok := btcr.docs[docID]; ok { - bodyMessagePair := &BodyMessagePair{message: msg} - btcr.docs[docID][revID] = bodyMessagePair + if doc, ok := btcr.docs[docID]; ok { + doc.addRevision(revID, nil, msg) } else { - bodyMessagePair := &BodyMessagePair{message: msg} - btcr.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} + btcr.docs[docID] = btcr.NewBlipTesterDoc(revID, nil, msg) } } @@ -505,6 +572,10 @@ func (btc *BlipTesterCollectionClient) TB() testing.TB { return btc.parent.rt.TB() } +func (btcc *BlipTesterCollectionClient) UseHLV() bool { + return btcc.parent.UseHLV() +} + // saveAttachment takes a content-type, and base64 encoded data and stores the attachment on the client func (btc *BlipTesterCollectionClient) saveAttachment(_, base64data string) (dataLength int, digest string, err error) { btc.attachmentsLock.Lock() @@ -539,32 +610,6 @@ func (btc *BlipTesterCollectionClient) getAttachment(digest string) (attachment return attachment, nil } -func (btc *BlipTesterCollectionClient) updateLastReplicatedRev(docID, revID string) { - btc.lastReplicatedRevLock.Lock() - defer btc.lastReplicatedRevLock.Unlock() - - currentRevID, ok := btc.lastReplicatedRev[docID] - if !ok { - btc.lastReplicatedRev[docID] = revID - return - } - - ctx := base.TestCtx(btc.parent.rt.TB()) - currentGen, _ := db.ParseRevID(ctx, currentRevID) - incomingGen, _ := db.ParseRevID(ctx, revID) - if incomingGen > currentGen { - btc.lastReplicatedRev[docID] = revID - } -} - -func (btc *BlipTesterCollectionClient) getLastReplicatedRev(docID string) (revID string, ok bool) { - btc.lastReplicatedRevLock.RLock() - defer btc.lastReplicatedRevLock.RUnlock() - - revID, ok = btc.lastReplicatedRev[docID] - return revID, ok -} - func newBlipTesterReplication(tb testing.TB, id string, btc *BlipTesterClient, skipCollectionsInitialization bool) (*BlipTesterReplicator, error) { bt, err := NewBlipTesterFromSpecWithRT(tb, &BlipTesterSpec{ connectingPassword: RestTesterDefaultUserPassword, @@ -591,9 +636,9 @@ func newBlipTesterReplication(tb testing.TB, id string, btc *BlipTesterClient, s // getCollectionsForBLIP returns collections configured by a single database instance on a restTester. If only default collection exists, it will skip returning it to test "legacy" blip mode. func getCollectionsForBLIP(_ testing.TB, rt *RestTester) []string { - db := rt.GetDatabase() + dbc := rt.GetDatabase() var collections []string - for _, collection := range db.CollectionByID { + for _, collection := range dbc.CollectionByID { if base.IsDefaultCollection(collection.ScopeName, collection.Name) { continue } @@ -605,12 +650,15 @@ func getCollectionsForBLIP(_ testing.TB, rt *RestTester) []string { } func (btcRunner *BlipTestClientRunner) NewBlipTesterClientOptsWithRT(rt *RestTester, opts *BlipTesterClientOpts) (client *BlipTesterClient) { - if !btcRunner.initialisedInsideRunnerCode { - require.FailNow(btcRunner.TB(), "must initialise BlipTesterClient inside Run() method") - } if opts == nil { opts = &BlipTesterClientOpts{} } + if !opts.AllowCreationWithoutBlipTesterClientRunner && !btcRunner.initialisedInsideRunnerCode { + require.FailNow(btcRunner.TB(), "must initialise BlipTesterClient inside Run() method") + } + if opts.SourceID == "" { + opts.SourceID = "blipclient" + } id, err := uuid.NewRandom() require.NoError(btcRunner.TB(), err) @@ -626,6 +674,11 @@ func (btcRunner *BlipTestClientRunner) NewBlipTesterClientOptsWithRT(rt *RestTes return client } +// ID returns the unique ID of the client. +func (btc *BlipTesterClient) ID() uint32 { + return btc.id +} + // TB returns testing.TB for the current test func (btc *BlipTesterClient) TB() testing.TB { return btc.rt.TB() @@ -646,22 +699,22 @@ func (btcRunner *BlipTestClientRunner) TB() testing.TB { return btcRunner.t } +// Add subtest to skip in runner code, if that is notes we skip the subtest. Remove skipnon hlv aware and version vector one func (btcRunner *BlipTestClientRunner) Run(test func(t *testing.T, SupportedBLIPProtocols []string)) { btcRunner.initialisedInsideRunnerCode = true // reset to protect against someone creating a new client after Run() is run defer func() { btcRunner.initialisedInsideRunnerCode = false }() - btcRunner.t.Run("revTree", func(t *testing.T) { - test(t, []string{db.CBMobileReplicationV3.SubprotocolString()}) - }) - // if test is not wanting version vector subprotocol to be run, return before we start this subtest - if btcRunner.SkipVersionVectorInitialization { - return + if !btcRunner.SkipSubtest[RevtreeSubtestName] { + btcRunner.t.Run(RevtreeSubtestName, func(t *testing.T) { + test(t, []string{db.CBMobileReplicationV3.SubprotocolString()}) + }) + } + if !btcRunner.SkipSubtest[VersionVectorSubtestName] { + btcRunner.t.Run(VersionVectorSubtestName, func(t *testing.T) { + // bump sub protocol version here + test(t, []string{db.CBMobileReplicationV4.SubprotocolString()}) + }) } - btcRunner.t.Run("versionVector", func(t *testing.T) { - t.Skip("skip VV subtest on master") - // bump sub protocol version here and pass into test function pending CBG-3253 - test(t, nil) - }) } func (btc *BlipTesterClient) tearDownBlipClientReplications() { @@ -692,10 +745,9 @@ func (btc *BlipTesterClient) createBlipTesterReplications() error { } } else { btc.nonCollectionAwareClient = &BlipTesterCollectionClient{ - docs: make(map[string]map[string]*BodyMessagePair), - attachments: make(map[string][]byte), - lastReplicatedRev: make(map[string]string), - parent: btc, + docs: make(map[string]*BlipTesterDoc), + attachments: make(map[string][]byte), + parent: btc, } } @@ -707,10 +759,9 @@ func (btc *BlipTesterClient) createBlipTesterReplications() error { func (btc *BlipTesterClient) initCollectionReplication(collection string, collectionIdx int) error { btcReplicator := &BlipTesterCollectionClient{ - docs: make(map[string]map[string]*BodyMessagePair), - attachments: make(map[string][]byte), - lastReplicatedRev: make(map[string]string), - parent: btc, + docs: make(map[string]*BlipTesterDoc), + attachments: make(map[string][]byte), + parent: btc, } btcReplicator.collection = collection @@ -840,13 +891,9 @@ func (btc *BlipTesterCollectionClient) UnsubPushChanges() (response []byte, err // Close will empty the stored docs and close the underlying replications. func (btc *BlipTesterCollectionClient) Close() { btc.docsLock.Lock() - btc.docs = make(map[string]map[string]*BodyMessagePair, 0) + btc.docs = make(map[string]*BlipTesterDoc, 0) btc.docsLock.Unlock() - btc.lastReplicatedRevLock.Lock() - btc.lastReplicatedRev = make(map[string]string, 0) - btc.lastReplicatedRevLock.Unlock() - btc.attachmentsLock.Lock() btc.attachments = make(map[string][]byte, 0) btc.attachmentsLock.Unlock() @@ -864,20 +911,84 @@ func (btr *BlipTesterReplicator) sendMsg(msg *blip.Message) (err error) { // PushRev creates a revision on the client, and immediately sends a changes request for it. // The rev ID is always: "N-abc", where N is rev generation for predictability. func (btc *BlipTesterCollectionClient) PushRev(docID string, parentVersion DocVersion, body []byte) (DocVersion, error) { - revid, err := btc.PushRevWithHistory(docID, parentVersion.RevID, body, 1, 0) - return DocVersion{RevID: revid}, err + + parentRev := parentVersion.GetRev(btc.UseHLV()) + revID, err := btc.PushRevWithHistory(docID, parentRev, body, 1, 0) + if err != nil { + return DocVersion{}, err + } + docVersion := btc.GetDocVersion(docID) + btc.requireRevID(docVersion, revID) + return docVersion, nil +} + +func (btc *BlipTesterCollectionClient) requireRevID(expected DocVersion, revID string) { + if btc.UseHLV() { + require.Equal(btc.parent.rt.TB(), expected.CV.String(), revID) + } else { + require.Equal(btc.parent.rt.TB(), expected.RevTreeID, revID) + } +} + +// GetDocVersion fetches revid and cv directly from the bucket. Used to support REST-based verification in btc tests +// even while REST only supports revTreeId +// TODO: This doesn't support multi-collection testing, btc.GetDocVersion uses +// +// GetSingleTestDatabaseCollection() +func (btcc *BlipTesterCollectionClient) GetDocVersion(docID string) DocVersion { + return btcc.parent.GetDocVersion(docID) +} + +// GetDocVersion fetches revid and cv directly from the bucket. Used to support REST-based verification in btc tests +// even while REST only supports revTreeId +func (btc *BlipTesterClient) GetDocVersion(docID string) DocVersion { + collection, ctx := btc.rt.GetSingleTestDatabaseCollection() + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalSync) + require.NoError(btc.rt.TB(), err) + if !btc.UseHLV() || doc.HLV == nil { + return DocVersion{RevTreeID: doc.CurrentRev} + } + return DocVersion{RevTreeID: doc.CurrentRev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } // PushRevWithHistory creates a revision on the client with history, and immediately sends a changes request for it. func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev string, body []byte, revCount, prunedRevCount int) (revID string, err error) { + ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB()), btc.parent.rt.GetDatabase().Name, nil) - parentRevGen, _ := db.ParseRevID(ctx, parentRev) - revGen := parentRevGen + revCount + prunedRevCount + revGen := 0 + newRevID := "" var revisionHistory []string - for i := revGen - 1; i > parentRevGen; i-- { - rev := fmt.Sprintf("%d-%s", i, "abc") - revisionHistory = append(revisionHistory, rev) + if btc.UseHLV() { + // When using version vectors: + // - source is "abc" + // - version value is simple counter + // - revisionHistory is just previous cv (parentRev) for changes response + startValue := uint64(0) + if parentRev != "" { + parentVersion, _ := db.ParseVersion(parentRev) + startValue = parentVersion.Value + revisionHistory = append(revisionHistory, parentRev) + } + newVersion := db.Version{SourceID: db.EncodeSource(btc.parent.SourceID), Value: startValue + uint64(revCount)} + newRevID = newVersion.String() + + } else { + // When using revtrees: + // - all revIDs are of the form [generation]-abc + // - [revCount] history entries are generated between the parent and the new rev + parentRevGen, _ := db.ParseRevID(ctx, parentRev) + revGen = parentRevGen + revCount + prunedRevCount + + for i := revGen - 1; i > parentRevGen; i-- { + rev := fmt.Sprintf("%d-%s", i, btc.parent.SourceID) + revisionHistory = append(revisionHistory, rev) + } + if parentRev != "" { + + revisionHistory = append(revisionHistory, parentRev) + } + newRevID = fmt.Sprintf("%d-%s", revGen, btc.parent.SourceID) } // Inline attachment processing @@ -887,19 +998,16 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin } var parentDocBody []byte - newRevID := fmt.Sprintf("%d-%s", revGen, "abc") btc.docsLock.Lock() if parentRev != "" { - revisionHistory = append(revisionHistory, parentRev) - if _, ok := btc.docs[docID]; ok { - // create new rev if doc and parent rev already exists - if parentDoc, okParent := btc.docs[docID][parentRev]; okParent { - parentDocBody = parentDoc.body - bodyMessagePair := &BodyMessagePair{body: body} - btc.docs[docID][newRevID] = bodyMessagePair + if doc, ok := btc.docs[docID]; ok { + // create new rev if doc exists and parent rev is current rev + if doc.getCurrentRevID() == parentRev { + parentDocBody = doc.body + doc.addRevision(newRevID, body, nil) } else { btc.docsLock.Unlock() - return "", fmt.Errorf("docID: %v with parent rev: %v was not found on the client", docID, parentRev) + return "", fmt.Errorf("docID: %v with current rev: %v was not found on the client", docID, parentRev) } } else { btc.docsLock.Unlock() @@ -908,8 +1016,7 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin } else { // create new doc + rev if _, ok := btc.docs[docID]; !ok { - bodyMessagePair := &BodyMessagePair{body: body} - btc.docs[docID] = map[string]*BodyMessagePair{newRevID: bodyMessagePair} + btc.docs[docID] = btc.NewBlipTesterDoc(newRevID, body, nil) } } btc.docsLock.Unlock() @@ -986,19 +1093,23 @@ func (btc *BlipTesterCollectionClient) PushRevWithHistory(docID, parentRev strin return "", fmt.Errorf("error %s %s from revResponse: %s", revResponse.Properties["Error-Domain"], revResponse.Properties["Error-Code"], rspBody) } - btc.updateLastReplicatedRev(docID, newRevID) return newRevID, nil } -func (btc *BlipTesterCollectionClient) StoreRevOnClient(docID, revID string, body []byte) error { +func (btc *BlipTesterCollectionClient) StoreRevOnClient(docID, rev string, body []byte) error { ctx := base.DatabaseLogCtx(base.TestCtx(btc.parent.rt.TB()), btc.parent.rt.GetDatabase().Name, nil) - revGen, _ := db.ParseRevID(ctx, revID) + + revGen := 0 + if !btc.UseHLV() { + revGen, _ = db.ParseRevID(ctx, rev) + } newBody, err := btc.ProcessInlineAttachments(body, revGen) if err != nil { return err } - bodyMessagePair := &BodyMessagePair{body: newBody} - btc.docs[docID] = map[string]*BodyMessagePair{revID: bodyMessagePair} + btc.docsLock.Lock() + defer btc.docsLock.Unlock() + btc.docs[docID] = btc.NewBlipTesterDoc(rev, newBody, nil) return nil } @@ -1054,17 +1165,57 @@ func (btc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte return inputBody, nil } -// GetVersion returns the data stored in the Client under the given docID and version -func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVersion) (data []byte, found bool) { +// GetCurrentRevID gets the current revID for the specified docID +func (btc *BlipTesterCollectionClient) GetCurrentRevID(docID string) (revID string, data []byte, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - if data, ok := rev[docVersion.RevID]; ok && data != nil { - return data.body, true + if doc, ok := btc.docs[docID]; ok { + return doc.getCurrentRevID(), doc.body, true + } + + return "", nil, false +} + +func (btc *BlipTesterClient) UseHLV() bool { + for _, protocol := range btc.SupportedBLIPProtocols { + subProtocol, err := db.ParseSubprotocolString(protocol) + require.NoError(btc.rt.TB(), err) + if subProtocol >= db.CBMobileReplicationV4 { + return true } } + return false +} +func (btc *BlipTesterClient) AssertOnBlipHistory(t *testing.T, msg *blip.Message, docVersion DocVersion) { + subProtocol, err := db.ParseSubprotocolString(btc.SupportedBLIPProtocols[0]) + require.NoError(t, err) + if subProtocol >= db.CBMobileReplicationV4 { // history could be empty a lot of the time in HLV messages as updates from the same source won't populate previous versions + if msg.Properties[db.RevMessageHistory] != "" { + assert.Equal(t, docVersion.CV.String(), msg.Properties[db.RevMessageHistory]) + } + } else { + assert.Equal(t, docVersion.RevTreeID, msg.Properties[db.RevMessageHistory]) + } +} + +// GetVersion returns the document body when the provided version matches the document's current revision +func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVersion) (data []byte, found bool) { + btc.docsLock.RLock() + defer btc.docsLock.RUnlock() + + if doc, ok := btc.docs[docID]; ok { + if doc.revMode == revModeHLV { + if doc.getCurrentRevID() == docVersion.CV.String() { + return doc.body, true + } + } else { + if doc.getCurrentRevID() == docVersion.RevTreeID { + return doc.body, true + } + } + } return nil, false } @@ -1081,15 +1232,13 @@ func (btc *BlipTesterCollectionClient) WaitForVersion(docID string, docVersion D return data } -// GetDoc returns a rev stored in the Client under the given docID. (if multiple revs are present, rev body returned is non-deterministic) +// GetDoc returns the current body stored in the Client for the given docID. func (btc *BlipTesterCollectionClient) GetDoc(docID string) (data []byte, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - for _, data := range rev { - return data.body, true - } + if doc, ok := btc.docs[docID]; ok { + return doc.body, true } return nil, false @@ -1101,11 +1250,12 @@ func (btc *BlipTesterCollectionClient) WaitForDoc(docID string) (data []byte) { if data, found := btc.GetDoc(docID); found { return data } + timeout := 10 * time.Second require.EventuallyWithT(btc.TB(), func(c *assert.CollectT) { var found bool data, found = btc.GetDoc(docID) assert.True(c, found, "Could not find docID:%+v", docID) - }, 10*time.Second, 50*time.Millisecond, "BlipTesterClient timed out waiting for doc %+v", docID) + }, timeout, 50*time.Millisecond, "BlipTesterClient timed out waiting for doc %+v after %s", docID, timeout) return data } @@ -1153,23 +1303,29 @@ func (btr *BlipTesterReplicator) storeMessage(msg *blip.Message) { } // WaitForBlipRevMessage blocks until the given doc ID and rev ID has been stored by the client, and returns the message when found. If not found after 10 seconds, test will fail. -func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, docVersion DocVersion) (msg *blip.Message) { +func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, version DocVersion) (msg *blip.Message) { + var revID string + if btc.UseHLV() { + revID = version.CV.String() + } else { + revID = version.RevTreeID + } + require.EventuallyWithT(btc.TB(), func(c *assert.CollectT) { var ok bool - msg, ok = btc.GetBlipRevMessage(docID, docVersion.RevID) - assert.True(c, ok, "Could not find docID:%+v, RevID: %+v", docID, docVersion.RevID) - }, 10*time.Second, 50*time.Millisecond, "BlipTesterReplicator timed out waiting for BLIP message") + msg, ok = btc.GetBlipRevMessage(docID, revID) + assert.True(c, ok, "Could not find docID:%+v, RevID: %+v", docID, revID) + }, 10*time.Second, 50*time.Millisecond, "BlipTesterClient timed out waiting for BLIP message docID: %v, revID: %v", docID, revID) return msg } -func (btc *BlipTesterCollectionClient) GetBlipRevMessage(docID, revID string) (msg *blip.Message, found bool) { +func (btc *BlipTesterCollectionClient) GetBlipRevMessage(docID string, revID string) (msg *blip.Message, found bool) { btc.docsLock.RLock() defer btc.docsLock.RUnlock() - if rev, ok := btc.docs[docID]; ok { - if pair, found := rev[revID]; found { - found = pair.message != nil - return pair.message, found + if doc, ok := btc.docs[docID]; ok { + if message, found := doc.revMessageHistory[revID]; found { + return message, found } } @@ -1191,8 +1347,8 @@ func (btcRunner *BlipTestClientRunner) WaitForDoc(clientID uint32, docID string) } // WaitForBlipRevMessage blocks until the given doc ID and rev ID has been stored by the client, and returns the message when found. If document is not found after 10 seconds, test will fail. -func (btcRunner *BlipTestClientRunner) WaitForBlipRevMessage(clientID uint32, docID string, docVersion DocVersion) *blip.Message { - return btcRunner.SingleCollection(clientID).WaitForBlipRevMessage(docID, docVersion) +func (btcRunner *BlipTestClientRunner) WaitForBlipRevMessage(clientID uint32, docID string, version DocVersion) *blip.Message { + return btcRunner.SingleCollection(clientID).WaitForBlipRevMessage(docID, version) } func (btcRunner *BlipTestClientRunner) StartOneshotPull(clientID uint32) { @@ -1215,8 +1371,8 @@ func (btcRunner *BlipTestClientRunner) StartPullSince(clientID uint32, options B btcRunner.SingleCollection(clientID).StartPullSince(options) } -func (btcRunner *BlipTestClientRunner) GetVersion(clientID uint32, docID string, docVersion DocVersion) ([]byte, bool) { - return btcRunner.SingleCollection(clientID).GetVersion(docID, docVersion) +func (btcRunner *BlipTestClientRunner) GetVersion(clientID uint32, docID string, version DocVersion) ([]byte, bool) { + return btcRunner.SingleCollection(clientID).GetVersion(docID, version) } func (btcRunner *BlipTestClientRunner) saveAttachment(clientID uint32, contentType string, attachmentData string) (int, string, error) { @@ -1294,3 +1450,36 @@ func (btc *BlipTesterCollectionClient) sendPushMsg(msg *blip.Message) error { btc.addCollectionProperty(msg) return btc.parent.pushReplication.sendMsg(msg) } + +// Wrappers for RT helpers that populate version information for use in HLV tests +// PutDoc will upsert the document with a given contents. +func (btc *BlipTesterClient) PutDoc(docID string, body string) DocVersion { + rt := btc.rt + version := rt.PutDoc(docID, body) + if btc.UseHLV() { + collection, _ := rt.GetSingleTestDatabaseCollection() + source, value := collection.GetDocumentCurrentVersion(rt.TB(), docID) + version.CV = db.Version{ + SourceID: source, + Value: value, + } + } + return version +} + +// RequireRev checks the current rev for the specified docID on the backend the BTC is replicating +// with (NOT in the btc store) +func (btc *BlipTesterClient) RequireRev(t *testing.T, expectedRev string, doc *db.Document) { + if btc.UseHLV() { + require.Equal(t, expectedRev, doc.HLV.GetCurrentVersionString()) + } else { + require.Equal(t, expectedRev, doc.CurrentRev) + } +} + +func (btc *BlipTesterClient) AssertDeltaSrcProperty(t *testing.T, msg *blip.Message, docVersion DocVersion) { + subProtocol, err := db.ParseSubprotocolString(btc.SupportedBLIPProtocols[0]) + require.NoError(t, err) + rev := docVersion.GetRev(subProtocol >= db.CBMobileReplicationV4) + assert.Equal(t, rev, msg.Properties[db.RevMessageDeltaSrc]) +} diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 81656727bc..4fd150d3f3 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -36,6 +36,11 @@ func (rt *RestTester) Run(name string, test func(*testing.T)) { }) } +func (rt *RestTester) UpdateTB(t *testing.T) { + var tb testing.TB = t + rt.testingTB.Store(&tb) +} + // GetDocBody returns the doc body for the given docID. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocBody(docID string) db.Body { rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID, "") @@ -55,12 +60,12 @@ func (rt *RestTester) GetDoc(docID string) (DocVersion, db.Body) { RevID *string `json:"_rev"` } require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &r)) - return DocVersion{RevID: *r.RevID}, body + return DocVersion{RevTreeID: *r.RevID}, body } // GetDocVersion returns the doc body and version for the given docID and version. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocVersion(docID string, version DocVersion) db.Body { - rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+version.RevID, "") + rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+version.RevTreeID, "") RequireStatus(rt.TB(), rawResponse, http.StatusOK) var body db.Body require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) @@ -83,13 +88,13 @@ func (rt *RestTester) PutDoc(docID string, body string) DocVersion { // UpdateDocRev updates a document at a specific revision and returns the new version. Deprecated for UpdateDoc. func (rt *RestTester) UpdateDocRev(docID, revID string, body string) string { - version := rt.UpdateDoc(docID, DocVersion{RevID: revID}, body) - return version.RevID + version := rt.UpdateDoc(docID, DocVersion{RevTreeID: revID}, body) + return version.RevTreeID } // UpdateDoc updates a document at a specific version and returns the new version. func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) DocVersion { - resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevID) + resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevTreeID) rawResponse := rt.SendAdminRequest(http.MethodPut, resource, body) RequireStatus(rt.TB(), rawResponse, http.StatusCreated) return DocVersionFromPutResponse(rt.TB(), rawResponse) @@ -103,14 +108,14 @@ func (rt *RestTester) DeleteDoc(docID string, docVersion DocVersion) { // DeleteDocReturnVersion deletes a document at a specific version. The test will fail if the revision does not exist. func (rt *RestTester) DeleteDocReturnVersion(docID string, docVersion DocVersion) DocVersion { resp := rt.SendAdminRequest(http.MethodDelete, - fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, docVersion.RevID), "") + fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, docVersion.RevTreeID), "") RequireStatus(rt.TB(), resp, http.StatusOK) return DocVersionFromPutResponse(rt.TB(), resp) } // DeleteDocRev removes a document at a specific revision. Deprecated for DeleteDoc. func (rt *RestTester) DeleteDocRev(docID, revID string) { - rt.DeleteDoc(docID, DocVersion{RevID: revID}) + rt.DeleteDoc(docID, DocVersion{RevTreeID: revID}) } func (rt *RestTester) GetDatabaseRoot(dbname string) DatabaseRoot { @@ -123,7 +128,7 @@ func (rt *RestTester) GetDatabaseRoot(dbname string) DatabaseRoot { // WaitForVersion retries a GET for a given document version until it returns 200 or 201 for a given document and revision. If version is not found, the test will fail. func (rt *RestTester) WaitForVersion(docID string, version DocVersion) error { - require.NotEqual(rt.TB(), "", version.RevID) + require.NotEqual(rt.TB(), "", version.RevTreeID) return rt.WaitForCondition(func() bool { rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID, "") if rawResponse.Code != 200 && rawResponse.Code != 201 { @@ -131,13 +136,13 @@ func (rt *RestTester) WaitForVersion(docID string, version DocVersion) error { } var body db.Body require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) - return body.ExtractRev() == version.RevID + return body.ExtractRev() == version.RevTreeID }) } // WaitForRev retries a GET until it returns 200 or 201. If revision is not found, the test will fail. This function is deprecated for RestTester.WaitForVersion func (rt *RestTester) WaitForRev(docID, revID string) error { - return rt.WaitForVersion(docID, DocVersion{RevID: revID}) + return rt.WaitForVersion(docID, DocVersion{RevTreeID: revID}) } func (rt *RestTester) WaitForCheckpointLastSequence(expectedName string) (string, error) { @@ -415,3 +420,51 @@ func (rt *RestTester) RequireDbOnline() { require.NoError(rt.TB(), base.JSONUnmarshal(response.Body.Bytes(), &body)) require.Equal(rt.TB(), "Online", body["state"].(string)) } + +// TEMPORARY HELPER METHODS FOR BLIP TEST CLIENT RUNNER +func (rt *RestTester) PutDocDirectly(docID string, body db.Body) DocVersion { + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + rev, doc, err := collection.Put(ctx, docID, body) + require.NoError(rt.TB(), err) + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} +} + +func (rt *RestTester) UpdateDocDirectly(docID string, version DocVersion, body db.Body) DocVersion { + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + body[db.BodyId] = docID + body[db.BodyRev] = version.RevTreeID + rev, doc, err := collection.Put(ctx, docID, body) + require.NoError(rt.TB(), err) + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} +} + +func (rt *RestTester) DeleteDocDirectly(docID string, version DocVersion) DocVersion { + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + rev, doc, err := collection.DeleteDoc(ctx, docID, version.RevTreeID) + require.NoError(rt.TB(), err) + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} +} + +func (rt *RestTester) PutDocDirectlyInCollection(collection *db.DatabaseCollection, docID string, body db.Body) DocVersion { + dbUser := &db.DatabaseCollectionWithUser{ + DatabaseCollection: collection, + } + ctx := base.UserLogCtx(collection.AddCollectionContext(rt.Context()), "gotest", base.UserDomainBuiltin, nil) + rev, doc, err := dbUser.Put(ctx, docID, body) + require.NoError(rt.TB(), err) + return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} +} + +// PutDocWithAttachment will upsert the document with a given contents and attachments. +func (rt *RestTester) PutDocWithAttachment(docID string, body string, attachmentName, attachmentBody string) DocVersion { + // create new body with a 1.x style inline attachment body like `{"_attachments": {"camera.txt": {"data": "Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}`. + require.NotEmpty(rt.TB(), attachmentName) + require.NotEmpty(rt.TB(), attachmentBody) + var rawBody db.Body + require.NoError(rt.TB(), base.JSONUnmarshal([]byte(body), &rawBody)) + require.NotContains(rt.TB(), rawBody, db.BodyAttachments) + rawBody[db.BodyAttachments] = map[string]any{ + attachmentName: map[string]any{"data": attachmentBody}, + } + return rt.PutDocDirectly(docID, rawBody) +} diff --git a/topologytest/couchbase_lite_mock_peer_test.go b/topologytest/couchbase_lite_mock_peer_test.go new file mode 100644 index 0000000000..1a918cc42f --- /dev/null +++ b/topologytest/couchbase_lite_mock_peer_test.go @@ -0,0 +1,223 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "context" + "fmt" + "testing" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// PeerBlipTesterClient is a wrapper around a BlipTesterClientRunner and BlipTesterClient, which need to match for a given Couchbase Lite interface. +type PeerBlipTesterClient struct { + btcRunner *rest.BlipTestClientRunner + btc *rest.BlipTesterClient +} + +// ID returns the unique ID of the blip client. +func (p *PeerBlipTesterClient) ID() uint32 { + return p.btc.ID() +} + +// CouchbaseLiteMockPeer represents an in-memory Couchbase Lite peer. This utilizes BlipTesterClient from the rest package to send and receive blip messages. +type CouchbaseLiteMockPeer struct { + t *testing.T + blipClients map[string]*PeerBlipTesterClient + name string +} + +func (p *CouchbaseLiteMockPeer) String() string { + return fmt.Sprintf("%s (sourceid:%s)", p.name, p.SourceID()) +} + +// GetDocument returns the latest version of a document. The test will fail the document does not exist. +func (p *CouchbaseLiteMockPeer) GetDocument(_ sgbucket.DataStoreName, _ string) (DocMetadata, db.Body) { + // this isn't yet collection aware, using single default collection + return DocMetadata{}, nil +} + +// getSingleBlipClient returns the single blip client for the peer. If there are multiple clients, or not clients it will fail the test. This is temporary to stub support for multiple Sync Gateway peers. +func (p *CouchbaseLiteMockPeer) getSingleBlipClient() *PeerBlipTesterClient { + // this isn't yet collection aware, using single default collection + if len(p.blipClients) != 1 { + require.Fail(p.t, "blipClients haven't been created for %s, a temporary limitation of CouchbaseLiteMockPeer", p) + } + for _, c := range p.blipClients { + return c + } + require.Fail(p.t, "no blipClients found for %s", p) + return nil +} + +// CreateDocument creates a document on the peer. The test will fail if the document already exists. +func (p *CouchbaseLiteMockPeer) CreateDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.t.Logf("%s: Creating document %s", p, docID) + return p.WriteDocument(dsName, docID, body) +} + +// WriteDocument writes a document to the peer. The test will fail if the write does not succeed. +func (p *CouchbaseLiteMockPeer) WriteDocument(_ sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.TB().Logf("%s: Writing document %s", p, docID) + // this isn't yet collection aware, using single default collection + client := p.getSingleBlipClient() + // set an HLV here. + docVersion, err := client.btcRunner.PushRev(client.ID(), docID, rest.EmptyDocVersion(), body) + require.NoError(client.btcRunner.TB(), err) + // FIXME: CBG-4257, this should read the existing HLV on doc, until this happens, pv is always missing + docMetadata := DocMetadataFromDocVersion(client.btc.TB(), docID, docVersion) + return BodyAndVersion{ + docMeta: docMetadata, + body: body, + updatePeer: p.name, + } +} + +// DeleteDocument deletes a document on the peer. The test will fail if the document does not exist. +func (p *CouchbaseLiteMockPeer) DeleteDocument(sgbucket.DataStoreName, string) DocMetadata { + return DocMetadata{} +} + +// WaitForDocVersion waits for a document to reach a specific version. The test will fail if the document does not reach the expected version in 20s. +func (p *CouchbaseLiteMockPeer) WaitForDocVersion(_ sgbucket.DataStoreName, docID string, docVersion DocMetadata) db.Body { + // this isn't yet collection aware, using single default collection + client := p.getSingleBlipClient() + var data []byte + require.EventuallyWithT(p.TB(), func(c *assert.CollectT) { + var found bool + data, found = client.btcRunner.GetVersion(client.ID(), docID, rest.DocVersion{CV: docVersion.CV(c)}) + if !assert.True(c, found, "Could not find docID:%+v on %p\nVersion %#v", docID, p, docVersion) { + return + } + }, totalWaitTime, pollInterval, "BlipTesterClient timed out waiting for doc %+v Version %#v", docID, docVersion) + var body db.Body + require.NoError(p.TB(), base.JSONUnmarshal(data, &body)) + return body +} + +// WaitForDeletion waits for a document to be deleted. This document must be a tombstone. The test will fail if the document still exists after 20s. +func (p *CouchbaseLiteMockPeer) WaitForDeletion(_ sgbucket.DataStoreName, _ string) { + require.Fail(p.TB(), "WaitForDeletion not yet implemented CBG-4257") +} + +// WaitForTombstoneVersion waits for a document to reach a specific version, this must be a tombstone. The test will fail if the document does not reach the expected version in 20s. +func (p *CouchbaseLiteMockPeer) WaitForTombstoneVersion(_ sgbucket.DataStoreName, _ string, _ DocMetadata) { + require.Fail(p.TB(), "WaitForTombstoneVersion not yet implemented CBG-4257") +} + +// RequireDocNotFound asserts that a document does not exist on the peer. +func (p *CouchbaseLiteMockPeer) RequireDocNotFound(sgbucket.DataStoreName, string) { + // not implemented yet in blip client tester + // _, err := p.btcRunner.GetDoc(p.btc.id, docID) + // base.RequireDocNotFoundError(p.btcRunner.TB(), err) +} + +// Close will shut down the peer and close any active replications on the peer. +func (p *CouchbaseLiteMockPeer) Close() { + for _, c := range p.blipClients { + c.btc.Close() + } +} + +// Type returns PeerTypeCouchbaseLite. +func (p *CouchbaseLiteMockPeer) Type() PeerType { + return PeerTypeCouchbaseLite +} + +// CreateReplication creates a replication instance +func (p *CouchbaseLiteMockPeer) CreateReplication(peer Peer, _ PeerReplicationConfig) PeerReplication { + sg, ok := peer.(*SyncGatewayPeer) + if !ok { + require.Fail(p.t, fmt.Sprintf("unsupported peer type %T for pull replication", peer)) + } + replication := &CouchbaseLiteMockReplication{ + activePeer: p, + passivePeer: peer, + btcRunner: rest.NewBlipTesterClientRunner(sg.rt.TB().(*testing.T)), + } + replication.btc = replication.btcRunner.NewBlipTesterClientOptsWithRT(sg.rt, &rest.BlipTesterClientOpts{ + Username: "user", + Channels: []string{"*"}, + SupportedBLIPProtocols: []string{db.CBMobileReplicationV4.SubprotocolString()}, + AllowCreationWithoutBlipTesterClientRunner: true, + SourceID: peer.SourceID(), + }, + ) + p.blipClients[sg.String()] = &PeerBlipTesterClient{ + btcRunner: replication.btcRunner, + btc: replication.btc, + } + return replication +} + +// SourceID returns the source ID for the peer used in @. +func (p *CouchbaseLiteMockPeer) SourceID() string { + return p.name +} + +// Context returns the context for the peer. +func (p *CouchbaseLiteMockPeer) Context() context.Context { + return base.TestCtx(p.TB()) +} + +// TB returns the testing.TB for the peer. +func (p *CouchbaseLiteMockPeer) TB() testing.TB { + return p.t +} + +// UpdateTB updates the testing.TB for the peer. +func (p *CouchbaseLiteMockPeer) UpdateTB(t *testing.T) { + p.t = t +} + +// GetBackingBucket returns the backing bucket for the peer. This is always nil. +func (p *CouchbaseLiteMockPeer) GetBackingBucket() base.Bucket { + return nil +} + +// CouchbaseLiteMockReplication represents a replication between Couchbase Lite and Sync Gateway. This can be a push or pull replication. +type CouchbaseLiteMockReplication struct { + activePeer Peer + passivePeer Peer + btc *rest.BlipTesterClient + btcRunner *rest.BlipTestClientRunner +} + +// ActivePeer returns the peer sending documents +func (r *CouchbaseLiteMockReplication) ActivePeer() Peer { + return r.activePeer +} + +// PassivePeer returns the peer receiving documents +func (r *CouchbaseLiteMockReplication) PassivePeer() Peer { + return r.passivePeer +} + +// Start starts the replication +func (r *CouchbaseLiteMockReplication) Start() { + r.btc.TB().Logf("starting CBL replication: %s", r) + r.btcRunner.StartPull(r.btc.ID()) +} + +// Stop halts the replication. The replication can be restarted after it is stopped. +func (r *CouchbaseLiteMockReplication) Stop() { + r.btc.TB().Logf("stopping CBL replication: %s", r) + _, err := r.btcRunner.UnsubPullChanges(r.btc.ID()) + require.NoError(r.btcRunner.TB(), err) +} + +func (r *CouchbaseLiteMockReplication) String() string { + return fmt.Sprintf("%s->%s", r.activePeer, r.passivePeer) +} diff --git a/topologytest/couchbase_server_peer_test.go b/topologytest/couchbase_server_peer_test.go new file mode 100644 index 0000000000..23d8dc4b48 --- /dev/null +++ b/topologytest/couchbase_server_peer_test.go @@ -0,0 +1,335 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/xdcr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// dummySystemXattr is created for XDCR testing. This prevents a document echo after an initial write. The dummy xattr also means that the document will always have xattrs when deleting it, which is necessary for WriteUpdateWithXattrs. +const dummySystemXattr = "_dummysystemxattr" + +var metadataXattrNames = []string{base.VvXattrName, base.MouXattrName, base.SyncXattrName, dummySystemXattr} + +// CouchbaseServerPeer represents an instance of a backing server (bucket). This is rosmar unless SG_TEST_BACKING_STORE=couchbase is set. +type CouchbaseServerPeer struct { + tb testing.TB + bucket base.Bucket + sourceID string + pullReplications map[Peer]xdcr.Manager + pushReplications map[Peer]xdcr.Manager + name string +} + +// CouchbaseServerReplication represents a unidirectional replication between two CouchbaseServerPeers. These are two buckets, using bucket to bucket XDCR. A rosmar implementation is used if SG_TEST_BACKING_STORE is unset. +type CouchbaseServerReplication struct { + t testing.TB + ctx context.Context + activePeer Peer + passivePeer Peer + direction PeerReplicationDirection + manager xdcr.Manager +} + +// ActivePeer returns the peer sending documents +func (r *CouchbaseServerReplication) ActivePeer() Peer { + return r.activePeer +} + +// PassivePeer returns the peer receiving documents +func (r *CouchbaseServerReplication) PassivePeer() Peer { + return r.passivePeer +} + +// Start starts the replication +func (r *CouchbaseServerReplication) Start() { + r.t.Logf("starting XDCR replication %s", r) + require.NoError(r.t, r.manager.Start(r.ctx)) +} + +// Stop halts the replication. The replication can be restarted after it is stopped. +func (r *CouchbaseServerReplication) Stop() { + r.t.Logf("stopping XDCR replication %s", r) + require.NoError(r.t, r.manager.Stop(r.ctx)) +} + +func (r *CouchbaseServerReplication) String() string { + switch r.direction { + case PeerReplicationDirectionPush: + return fmt.Sprintf("%s->%s", r.activePeer, r.passivePeer) + case PeerReplicationDirectionPull: + return fmt.Sprintf("%s->%s", r.passivePeer, r.activePeer) + } + return fmt.Sprintf("%s-%s (direction unknown)", r.activePeer, r.passivePeer) +} + +func (p *CouchbaseServerPeer) String() string { + return fmt.Sprintf("%s (bucket:%s,sourceid:%s)", p.name, p.bucket.GetName(), p.sourceID) +} + +// Context returns the context for the peer. +func (p *CouchbaseServerPeer) Context() context.Context { + return base.TestCtx(p.tb) +} + +func (p *CouchbaseServerPeer) getCollection(dsName sgbucket.DataStoreName) sgbucket.DataStore { + collection, err := p.bucket.NamedDataStore(dsName) + require.NoError(p.tb, err) + return collection +} + +// GetDocument returns the latest version of a document. The test will fail the document does not exist. +func (p *CouchbaseServerPeer) GetDocument(dsName sgbucket.DataStoreName, docID string) (DocMetadata, db.Body) { + return getBodyAndVersion(p, p.getCollection(dsName), docID) +} + +// CreateDocument creates a document on the peer. The test will fail if the document already exists. +func (p *CouchbaseServerPeer) CreateDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.tb.Logf("%s: Creating document %s", p, docID) + // create document with xattrs to prevent XDCR from doing a round trip replication in this scenario: + // CBS1: write document (cas1, no _vv) + // CBS1->CBS2: XDCR replication + // CBS2->CBS1: XDCR replication, creates a new _vv + cas, err := p.getCollection(dsName).WriteWithXattrs(p.Context(), docID, 0, 0, body, map[string][]byte{dummySystemXattr: []byte(`{"dummy": "xattr"}`)}, nil, nil) + require.NoError(p.tb, err) + implicitHLV := db.NewHybridLogicalVector() + require.NoError(p.tb, implicitHLV.AddVersion(db.Version{SourceID: p.SourceID(), Value: cas})) + docMetadata := DocMetadata{ + DocID: docID, + Cas: cas, + ImplicitHLV: implicitHLV, + } + return BodyAndVersion{ + docMeta: docMetadata, + body: body, + updatePeer: p.name, + } +} + +// WriteDocument writes a document to the peer. The test will fail if the write does not succeed. +func (p *CouchbaseServerPeer) WriteDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.tb.Logf("%s: Writing document %s", p, docID) + var lastXattrs map[string][]byte + // write the document LWW, ignoring any in progress writes + callback := func(_ []byte, xattrs map[string][]byte, _ uint64) (sgbucket.UpdatedDoc, error) { + lastXattrs = xattrs + return sgbucket.UpdatedDoc{Doc: body}, nil + } + cas, err := p.getCollection(dsName).WriteUpdateWithXattrs(p.Context(), docID, metadataXattrNames, 0, nil, nil, callback) + require.NoError(p.tb, err) + return BodyAndVersion{ + docMeta: getDocVersion(docID, p, cas, lastXattrs), + body: body, + updatePeer: p.name, + } +} + +// DeleteDocument deletes a document on the peer. The test will fail if the document does not exist. +func (p *CouchbaseServerPeer) DeleteDocument(dsName sgbucket.DataStoreName, docID string) DocMetadata { + // delete the document, ignoring any in progress writes. We are allowed to delete a document that does not exist. + var lastXattrs map[string][]byte + // write the document LWW, ignoring any in progress writes + callback := func(_ []byte, xattrs map[string][]byte, _ uint64) (sgbucket.UpdatedDoc, error) { + lastXattrs = xattrs + return sgbucket.UpdatedDoc{Doc: nil, IsTombstone: true, Xattrs: xattrs}, nil + } + cas, err := p.getCollection(dsName).WriteUpdateWithXattrs(p.Context(), docID, metadataXattrNames, 0, nil, nil, callback) + require.NoError(p.tb, err) + return getDocVersion(docID, p, cas, lastXattrs) +} + +// WaitForDocVersion waits for a document to reach a specific version. The test will fail if the document does not reach the expected version in 20s. +func (p *CouchbaseServerPeer) WaitForDocVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) db.Body { + docBytes := p.waitForDocVersion(dsName, docID, expected) + var body db.Body + require.NoError(p.tb, base.JSONUnmarshal(docBytes, &body), "couldn't unmarshal docID %s: %s", docID, docBytes) + return body +} + +// WaitForDeletion waits for a document to be deleted. This document must be a tombstone. The test will fail if the document still exists after 20s. +func (p *CouchbaseServerPeer) WaitForDeletion(dsName sgbucket.DataStoreName, docID string) { + require.EventuallyWithT(p.tb, func(c *assert.CollectT) { + _, err := p.getCollection(dsName).Get(docID, nil) + assert.True(c, base.IsDocNotFoundError(err), "expected docID %s to be deleted from peer %s, found err=%v", docID, p.name, err) + }, totalWaitTime, pollInterval) +} + +// WaitForTombstoneVersion waits for a document to reach a specific version, this must be a tombstone. The test will fail if the document does not reach the expected version in 20s. +func (p *CouchbaseServerPeer) WaitForTombstoneVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) { + docBytes := p.waitForDocVersion(dsName, docID, expected) + require.Nil(p.tb, docBytes, "expected tombstone for docID %s, got %s", docID, docBytes) +} + +// waitForDocVersion waits for a document to reach a specific version and returns the body in bytes. The bytes will be nil if the document is a tombstone. The test will fail if the document does not reach the expected version in 20s. +func (p *CouchbaseServerPeer) waitForDocVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) []byte { + var docBytes []byte + var version DocMetadata + require.EventuallyWithT(p.tb, func(c *assert.CollectT) { + var err error + var xattrs map[string][]byte + var cas uint64 + docBytes, xattrs, cas, err = p.getCollection(dsName).GetWithXattrs(p.Context(), docID, metadataXattrNames) + if !assert.NoError(c, err) { + return + } + // have to use p.tb instead of c because of the assert.CollectT doesn't implement TB + version = getDocVersion(docID, p, cas, xattrs) + + assert.Equal(c, expected.CV(c), version.CV(c), "Could not find matching CV on %s for peer %s\nexpected: %#v\nactual: %#v\n body: %#v\n", docID, p, expected, version, string(docBytes)) + + }, totalWaitTime, pollInterval) + return docBytes +} + +// RequireDocNotFound asserts that a document does not exist on the peer. +func (p *CouchbaseServerPeer) RequireDocNotFound(dsName sgbucket.DataStoreName, docID string) { + _, err := p.getCollection(dsName).Get(docID, nil) + base.RequireDocNotFoundError(p.tb, err) +} + +// Close will shut down the peer and close any active replications on the peer. +func (p *CouchbaseServerPeer) Close() { + for _, r := range p.pullReplications { + assert.NoError(p.tb, r.Stop(p.Context())) + } + for _, r := range p.pushReplications { + assert.NoError(p.tb, r.Stop(p.Context())) + } +} + +// Type returns PeerTypeCouchbaseServer. +func (p *CouchbaseServerPeer) Type() PeerType { + return PeerTypeCouchbaseServer +} + +// CreateReplication creates an XDCR manager. +func (p *CouchbaseServerPeer) CreateReplication(passivePeer Peer, config PeerReplicationConfig) PeerReplication { + switch config.direction { + case PeerReplicationDirectionPull: + _, ok := p.pullReplications[passivePeer] + if ok { + require.Fail(p.tb, fmt.Sprintf("pull replication already exists for %s-%s", p, passivePeer)) + } + r, err := xdcr.NewXDCR(p.Context(), passivePeer.GetBackingBucket(), p.bucket, xdcr.XDCROptions{Mobile: xdcr.MobileOn}) + require.NoError(p.tb, err) + p.pullReplications[passivePeer] = r + return &CouchbaseServerReplication{ + activePeer: p, + passivePeer: passivePeer, + direction: config.direction, + t: p.tb.(*testing.T), + ctx: p.Context(), + manager: r, + } + case PeerReplicationDirectionPush: + _, ok := p.pushReplications[passivePeer] + if ok { + require.Fail(p.tb, fmt.Sprintf("pull replication already exists for %s-%s", p, passivePeer)) + } + r, err := xdcr.NewXDCR(p.Context(), p.bucket, passivePeer.GetBackingBucket(), xdcr.XDCROptions{Mobile: xdcr.MobileOn}) + require.NoError(p.tb, err) + p.pushReplications[passivePeer] = r + return &CouchbaseServerReplication{ + activePeer: p, + passivePeer: passivePeer, + direction: config.direction, + t: p.tb.(*testing.T), + ctx: p.Context(), + manager: r, + } + default: + require.Fail(p.tb, fmt.Sprintf("unsupported replication direction %d for %s-%s", config.direction, p, passivePeer)) + } + return nil +} + +// SourceID returns the source ID for the peer used in @sourceID. +func (p *CouchbaseServerPeer) SourceID() string { + return p.sourceID +} + +// GetBackingBucket returns the backing bucket for the peer. +func (p *CouchbaseServerPeer) GetBackingBucket() base.Bucket { + return p.bucket +} + +// TB returns the testing.TB for the peer. +func (p *CouchbaseServerPeer) TB() testing.TB { + return p.tb +} + +func (p *CouchbaseServerPeer) UpdateTB(tb *testing.T) { + p.tb = tb +} + +// useImplicitHLV returns true if the document's HLV is not up to date and an HLV should be composed of current sourceID and cas. +func useImplicitHLV(doc DocMetadata) bool { + if doc.HLV == nil { + return true + } + if doc.HLV.CurrentVersionCAS == doc.Cas { + return false + } + if doc.Mou == nil { + return true + } + return doc.Mou.CAS() != doc.Cas +} + +// getDocVersion returns a DocVersion from a cas and xattrs with _vv (hlv) and _sync (RevTreeID). +func getDocVersion(docID string, peer Peer, cas uint64, xattrs map[string][]byte) DocMetadata { + docVersion := DocMetadata{ + DocID: docID, + Cas: cas, + } + mouBytes, ok := xattrs[base.MouXattrName] + if ok { + require.NoError(peer.TB(), json.Unmarshal(mouBytes, &docVersion.Mou)) + } + hlvBytes, ok := xattrs[base.VvXattrName] + if ok { + require.NoError(peer.TB(), json.Unmarshal(hlvBytes, &docVersion.HLV)) + } + if useImplicitHLV(docVersion) { + if docVersion.HLV == nil { + docVersion.ImplicitHLV = db.NewHybridLogicalVector() + } else { + require.NoError(peer.TB(), json.Unmarshal(hlvBytes, &docVersion.ImplicitHLV)) + docVersion.ImplicitHLV = docVersion.HLV + } + require.NoError(peer.TB(), docVersion.ImplicitHLV.AddVersion(db.Version{SourceID: peer.SourceID(), Value: cas})) + } + sync, ok := xattrs[base.SyncXattrName] + if ok { + var syncData *db.SyncData + require.NoError(peer.TB(), json.Unmarshal(sync, &syncData)) + docVersion.RevTreeID = syncData.CurrentRev + } + return docVersion +} + +// getBodyAndVersion returns the body and version of a document from a sgbucket.DataStore. +func getBodyAndVersion(peer Peer, collection sgbucket.DataStore, docID string) (DocMetadata, db.Body) { + docBytes, xattrs, cas, err := collection.GetWithXattrs(peer.Context(), docID, metadataXattrNames) + require.NoError(peer.TB(), err) + // get hlv to construct DocVersion + var body db.Body + require.NoError(peer.TB(), base.JSONUnmarshal(docBytes, &body)) + return getDocVersion(docID, peer, cas, xattrs), body +} diff --git a/topologytest/hlv_test.go b/topologytest/hlv_test.go new file mode 100644 index 0000000000..63172ef770 --- /dev/null +++ b/topologytest/hlv_test.go @@ -0,0 +1,158 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "fmt" + "strings" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/stretchr/testify/require" +) + +// getSingleDsName returns the default scope and collection name for tests +func getSingleDsName() base.ScopeAndCollectionName { + if base.TestsUseNamedCollections() { + return base.ScopeAndCollectionName{Scope: "sg_test_0", Collection: "sg_test_0"} + } + return base.DefaultScopeAndCollectionName() +} + +// BodyAndVersion struct to hold doc update information to assert on +type BodyAndVersion struct { + docMeta DocMetadata + body []byte // expected body for version + updatePeer string // the peer this particular document version mutation originated from +} + +func (b BodyAndVersion) GoString() string { + return fmt.Sprintf("%#v body:%s, updatePeer:%s", b.docMeta, string(b.body), b.updatePeer) +} + +// requireBodyEqual compares bodies, removing private properties that might exist. +func requireBodyEqual(t *testing.T, expected []byte, actual db.Body) { + actual = actual.DeepCopy(base.TestCtx(t)) + stripInternalProperties(actual) + require.JSONEq(t, string(expected), string(base.MustJSONMarshal(t, actual))) +} + +func stripInternalProperties(body db.Body) { + delete(body, "_rev") + delete(body, "_id") +} + +// waitForVersionAndBody waits for a document to reach a specific version on all peers. +func waitForVersionAndBody(t *testing.T, dsName base.ScopeAndCollectionName, peers Peers, docID string, expectedVersion BodyAndVersion) { + for _, peer := range peers.SortedPeers() { + t.Logf("waiting for doc version %#v on %s, written from %s", expectedVersion, peer, expectedVersion.updatePeer) + body := peer.WaitForDocVersion(dsName, docID, expectedVersion.docMeta) + requireBodyEqual(t, expectedVersion.body, body) + } +} + +func waitForDeletion(t *testing.T, dsName base.ScopeAndCollectionName, peers Peers, docID string, deleteActor string) { + for peerName, peer := range peers { + if peer.Type() == PeerTypeCouchbaseLite { + t.Logf("skipping deletion check for Couchbase Lite peer %s, CBG-4257", peerName) + continue + } + t.Logf("waiting for doc to be deleted on %s, written from %s", peer, deleteActor) + peer.WaitForDeletion(dsName, docID) + } +} + +// removeSyncGatewayBackingPeers will check if there is sync gateway in topology, if so will track the backing CBS +// so we can skip creating docs on these peers (avoiding conflicts between docs created on the SGW and cbs) +func removeSyncGatewayBackingPeers(peers map[string]Peer) map[string]bool { + peersToRemove := make(map[string]bool) + if peers["sg1"] != nil { + // remove the backing store from doc update cycle to avoid conflicts on creating the document in bucket + peersToRemove["cbs1"] = true + } + if peers["sg2"] != nil { + // remove the backing store from doc update cycle to avoid conflicts on creating the document in bucket + peersToRemove["cbs2"] = true + } + return peersToRemove +} + +// createConflictingDocs will create a doc on each peer of the same doc ID to create conflicting documents, then +// returns the last peer to have a doc created on it +func createConflictingDocs(t *testing.T, dsName base.ScopeAndCollectionName, peers Peers, docID, topologyDescription string) (lastWrite BodyAndVersion) { + backingPeers := removeSyncGatewayBackingPeers(peers) + documentVersion := make([]BodyAndVersion, 0, len(peers)) + for peerName, peer := range peers { + if backingPeers[peerName] { + continue + } + if peer.Type() == PeerTypeCouchbaseLite { + // FIXME: Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257 + continue + } + docBody := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "create"}`, peerName, topologyDescription)) + docVersion := peer.CreateDocument(dsName, docID, docBody) + t.Logf("createVersion: %+v", docVersion.docMeta) + documentVersion = append(documentVersion, docVersion) + } + index := len(documentVersion) - 1 + lastWrite = documentVersion[index] + + return lastWrite +} + +// updateConflictingDocs will update a doc on each peer of the same doc ID to create conflicting document mutations, then +// returns the last peer to have a doc updated on it. +func updateConflictingDocs(t *testing.T, dsName base.ScopeAndCollectionName, peers Peers, docID, topologyDescription string) (lastWrite BodyAndVersion) { + backingPeers := removeSyncGatewayBackingPeers(peers) + var documentVersion []BodyAndVersion + for peerName, peer := range peers { + if backingPeers[peerName] { + continue + } + docBody := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "update"}`, peerName, topologyDescription)) + docVersion := peer.WriteDocument(dsName, docID, docBody) + t.Logf("updateVersion: %+v", docVersion.docMeta) + documentVersion = append(documentVersion, docVersion) + } + index := len(documentVersion) - 1 + lastWrite = documentVersion[index] + + return lastWrite +} + +// deleteConflictDocs will delete a doc on each peer of the same doc ID to create conflicting document deletions, then +// returns the last peer to have a doc deleted on it +func deleteConflictDocs(t *testing.T, dsName base.ScopeAndCollectionName, peers Peers, docID string) (lastWrite BodyAndVersion) { + backingPeers := removeSyncGatewayBackingPeers(peers) + var documentVersion []BodyAndVersion + for peerName, peer := range peers { + if backingPeers[peerName] { + continue + } + deleteVersion := peer.DeleteDocument(dsName, docID) + t.Logf("deleteVersion: %+v", deleteVersion) + documentVersion = append(documentVersion, BodyAndVersion{docMeta: deleteVersion, updatePeer: peerName}) + } + index := len(documentVersion) - 1 + lastWrite = documentVersion[index] + + return lastWrite +} + +// getDocID returns a unique doc ID for the test case. Note: when running with Couchbase Server and -count > 1, this will return duplicate IDs for count 2 and higher and they can conflict due to the way bucket pool works. +func getDocID(t *testing.T) string { + name := strings.TrimPrefix(t.Name(), "Test") // shorten doc name + replaceChars := []string{" ", "/"} + for _, char := range replaceChars { + name = strings.ReplaceAll(name, char, "_") + } + return fmt.Sprintf("doc_%s", name) +} diff --git a/topologytest/main_test.go b/topologytest/main_test.go new file mode 100644 index 0000000000..00b6348e8e --- /dev/null +++ b/topologytest/main_test.go @@ -0,0 +1,33 @@ +/* +Copyright 2020-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package topologytest + +import ( + "context" + "os" + "strconv" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" +) + +func TestMain(m *testing.M) { + ctx := context.Background() // start of test process + runTests, _ := strconv.ParseBool(os.Getenv(base.TbpEnvTopologyTests)) + if !base.UnitTestUrlIsWalrus() && !runTests { + base.SkipTestMain(m, "Tests are disabled for Couchbase Server by default, to enable set %s=true environment variable", base.TbpEnvTopologyTests) + return + } + tbpOptions := base.TestBucketPoolOptions{MemWatermarkThresholdMB: 2048} + // Do not create indexes for this test, so they are built by server_context.go + db.TestBucketPoolWithIndexes(ctx, m, tbpOptions) +} diff --git a/topologytest/multi_actor_conflict_test.go b/topologytest/multi_actor_conflict_test.go new file mode 100644 index 0000000000..b30fca941f --- /dev/null +++ b/topologytest/multi_actor_conflict_test.go @@ -0,0 +1,160 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "strings" + "testing" + + "github.com/couchbase/sync_gateway/base" +) + +// TestMultiActorConflictCreate +// 1. create document on each peer with different contents +// 2. start replications +// 3. wait for documents to exist with hlv sources equal to the number of active peers +func TestMultiActorConflictCreate(t *testing.T) { + if !base.UnitTestUrlIsWalrus() { + t.Skip("Flakey failures on multi actor conflicting writes, CBG-4379") + } + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, replications := setupTests(t, topology) + replications.Stop() + + docID := getDocID(t) + docVersion := createConflictingDocs(t, collectionName, peers, docID, topology.description) + replications.Start() + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + + }) + } +} + +// TestMultiActorConflictUpdate +// 1. create document on each peer with different contents +// 2. start replications +// 3. wait for documents to exist with hlv sources equal to the number of active peers +// 4. stop replications +// 5. update documents on all peers +// 6. start replications +// 7. assert that the documents are deleted on all peers and have hlv sources equal to the number of active peers +func TestMultiActorConflictUpdate(t *testing.T) { + if !base.UnitTestUrlIsWalrus() { + t.Skip("Flakey failures on multi actor conflicting writes, CBG-4379") + } + for _, topology := range append(simpleTopologies, Topologies...) { + if strings.Contains(topology.description, "CBL") { + // Test case flakes given the WaitForDocVersion function only waits for a docID on the cbl peer. We need to be + // able to wait for a specific version to arrive over pull replication + t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") + } + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, replications := setupTests(t, topology) + replications.Stop() + + docID := getDocID(t) + docVersion := createConflictingDocs(t, collectionName, peers, docID, topology.description) + + replications.Start() + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + + replications.Stop() + + docVersion = updateConflictingDocs(t, collectionName, peers, docID, topology.description) + replications.Start() + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + }) + } +} + +// TestMultiActorConflictDelete +// 1. create document on each peer with different contents +// 2. start replications +// 3. wait for documents to exist with hlv sources equal to the number of active peers +// 4. stop replications +// 5. delete documents on all peers +// 6. start replications +// 7. assert that the documents are deleted on all peers and have hlv sources equal to the number of active peers +func TestMultiActorConflictDelete(t *testing.T) { + if !base.UnitTestUrlIsWalrus() { + t.Skip("Flakey failures on multi actor conflicting writes, CBG-4379") + } + for _, topology := range append(simpleTopologies, Topologies...) { + if strings.Contains(topology.description, "CBL") { + // Test case flakes given the WaitForDocVersion function only waits for a docID on the cbl peer. We need to be + // able to wait for a specific version to arrive over pull replication + t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") + } + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, replications := setupTests(t, topology) + replications.Stop() + + docID := getDocID(t) + docVersion := createConflictingDocs(t, collectionName, peers, docID, topology.description) + + replications.Start() + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + + replications.Stop() + lastWrite := deleteConflictDocs(t, collectionName, peers, docID) + + replications.Start() + waitForDeletion(t, collectionName, peers, docID, lastWrite.updatePeer) + }) + } +} + +// TestMultiActorConflictResurrect +// 1. create document on each peer with different contents +// 2. start replications +// 3. wait for documents to exist with hlv sources equal to the number of active peers and the document body is equivalent to the last write +// 4. stop replications +// 5. delete documents on all peers +// 6. start replications +// 7. assert that the documents are deleted on all peers and have hlv sources equal to the number of active peers +// 8. stop replications +// 9. resurrect documents on all peers with unique contents +// 10. start replications +// 11. assert that the documents are resurrected on all peers and have hlv sources equal to the number of active peers and the document body is equivalent to the last write +func TestMultiActorConflictResurrect(t *testing.T) { + if !base.UnitTestUrlIsWalrus() { + t.Skip("Flakey failures on multi actor conflicting writes, CBG-4379") + } + for _, topology := range append(simpleTopologies, Topologies...) { + if strings.Contains(topology.description, "CBL") { + // Test case flakes given the WaitForDocVersion function only waits for a docID on the cbl peer. We need to be + // able to wait for a specific version to arrive over pull replication + t.Skip("We need to be able to wait for a specific version to arrive over pull replication + unexpected body in proposeChanges: [304] issue, CBG-4257") + } + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, replications := setupTests(t, topology) + replications.Stop() + + docID := getDocID(t) + docVersion := createConflictingDocs(t, collectionName, peers, docID, topology.description) + + replications.Start() + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + + replications.Stop() + lastWrite := deleteConflictDocs(t, collectionName, peers, docID) + + replications.Start() + + waitForDeletion(t, collectionName, peers, docID, lastWrite.updatePeer) + replications.Stop() + + lastWriteVersion := updateConflictingDocs(t, collectionName, peers, docID, topology.description) + replications.Start() + + waitForVersionAndBody(t, collectionName, peers, docID, lastWriteVersion) + }) + } +} diff --git a/topologytest/multi_actor_no_conflict_test.go b/topologytest/multi_actor_no_conflict_test.go new file mode 100644 index 0000000000..1aa1a7cc82 --- /dev/null +++ b/topologytest/multi_actor_no_conflict_test.go @@ -0,0 +1,120 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "fmt" + "testing" + + "github.com/couchbase/sync_gateway/base" +) + +// TestMultiActorUpdate tests that a single actor can update a document that was created on a different peer. +// 1. start replications +// 2. create documents on each peer, to be updated by each other peer +// 3. wait for all documents to be replicated +// 4. update each document on a single peer, documents exist in pairwise create peer and update peer +// 5. wait for the hlv for updated documents to synchronized +func TestMultiActorUpdate(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + + for createPeerName, createPeer := range peers { + for updatePeerName, updatePeer := range peers { + if updatePeer.Type() == PeerTypeCouchbaseLite { + continue + } + + docID := getDocID(t) + "_create=" + createPeerName + ",update=" + updatePeerName + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "createPeer": "%s", "updatePeer": "%s", "topology": "%s", "action": "create"}`, createPeerName, createPeerName, updatePeer, topology.description)) + createVersion := createPeer.CreateDocument(collectionName, docID, body1) + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + newBody := []byte(fmt.Sprintf(`{"activePeer": "%s", "createPeer": "%s", "updatePeer": "%s", "topology": "%s", "action": "update"}`, updatePeerName, createPeerName, updatePeerName, topology.description)) + updateVersion := updatePeer.WriteDocument(collectionName, docID, newBody) + + waitForVersionAndBody(t, collectionName, peers, docID, updateVersion) + } + } + }) + } +} + +// TestMultiActorDelete tests that a single actor can update a document that was created on a different peer. +// 1. start replications +// 2. create documents on each peer, to be updated by each other peer +// 3. wait for all documents to be replicated +// 4. delete each document on a single peer, documents exist in pairwise create peer and update peer +// 5. wait for the hlv for updated documents to synchronized +func TestMultiActorDelete(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + + for createPeerName, createPeer := range peers { + for deletePeerName, deletePeer := range peers { + if deletePeer.Type() == PeerTypeCouchbaseLite { + continue + } + + docID := getDocID(t) + "_create=" + createPeerName + ",update=" + deletePeerName + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "createPeer": "%s", "deletePeer": "%s", "topology": "%s", "action": "create"}`, createPeerName, createPeerName, deletePeer, topology.description)) + createVersion := createPeer.CreateDocument(collectionName, docID, body1) + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + deleteVersion := deletePeer.DeleteDocument(collectionName, docID) + t.Logf("deleteVersion: %+v\n", deleteVersion) // FIXME: verify hlv in CBG-4416 + waitForDeletion(t, collectionName, peers, docID, deletePeerName) + } + } + }) + } +} + +// TestMultiActorResurrect tests that a single actor can update a document that was created on a different peer. +// 1. start replications +// 2. create documents on each peer, to be updated by each other peer +// 3. wait for all documents to be replicated +// 4. delete each document on a single peer, documents exist in pairwise create peer and update peer +// 5. wait for the hlv for updated documents to synchronized +// 6. resurrect each document on a single peer +// 7. wait for the hlv for updated documents to be synchronized +func TestMultiActorResurrect(t *testing.T) { + if base.UnitTestUrlIsWalrus() { + t.Skip("CBG-4419: this test fails xdcr with: could not write doc: cas mismatch: expected 0, really 1 -- xdcr.(*rosmarManager).processEvent() at rosmar_xdcr.go:201") + } + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + + for createPeerName, createPeer := range peers { + for deletePeerName, deletePeer := range peers { + if deletePeer.Type() == PeerTypeCouchbaseLite { + continue + } + for resurrectPeerName, resurrectPeer := range peers { + docID := getDocID(t) + "_create=" + createPeerName + ",delete=" + deletePeerName + ",resurrect=" + resurrectPeerName + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "createPeer": "%s", "deletePeer": "%s", "resurrectPeer": "%s", "topology": "%s", "action": "create"}`, createPeerName, createPeerName, deletePeer, resurrectPeer, topology.description)) + createVersion := createPeer.CreateDocument(collectionName, docID, body1) + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + deleteVersion := deletePeer.DeleteDocument(collectionName, docID) + t.Logf("deleteVersion: %+v\n", deleteVersion) // FIXME: verify hlv in CBG-4416 + waitForDeletion(t, collectionName, peers, docID, deletePeerName) + + resBody := []byte(fmt.Sprintf(`{"activePeer": "%s", "createPeer": "%s", "deletePeer": "%s", "resurrectPeer": "%s", "topology": "%s", "action": "resurrect"}`, resurrectPeerName, createPeerName, deletePeer, resurrectPeer, topology.description)) + resurrectVersion := resurrectPeer.WriteDocument(collectionName, docID, resBody) + waitForVersionAndBody(t, collectionName, peers, docID, resurrectVersion) + } + } + } + }) + } +} diff --git a/topologytest/peer_test.go b/topologytest/peer_test.go new file mode 100644 index 0000000000..ca0cc48eb8 --- /dev/null +++ b/topologytest/peer_test.go @@ -0,0 +1,410 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +// Package topologytest implements code to be able to test with Couchbase Server, Sync Gateway, and Couchbase Lite from a go test. This can be with Couchbase Server or rosmar depending on SG_TEST_BACKING_STORE. Couchbase Lite can either be an in memory implementation of a Couchbase Lite peer, or a real Couchbase Lite peer. +package topologytest + +import ( + "context" + "fmt" + "iter" + "maps" + "slices" + "testing" + "time" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/xdcr" + "github.com/stretchr/testify/require" +) + +// totalWaitTime is the time to wait for a document on a peer. This time is low for rosmar and high for Couchbase Server. +var totalWaitTime = 3 * time.Second + +// pollInterval is the time to poll to see if a document is updated on a peer +var pollInterval = 50 * time.Millisecond + +func init() { + if !base.UnitTestUrlIsWalrus() { + totalWaitTime = 40 * time.Second + } +} + +// Peer represents a peer in an Mobile workflow. The types of Peers are Couchbase Server, Sync Gateway, or Couchbase Lite. +type Peer interface { + // GetDocument returns the latest version of a document. The test will fail the document does not exist. + GetDocument(dsName sgbucket.DataStoreName, docID string) (DocMetadata, db.Body) + // CreateDocument creates a document on the peer. The test will fail if the document already exists. + CreateDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion + // WriteDocument upserts a document to the peer. The test will fail if the write does not succeed. Reasons for failure might be sync function rejections for Sync Gateway rejections. + WriteDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion + // DeleteDocument deletes a document on the peer. The test will fail if the document does not exist. + DeleteDocument(dsName sgbucket.DataStoreName, docID string) DocMetadata + + // WaitForDocVersion waits for a document to reach a specific version. Returns the state of the document at that version. The test will fail if the document does not reach the expected version in 20s. + WaitForDocVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) db.Body + + // WaitForDeletion waits for a document to be deleted. This document must be a tombstone. The test will fail if the document still exists after 20s. + WaitForDeletion(dsName sgbucket.DataStoreName, docID string) + + // WaitForTombstoneVersion waits for a document to reach a specific version. This document must be a tombstone. The test will fail if the document does not reach the expected version in 20s. + WaitForTombstoneVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) + + // RequireDocNotFound asserts that a document does not exist on the peer. + RequireDocNotFound(dsName sgbucket.DataStoreName, docID string) + + // CreateReplication creates a replication instance + CreateReplication(Peer, PeerReplicationConfig) PeerReplication + + // Close will shut down the peer and close any active replications on the peer. + Close() + + internalPeer +} + +// internalPeer represents Peer interface that are only intdeded to be used from within a Peer or Replication class, but not by tests themselves. +type internalPeer interface { + // SourceID returns the source ID for the peer used in @. + SourceID() string + + // GetBackingBucket returns the backing bucket for the peer. This is nil when the peer is a Couchbase Lite peer. + GetBackingBucket() base.Bucket + + // TB returns the testing.TB for the peer. + TB() testing.TB + + // UpdateTB updates the testing.TB for the peer. + UpdateTB(*testing.T) + + // Context returns the context for the peer. + Context() context.Context + + // Type returns the type of the peer. + Type() PeerType +} + +// PeerReplication represents a replication between two peers. This replication is unidirectional since all bi-directional replications are represented by two unidirectional instances. +type PeerReplication interface { + // ActivePeer returns the peer sending documents + ActivePeer() Peer + // PassivePeer returns the peer receiving documents + PassivePeer() Peer + // Start starts the replication + Start() + // Stop halts the replication. The replication can be restarted after it is stopped. + Stop() +} + +// Peers represents a set of peers. The peers are indexed by name. +type Peers map[string]Peer + +// SortedPeers returns a sorted list of peers by name, for deterministic output. +func (p Peers) SortedPeers() iter.Seq2[string, Peer] { + keys := slices.Collect(maps.Keys(p)) + slices.Sort(keys) + return func(yield func(k string, v Peer) bool) { + for _, peerName := range keys { + if !yield(peerName, p[peerName]) { + return + } + } + } +} + +var _ PeerReplication = &CouchbaseLiteMockReplication{} +var _ PeerReplication = &CouchbaseServerReplication{} +var _ PeerReplication = &CouchbaseServerReplication{} + +// Replications are a collection of PeerReplications. +type Replications []PeerReplication + +// Stop stops all replications. +func (r Replications) Stop() { + for _, replication := range r { + replication.Stop() + } +} + +// Start starts all replications. +func (r Replications) Start() { + for _, replication := range r { + replication.Start() + } +} + +// PeerReplicationDirection represents the direction of a replication from the active peer. +type PeerReplicationDirection int + +const ( + // PeerReplicationDirectionPush pushes data from an active peer to a passive peer. + PeerReplicationDirectionPush PeerReplicationDirection = iota + // PeerReplicationDirectionPull pulls data from an active peer to a passive peer. + PeerReplicationDirectionPull +) + +// PeerReplicationConfig represents the configuration for a given replication. +type PeerReplicationConfig struct { + direction PeerReplicationDirection + // oneShot bool // not implemented, would only be supported for SG <-> CBL, XDCR is always continuous +} + +// PeerReplicationDefinition defines a pair of peers and a configuration. +type PeerReplicationDefinition struct { + activePeer string + passivePeer string + config PeerReplicationConfig +} + +var _ Peer = &CouchbaseServerPeer{} +var _ Peer = &CouchbaseLiteMockPeer{} +var _ Peer = &SyncGatewayPeer{} + +// PeerType represents the type of a peer. These will be: +// +// - Couchbase Server (backed by TestBucket) +// - rosmar default +// - Couchbase Server based on SG_TEST_BACKING_STORE=couchbase +// +// - Sync Gateway (backed by RestTester) +// +// - Couchbase Lite +// - CouchbaseLiteMockPeer is in memory backed by BlipTesterClient +// - CouchbaseLitePeer (backed by Test Server) Not Yet Implemented +type PeerType int + +const ( + // PeerTypeCouchbaseServer represents a Couchbase Server peer. This can be backed by rosmar or couchbase server (controlled by SG_TEST_BACKING_STORE). + PeerTypeCouchbaseServer PeerType = iota + // PeerTypeCouchbaseLite represents a Couchbase Lite peer. This is currently backed in memory but will be backed by in memory structure that will send and receive blip messages. Future expansion to real Couchbase Lite peer in CBG-4260. + PeerTypeCouchbaseLite + // PeerTypeSyncGateway represents a Sync Gateway peer backed by a RestTester. + PeerTypeSyncGateway +) + +// PeerBucketID represents a specific bucket for a test. This allows multiple Sync Gateway instances to point to the same bucket, or a different buckets. There is no significance to the numbering of the buckets. We can use as many buckets as the MainTestBucketPool allows. +type PeerBucketID int + +const ( + // PeerBucketNoBackingBucket represents a peer that does not have a backing bucket. This is used for Couchbase Lite peers. + PeerBucketNoBackingBucket PeerBucketID = iota + // PeerBucketID1 represents the first bucket in the test. + PeerBucketID1 // start at 1 to avoid 0 value being accidentally used + // PeerBucketID2 represents the second bucket in the test. + PeerBucketID2 +) + +// PeerOptions are options to create a peer. +type PeerOptions struct { + Type PeerType + BucketID PeerBucketID // BucketID is used to identify the bucket for a Couchbase Server or Sync Gateway peer. This option is ignored for Couchbase Lite peers. +} + +// NewPeer creates a new peer for replication. The buckets must be created before the peers are created. +func NewPeer(t *testing.T, name string, buckets map[PeerBucketID]*base.TestBucket, opts PeerOptions) Peer { + switch opts.Type { + case PeerTypeCouchbaseServer: + bucket, ok := buckets[opts.BucketID] + require.True(t, ok, "bucket not found for bucket ID %d", opts.BucketID) + sourceID, err := xdcr.GetSourceID(base.TestCtx(t), bucket) + require.NoError(t, err) + return &CouchbaseServerPeer{ + name: name, + tb: t, + bucket: bucket, + sourceID: sourceID, + pullReplications: make(map[Peer]xdcr.Manager), + pushReplications: make(map[Peer]xdcr.Manager), + } + case PeerTypeCouchbaseLite: + require.Equal(t, PeerBucketNoBackingBucket, opts.BucketID, "bucket should not be specified for Couchbase Lite peer %+v", opts) + _, ok := buckets[opts.BucketID] + require.False(t, ok, "bucket should not be specified for Couchbase Lite peer") + return &CouchbaseLiteMockPeer{ + t: t, + name: name, + blipClients: make(map[string]*PeerBlipTesterClient), + } + case PeerTypeSyncGateway: + bucket, ok := buckets[opts.BucketID] + require.True(t, ok, "bucket not found for bucket ID %d", opts.BucketID) + return newSyncGatewayPeer(t, name, bucket) + default: + require.Fail(t, fmt.Sprintf("unsupported peer type %T", opts.Type)) + } + return nil +} + +// createPeerReplications creates a list of peers and replications. The replications will not have started. +func createPeerReplications(t *testing.T, peers map[string]Peer, configs []PeerReplicationDefinition) []PeerReplication { + replications := make([]PeerReplication, 0, len(configs)) + for _, config := range configs { + activePeer, ok := peers[config.activePeer] + require.True(t, ok, "active peer %s not found", config.activePeer) + passivePeer, ok := peers[config.passivePeer] + require.True(t, ok, "passive peer %s not found", config.passivePeer) + replications = append(replications, activePeer.CreateReplication(passivePeer, config.config)) + } + return replications +} + +// getPeerBuckets returns a map of bucket IDs to buckets for a list of peers. This requires sufficient number of buckets in the bucket pool. The buckets will be released with a testing.T.Cleanup function. +func getPeerBuckets(t *testing.T, peerOptions map[string]PeerOptions) map[PeerBucketID]*base.TestBucket { + buckets := make(map[PeerBucketID]*base.TestBucket) + for _, p := range peerOptions { + if p.BucketID == PeerBucketNoBackingBucket { + continue + } + _, ok := buckets[p.BucketID] + if !ok { + bucket := base.GetTestBucket(t) + buckets[p.BucketID] = bucket + t.Cleanup(func() { + bucket.Close(base.TestCtx(t)) + }) + } + } + return buckets +} + +// createPeers will create a sets of peers. The underlying buckets will be created. The peers will be closed and the buckets will be destroyed. +func createPeers(t *testing.T, peersOptions map[string]PeerOptions) Peers { + buckets := getPeerBuckets(t, peersOptions) + peers := make(Peers, len(peersOptions)) + for id, peerOptions := range peersOptions { + peer := NewPeer(t, id, buckets, peerOptions) + t.Logf("TopologyTest: created peer %s", peer) + t.Cleanup(func() { + peer.Close() + }) + peers[id] = peer + } + return peers +} + +func updatePeersT(t *testing.T, peers map[string]Peer) { + for _, peer := range peers { + oldTB := peer.TB().(*testing.T) + t.Cleanup(func() { peer.UpdateTB(oldTB) }) + peer.UpdateTB(t) + } +} + +// setupTests returns a map of peers and a list of replications. The peers will be closed and the buckets will be destroyed by t.Cleanup. +func setupTests(t *testing.T, topology Topology) (base.ScopeAndCollectionName, Peers, Replications) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyImport, base.KeyVV) + peers := createPeers(t, topology.peers) + replications := createPeerReplications(t, peers, topology.replications) + + for _, replication := range replications { + // temporarily start the replication before writing the document, limitation of CouchbaseLiteMockPeer as active peer since WriteDocument is calls PushRev + replication.Start() + } + return getSingleDsName(), peers, replications +} + +func TestPeerImplementation(t *testing.T) { + testCases := []struct { + name string + peerOption PeerOptions + }{ + { + name: "cbs", + peerOption: PeerOptions{ + Type: PeerTypeCouchbaseServer, + BucketID: PeerBucketID1, + }, + }, + { + name: "sg", + peerOption: PeerOptions{ + Type: PeerTypeSyncGateway, + BucketID: PeerBucketID1, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + peers := createPeers(t, map[string]PeerOptions{tc.name: tc.peerOption}) + peer := peers[tc.name] + + docID := t.Name() + collectionName := getSingleDsName() + + peer.RequireDocNotFound(collectionName, docID) + // Create + createBody := []byte(`{"op": "creation"}`) + createVersion := peer.CreateDocument(collectionName, docID, []byte(`{"op": "creation"}`)) + require.NotEmpty(t, createVersion.docMeta.CV) + if tc.peerOption.Type == PeerTypeCouchbaseServer { + require.Empty(t, createVersion.docMeta.RevTreeID) + } else { + require.NotEmpty(t, createVersion.docMeta.RevTreeID) + } + + peer.WaitForDocVersion(collectionName, docID, createVersion.docMeta) + // Check Get after creation + roundtripGetVersion, roundtripGetbody := peer.GetDocument(collectionName, docID) + require.Equal(t, createVersion.docMeta, roundtripGetVersion) + require.JSONEq(t, string(createBody), string(base.MustJSONMarshal(t, roundtripGetbody))) + + // Update + updateBody := []byte(`{"op": "update"}`) + updateVersion := peer.WriteDocument(collectionName, docID, updateBody) + require.NotEmpty(t, updateVersion.docMeta.CV) + require.NotEqual(t, updateVersion.docMeta.CV(t), createVersion.docMeta.CV(t)) + if tc.peerOption.Type == PeerTypeCouchbaseServer { + require.Empty(t, updateVersion.docMeta.RevTreeID) + } else { + require.NotEmpty(t, updateVersion.docMeta.RevTreeID) + require.NotEqual(t, updateVersion.docMeta.RevTreeID, createVersion.docMeta.RevTreeID) + } + peer.WaitForDocVersion(collectionName, docID, updateVersion.docMeta) + + // Check Get after update + roundtripGetVersion, roundtripGetbody = peer.GetDocument(collectionName, docID) + require.Equal(t, updateVersion.docMeta, roundtripGetVersion) + require.JSONEq(t, string(updateBody), string(base.MustJSONMarshal(t, roundtripGetbody))) + + // Delete + deleteVersion := peer.DeleteDocument(collectionName, docID) + require.NotEmpty(t, deleteVersion.CV(t)) + require.NotEqual(t, deleteVersion.CV(t), updateVersion.docMeta.CV(t)) + require.NotEqual(t, deleteVersion.CV(t), createVersion.docMeta.CV(t)) + if tc.peerOption.Type == PeerTypeCouchbaseServer { + require.Empty(t, deleteVersion.RevTreeID) + } else { + require.NotEmpty(t, deleteVersion.RevTreeID) + require.NotEqual(t, deleteVersion.RevTreeID, createVersion.docMeta.RevTreeID) + require.NotEqual(t, deleteVersion.RevTreeID, updateVersion.docMeta.RevTreeID) + } + peer.RequireDocNotFound(collectionName, docID) + + // Resurrection + + resurrectionBody := []byte(`{"op": "resurrection"}`) + resurrectionVersion := peer.WriteDocument(collectionName, docID, resurrectionBody) + require.NotEmpty(t, resurrectionVersion.docMeta.CV(t)) + require.NotEqual(t, resurrectionVersion.docMeta.CV(t), deleteVersion.CV(t)) + require.NotEqual(t, resurrectionVersion.docMeta.CV(t), updateVersion.docMeta.CV(t)) + require.NotEqual(t, resurrectionVersion.docMeta.CV(t), createVersion.docMeta.CV(t)) + if tc.peerOption.Type == PeerTypeCouchbaseServer { + require.Empty(t, resurrectionVersion.docMeta.RevTreeID) + } else { + require.NotEmpty(t, resurrectionVersion.docMeta.RevTreeID) + require.NotEqual(t, resurrectionVersion.docMeta.RevTreeID, createVersion.docMeta.RevTreeID) + require.NotEqual(t, resurrectionVersion.docMeta.RevTreeID, updateVersion.docMeta.RevTreeID) + require.NotEqual(t, resurrectionVersion.docMeta.RevTreeID, deleteVersion.RevTreeID) + } + peer.WaitForDocVersion(collectionName, docID, resurrectionVersion.docMeta) + + }) + } + +} diff --git a/topologytest/single_actor_test.go b/topologytest/single_actor_test.go new file mode 100644 index 0000000000..8bd05dc1a8 --- /dev/null +++ b/topologytest/single_actor_test.go @@ -0,0 +1,144 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "fmt" + "testing" +) + +// TestSingleActorCreate tests creating a document with a single actor in different topologies. +// 1. start replications +// 2. create document on a single active peer (version1) +// 3. wait for convergence on other peers +func TestSingleActorCreate(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + for activePeerID, activePeer := range peers.SortedPeers() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + docID := getDocID(t) + docBody := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "create"}`, activePeerID, topology.description)) + docVersion := activePeer.CreateDocument(collectionName, docID, docBody) + waitForVersionAndBody(t, collectionName, peers, docID, docVersion) + }) + } + }) + } +} + +// TestSingleActorUpdate tests updating a document on a single actor and ensuring the matching hlv exists on all peers. +// 1. start replications +// 2. create document on a single active peer (version1) +// 3. wait for convergence on other peers +// 4. update document on a single active peer (version2) +// 5. wait for convergence on other peers +func TestSingleActorUpdate(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + for activePeerID, activePeer := range peers { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + if activePeer.Type() == PeerTypeCouchbaseLite { + t.Skip("Skipping Couchbase Lite test, returns unexpected body in proposeChanges: [304], CBG-4257") + } + + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "create"}`, activePeerID, topology.description)) + createVersion := activePeer.CreateDocument(collectionName, docID, body1) + + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + body2 := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "update"}`, activePeerID, topology.description)) + updateVersion := activePeer.WriteDocument(collectionName, docID, body2) + t.Logf("createVersion: %+v, updateVersion: %+v", createVersion.docMeta, updateVersion.docMeta) + t.Logf("waiting for document version 2 on all peers") + + waitForVersionAndBody(t, collectionName, peers, docID, updateVersion) + }) + } + }) + } +} + +// TestSingleActorDelete tests deletion of a documents on an active peer and makes sure the deletion and hlv matches on all peers. +// 1. start replications +// 2. create document on a single active peer (version1) +// 3. wait for convergence on other peers +// 4. delete document on a single active peer (version2) +// 5. wait for convergence on other peers for a deleted document with correct hlv +func TestSingleActorDelete(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + for activePeerID, activePeer := range peers { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + if activePeer.Type() == PeerTypeCouchbaseLite { + t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") + } + + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "create"}`, activePeerID, topology.description)) + createVersion := activePeer.CreateDocument(collectionName, docID, body1) + + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + deleteVersion := activePeer.DeleteDocument(collectionName, docID) + t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion.docMeta, deleteVersion) + t.Logf("waiting for document deletion on all peers") + waitForDeletion(t, collectionName, peers, docID, activePeerID) + }) + } + }) + } +} + +// TestSingleActorResurrect tests resurrect a document with a single actor in different topologies. +// 1. start replications +// 2. create document on a single active peer (version1) +// 3. wait for convergence on other peers +// 4. delete document on a single active peer (version2) +// 5. wait for convergence on other peers for a deleted document with correct hlv +// 6. resurrect document on a single active peer (version3) +// 7. wait for convergence on other peers for a resurrected document with correct hlv +func TestSingleActorResurrect(t *testing.T) { + for _, topology := range append(simpleTopologies, Topologies...) { + t.Run(topology.description, func(t *testing.T) { + collectionName, peers, _ := setupTests(t, topology) + for activePeerID, activePeer := range peers.SortedPeers() { + t.Run(fmt.Sprintf("actor=%s", activePeerID), func(t *testing.T) { + updatePeersT(t, peers) + if activePeer.Type() == PeerTypeCouchbaseLite { + t.Skip("Skipping Couchbase Lite test, does not know how to push a deletion yet CBG-4257") + } + + docID := getDocID(t) + body1 := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "create"}`, activePeerID, topology.description)) + createVersion := activePeer.CreateDocument(collectionName, docID, body1) + waitForVersionAndBody(t, collectionName, peers, docID, createVersion) + + deleteVersion := activePeer.DeleteDocument(collectionName, docID) + t.Logf("createVersion: %+v, deleteVersion: %+v", createVersion, deleteVersion) + t.Logf("waiting for document deletion on all peers") + waitForDeletion(t, collectionName, peers, docID, activePeerID) + + body2 := []byte(fmt.Sprintf(`{"activePeer": "%s", "topology": "%s", "action": "resurrect"}`, activePeerID, topology.description)) + resurrectVersion := activePeer.WriteDocument(collectionName, docID, body2) + t.Logf("createVersion: %+v, deleteVersion: %+v, resurrectVersion: %+v", createVersion.docMeta, deleteVersion, resurrectVersion.docMeta) + t.Logf("waiting for document resurrection on all peers") + + waitForVersionAndBody(t, collectionName, peers, docID, resurrectVersion) + }) + } + }) + } +} diff --git a/topologytest/sync_gateway_peer_test.go b/topologytest/sync_gateway_peer_test.go new file mode 100644 index 0000000000..42d43fceb8 --- /dev/null +++ b/topologytest/sync_gateway_peer_test.go @@ -0,0 +1,216 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type SyncGatewayPeer struct { + rt *rest.RestTester + name string +} + +func newSyncGatewayPeer(t *testing.T, name string, bucket *base.TestBucket) Peer { + rt := rest.NewRestTester(t, &rest.RestTesterConfig{ + PersistentConfig: true, + CustomTestBucket: bucket.NoCloseClone(), + }) + config := rt.NewDbConfig() + config.AutoImport = base.BoolPtr(true) + rest.RequireStatus(t, rt.CreateDatabase(rest.SafeDatabaseName(t, name), config), http.StatusCreated) + return &SyncGatewayPeer{ + name: name, + rt: rt, + } +} + +func (p *SyncGatewayPeer) String() string { + return fmt.Sprintf("%s (bucket:%s,sourceid:%s)", p.name, p.rt.Bucket().GetName(), p.SourceID()) +} + +// getCollection returns the collection for the given data store name and a related context. The special context is needed to add fields for audit logging, required by build tag cb_sg_devmode. +func (p *SyncGatewayPeer) getCollection(dsName sgbucket.DataStoreName) (*db.DatabaseCollectionWithUser, context.Context) { + dbCtx, err := db.GetDatabase(p.rt.GetDatabase(), nil) + require.NoError(p.TB(), err) + collection, err := dbCtx.GetDatabaseCollectionWithUser(dsName.ScopeName(), dsName.CollectionName()) + require.NoError(p.TB(), err) + ctx := base.UserLogCtx(collection.AddCollectionContext(p.Context()), "gotest", base.UserDomainBuiltin, nil) + return collection, ctx +} + +// GetDocument returns the latest version of a document. The test will fail the document does not exist. +func (p *SyncGatewayPeer) GetDocument(dsName sgbucket.DataStoreName, docID string) (DocMetadata, db.Body) { + collection, ctx := p.getCollection(dsName) + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + require.NoError(p.TB(), err) + return DocMetadataFromDocument(doc), doc.Body(ctx) +} + +// CreateDocument creates a document on the peer. The test will fail if the document already exists. +func (p *SyncGatewayPeer) CreateDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.TB().Logf("%s: Creating document %s", p, docID) + return p.WriteDocument(dsName, docID, body) +} + +// writeDocument writes a document to the peer. The test will fail if the write does not succeed. +func (p *SyncGatewayPeer) writeDocument(dsName sgbucket.DataStoreName, docID string, body []byte) DocMetadata { + collection, ctx := p.getCollection(dsName) + + var doc *db.Document + // loop to write document in the case that there is a conflict while writing the document + err, _ := base.RetryLoop(ctx, "write document", func() (shouldRetry bool, err error, value any) { + var bodyMap db.Body + err = base.JSONUnmarshal(body, &bodyMap) + require.NoError(p.TB(), err) + + // allow upsert rev + existingDoc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + if err == nil { + bodyMap[db.BodyRev] = existingDoc.CurrentRev + } + _, doc, err = collection.Put(ctx, docID, bodyMap) + if err != nil { + var httpError *base.HTTPError + if errors.As(err, &httpError) && httpError.Status == http.StatusConflict { + return true, err, nil + } + require.NoError(p.TB(), err) + } + return false, nil, nil + }, base.CreateSleeperFunc(5, 100)) + require.NoError(p.TB(), err) + return DocMetadataFromDocument(doc) +} + +// WriteDocument writes a document to the peer. The test will fail if the write does not succeed. +func (p *SyncGatewayPeer) WriteDocument(dsName sgbucket.DataStoreName, docID string, body []byte) BodyAndVersion { + p.TB().Logf("%s: Writing document %s", p, docID) + docMetadata := p.writeDocument(dsName, docID, body) + return BodyAndVersion{ + docMeta: docMetadata, + body: body, + updatePeer: p.name, + } +} + +// DeleteDocument deletes a document on the peer. The test will fail if the document does not exist. +func (p *SyncGatewayPeer) DeleteDocument(dsName sgbucket.DataStoreName, docID string) DocMetadata { + collection, ctx := p.getCollection(dsName) + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + var revID string + if err == nil { + revID = doc.CurrentRev + } + _, doc, err = collection.DeleteDoc(ctx, docID, revID) + require.NoError(p.TB(), err) + return DocMetadataFromDocument(doc) +} + +// WaitForDocVersion waits for a document to reach a specific version. The test will fail if the document does not reach the expected version in 20s. +func (p *SyncGatewayPeer) WaitForDocVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) db.Body { + collection, ctx := p.getCollection(dsName) + var doc *db.Document + require.EventuallyWithT(p.TB(), func(c *assert.CollectT) { + var err error + doc, err = collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + assert.NoError(c, err) + if doc == nil { + return + } + version := DocMetadataFromDocument(doc) + // Only assert on CV since RevTreeID might not be present if this was a Couchbase Server write + bodyBytes, err := doc.BodyBytes(ctx) + assert.NoError(c, err) + assert.Equal(c, expected.CV(c), version.CV(c), "Could not find matching CV on %s for peer %s (sourceID:%s)\nexpected: %#v\nactual: %#v\n body: %+v\n", docID, p, p.SourceID(), expected, version, string(bodyBytes)) + }, totalWaitTime, pollInterval) + return doc.Body(ctx) +} + +// WaitForDeletion waits for a document to be deleted. This document must be a tombstone. The test will fail if the document still exists after 20s. +func (p *SyncGatewayPeer) WaitForDeletion(dsName sgbucket.DataStoreName, docID string) { + collection, ctx := p.getCollection(dsName) + require.EventuallyWithT(p.TB(), func(c *assert.CollectT) { + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + if err == nil { + assert.True(c, doc.IsDeleted(), "expected %+v on %s to be deleted", doc, p) + return + } + assert.True(c, base.IsDocNotFoundError(err), "expected docID %s on %s to be deleted, found doc=%#v err=%v", docID, p, doc, err) + }, totalWaitTime, pollInterval) +} + +// WaitForTombstoneVersion waits for a document to reach a specific version, this must be a tombstone. The test will fail if the document does not reach the expected version in 20s. +func (p *SyncGatewayPeer) WaitForTombstoneVersion(dsName sgbucket.DataStoreName, docID string, expected DocMetadata) { + docBytes := p.WaitForDocVersion(dsName, docID, expected) + require.Nil(p.TB(), docBytes, "expected tombstone for docID %s, got %s", docID, docBytes) +} + +// RequireDocNotFound asserts that a document does not exist on the peer. +func (p *SyncGatewayPeer) RequireDocNotFound(dsName sgbucket.DataStoreName, docID string) { + collection, ctx := p.getCollection(dsName) + doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) + if err == nil { + require.True(p.TB(), doc.IsDeleted(), "expected %s to be deleted", doc) + return + } + base.RequireDocNotFoundError(p.TB(), err) +} + +// Close will shut down the peer and close any active replications on the peer. +func (p *SyncGatewayPeer) Close() { + p.rt.Close() +} + +// Type returns PeerTypeSyncGateway. +func (p *SyncGatewayPeer) Type() PeerType { + return PeerTypeSyncGateway +} + +// CreateReplication creates a replication instance. This is currently not supported for Sync Gateway peers. A future ISGR implementation will support this. +func (p *SyncGatewayPeer) CreateReplication(_ Peer, _ PeerReplicationConfig) PeerReplication { + require.Fail(p.rt.TB(), "can not create a replication with Sync Gateway as an active peer") + return nil +} + +// SourceID returns the source ID for the peer used in @. +func (p *SyncGatewayPeer) SourceID() string { + return p.rt.GetDatabase().EncodedSourceID +} + +// Context returns the context for the peer. +func (p *SyncGatewayPeer) Context() context.Context { + return p.rt.Context() +} + +// TB returns the testing.TB for the peer. +func (p *SyncGatewayPeer) TB() testing.TB { + return p.rt.TB() +} + +// UpdateTB updates the testing.TB for the peer. +func (p *SyncGatewayPeer) UpdateTB(t *testing.T) { + p.rt.UpdateTB(t) +} + +// GetBackingBucket returns the backing bucket for the peer. +func (p *SyncGatewayPeer) GetBackingBucket() base.Bucket { + return p.rt.Bucket() +} diff --git a/topologytest/topologies_test.go b/topologytest/topologies_test.go new file mode 100644 index 0000000000..98a31a9573 --- /dev/null +++ b/topologytest/topologies_test.go @@ -0,0 +1,343 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +// Topology defines a topology for a set of peers and replications. This can include Couchbase Server, Sync Gateway, and Couchbase Lite peers, with push or pull replications between them. +type Topology struct { + description string + peers map[string]PeerOptions + replications []PeerReplicationDefinition +} + +// Topologies represents user configurations of replications. +var Topologies = []Topology{ + { + /* + + - - - - - - + + ' +---------+ ' + ' | cbs1 | ' + ' +---------+ ' + ' +---------+ ' + ' | sg1 | ' + ' +---------+ ' + + - - - - - - + + ^ + | + | + v + +---------+ + | cbl1 | + +---------+ + */ + description: "CBL<->SG<->CBS 1.1", + peers: map[string]PeerOptions{ + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + "sg1": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID1}, + "cbl1": {Type: PeerTypeCouchbaseLite}, + }, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + }, + }, + { + /* + Test topology 1.2 + + + - - - - - - + +- - - - - - -+ + ' cluster A ' ' cluster B ' + ' +---------+ ' ' +---------+ ' + ' | cbs1 | ' <--> ' | cbs2 | ' + ' +---------+ ' ' +---------+ ' + ' +---------+ ' + - - - - - - + + ' | sg1 | ' + ' +---------+ ' + + - - - - - - + + ^ + | + | + v + +---------+ + | cbl1 | + +---------+ + */ + description: "CBL<->SG<->CBS1 CBS1<->CBS2 1.2", + peers: map[string]PeerOptions{ + "sg1": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID1}, + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + + "cbs2": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID2}, + "cbl1": {Type: PeerTypeCouchbaseLite}, + }, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + }, + }, + { + /* + Test topology 1.3 + + + - - - - - - + +- - - - - - -+ + ' cluster A ' ' cluster B ' + ' +---------+ ' ' +---------+ ' + ' | cbs1 | ' <--> ' | cbs2 | ' + ' +---------+ ' ' +---------+ ' + ' +---------+ ' ' +---------+ ' + ' | sg1 | ' ' | sg2 | ' + ' +---------+ ' ' +---------+ ' + + - - - - - - + +- - - - - - -+ + ^ ^ + | | + | | + v v + +---------+ +---------+ + | cbl1 | | cbl2 | + +---------+ +---------+ + */ + description: "2x CBL<->SG<->CBS XDCR only 1.3", + peers: map[string]PeerOptions{ + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + "cbs2": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID2}, + "sg1": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID1}, + "sg2": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID2}, + "cbl1": {Type: PeerTypeCouchbaseLite}, + // TODO: CBG-4270, push replication only exists empemerally + "cbl2": {Type: PeerTypeCouchbaseLite}, + }, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbl2", + passivePeer: "sg2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl2", + passivePeer: "sg2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + }, + }, + // topology 1.4 not present, no P2P supported yet + /* + { + Test topology 1.5 + + + - - - - - - + +- - - - - - -+ + ' cluster A ' ' cluster B ' + ' +---------+ ' ' +---------+ ' + ' | cbs1 | ' <--> ' | cbs2 | ' + ' +---------+ ' ' +---------+ ' + ' +---------+ ' ' +---------+ ' + ' | sg1 | ' ' | sg2 | ' + ' +---------+ ' ' +---------+ ' + + - - - - - - + +- - - - - - -+ + ^ ^ + | | + | | + | | + | +------+ | + +---> | cbl1 | <---+ + +------+ + */ + /* This test doesn't work yet, CouchbaseLiteMockPeer doesn't support writing data to multiple Sync Gateway peers yet + description: "Sync Gateway -> Couchbase Server -> Couchbase Server", + peers: map[string]PeerOptions{ + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + "cbs2": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID2}, + "sg1": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID1}, + "sg2": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID2}, + "cbl1": {Type: PeerTypeCouchbaseLite}, + }, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbs2", + passivePeer: "cbs1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg1", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + { + activePeer: "cbl1", + passivePeer: "sg2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + }, + }, + */ +} + +// simpleTopologies represents simplified topologies to make testing the integration test code easier. +// nolint: unused +var simpleTopologies = []Topology{ + { + + /* + +------+ +------+ + | cbs1 | <--> | cbs2 | + +------+ +------+ + */ + description: "CBS<->CBS", + peers: map[string]PeerOptions{ + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + "cbs2": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID2}}, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbs1", + passivePeer: "cbs2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbs1", + passivePeer: "cbs2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + }, + }, + { + /* + + - - - - - - + +- - - - - - -+ + ' cluster A ' ' cluster B ' + ' +---------+ ' ' +---------+ ' + ' | cbs1 | ' <--> ' | cbs2 | ' + ' +---------+ ' ' +---------+ ' + ' +---------+ ' + - - - - - - + + ' | sg1 | ' + ' +---------+ ' + + - - - - - - + + */ + description: "CBS+SG<->CBS", + peers: map[string]PeerOptions{ + "cbs1": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID1}, + "sg1": {Type: PeerTypeSyncGateway, BucketID: PeerBucketID1}, + "cbs2": {Type: PeerTypeCouchbaseServer, BucketID: PeerBucketID2}, + }, + replications: []PeerReplicationDefinition{ + { + activePeer: "cbs1", + passivePeer: "cbs2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPush, + }, + }, + { + activePeer: "cbs1", + passivePeer: "cbs2", + config: PeerReplicationConfig{ + direction: PeerReplicationDirectionPull, + }, + }, + }, + }, +} diff --git a/topologytest/version_test.go b/topologytest/version_test.go new file mode 100644 index 0000000000..0ae24fa70a --- /dev/null +++ b/topologytest/version_test.go @@ -0,0 +1,66 @@ +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package topologytest + +import ( + "fmt" + "testing" + + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/require" +) + +// DocMetadata is a struct that contains metadata about a document. It contains the relevant information for testing versions of documents, as well as debugging information. +type DocMetadata struct { + DocID string // DocID is the document ID + RevTreeID string // RevTreeID is the rev treee ID of a document, may be empty not present + HLV *db.HybridLogicalVector // HLV is the hybrid logical vector of the document, may not be present + Mou *db.MetadataOnlyUpdate // Mou is the metadata only update of the document, may not be present + Cas uint64 // Cas is the cas value of the document + ImplicitHLV *db.HybridLogicalVector // ImplicitHLV is the version of the document, if there was no HLV +} + +// CV returns the current version of the document. +func (v DocMetadata) CV(t require.TestingT) db.Version { + if v.ImplicitHLV != nil { + return *v.ImplicitHLV.ExtractCurrentVersionFromHLV() + } else if v.HLV != nil { + return *v.HLV.ExtractCurrentVersionFromHLV() + } + require.FailNow(t, "no hlv available %#v", v) + return db.Version{} +} + +// DocMetadataFromDocument returns a DocVersion from the given document. +func DocMetadataFromDocument(doc *db.Document) DocMetadata { + return DocMetadata{ + DocID: doc.ID, + RevTreeID: doc.CurrentRev, + Mou: doc.MetadataOnlyUpdate, + Cas: doc.Cas, + HLV: doc.HLV, + } +} + +func (v DocMetadata) GoString() string { + return fmt.Sprintf("DocMetadata{\nDocID:%s\n\tRevTreeID:%s\n\tHLV:%+v\n\tMou:%+v\n\tCas:%d\n\tImplicitHLV:%+v\n}", v.DocID, v.RevTreeID, v.HLV, v.Mou, v.Cas, v.ImplicitHLV) +} + +// DocMetadataFromDocVersion returns metadata DocVersion from the given document and version. +func DocMetadataFromDocVersion(t testing.TB, docID string, version rest.DocVersion) DocMetadata { + // FIXME: CBG-4257, this should read the existing HLV on doc, until this happens, pv is always missing + hlv := db.NewHybridLogicalVector() + require.NoError(t, hlv.AddVersion(version.CV)) + return DocMetadata{ + DocID: docID, + RevTreeID: version.RevTreeID, + ImplicitHLV: hlv, + } +} diff --git a/xdcr/cbs_xdcr.go b/xdcr/cbs_xdcr.go index 02d620a72f..e09c85cd9a 100644 --- a/xdcr/cbs_xdcr.go +++ b/xdcr/cbs_xdcr.go @@ -10,6 +10,7 @@ package xdcr import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -20,12 +21,15 @@ import ( ) const ( - cbsRemoteClustersEndpoint = "/pools/default/remoteClusters" - xdcrClusterName = "sync_gateway_xdcr" - totalDocsFilteredStat = "xdcr_docs_filtered_total" - totalDocsWrittenStat = "xdcr_docs_written_total" + cbsRemoteClustersEndpoint = "/pools/default/remoteClusters" + xdcrClusterName = "sync_gateway_xdcr" // this is a hardcoded name for the local XDCR cluster + totalMobileDocsFiltered = "xdcr_mobile_docs_filtered_total" + totalDocsWrittenStat = "xdcr_docs_written_total" + totalDocsConflictResolutionRejected = "xdcr_docs_failed_cr_source_total" ) +var errNoXDCRMetrics = errors.New("No metric found") + // couchbaseServerManager implements a XDCR setup cluster on Couchbase Server. type couchbaseServerManager struct { fromBucket *base.GocbV2Bucket @@ -62,22 +66,7 @@ func isClusterPresent(ctx context.Context, bucket *base.GocbV2Bucket) (bool, err return false, nil } -// deleteCluster deletes an XDCR cluster. The cluster must be present in order to delete it. -func deleteCluster(ctx context.Context, bucket *base.GocbV2Bucket) error { - method := http.MethodDelete - url := "/pools/default/remoteClusters/" + xdcrClusterName - output, statusCode, err := bucket.MgmtRequest(ctx, method, url, "application/x-www-form-urlencoded", nil) - if err != nil { - return err - } - - if statusCode != http.StatusOK { - return fmt.Errorf("Could not delete xdcr cluster: %s. %s %s -> (%d) %s", xdcrClusterName, http.MethodDelete, method, statusCode, output) - } - return nil -} - -// createCluster deletes an XDCR cluster. The cluster must be present in order to delete it. +// createCluster creates an XDCR cluster. func createCluster(ctx context.Context, bucket *base.GocbV2Bucket) error { serverURL, err := url.Parse(base.UnitTestUrl()) if err != nil { @@ -105,21 +94,18 @@ func createCluster(ctx context.Context, bucket *base.GocbV2Bucket) error { // newCouchbaseServerManager creates an instance of XDCR backed by Couchbase Server. This is not started until Start is called. func newCouchbaseServerManager(ctx context.Context, fromBucket *base.GocbV2Bucket, toBucket *base.GocbV2Bucket, opts XDCROptions) (*couchbaseServerManager, error) { + // there needs to be a global cluster present, this is a hostname + username + password. There can be only one per hostname, so create it lazily. isPresent, err := isClusterPresent(ctx, fromBucket) if err != nil { return nil, err } - if isPresent { - err := deleteCluster(ctx, fromBucket) + if !isPresent { + err := createCluster(ctx, fromBucket) if err != nil { return nil, err } } - err = createCluster(ctx, fromBucket) - if err != nil { - return nil, err - } return &couchbaseServerManager{ fromBucket: fromBucket, toBucket: toBucket, @@ -130,9 +116,12 @@ func newCouchbaseServerManager(ctx context.Context, fromBucket *base.GocbV2Bucke // Start starts the XDCR replication. func (x *couchbaseServerManager) Start(ctx context.Context) error { + if x.replicationID != "" { + return ErrReplicationAlreadyRunning + } method := http.MethodPost body := url.Values{} - body.Add("name", xdcrClusterName) + body.Add("name", fmt.Sprintf("%s_%s", x.fromBucket.GetName(), x.toBucket.GetName())) body.Add("fromBucket", x.fromBucket.GetName()) body.Add("toBucket", x.toBucket.GetName()) body.Add("toCluster", xdcrClusterName) @@ -149,7 +138,7 @@ func (x *couchbaseServerManager) Start(ctx context.Context) error { return err } if statusCode != http.StatusOK { - return fmt.Errorf("Could not create xdcr cluster: %s. %s %s -> (%d) %s", xdcrClusterName, method, url, statusCode, output) + return fmt.Errorf("Could not create xdcr replication: %s. %s %s -> (%d) %s", xdcrClusterName, method, url, statusCode, output) } type replicationOutput struct { ID string `json:"id"` @@ -168,11 +157,15 @@ func (x *couchbaseServerManager) Start(ctx context.Context) error { // Stop starts the XDCR replication and deletes the replication from Couchbase Server. func (x *couchbaseServerManager) Stop(ctx context.Context) error { + // replication is not started + if x.replicationID == "" { + return ErrReplicationNotRunning + } method := http.MethodDelete url := "/controller/cancelXDCR/" + url.PathEscape(x.replicationID) output, statusCode, err := x.fromBucket.MgmtRequest(ctx, method, url, "application/x-www-form-urlencoded", nil) if err != nil { - return err + return fmt.Errorf("Could not %s to %s: %w", method, url, err) } if statusCode != http.StatusOK { return fmt.Errorf("Could not cancel XDCR replication: %s. %s %s -> (%d) %s", x.replicationID, method, url, statusCode, output) @@ -188,15 +181,27 @@ func (x *couchbaseServerManager) Stats(ctx context.Context) (*Stats, error) { return nil, err } stats := &Stats{} - stats.DocsFiltered, err = x.getValue(mf[totalDocsFilteredStat]) - if err != nil { - return stats, err - } - stats.DocsWritten, err = x.getValue(mf[totalDocsWrittenStat]) - if err != nil { - return stats, err + + statMap := map[string]*uint64{ + totalMobileDocsFiltered: &stats.MobileDocsFiltered, + totalDocsWrittenStat: &stats.DocsWritten, + totalDocsConflictResolutionRejected: &stats.TargetNewerDocs, + } + var errs *base.MultiError + for metricName, stat := range statMap { + metricFamily, ok := mf[metricName] + if !ok { + errs = errs.Append(fmt.Errorf("Could not find %s metric: %+v", metricName, mf)) + continue + } + var err error + *stat, err = x.getValue(metricFamily) + if err != nil { + errs = errs.Append(err) + } } - return stats, nil + stats.DocsProcessed = stats.DocsWritten + stats.MobileDocsFiltered + stats.TargetNewerDocs + return stats, errs.ErrorOrNil() } func (x *couchbaseServerManager) getValue(metrics *dto.MetricFamily) (uint64, error) { @@ -222,7 +227,7 @@ outer: return 0, fmt.Errorf("Do not have a relevant type for %v", metrics.Type) } } - return 0, fmt.Errorf("Could not find relevant value for metrics %v", metrics) + return 0, errNoXDCRMetrics } var _ Manager = &couchbaseServerManager{} diff --git a/xdcr/replication.go b/xdcr/replication.go index 2001ce0bae..3f1c4bcc21 100644 --- a/xdcr/replication.go +++ b/xdcr/replication.go @@ -14,9 +14,13 @@ import ( "fmt" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" "github.com/couchbaselabs/rosmar" ) +var ErrReplicationNotRunning = fmt.Errorf("Replication is not running") +var ErrReplicationAlreadyRunning = fmt.Errorf("Replication is already running") + // Manager represents a bucket to bucket replication. type Manager interface { // Start starts the replication. @@ -75,3 +79,16 @@ func NewXDCR(ctx context.Context, fromBucket, toBucket base.Bucket, opts XDCROpt } return newCouchbaseServerManager(ctx, gocbFromBucket, gocbToBucket, opts) } + +// GetSourceID returns the source ID for a bucket. +func GetSourceID(ctx context.Context, bucket base.Bucket) (string, error) { + serverUUID, err := db.GetServerUUID(ctx, bucket) + if err != nil { + return "", err + } + bucketUUID, err := bucket.UUID() + if err != nil { + return "", err + } + return db.CreateEncodedSourceID(bucketUUID, serverUUID) +} diff --git a/xdcr/rosmar_xdcr.go b/xdcr/rosmar_xdcr.go index 6daa60d158..1cbb46d58c 100644 --- a/xdcr/rosmar_xdcr.go +++ b/xdcr/rosmar_xdcr.go @@ -14,91 +14,152 @@ import ( "errors" "fmt" "strings" + "sync" "sync/atomic" + "golang.org/x/exp/maps" + sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" "github.com/couchbaselabs/rosmar" ) +// replicatedDocLocation represents whether a document is from the source or target bucket. +type replicatedDocLocation uint8 + +const ( + sourceDoc replicatedDocLocation = iota + targetDoc +) + +func (r replicatedDocLocation) String() string { + switch r { + case sourceDoc: + return "source" + case targetDoc: + return "target" + default: + return "unknown" + } +} + // rosmarManager implements a XDCR bucket to bucket replication within rosmar. type rosmarManager struct { filterFunc xdcrFilterFunc terminator chan bool + collectionsLock sync.RWMutex + fromBucketKeyspaces map[uint32]string toBucketCollections map[uint32]*rosmar.Collection fromBucket *rosmar.Bucket + fromBucketSourceID string toBucket *rosmar.Bucket replicationID string - docsFiltered atomic.Uint64 + mobileDocsFiltered atomic.Uint64 docsWritten atomic.Uint64 errorCount atomic.Uint64 targetNewerDocs atomic.Uint64 } // newRosmarManager creates an instance of XDCR backed by rosmar. This is not started until Start is called. -func newRosmarManager(_ context.Context, fromBucket, toBucket *rosmar.Bucket, opts XDCROptions) (Manager, error) { +func newRosmarManager(ctx context.Context, fromBucket, toBucket *rosmar.Bucket, opts XDCROptions) (Manager, error) { if opts.Mobile != MobileOn { return nil, errors.New("Only sgbucket.XDCRMobileOn is supported in rosmar") } + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + if err != nil { + return nil, fmt.Errorf("Could not get source ID for %s: %w", fromBucket.GetName(), err) + } return &rosmarManager{ fromBucket: fromBucket, + fromBucketSourceID: fromBucketSourceID, toBucket: toBucket, replicationID: fmt.Sprintf("%s-%s", fromBucket.GetName(), toBucket.GetName()), toBucketCollections: make(map[uint32]*rosmar.Collection), - terminator: make(chan bool), + fromBucketKeyspaces: make(map[uint32]string), filterFunc: mobileXDCRFilter, }, nil } -// processEvent processes a DCP event coming from a toBucket and replicates it to the target datastore. +// processEvent processes a DCP event coming from a source bucket and replicates it to the target datastore. func (r *rosmarManager) processEvent(ctx context.Context, event sgbucket.FeedEvent) bool { docID := string(event.Key) - base.TracefCtx(ctx, base.KeyWalrus, "Got event %s, opcode: %s", docID, event.Opcode) + base.TracefCtx(ctx, base.KeyVV, "Got event %s, opcode: %s", docID, event.Opcode) + r.collectionsLock.RLock() + defer r.collectionsLock.RUnlock() col, ok := r.toBucketCollections[event.CollectionID] if !ok { base.ErrorfCtx(ctx, "This violates the assumption that all collections are mapped to a target collection. This should not happen. Found event=%+v", event) r.errorCount.Add(1) return false } + ctx = base.CorrelationIDLogCtx(context.Background(), fmt.Sprintf("%s->%s\n\t", r.fromBucketKeyspaces[event.CollectionID], col.GetName())) // use context.Background() to drop test information since context is too long switch event.Opcode { case sgbucket.FeedOpDeletion, sgbucket.FeedOpMutation: // Filter out events if we have a non XDCR filter if r.filterFunc != nil && !r.filterFunc(&event) { - base.TracefCtx(ctx, base.KeyWalrus, "Filtering doc %s", docID) - r.docsFiltered.Add(1) + base.TracefCtx(ctx, base.KeyVV, "Filtering doc %s", docID) + r.mobileDocsFiltered.Add(1) return true } - toCas, err := col.Get(docID, nil) + // Have to use GetWithXattrs to get a cas value back if there are no xattrs (GetWithXattrs will not return a cas if there are no xattrs) + _, targetXattrs, actualTargetCas, err := col.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName, base.SyncXattrName}) if err != nil && !base.IsDocNotFoundError(err) { base.WarnfCtx(ctx, "Skipping replicating doc %s, could not perform a kv op get doc in toBucket: %s", event.Key, err) r.errorCount.Add(1) return false } - /* full LWW conflict resolution is not implemented in rosmar yet + sourceHLV, sourceMou, nonMobileXattrs, body, err := processDCPEvent(&event) + if err != nil { + base.WarnfCtx(ctx, "Replicating doc %s, could not get body, hlv, and mou: %s", event.Key, err) + r.errorCount.Add(1) + return false + } - CBS algorithm is: + actualSourceCas := event.Cas + conflictResolutionSourceCas := getConflictResolutionCas(ctx, docID, sourceDoc, actualSourceCas, sourceHLV, sourceMou) - if (command.CAS > document.CAS) - command succeeds - else if (command.CAS == document.CAS) - // Check the RevSeqno - if (command.RevSeqno > document.RevSeqno) - command succeeds - else if (command.RevSeqno == document.RevSeqno) - // Check the expiry time - if (command.Expiry > document.Expiry) - command succeeds - else if (command.Expiry == document.Expiry) - // Finally check flags - if (command.Flags < document.Flags) - command succeeds + targetHLV, targetMou, err := getHLVAndMou(targetXattrs) + if err != nil { + base.WarnfCtx(ctx, "Replicating doc %s, could not get target hlv and mou: %s", event.Key, err) + r.errorCount.Add(1) + return false + } + conflictResolutionTargetCas := getConflictResolutionCas(ctx, docID, targetDoc, actualTargetCas, targetHLV, targetMou) + /* full LWW conflict resolution is implemented in rosmar. There is no need to implement this since CAS will always be unique due to rosmar limitations. - command fails + CBS algorithm is, return true when a document should be copied: + + if source.CAS > target.CAS { + return true + } else if source.CAS < target.CAS { + return false + } + // Check the RevSeqno + if source.RevSeqno > target.RevSeqno { + return true + } else if source.RevSeqno < target.RevSeqno { + return false + } + // Check the expiry time + if source.Expiry > target.Expiry { + return true + } else if source.Expiry < target.Expiry { + return false + } + // Check flags + if source.Flags > target.Flags { + return true + } else if source.Flags < target.Flags { + return false + } + // Check xattrs + return source_has_xattrs && !target_has_xattrs In the current state of rosmar: @@ -106,16 +167,36 @@ func (r *rosmarManager) processEvent(ctx context.Context, event sgbucket.FeedEve 2. RevSeqno is not implemented 3. Expiry is implemented and could be compared except all CAS values are unique. 4. Flags are not implemented + 5. Presence of xattrs on the source and not the target. (CBG-4334 is not implemented.) */ - if event.Cas <= toCas { + if conflictResolutionSourceCas <= conflictResolutionTargetCas { + base.InfofCtx(ctx, base.KeyVV, "XDCR doc:%s skipping replication since sourceCas (%d) < targetCas (%d)", docID, conflictResolutionSourceCas, conflictResolutionTargetCas) r.targetNewerDocs.Add(1) - base.TracefCtx(ctx, base.KeyWalrus, "Skipping replicating doc %s, cas %d <= %d", docID, event.Cas, toCas) return true + } /* else if sourceCas == targetCas { + // CBG-4334, check datatype for targetXattrs to see if there are any xattrs present + hasSourceXattrs := event.DataType&sgbucket.FeedDataTypeXattr != 0 + hasTargetXattrs := len(targetXattrs) > 0 + if !(hasSourceXattrs && !hasTargetXattrs) { + base.InfofCtx(ctx, base.KeyVV, "skipping %q skipping replication since sourceCas (%d) < targetCas (%d)", docID, sourceCas, targetCas) + return true + } } - - err = opWithMeta(ctx, col, toCas, event) + */ + newXattrs := nonMobileXattrs + if targetSyncXattr, ok := targetXattrs[base.SyncXattrName]; ok { + newXattrs[base.SyncXattrName] = targetSyncXattr + } + err = updateHLV(newXattrs, sourceHLV, sourceMou, r.fromBucketSourceID, actualSourceCas) + if err != nil { + base.WarnfCtx(ctx, "Replicating doc %s, could not update hlv: %s", event.Key, err) + r.errorCount.Add(1) + return false + } + base.InfofCtx(ctx, base.KeyVV, "Replicating doc %q, with cas (%d), body %s, xattrsKeys: %+v", event.Key, actualSourceCas, string(body), maps.Keys(newXattrs)) + err = opWithMeta(ctx, col, actualTargetCas, newXattrs, body, &event) if err != nil { base.WarnfCtx(ctx, "Replicating doc %s, could not write doc: %s", event.Key, err) r.errorCount.Add(1) @@ -131,6 +212,12 @@ func (r *rosmarManager) processEvent(ctx context.Context, event sgbucket.FeedEve // Start starts the replication for all existing replications. Errors if there aren't corresponding named collections on each bucket. func (r *rosmarManager) Start(ctx context.Context) error { + if r.terminator != nil { + return ErrReplicationAlreadyRunning + } + r.collectionsLock.Lock() + defer r.collectionsLock.Unlock() + r.terminator = make(chan bool) // set up replication to target all existing collections, and map to other collections scopes := make(map[string][]string) fromDataStores, err := r.fromBucket.ListDataStores() @@ -160,6 +247,7 @@ func (r *rosmarManager) Start(ctx context.Context) error { return fmt.Errorf("DataStore %s is not of rosmar.Collection: %T", toDataStore, toDataStore) } r.toBucketCollections[collectionID] = col + r.fromBucketKeyspaces[collectionID] = fromDataStore.GetName() scopes[fromName.ScopeName()] = append(scopes[fromName.ScopeName()], fromName.CollectionName()) break } @@ -167,7 +255,6 @@ func (r *rosmarManager) Start(ctx context.Context) error { args := sgbucket.FeedArguments{ ID: "xdcr-" + r.replicationID, - Backfill: sgbucket.FeedNoBackfill, Terminator: r.terminator, Scopes: scopes, } @@ -181,47 +268,39 @@ func (r *rosmarManager) Start(ctx context.Context) error { // Stop terminates the replication. func (r *rosmarManager) Stop(_ context.Context) error { + if r.terminator == nil { + return ErrReplicationNotRunning + } close(r.terminator) r.terminator = nil return nil } -// opWithMeta writes a document to the target datastore given a type of Deletion or Mutation event with a specific cas. -func opWithMeta(ctx context.Context, collection *rosmar.Collection, originalCas uint64, event sgbucket.FeedEvent) error { - var xattrs []byte - var body []byte - if event.DataType&sgbucket.FeedDataTypeXattr != 0 { - var err error - var dcpXattrs map[string][]byte - body, dcpXattrs, err = sgbucket.DecodeValueWithAllXattrs(event.Value) - if err != nil { - return err - } - xattrs, err = xattrToBytes(dcpXattrs) - if err != nil { - return err - } - } else { - body = event.Value +// opWithMeta writes a document to the target datastore given a type of Deletion or Mutation event with a specific cas, xattrs, and body. +func opWithMeta(ctx context.Context, collection *rosmar.Collection, originalCas uint64, xattrs map[string][]byte, body []byte, event *sgbucket.FeedEvent) error { + xattrBytes, err := xattrToBytes(xattrs) + if err != nil { + return err } if event.Opcode == sgbucket.FeedOpDeletion { - return collection.DeleteWithMeta(ctx, string(event.Key), originalCas, event.Cas, event.Expiry, xattrs) + return collection.DeleteWithMeta(ctx, string(event.Key), originalCas, event.Cas, event.Expiry, xattrBytes) } - return collection.SetWithMeta(ctx, string(event.Key), originalCas, event.Cas, event.Expiry, xattrs, body, event.DataType) + return collection.SetWithMeta(ctx, string(event.Key), originalCas, event.Cas, event.Expiry, xattrBytes, body, event.DataType) } // Stats returns the stats of the XDCR replication. func (r *rosmarManager) Stats(context.Context) (*Stats, error) { - - return &Stats{ - DocsWritten: r.docsWritten.Load(), - DocsFiltered: r.docsFiltered.Load(), - ErrorCount: r.errorCount.Load(), - TargetNewerDocs: r.targetNewerDocs.Load(), - }, nil + stats := &Stats{ + DocsWritten: r.docsWritten.Load(), + MobileDocsFiltered: r.mobileDocsFiltered.Load(), + ErrorCount: r.errorCount.Load(), + TargetNewerDocs: r.targetNewerDocs.Load(), + } + stats.DocsProcessed = stats.DocsWritten + stats.MobileDocsFiltered + stats.TargetNewerDocs + return stats, nil } // xattrToBytes converts a map of xattrs of marshalled json. @@ -240,3 +319,106 @@ type xdcrFilterFunc func(event *sgbucket.FeedEvent) bool func mobileXDCRFilter(event *sgbucket.FeedEvent) bool { return !(strings.HasPrefix(string(event.Key), base.SyncDocPrefix) && !strings.HasPrefix(string(event.Key), base.Att2Prefix)) } + +// processDCPEvent gets the body, non mobile, xattrs, vv, and mou from the event. +func processDCPEvent(event *sgbucket.FeedEvent) (*db.HybridLogicalVector, *db.MetadataOnlyUpdate, map[string][]byte, []byte, error) { + if event.DataType&sgbucket.FeedDataTypeXattr == 0 { + xattrs := make(map[string][]byte) + return nil, nil, xattrs, event.Value, nil + } + body, xattrs, err := sgbucket.DecodeValueWithAllXattrs(event.Value) + if err != nil { + return nil, nil, nil, nil, err + } + if xattrs == nil { + xattrs = make(map[string][]byte) + } + hlv, mou, err := getHLVAndMou(xattrs) + if err != nil { + return nil, nil, nil, nil, err + } + for _, xattrName := range []string{base.VvXattrName, base.MouXattrName, base.SyncXattrName} { + delete(xattrs, xattrName) + } + return hlv, mou, xattrs, body, nil +} + +// getHLVAndMou gets the hlv and mou from the xattrs. +func getHLVAndMou(xattrs map[string][]byte) (*db.HybridLogicalVector, *db.MetadataOnlyUpdate, error) { + var hlv *db.HybridLogicalVector + if bytes, ok := xattrs[base.VvXattrName]; ok { + err := json.Unmarshal(bytes, &hlv) + if err != nil { + return nil, nil, fmt.Errorf("Could not unmarshal the vv xattr %s: %w", string(bytes), err) + } + } + var mou *db.MetadataOnlyUpdate + if bytes, ok := xattrs[base.MouXattrName]; ok { + err := json.Unmarshal(bytes, &mou) + if err != nil { + return nil, nil, fmt.Errorf("Could not unmarshal the mou xattr %s: %w", string(bytes), err) + } + } + return hlv, mou, nil +} + +// updateHLV will update the xattrs on the target document considering the source's HLV, _mou, sourceID and cas. +func updateHLV(xattrs map[string][]byte, sourceHLV *db.HybridLogicalVector, sourceMou *db.MetadataOnlyUpdate, sourceID string, sourceCas uint64) error { + + targetHLV := db.NewHybridLogicalVector() + if sourceHLV != nil { + targetHLV = sourceHLV + } + + // If source vv.cvCas == cas, the _vv.cv, _vv.cvCAS from the source already includes the latest mutation and we can use it directly. + // Otherwise we need to add the current mutation (sourceID, sourceCas) to the HLV before writing to the target + sourcecvCASMatch := sourceHLV != nil && sourceHLV.CurrentVersionCAS == sourceCas + sourceWasImport := sourceMou != nil && sourceMou.CAS() == sourceCas + if !(sourceWasImport || sourcecvCASMatch) { + err := targetHLV.AddVersion(db.Version{ + SourceID: sourceID, + Value: sourceCas, + }) + if err != nil { + return err + } + targetHLV.CurrentVersionCAS = sourceCas + } + var err error + xattrs[base.VvXattrName], err = json.Marshal(targetHLV) + if err != nil { + return err + } + if sourceMou != nil { + // removing _mou.cas and _mou.pRev matches cbs xdcr behavior. + // CBS xdcr maybe should clear _mou.pCas as well, but it is not a problem since all checks for _mou.cas should check current cas for _mou being up to date. + sourceMou.HexCAS = "" + sourceMou.PreviousRevSeqNo = 0 + var err error + xattrs[base.MouXattrName], err = json.Marshal(sourceMou) + if err != nil { + return err + } + } + return nil +} + +// getConflictResolutionCas returns cas for conflict resolution. +// If _mou.cas == actualCas, assume _vv is up to date and use _vv.cvCAS +// Otherwise, return actualCas +func getConflictResolutionCas(ctx context.Context, docID string, location replicatedDocLocation, actualCas uint64, hlv *db.HybridLogicalVector, mou *db.MetadataOnlyUpdate) uint64 { + if mou == nil { + return actualCas + } + // _mou.CAS is out of date, ignoring + if mou.CAS() != actualCas { + return actualCas + } + if hlv == nil { + base.InfofCtx(ctx, base.KeyVV, "XDCR doc:%s %s _mou.cas=cas (%d), but there is no HLV, using 0 for conflict resolution to match behavior of Couchbase Server", docID, location, actualCas) + return 0 + } + // _mou.CAS matches the CAS value, use the _vv.cvCAS for conflict resolution + base.InfofCtx(ctx, base.KeyVV, "XDCR doc:%s %s _mou.cas=cas (%d), using _vv.cvCAS (%d) for conflict resolution", docID, location, actualCas, hlv.CurrentVersionCAS) + return hlv.CurrentVersionCAS +} diff --git a/xdcr/stats.go b/xdcr/stats.go index ea5caa1e83..7363384427 100644 --- a/xdcr/stats.go +++ b/xdcr/stats.go @@ -10,10 +10,12 @@ package xdcr // Stats represents the stats of a replication. type Stats struct { - // DocsFiltered is the number of documents that have been filtered out and have not been replicated to the target cluster. - DocsFiltered uint64 + // MobileDocsFiltered is the number of documents that have been filtered out and have not been replicated to the target cluster. + MobileDocsFiltered uint64 // DocsWritten is the number of documents written to the destination cluster, since the start or resumption of the current replication. DocsWritten uint64 + // DocsProcessed is the number of documents that have been processed by the replication. + DocsProcessed uint64 // ErrorCount is the number of errors that have occurred during the replication. ErrorCount uint64 diff --git a/xdcr/xdcr_test.go b/xdcr/xdcr_test.go index b861937340..b47d4ab7cf 100644 --- a/xdcr/xdcr_test.go +++ b/xdcr/xdcr_test.go @@ -10,10 +10,15 @@ package xdcr import ( "fmt" + "slices" "testing" "time" + "golang.org/x/exp/maps" + + sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -40,84 +45,729 @@ func TestMobileXDCRNoSyncDataCopied(t *testing.T) { } xdcr, err := NewXDCR(ctx, fromBucket, toBucket, opts) require.NoError(t, err) - err = xdcr.Start(ctx) - require.NoError(t, err) + require.NoError(t, xdcr.Start(ctx)) + defer func() { - assert.NoError(t, xdcr.Stop(ctx)) + // stop XDCR, will already be stopped if test doesn't fail early + err := xdcr.Stop(ctx) + if err != nil { + assert.Equal(t, ErrReplicationNotRunning, err) + } }() + require.ErrorIs(t, xdcr.Start(ctx), ErrReplicationAlreadyRunning) const ( syncDoc = "_sync:doc1doc2" attachmentDoc = "_sync:att2:foo" normalDoc = "doc2" exp = 0 - body = `{"key":"value"}` - version = "ver" - source = "src" - curCAS = "cvCas" + startingBody = `{"key":"value"}` ) dataStores := map[base.DataStore]base.DataStore{ fromBucket.DefaultDataStore(): toBucket.DefaultDataStore(), } + var fromDs base.DataStore + var toDs base.DataStore if base.TestsUseNamedCollections() { - fromDs, err := fromBucket.GetNamedDataStore(0) + fromDs, err = fromBucket.GetNamedDataStore(0) require.NoError(t, err) - toDs, err := toBucket.GetNamedDataStore(0) + toDs, err = toBucket.GetNamedDataStore(0) require.NoError(t, err) dataStores[fromDs] = toDs + } else { + fromDs = fromBucket.DefaultDataStore() + toDs = toBucket.DefaultDataStore() + } + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + docCas := make(map[string]uint64) + for _, doc := range []string{syncDoc, attachmentDoc, normalDoc} { + var inputCas uint64 + var err error + docCas[doc], err = fromDs.WriteCas(doc, exp, inputCas, []byte(startingBody), 0) + require.NoError(t, err) + _, _, err = fromDs.GetXattrs(ctx, doc, []string{base.VvXattrName}) + // make sure that the doc does not have a version vector + base.RequireXattrNotFoundError(t, err) + } + + // make sure attachments are copied + for _, doc := range []string{normalDoc, attachmentDoc} { + var body []byte + var xattrs map[string][]byte + var cas uint64 + require.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + body, xattrs, cas, err = toDs.GetWithXattrs(ctx, doc, []string{base.VvXattrName, base.MouXattrName}) + assert.NoError(c, err, "Could not get doc %s", doc) + }, time.Second*5, time.Millisecond*100) + require.Equal(t, docCas[doc], cas) + require.JSONEq(t, startingBody, string(body)) + require.NotContains(t, xattrs, base.MouXattrName) + if !base.TestSupportsMobileXDCR() { + require.Len(t, xattrs, 0) + continue + } + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, cas) } + + _, err = toDs.Get(syncDoc, nil) + base.RequireDocNotFoundError(t, err) + var totalDocsWritten uint64 var totalDocsFiltered uint64 - for fromDs, toDs := range dataStores { - for _, doc := range []string{syncDoc, attachmentDoc, normalDoc} { - _, err = fromDs.Add(doc, exp, body) - require.NoError(t, err) - } - // make sure attachments are copied - for _, doc := range []string{normalDoc, attachmentDoc} { - require.EventuallyWithT(t, func(c *assert.CollectT) { - var value string - _, err = toDs.Get(doc, &value) - assert.NoError(c, err, "Could not get doc %s", doc) - assert.Equal(c, body, value) - }, time.Second*5, time.Millisecond*100) + // stats are not updated in real time, so we need to wait a bit + require.EventuallyWithT(t, func(c *assert.CollectT) { + stats, err := xdcr.Stats(ctx) + if !assert.NoError(c, err) { + assert.NoError(c, err) } + assert.Equal(c, totalDocsFiltered+1, stats.MobileDocsFiltered) + assert.Equal(c, totalDocsWritten+2, stats.DocsWritten) - var value any - _, err = toDs.Get(syncDoc, &value) - base.RequireDocNotFoundError(t, err) + }, time.Second*5, time.Millisecond*100) - // stats are not updated in real time, so we need to wait a bit - require.EventuallyWithT(t, func(c *assert.CollectT) { - stats, err := xdcr.Stats(ctx) - assert.NoError(t, err) - assert.Equal(c, totalDocsFiltered+1, stats.DocsFiltered) - assert.Equal(c, totalDocsWritten+2, stats.DocsWritten) + require.NoError(t, xdcr.Stop(ctx)) + require.ErrorIs(t, xdcr.Stop(ctx), ErrReplicationNotRunning) +} - }, time.Second*5, time.Millisecond*100) - totalDocsWritten += 2 - totalDocsFiltered++ - if base.UnitTestUrlIsWalrus() { - // TODO: CBG-3861 implement _vv support in rosmar - continue - } - // in mobile xdcr mode a version vector will be written - if base.TestSupportsMobileXDCR() { - // verify VV is written to docs that are replicated - for _, doc := range []string{normalDoc, attachmentDoc} { - require.EventuallyWithT(t, func(c *assert.CollectT) { - xattrs, _, err := toDs.GetXattrs(ctx, doc, []string{"_vv"}) - assert.NoError(c, err, "Could not get doc %s", doc) - vvXattrBytes, ok := xattrs["_vv"] - require.True(t, ok) - var vvXattrVal map[string]any - require.NoError(t, base.JSONUnmarshal(vvXattrBytes, &vvXattrVal)) - assert.NotNil(c, vvXattrVal[version]) - assert.NotNil(c, vvXattrVal[source]) - assert.NotNil(c, vvXattrVal[curCAS]) - - }, time.Second*5, time.Millisecond*100) +// getTwoBucketDataStores creates two data stores in separate buckets to run xdcr within. Returns a named collection or a default collection based on the global test configuration. +func getTwoBucketDataStores(t *testing.T) (base.Bucket, sgbucket.DataStore, base.Bucket, sgbucket.DataStore) { + ctx := base.TestCtx(t) + base.RequireNumTestBuckets(t, 2) + fromBucket := base.GetTestBucket(t) + t.Cleanup(func() { + fromBucket.Close(ctx) + }) + toBucket := base.GetTestBucket(t) + t.Cleanup(func() { + toBucket.Close(ctx) + }) + var fromDs base.DataStore + var toDs base.DataStore + if base.TestsUseNamedCollections() { + var err error + fromDs, err = fromBucket.GetNamedDataStore(0) + require.NoError(t, err) + toDs, err = toBucket.GetNamedDataStore(0) + require.NoError(t, err) + } else { + fromDs = fromBucket.DefaultDataStore() + toDs = toBucket.DefaultDataStore() + } + return fromBucket, fromDs, toBucket, toDs +} + +func TestReplicateVV(t *testing.T) { + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + + hlvAgent := db.NewHLVAgent(t, fromDs, "fakeHLVSourceID", base.VvXattrName) + + testCases := []struct { + name string + docID string + body string + HLV func(fromCas uint64) *db.HybridLogicalVector + hasHLV bool + preXDCRFunc func(t *testing.T, docID string) uint64 + }{ + { + name: "normal doc", + docID: "doc1", + body: `{"key":"value"}`, + HLV: func(fromCas uint64) *db.HybridLogicalVector { + return &db.HybridLogicalVector{ + CurrentVersionCAS: fromCas, + SourceID: fromBucketSourceID, + Version: fromCas, + } + }, + hasHLV: true, + preXDCRFunc: func(t *testing.T, docID string) uint64 { + cas, err := fromDs.WriteCas(docID, 0, 0, []byte(`{"key":"value"}`), 0) + require.NoError(t, err) + return cas + }, + }, + { + name: "dest doc older, expect overwrite", + docID: "doc2", + body: `{"datastore":"fromDs"}`, + HLV: func(fromCas uint64) *db.HybridLogicalVector { + return &db.HybridLogicalVector{ + CurrentVersionCAS: fromCas, + SourceID: fromBucketSourceID, + Version: fromCas, + } + }, + hasHLV: true, + preXDCRFunc: func(t *testing.T, docID string) uint64 { + _, err := toDs.WriteCas(docID, 0, 0, []byte(`{"datastore":"toDs"}`), 0) + require.NoError(t, err) + cas, err := fromDs.WriteCas(docID, 0, 0, []byte(`{"datastore":"fromDs"}`), 0) + require.NoError(t, err) + return cas + }, + }, + { + name: "dest doc newer, expect keep same dest doc", + docID: "doc3", + body: `{"datastore":"toDs"}`, + hasHLV: false, + preXDCRFunc: func(t *testing.T, docID string) uint64 { + _, err := fromDs.WriteCas(docID, 0, 0, []byte(`{"datastore":"fromDs"}`), 0) + require.NoError(t, err) + cas, err := toDs.WriteCas(docID, 0, 0, []byte(`{"datastore":"toDs"}`), 0) + require.NoError(t, err) + return cas + }, + }, + { + name: "src doc has hlv", + docID: "doc4", + body: hlvAgent.GetHelperBody(), + HLV: func(fromCas uint64) *db.HybridLogicalVector { + return &db.HybridLogicalVector{ + CurrentVersionCAS: fromCas, + SourceID: hlvAgent.SourceID(), + Version: fromCas, + } + }, + hasHLV: true, + preXDCRFunc: func(t *testing.T, docID string) uint64 { + ctx := base.TestCtx(t) + return hlvAgent.InsertWithHLV(ctx, docID) + }, + }, + } + // tests write a document + // start xdcr + // verify result + + var totalDocsProcessed uint64 // totalDocsProcessed will be incremented in each subtest + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + fromCAS := testCase.preXDCRFunc(t, testCase.docID) + + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + stats, err := xdcr.Stats(ctx) + assert.NoError(t, err) + totalDocsProcessed = stats.DocsProcessed + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1+totalDocsProcessed) + + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, testCase.docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err, "Could not get doc %s", testCase.docID) + require.Equal(t, fromCAS, destCas) + require.JSONEq(t, testCase.body, string(body)) + require.NotContains(t, xattrs, base.MouXattrName) + if !testCase.hasHLV { + require.NotContains(t, xattrs, base.VvXattrName) + return } - } + require.Contains(t, xattrs, base.VvXattrName) + + var hlv db.HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &hlv)) + require.Equal(t, *testCase.HLV(fromCAS), hlv) + }) } } + +func TestVVWriteTwice(t *testing.T) { + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + + docID := "doc1" + ver1Body := `{"ver":1}` + fromCAS, err := fromDs.WriteCas(docID, 0, 0, []byte(ver1Body), 0) + require.NoError(t, err) + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1) + + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCAS, destCas) + require.JSONEq(t, ver1Body, string(body)) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) + + fromCAS2, err := fromDs.WriteCas(docID, 0, fromCAS, []byte(`{"ver":2}`), 0) + require.NoError(t, err) + requireWaitForXDCRDocsProcessed(t, xdcr, 2) + + body, xattrs, destCas, err = toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCAS2, destCas) + require.JSONEq(t, `{"ver":2}`, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS2) +} + +func TestVVObeyMou(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeySGTest) + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + + docID := "doc1" + hlvAgent := db.NewHLVAgent(t, fromDs, fromBucketSourceID, base.VvXattrName) + fromCas1 := hlvAgent.InsertWithHLV(ctx, "doc1") + + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1) + + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName, base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + require.Equal(t, fromCas1, destCas) + require.JSONEq(t, hlvAgent.GetHelperBody(), string(body)) + require.NotContains(t, xattrs, base.MouXattrName) + require.Contains(t, xattrs, base.VvXattrName) + var vv db.HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &vv)) + expectedVV := db.HybridLogicalVector{ + CurrentVersionCAS: fromCas1, + SourceID: hlvAgent.SourceID(), + Version: fromCas1, + } + + require.Equal(t, expectedVV, vv) + + stats, err := xdcr.Stats(ctx) + require.NoError(t, err) + require.Equal(t, Stats{ + DocsWritten: 1, + DocsProcessed: 1, + }, *stats) + + mou := &db.MetadataOnlyUpdate{ + PreviousHexCAS: base.CasToString(fromCas1), + PreviousRevSeqNo: db.RetrieveDocRevSeqNo(t, xattrs[base.VirtualXattrRevSeqNo]), + } + + opts := &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec(db.XattrMouCasPath(), sgbucket.MacroCas), + }, + } + const userXattrKey = "extra_xattr" + fromCas2, err := fromDs.UpdateXattrs(ctx, docID, 0, fromCas1, map[string][]byte{ + base.MouXattrName: base.MustJSONMarshal(t, mou), + userXattrKey: []byte(`{"key":"value"}`), + }, opts) + require.NoError(t, err) + require.NotEqual(t, fromCas1, fromCas2) + + requireWaitForXDCRDocsProcessed(t, xdcr, 2) + stats, err = xdcr.Stats(ctx) + require.NoError(t, err) + require.Equal(t, Stats{ + TargetNewerDocs: 1, + DocsWritten: 1, + DocsProcessed: 2, + }, *stats) + + body, xattrs, destCas, err = toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName, userXattrKey}) + require.NoError(t, err) + require.Equal(t, fromCas1, destCas) + require.JSONEq(t, hlvAgent.GetHelperBody(), string(body)) + require.NotContains(t, xattrs, base.MouXattrName) + require.Contains(t, xattrs, base.VvXattrName) + require.NotContains(t, xattrs, userXattrKey) + vv = db.HybridLogicalVector{} + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &vv)) + require.Equal(t, expectedVV, vv) +} + +func TestVVMouImport(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeySGTest) + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + + docID := "doc1" + ver1Body := `{"ver":1}` + fromCas1, err := fromDs.WriteWithXattrs(ctx, docID, 0, 0, []byte(ver1Body), map[string][]byte{"ver1": []byte(`{}`)}, nil, + &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec("ver1.cas", sgbucket.MacroCas), + }, + }) + require.NoError(t, err) + + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1) + + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName, base.VirtualXattrRevSeqNo}) + require.NoError(t, err) + require.Equal(t, fromCas1, destCas) + require.JSONEq(t, ver1Body, string(body)) + require.NotContains(t, xattrs, base.MouXattrName) + require.Contains(t, xattrs, base.VvXattrName) + var vv db.HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &vv)) + expectedVV := db.HybridLogicalVector{ + CurrentVersionCAS: fromCas1, + SourceID: fromBucketSourceID, + Version: fromCas1, + } + + require.Equal(t, expectedVV, vv) + + stats, err := xdcr.Stats(ctx) + require.NoError(t, err) + require.Equal(t, Stats{ + DocsWritten: 1, + DocsProcessed: 1, + }, *stats) + + mou := &db.MetadataOnlyUpdate{ + HexCAS: "expand", + PreviousHexCAS: base.CasToString(fromCas1), + PreviousRevSeqNo: db.RetrieveDocRevSeqNo(t, xattrs[base.VirtualXattrRevSeqNo]), + } + + opts := &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec(db.XattrMouCasPath(), sgbucket.MacroCas), + sgbucket.NewMacroExpansionSpec("ver2.cas", sgbucket.MacroCas)}, + } + fromCas2, err := fromDs.UpdateXattrs(ctx, docID, 0, fromCas1, map[string][]byte{ + base.MouXattrName: base.MustJSONMarshal(t, mou), + "ver2": []byte(`{}`), + }, opts) + require.NoError(t, err) + require.NotEqual(t, fromCas1, fromCas2) + + requireWaitForXDCRDocsProcessed(t, xdcr, 2) + stats, err = xdcr.Stats(ctx) + require.NoError(t, err) + require.Equal(t, Stats{ + TargetNewerDocs: 1, + DocsWritten: 1, + DocsProcessed: 2, + }, *stats) + + ver3Body := `{"ver":3}` + fromCas3, err := fromDs.WriteWithXattrs(ctx, docID, 0, fromCas2, []byte(ver3Body), map[string][]byte{"ver3": []byte(`{}`)}, nil, + &sgbucket.MutateInOptions{ + MacroExpansion: []sgbucket.MacroExpansionSpec{ + sgbucket.NewMacroExpansionSpec("ver3.cas", sgbucket.MacroCas), + }, + }) + require.NoError(t, err) + requireWaitForXDCRDocsProcessed(t, xdcr, 3) + + stats, err = xdcr.Stats(ctx) + require.NoError(t, err) + require.Equal(t, Stats{ + TargetNewerDocs: 1, + DocsWritten: 2, + DocsProcessed: 3, + }, *stats) + + body, xattrs, destCas, err = toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCas3, destCas) + require.JSONEq(t, ver3Body, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + vv = db.HybridLogicalVector{} + require.NoError(t, base.JSONUnmarshal(xattrs[base.VvXattrName], &vv)) + require.Equal(t, db.HybridLogicalVector{ + CurrentVersionCAS: fromCas3, + SourceID: fromBucketSourceID, + Version: fromCas3}, vv) + require.Contains(t, xattrs, base.MouXattrName) + var actualMou *db.MetadataOnlyUpdate + require.NoError(t, base.JSONUnmarshal(xattrs[base.MouXattrName], &actualMou)) + // it is weird that couchbase server XDCR doesn't clear _mou but only _mou.cas and _mou.pRev but this is not a problem since eventing and couchbase server read _mou.cas to determine if _mou should be used + require.Equal(t, db.MetadataOnlyUpdate{ + PreviousHexCAS: mou.PreviousHexCAS}, + *actualMou) +} + +func TestLWWAfterInitialReplication(t *testing.T) { + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + + docID := "doc1" + ver1Body := `{"ver":1}` + fromCAS, err := fromDs.WriteCas(docID, 0, 0, []byte(ver1Body), 0) + require.NoError(t, err) + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1) + + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCAS, destCas) + require.JSONEq(t, ver1Body, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) + + // write to dest bucket again + toCas2, err := toDs.WriteCas(docID, 0, fromCAS, []byte(`{"ver":3}`), 0) + require.NoError(t, err) + + body, xattrs, destCas, err = toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, toCas2, destCas) + require.JSONEq(t, `{"ver":3}`, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) +} + +func TestReplicateXattrs(t *testing.T) { + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + + testCases := []struct { + name string + startingSourceXattrs map[string][]byte + startingDestXattrs map[string][]byte + finalXattrs map[string][]byte + }{ + { + name: "_sync on source only", + startingSourceXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"fromDs"}`), + }, + finalXattrs: map[string][]byte{}, + }, + { + name: "_sync on dest only", + startingDestXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"toDs"}`), + }, + finalXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"toDs"}`), + }, + }, + { + name: "_sync on both", + startingSourceXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"fromDs"}`), + }, + startingDestXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"toDs"}`), + }, + finalXattrs: map[string][]byte{ + base.SyncXattrName: []byte(`{"source":"toDs"}`), + }, + }, + { + name: "_globalSync on source only", + startingSourceXattrs: map[string][]byte{ + base.GlobalXattrName: []byte(`{"source":"fromDs"}`), + }, + finalXattrs: map[string][]byte{ + base.GlobalXattrName: []byte(`{"source":"fromDs"}`), + }, + }, + { + name: "_globalSync on overwrite dest", + startingSourceXattrs: map[string][]byte{ + base.GlobalXattrName: []byte(`{"source":"fromDs"}`), + }, + startingDestXattrs: map[string][]byte{ + base.GlobalXattrName: []byte(`{"source":"toDs"}`), + }, + finalXattrs: map[string][]byte{ + base.GlobalXattrName: []byte(`{"source":"fromDs"}`), + }, + }, + } + + var totalDocsProcessed uint64 // totalDocsProcessed will be incremented in each subtest + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + docID := testCase.name + + ctx := base.TestCtx(t) + body := []byte(`{"key":"value"}`) + if testCase.startingDestXattrs != nil { + _, err := toDs.WriteWithXattrs(ctx, docID, 0, 0, body, testCase.startingDestXattrs, nil, nil) + require.NoError(t, err) + } + fromCas, err := fromDs.WriteWithXattrs(ctx, docID, 0, 0, body, testCase.startingSourceXattrs, nil, nil) + require.NoError(t, err) + xdcr := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + stats, err := xdcr.Stats(ctx) + assert.NoError(t, err) + totalDocsProcessed = stats.DocsProcessed + assert.NoError(t, xdcr.Stop(ctx)) + }() + requireWaitForXDCRDocsProcessed(t, xdcr, 1+totalDocsProcessed) + + allXattrKeys := slices.Concat(maps.Keys(testCase.startingSourceXattrs), maps.Keys(testCase.finalXattrs)) + _, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, allXattrKeys) + require.NoError(t, err) + require.Equal(t, fromCas, destCas) + require.Equal(t, testCase.finalXattrs, xattrs) + }) + } +} + +// TestVVMultiActor verifies that updates by multiple actors (updates to different clusters/buckets) are properly +// reflected in the HLV (cv and pv). +func TestVVMultiActor(t *testing.T) { + if !base.UnitTestUrlIsWalrus() { + t.Skip("This test can fail with CBS due to CBS-4334 since a document without xattrs will be written to the target bucket, even if it is otherwise up to date") + } + fromBucket, fromDs, toBucket, toDs := getTwoBucketDataStores(t) + ctx := base.TestCtx(t) + fromBucketSourceID, err := GetSourceID(ctx, fromBucket) + require.NoError(t, err) + toBucketSourceID, err := GetSourceID(ctx, toBucket) + require.NoError(t, err) + + // Create document on source + docID := "doc1" + ver1Body := `{"ver":1}` + fromCAS, err := fromDs.WriteCas(docID, 0, 0, []byte(ver1Body), 0) + require.NoError(t, err) + + // start bidirectional XDCR + xdcrSource := startXDCR(t, fromBucket, toBucket, XDCROptions{Mobile: MobileOn}) + xdcrTarget := startXDCR(t, toBucket, fromBucket, XDCROptions{Mobile: MobileOn}) + defer func() { + assert.NoError(t, xdcrSource.Stop(ctx)) + assert.NoError(t, xdcrTarget.Stop(ctx)) + }() + requireWaitForXDCRDocsWritten(t, xdcrSource, 1) + + // Verify HLV on remote. + // expected HLV: + // cv: fromCAS@source + body, xattrs, destCas, err := toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCAS, destCas) + require.JSONEq(t, ver1Body, string(body)) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) + + // Update document on remote + toCAS, err := toDs.WriteCas(docID, 0, fromCAS, []byte(`{"ver":2}`), 0) + require.NoError(t, err) + requireWaitForXDCRDocsWritten(t, xdcrTarget, 1) + + // Verify HLV on source. + // expected HLV: + // cv: toCAS@remote + // pv: fromCAS@source + body, xattrs, destCas, err = fromDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, toCAS, destCas) + require.JSONEq(t, `{"ver":2}`, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], toBucketSourceID, toCAS) + requirePV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) + + // Update document on remote again. Verifies that another update to cv doesn't affect pv. + toCAS2, err := toDs.WriteCas(docID, 0, toCAS, []byte(`{"ver":3}`), 0) + require.NoError(t, err) + requireWaitForXDCRDocsWritten(t, xdcrTarget, 2) + + // Verify HLV on source bucket. + // expected HLV: + // cv: toCAS2@remote + // pv: fromCAS@source + body, xattrs, destCas, err = fromDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, toCAS2, destCas) + require.JSONEq(t, `{"ver":3}`, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], toBucketSourceID, toCAS2) + requirePV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS) + + // Update document on source bucket. Verifies that local source is moved from pv to cv, target source from cv to pv. + fromCAS2, err := fromDs.WriteCas(docID, 0, toCAS2, []byte(`{"ver":4}`), 0) + require.NoError(t, err) + requireWaitForXDCRDocsWritten(t, xdcrTarget, 2) + + // Verify HLV on target + // expected HLV: + // cv: fromCAS2@source + // pv: toCAS2@remote + body, xattrs, destCas, err = toDs.GetWithXattrs(ctx, docID, []string{base.VvXattrName, base.MouXattrName}) + require.NoError(t, err) + require.Equal(t, fromCAS2, destCas) + require.JSONEq(t, `{"ver":4}`, string(body)) + require.Contains(t, xattrs, base.VvXattrName) + requireCV(t, xattrs[base.VvXattrName], fromBucketSourceID, fromCAS2) + requirePV(t, xattrs[base.VvXattrName], toBucketSourceID, toCAS2) + +} + +// startXDCR will create a new XDCR manager and start it. This must be closed by the caller. +func startXDCR(t *testing.T, fromBucket base.Bucket, toBucket base.Bucket, opts XDCROptions) Manager { + ctx := base.TestCtx(t) + xdcr, err := NewXDCR(ctx, fromBucket, toBucket, opts) + require.NoError(t, err) + err = xdcr.Start(ctx) + require.NoError(t, err) + return xdcr +} + +// requireWaitForXDCRDocsProcessed waits for the replication to process the exact number of documents. If more than the expected number of documents are processed, this will fail. +func requireWaitForXDCRDocsProcessed(t *testing.T, xdcr Manager, expectedDocsProcessed uint64) { + ctx := base.TestCtx(t) + require.EventuallyWithT(t, func(c *assert.CollectT) { + stats, err := xdcr.Stats(ctx) + if !assert.NoError(c, err) { + return + } + assert.Equal(c, expectedDocsProcessed, stats.DocsProcessed, "all stats=%+v", stats) + }, time.Second*5, time.Millisecond*100) +} + +// requireWaitForXDCRDocsWritten waits for the replication to write the exact number of documents. +func requireWaitForXDCRDocsWritten(t *testing.T, xdcr Manager, expectedDocsWritten uint64) { + ctx := base.TestCtx(t) + require.EventuallyWithT(t, func(c *assert.CollectT) { + stats, err := xdcr.Stats(ctx) + if !assert.NoError(c, err) { + return + } + assert.Equal(c, expectedDocsWritten, stats.DocsWritten, "all stats=%+v", stats) + }, time.Second*5, time.Millisecond*100) +} + +// requireCV requires tests that a given hlv from server has sourceID and cas matching the current version. +func requireCV(t *testing.T, vvBytes []byte, sourceID string, cas uint64) { + var vv *db.HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(vvBytes, &vv)) + require.Equal(t, cas, vv.CurrentVersionCAS) + require.Equal(t, sourceID, vv.SourceID) +} + +// requirePV requires tests that a given hlv from server has an entry in the PV with sourceID and cas matching the provided values. +func requirePV(t *testing.T, vvBytes []byte, sourceID string, cas uint64) { + var vv *db.HybridLogicalVector + require.NoError(t, base.JSONUnmarshal(vvBytes, &vv)) + require.NotNil(t, vv.PreviousVersions) + pvValue, ok := vv.PreviousVersions[sourceID] + require.True(t, ok) + require.Equal(t, cas, pvValue) +}