This repository has been archived by the owner on Oct 9, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add input hash to workItemID Signed-off-by: Bernhard Stadlbauer <[email protected]> * Add test to ensure different IDs Signed-off-by: Bernhard Stadlbauer <[email protected]> * Cleanup `make lint` Signed-off-by: Bernhard Stadlbauer <[email protected]> * Move `emptyLiteralMap` to `hashing.go` Signed-off-by: Bernhard Stadlbauer <[email protected]> * Fix import ordering Signed-off-by: Bernhard Stadlbauer <[email protected]> --------- Signed-off-by: Bernhard Stadlbauer <[email protected]>
- Loading branch information
1 parent
1d0f3e9
commit 5a5ad82
Showing
6 changed files
with
788 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package catalog | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/base64" | ||
|
|
||
| "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" | ||
| "github.com/flyteorg/flytestdlib/pbhash" | ||
| ) | ||
|
|
||
| var emptyLiteralMap = core.LiteralMap{Literals: map[string]*core.Literal{}} | ||
|
|
||
| // Hashify a literal, in other words, produce a new literal where the corresponding value is removed in case | ||
| // the literal hash is set. | ||
| func hashify(literal *core.Literal) *core.Literal { | ||
| // Two recursive cases: | ||
| // 1. A collection of literals or | ||
| // 2. A map of literals | ||
|
|
||
| if literal.GetCollection() != nil { | ||
| literals := literal.GetCollection().Literals | ||
| literalsHash := make([]*core.Literal, 0) | ||
| for _, lit := range literals { | ||
| literalsHash = append(literalsHash, hashify(lit)) | ||
| } | ||
| return &core.Literal{ | ||
| Value: &core.Literal_Collection{ | ||
| Collection: &core.LiteralCollection{ | ||
| Literals: literalsHash, | ||
| }, | ||
| }, | ||
| } | ||
| } | ||
| if literal.GetMap() != nil { | ||
| literalsMap := make(map[string]*core.Literal) | ||
| for key, lit := range literal.GetMap().Literals { | ||
| literalsMap[key] = hashify(lit) | ||
| } | ||
| return &core.Literal{ | ||
| Value: &core.Literal_Map{ | ||
| Map: &core.LiteralMap{ | ||
| Literals: literalsMap, | ||
| }, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| // And a base case that consists of a scalar, where the hash might be set | ||
| if literal.GetHash() != "" { | ||
| return &core.Literal{ | ||
| Hash: literal.GetHash(), | ||
| } | ||
| } | ||
| return literal | ||
| } | ||
|
|
||
| func HashLiteralMap(ctx context.Context, literalMap *core.LiteralMap) (string, error) { | ||
| if literalMap == nil || len(literalMap.Literals) == 0 { | ||
| literalMap = &emptyLiteralMap | ||
| } | ||
|
|
||
| // Hashify, i.e. generate a copy of the literal map where each literal value is removed | ||
| // in case the corresponding hash is set. | ||
| hashifiedLiteralMap := make(map[string]*core.Literal, len(literalMap.Literals)) | ||
| for name, literal := range literalMap.Literals { | ||
| hashifiedLiteralMap[name] = hashify(literal) | ||
| } | ||
| hashifiedInputs := &core.LiteralMap{ | ||
| Literals: hashifiedLiteralMap, | ||
| } | ||
|
|
||
| inputsHash, err := pbhash.ComputeHash(ctx, hashifiedInputs) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| return base64.RawURLEncoding.EncodeToString(inputsHash), nil | ||
| } |
Oops, something went wrong.