Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement product and group IDs #8

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions docs/templates.md
Original file line number Diff line number Diff line change
Expand Up @@ -293,3 +293,31 @@ require custom text.
type = "book"
path = "moby-dick.txt"
```

#### `id` and `ref`

Together, these are used to generate IDs in the document, such as
product IDs and group IDs, that are defined in the document in one place
and referenced in other places in the document. To support multiple
kinds of IDs, there can be any number of namespaces for IDs. Both `id`
and `ref` have a `namespace` attribute that indicates which namespace to
use. Which namespaces exist is implicitly defined by which namespaces
are mentioned in the `id` templates.


##### Attributes

* `namespace`: String with the name of the namespace of the IDs.


##### Example

``` toml
[types."fakedoc:product_id_generator"]
namespace = "product_id"
type = "id"

[types."csaf:#/$defs/product_id_t"]
namespace = "product_id"
type = "ref"
```
248 changes: 222 additions & 26 deletions pkg/fakedoc/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package fakedoc

import (
"encoding/json"
"errors"
"fmt"
"io"
Expand All @@ -24,18 +25,26 @@ import (
"github.com/go-loremipsum/loremipsum"
)

// ErrBranchAbandoned is the base errors that indicate that the
// generator should abandon a recursive descent and try again with a
// different branch.
//
// This error is mostly used internally in the generator and is unlikely
// to be returned from Generate method.
var ErrBranchAbandoned = errors.New("branch abandoned")

// ErrDepthExceeded is returned as error by the generator if exceeding
// the maximum depth of the generated document could not be avoided.
// This is mostly used internally in the generator and is unlikely to be
// returned from Generate method.
var ErrDepthExceeded = errors.New("maximum depth exceeded")
// It is based on ErrBranchAbandoned
var ErrDepthExceeded = fmt.Errorf("%w: maximum depth exceeded", ErrBranchAbandoned)

// ErrNoValidValue is returned as error by the generator if no value
// that conforms to the constraints given in the template could be
// generated. This can happen for arrays where UniqueItems is true, for
// instance, if the minimum number of items is large compared to number
// of different valid items.
var ErrNoValidValue = errors.New("could not generate valid value")
// It is based on ErrBranchAbandoned
var ErrNoValidValue = fmt.Errorf("%w: could not generate valid value", ErrBranchAbandoned)

// ErrInvalidString is returned as error by the generator if the input
// text is not valid UTF-8. This can happen if the input is a binary
Expand All @@ -44,9 +53,54 @@ var ErrInvalidString = errors.New("not valid utf-8")

// Generator is the type of CSAF document generators
type Generator struct {
Template *Template
Rand *rand.Rand
FileCache map[string]string
Template *Template
Rand *rand.Rand
FileCache map[string]string
NameSpaces map[string]*NameSpace
}

// NameSpace helps implement TmplID and TmplRef by collecting the IDs
// and references for a name space. It holds both values and references
// so that the references can be set to actually existing IDs once all
// IDs have been generated
type NameSpace struct {
Values []string
Refs []*reference
}

func (ns *NameSpace) addValue(v string) {
ns.Values = append(ns.Values, v)
}

func (ns *NameSpace) addRef(r *reference) {
ns.Refs = append(ns.Refs, r)
}

func (ns *NameSpace) snapshot() *NameSpace {
return &NameSpace{
Values: ns.Values,
Refs: ns.Refs,
}
}

// reference is the value of a node created for TmplRef or arrays of
// TmplRef during generation. In the former case it represents a single
// reference serialized to JSON as a JSON string. In the latter case
// it's a slice of references serialized as a JSON array of strings.
// The length field indicates which variant it is.
type reference struct {
namespace string
// length is less than zero to indicate a single reference, greater
// or equal to zero to indicate an array
length int
values []string
}

func (ref *reference) MarshalJSON() ([]byte, error) {
if ref.length < 0 {
return json.Marshal(ref.values[0])
}
return json.Marshal(ref.values)
}

// NewGenerator creates a new Generator based on a Template and an
Expand All @@ -57,18 +111,65 @@ func NewGenerator(tmpl *Template, rng *rand.Rand) *Generator {
rng = rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))
}
return &Generator{
Template: tmpl,
Rand: rng,
FileCache: make(map[string]string),
Template: tmpl,
Rand: rng,
FileCache: make(map[string]string),
NameSpaces: make(map[string]*NameSpace),
}
}

func (gen *Generator) getNamespace(namespace string) *NameSpace {
if _, ok := gen.NameSpaces[namespace]; !ok {
gen.NameSpaces[namespace] = &NameSpace{}
}
return gen.NameSpaces[namespace]
}

// addNSValue adds a value to a namespace
func (gen *Generator) addNSValue(namespace, v string) {
gen.getNamespace(namespace).addValue(v)
}

// adNSRef adds a reference to a namespace
func (gen *Generator) adNSRef(namespace string, r *reference) {
gen.getNamespace(namespace).addRef(r)
}

func (gen *Generator) hasNSValues(namespace string) bool {
return len(gen.getNamespace(namespace).Values) > 0
}

func (gen *Generator) numNSValues(namespace string) int {
return len(gen.getNamespace(namespace).Values)
}

func (gen *Generator) snapshotNamespaces() map[string]*NameSpace {
snap := make(map[string]*NameSpace, len(gen.NameSpaces))
for name, ns := range gen.NameSpaces {
snap[name] = ns.snapshot()
}
return snap
}

func (gen *Generator) restoreSnapshot(snapshot map[string]*NameSpace) {
gen.NameSpaces = snapshot
}

// Generate generates a document
func (gen *Generator) Generate() (any, error) {
return gen.generateNode(gen.Template.Root, 25)
doc, err := gen.generateNode(gen.Template.Root, 25)
if err != nil {
return nil, err
}

if err = gen.fixupReferences(); err != nil {
return nil, err
}

return doc, nil
}

func (gen *Generator) generateNode(typename string, depth int) (any, error) {
func (gen *Generator) generateNode(typename string, depth int) (_ any, err error) {
if depth <= 0 {
return nil, ErrDepthExceeded
}
Expand All @@ -77,14 +178,24 @@ func (gen *Generator) generateNode(typename string, depth int) (any, error) {
if !ok {
return nil, fmt.Errorf("unknown type '%s'", typename)
}

// make sure IDs generated in abandoned branches are discarded so
// that we don't end up with e.g. references to group IDs that are
// not actually there.
snapshot := gen.snapshotNamespaces()
defer func() {
if errors.Is(err, ErrBranchAbandoned) {
gen.restoreSnapshot(snapshot)
}
}()

switch node := nodeTmpl.(type) {
case *TmplObject:
return gen.generateObject(node, depth)
case *TmplArray:
return gen.randomArray(node, depth)
case *TmplOneOf:
typename := choose(gen.Rand, node.OneOf)
return gen.generateNode(typename, depth-1)
return gen.randomOneOf(node.OneOf, depth)
case *TmplString:
if len(node.Enum) > 0 {
return choose(gen.Rand, node.Enum), nil
Expand All @@ -97,6 +208,10 @@ func (gen *Generator) generateNode(typename string, depth int) (any, error) {
return gen.loremIpsum(node.MinLength, node.MaxLength, node.Unit), nil
case *TmplBook:
return gen.book(node.MinLength, node.MaxLength, node.Path)
case *TmplID:
return gen.generateID(node.Namespace), nil
case *TmplRef:
return gen.generateReference(node.Namespace)
case *TmplNumber:
return gen.randomNumber(node.Minimum, node.Maximum), nil
case *TmplDateTime:
Expand Down Expand Up @@ -135,6 +250,19 @@ func (gen *Generator) randomArray(tmpl *TmplArray, depth int) (any, error) {
maxitems = minitems + 2
}

if refnode, ok := gen.Template.Types[tmpl.Items].(*TmplRef); ok {
known := gen.numNSValues(refnode.Namespace)
if known >= minitems && tmpl.UniqueItems {
ref := &reference{
namespace: refnode.Namespace,
length: minitems + gen.Rand.IntN(known-minitems+1),
values: nil,
}
gen.adNSRef(refnode.Namespace, ref)
return ref, nil
}
}

length := minitems + gen.Rand.IntN(maxitems-minitems+1)
items := make([]any, 0, length)
notInItems := func(v any) bool {
Expand Down Expand Up @@ -201,22 +329,44 @@ generateItem:
return item, nil
}

func (gen *Generator) randomOneOf(oneof []string, depth int) (any, error) {
shuffled := shuffle(gen.Rand, oneof)
var abandoned error
for _, typename := range shuffled {
value, err := gen.generateNode(typename, depth-1)
if errors.Is(err, ErrBranchAbandoned) {
abandoned = err
continue
}
return value, err
}

if abandoned != nil {
return nil, abandoned
}
return nil, fmt.Errorf("could not generate any of %v", oneof)
}

func (gen *Generator) generateObject(node *TmplObject, depth int) (any, error) {
properties := make(map[string]any)
optional := make([]*Property, 0, len(node.Properties))
var optional, required []*Property
for _, prop := range node.Properties {
switch {
case prop.Required:
value, err := gen.generateNode(prop.Type, depth-1)
if err != nil {
return nil, err
}
properties[prop.Name] = value
required = append(required, prop)
default:
optional = append(optional, prop)
}
}

properties := make(map[string]any)
for _, prop := range required {
value, err := gen.generateNode(prop.Type, depth-1)
if err != nil {
return nil, err
}
properties[prop.Name] = value
}

// Choose a value for extraProps, the number of optional properties
// to add based on how many we need at least, node.MinProperties,
// and how many we may have at most, node.MaxProperties. Both of
Expand Down Expand Up @@ -244,15 +394,15 @@ func (gen *Generator) generateObject(node *TmplObject, depth int) (any, error) {
// try. Generating a property may fail because the maximum depth
// would be exceeded in which case we just try again with a
// different property.
depthExceeded := false
var branchAbandoned error
for extraProps > 0 && len(optional) > 0 {
i := gen.Rand.IntN(len(optional))
prop := optional[i]
optional = slices.Delete(optional, i, i+1)
value, err := gen.generateNode(prop.Type, depth-1)
switch {
case errors.Is(err, ErrDepthExceeded):
depthExceeded = true
case errors.Is(err, ErrBranchAbandoned):
branchAbandoned = err
continue
case err != nil:
return nil, err
Expand All @@ -266,8 +416,8 @@ func (gen *Generator) generateObject(node *TmplObject, depth int) (any, error) {
// failure is due to exceeding the maximum depth we report that to
// the caller so that it can try something else.
if len(properties) < minProps {
if depthExceeded {
return nil, ErrDepthExceeded
if branchAbandoned != nil {
return nil, branchAbandoned
}
return nil, fmt.Errorf("could not generate at least %d properties", minProps)
}
Expand Down Expand Up @@ -364,3 +514,49 @@ func (gen *Generator) book(minlength, maxlength int, path string) (string, error
trimmed = trimmed[:length]
return string(trimmed), nil
}

func (gen *Generator) generateID(namespace string) string {
id := gen.randomString(1, 20)
gen.addNSValue(namespace, id)
return id
}

func (gen *Generator) generateReference(namespace string) (any, error) {
if !gen.hasNSValues(namespace) {
return nil, fmt.Errorf(
"%w: no IDs in namespace %q", ErrBranchAbandoned, namespace,
)
}

ref := &reference{
namespace: namespace,
length: -1,
values: nil,
}
gen.adNSRef(namespace, ref)
return ref, nil
}

func (gen *Generator) fixupReferences() error {
for name, ns := range gen.NameSpaces {
if len(ns.Values) == 0 && len(ns.Refs) > 0 {
// this should never happen because references should
// only be generated if there are values available
return fmt.Errorf(
"no IDs when filling references in namespace %q",
name,
)
}
for _, ref := range ns.Refs {
switch {
case ref.length < 0:
ref.values = []string{choose(gen.Rand, ns.Values)}
case ref.length == 0:
ref.values = nil
default:
ref.values = chooseK(gen.Rand, ref.length, ns.Values)
}
}
}
return nil
}
Loading