diff --git a/src/event_model/__init__.py b/src/event_model/__init__.py index 3d45ec83..305fa162 100644 --- a/src/event_model/__init__.py +++ b/src/event_model/__init__.py @@ -49,7 +49,16 @@ from .documents.partial_event_page import PartialEventPage from .documents.partial_resource import PartialResource from .documents.resource import Resource -from .documents.run_start import Calculation, Hints, Projection, Projections, RunStart +from .documents.run_start import ( + CalculatedEventProjection, + Calculation, + ConfigurationProjection, + Hints, + LinkedEventProjection, + Projections, + RunStart, + StaticProjection, +) from .documents.run_stop import RunStop from .documents.stream_datum import StreamDatum, StreamRange from .documents.stream_resource import StreamResource @@ -79,6 +88,10 @@ "Resource", "Calculation", "Hints", + "LinkedEventProjection", + "StaticProjection", + "CalculatedEventProjection", + "ConfigurationProjection", "Projection", "Projections", "RunStart", @@ -1822,6 +1835,8 @@ class MismatchedDataKeys(InvalidData): DocumentNames.resource: "jsonschemas/resource.json", DocumentNames.stream_datum: "jsonschemas/stream_datum.json", DocumentNames.stream_resource: "jsonschemas/stream_resource.json", + DocumentNames.bulk_datum: "jsonschemas/bulk_datum.json", + DocumentNames.bulk_events: "jsonschemas/bulk_events.json", } schemas = {} for name, filename in SCHEMA_NAMES.items(): diff --git a/src/event_model/basemodels/run_start.py b/src/event_model/basemodels/run_start.py index a7a9d7a2..3113ddb1 100644 --- a/src/event_model/basemodels/run_start.py +++ b/src/event_model/basemodels/run_start.py @@ -39,101 +39,92 @@ class Calculation(BaseModel): ] -class Projection(BaseModel): - """Where to get the data from""" - - model_config = ConfigDict(extra="forbid") +class ConfigurationProjection(BaseModel): + location: Annotated[ + Literal["configuration"], + Field( + description="Projection comes from configuration " + "fields in the event_descriptor document", + ), + ] - calculation: Annotated[ - Optional[Calculation], + type: Annotated[ + Literal["linked"], Field( - description="required fields if type is calculated", - title="calculation properties", - default=None, + description=( + "Projection is of type linked, a value linked from the data set." + ) ), ] - config_index: Annotated[Optional[int], Field(default=None)] - config_device: Annotated[Optional[str], Field(default=None)] - field: Annotated[Optional[str], Field(default=None)] + config_index: Annotated[int, Field()] + config_device: Annotated[str, Field()] + field: Annotated[str, Field()] + stream: Annotated[str, Field()] + + +class LinkedEventProjection(BaseModel): location: Annotated[ - Optional[Literal["start", "event", "configuration"]], + Literal["event"], + Field(description="Projection comes and event"), + ] + + type: Annotated[ + Literal["linked"], Field( - description="start comes from metadata fields in the start document, " - "event comes from event, configuration comes from configuration " - "fields in the event_descriptor document", - default=None, + description=( + "Projection is of type linked, a value linked from the data set." + ) ), ] - stream: Annotated[Optional[str], Field(default=None)] + field: Annotated[str, Field()] + stream: Annotated[str, Field()] + + +class CalculatedEventProjection(BaseModel): + location: Annotated[ + Literal["event"], + Field(description="Projection comes and event"), + ] + type: Annotated[ - Optional[Literal["linked", "calculated", "static"]], + Literal["calculated"], Field( - description="linked: a value linked from the data set, " - "calculated: a value that requires calculation, " - "static: a value defined here in the projection ", - default=None, + description=( + "Projection is of type calculated, a value that requires calculation." + ) ), ] - value: Annotated[ - Optional[Any], + field: Annotated[str, Field()] + stream: Annotated[str, Field()] + calculation: Annotated[ + Calculation, Field( - description="value explicitely defined in the projection " - "when type==static.", - default=None, + description="required fields if type is calculated", + title="calculation properties", ), ] +class StaticProjection(BaseModel): + type: Annotated[ + Literal["static"], + Field( + description=( + "Projection is of type static, a value defined here in the projection" + ) + ), + ] + value: Annotated[ + Any, Field(description="value explicitely defined in the static projection") + ] + + RUN_START_EXTRA_SCHEMA = { "$defs": { "DataType": { "patternProperties": {"^([^./]+)$": {"$ref": "#/$defs/DataType"}}, "additionalProperties": False, }, - "Projection": { - "allOf": [ - { - "if": { - "allOf": [ - {"properties": {"location": {"enum": ["configuration"]}}}, - {"properties": {"type": {"enum": ["linked"]}}}, - ] - }, - "then": { - "required": [ - "type", - "location", - "config_index", - "config_device", - "field", - "stream", - ] - }, - }, - { - "if": { - "allOf": [ - {"properties": {"location": {"enum": ["event"]}}}, - {"properties": {"type": {"enum": ["linked"]}}}, - ] - }, - "then": {"required": ["type", "location", "field", "stream"]}, - }, - { - "if": { - "allOf": [ - {"properties": {"location": {"enum": ["event"]}}}, - {"properties": {"type": {"enum": ["calculated"]}}}, - ] - }, - "then": {"required": ["type", "field", "stream", "calculation"]}, - }, - { - "if": {"properties": {"type": {"enum": ["static"]}}}, - "then": {"required": ["type", "value"]}, - }, - ], - }, }, "properties": { "hints": { @@ -155,7 +146,18 @@ class Projections(BaseModel): name: Annotated[ Optional[str], Field(description="The name of the projection", default=None) ] - projection: Annotated[Dict[Any, Projection], Field(description="")] + projection: Annotated[ + Dict[ + Any, + Union[ + ConfigurationProjection, + LinkedEventProjection, + CalculatedEventProjection, + StaticProjection, + ], + ], + Field(description=""), + ] version: Annotated[ str, Field( diff --git a/src/event_model/documents/run_start.py b/src/event_model/documents/run_start.py index 84fbb698..2c9f5388 100644 --- a/src/event_model/documents/run_start.py +++ b/src/event_model/documents/run_start.py @@ -1,11 +1,11 @@ # ruff: noqa # generated by datamodel-codegen: # filename: run_start.json -# timestamp: 2024-12-18T11:01:13+00:00 +# timestamp: 2024-12-18T12:11:28+00:00 from __future__ import annotations -from typing import Any, Dict, List, Literal, Optional, TypedDict, Union +from typing import Any, Dict, List, Optional, TypedDict, Union from typing_extensions import NotRequired @@ -22,6 +22,21 @@ class Calculation(TypedDict): """ +class ConfigurationProjection(TypedDict): + config_device: str + config_index: int + field: str + location: NotRequired[str] + """ + Projection comes from configuration fields in the event_descriptor document + """ + stream: str + type: NotRequired[str] + """ + Projection is of type linked, a value linked from the data set. + """ + + DataType = Any @@ -36,30 +51,44 @@ class Hints(TypedDict): """ -class Projection(TypedDict): +class LinkedEventProjection(TypedDict): + field: str + location: NotRequired[str] + """ + Projection comes and event + """ + stream: str + type: NotRequired[str] """ - Where to get the data from + Projection is of type linked, a value linked from the data set. """ - calculation: NotRequired[Optional[Calculation]] + +class StaticProjection(TypedDict): + type: NotRequired[str] """ - required fields if type is calculated + Projection is of type static, a value defined here in the projection """ - config_device: NotRequired[Optional[str]] - config_index: NotRequired[Optional[int]] - field: NotRequired[Optional[str]] - location: NotRequired[Optional[Literal['start', 'event', 'configuration']]] + value: Any """ - start comes from metadata fields in the start document, event comes from event, configuration comes from configuration fields in the event_descriptor document + value explicitely defined in the static projection + """ + + +class CalculatedEventProjection(TypedDict): + calculation: Calculation + """ + required fields if type is calculated """ - stream: NotRequired[Optional[str]] - type: NotRequired[Optional[Literal['linked', 'calculated', 'static']]] + field: str + location: NotRequired[str] """ - linked: a value linked from the data set, calculated: a value that requires calculation, static: a value defined here in the projection + Projection comes and event """ - value: NotRequired[Any] + stream: str + type: NotRequired[str] """ - value explicitely defined in the projection when type==static. + Projection is of type calculated, a value that requires calculation. """ @@ -76,7 +105,15 @@ class Projections(TypedDict): """ The name of the projection """ - projection: Dict[str, Projection] + projection: Dict[ + str, + Union[ + ConfigurationProjection, + LinkedEventProjection, + CalculatedEventProjection, + StaticProjection, + ], + ] version: str """ The version of the projection spec. Can specify the version of an external specification. diff --git a/src/event_model/jsonschemas/bulk_datum.json b/src/event_model/jsonschemas/bulk_datum.json new file mode 100644 index 00000000..f77f65ce --- /dev/null +++ b/src/event_model/jsonschemas/bulk_datum.json @@ -0,0 +1,27 @@ +{ + "properties": { + "datum_kwarg_list": { + "type": "array", + "items": {"type": "object"}, + "description": "Array of arguments to pass to the Handler to retrieve one quanta of data" + }, + "resource": { + "type": "string", + "description": "UID of the Resource to which all these Datum documents belong" + }, + "datum_ids": { + "type": "array", + "items": {"type": "string"}, + "description": "Globally unique identifiers for each Datum (akin to 'uid' for other Document types), typically formatted as '/'" + } + }, + "required": [ + "datum_kwarg_list", + "resource", + "datum_ids" + ], + "additionalProperties": false, + "type": "object", + "title": "bulk_datum", + "description": "Document to reference a quanta of externally-stored data" +} diff --git a/src/event_model/jsonschemas/bulk_events.json b/src/event_model/jsonschemas/bulk_events.json new file mode 100644 index 00000000..267efd12 --- /dev/null +++ b/src/event_model/jsonschemas/bulk_events.json @@ -0,0 +1,52 @@ +{ + "patternProperties": { + "^.*$": { + "type": "array", + "items": { + "type": "object", + "properties": { + "data": { + "type": "object", + "description": "The actual measurement data" + }, + "timestamps": { + "type": "object", + "description": "The timestamps of the individual measurement data" + }, + "filled": { + "type": "object", + "description": "Mapping the keys of externally-stored data to a boolean indicating whether that data has yet been loaded" + }, + "descriptor": { + "type": "string", + "description": "UID to point back to Descriptor for this event stream" + }, + "seq_num": { + "type": "integer", + "description": "Sequence number to identify the location of this Event in the Event stream" + }, + "time": { + "type": "number", + "description": "The event time. This maybe different than the timestamps on each of the data entries" + }, + "uid": { + "type": "string", + "description": "Globally unique identifier for this Event" + } + }, + "required": [ + "uid", + "data", + "timestamps", + "time", + "descriptor", + "seq_num" + ], + "additionalProperties": false, + "type": "object", + "title": "bulk_events", + "description": "Document to record a quanta of collected data" + } + } + } +} diff --git a/src/event_model/jsonschemas/run_start.json b/src/event_model/jsonschemas/run_start.json index 0e53985a..134fd9d5 100644 --- a/src/event_model/jsonschemas/run_start.json +++ b/src/event_model/jsonschemas/run_start.json @@ -3,6 +3,44 @@ "description": "Document created at the start of run. Provides a seach target and\nlater documents link to it", "type": "object", "$defs": { + "CalculatedEventProjection": { + "title": "CalculatedEventProjection", + "type": "object", + "properties": { + "calculation": { + "title": "calculation properties", + "description": "required fields if type is calculated", + "$ref": "#/$defs/Calculation" + }, + "field": { + "title": "Field", + "type": "string" + }, + "location": { + "title": "Location", + "description": "Projection comes and event", + "type": "string", + "const": "event" + }, + "stream": { + "title": "Stream", + "type": "string" + }, + "type": { + "title": "Type", + "description": "Projection is of type calculated, a value that requires calculation.", + "type": "string", + "const": "calculated" + } + }, + "required": [ + "calculation", + "field", + "location", + "stream", + "type" + ] + }, "Calculation": { "title": "Calculation", "type": "object", @@ -44,6 +82,48 @@ ], "additionalProperties": false }, + "ConfigurationProjection": { + "title": "ConfigurationProjection", + "type": "object", + "properties": { + "config_device": { + "title": "Config Device", + "type": "string" + }, + "config_index": { + "title": "Config Index", + "type": "integer" + }, + "field": { + "title": "Field", + "type": "string" + }, + "location": { + "title": "Location", + "description": "Projection comes from configuration fields in the event_descriptor document", + "type": "string", + "const": "configuration" + }, + "stream": { + "title": "Stream", + "type": "string" + }, + "type": { + "title": "Type", + "description": "Projection is of type linked, a value linked from the data set.", + "type": "string", + "const": "linked" + } + }, + "required": [ + "config_device", + "config_index", + "field", + "location", + "stream", + "type" + ] + }, "DataType": { "title": "DataType", "patternProperties": { @@ -90,237 +170,36 @@ }, "additionalProperties": false }, - "Projection": { - "title": "Projection", - "description": "Where to get the data from", + "LinkedEventProjection": { + "title": "LinkedEventProjection", "type": "object", "properties": { - "calculation": { - "title": "calculation properties", - "description": "required fields if type is calculated", - "anyOf": [ - { - "$ref": "#/$defs/Calculation" - }, - { - "type": "null" - } - ], - "default": null - }, - "config_device": { - "title": "Config Device", - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null - }, - "config_index": { - "title": "Config Index", - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null - }, "field": { "title": "Field", - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null + "type": "string" }, "location": { "title": "Location", - "description": "start comes from metadata fields in the start document, event comes from event, configuration comes from configuration fields in the event_descriptor document", - "anyOf": [ - { - "enum": [ - "start", - "event", - "configuration" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null + "description": "Projection comes and event", + "type": "string", + "const": "event" }, "stream": { "title": "Stream", - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null + "type": "string" }, "type": { "title": "Type", - "description": "linked: a value linked from the data set, calculated: a value that requires calculation, static: a value defined here in the projection ", - "anyOf": [ - { - "enum": [ - "linked", - "calculated", - "static" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null - }, - "value": { - "title": "Value", - "description": "value explicitely defined in the projection when type==static.", - "anyOf": [ - {}, - { - "type": "null" - } - ], - "default": null + "description": "Projection is of type linked, a value linked from the data set.", + "type": "string", + "const": "linked" } }, - "additionalProperties": false, - "allOf": [ - { - "if": { - "allOf": [ - { - "properties": { - "location": { - "enum": [ - "configuration" - ] - } - } - }, - { - "properties": { - "type": { - "enum": [ - "linked" - ] - } - } - } - ] - }, - "then": { - "required": [ - "type", - "location", - "config_index", - "config_device", - "field", - "stream" - ] - } - }, - { - "if": { - "allOf": [ - { - "properties": { - "location": { - "enum": [ - "event" - ] - } - } - }, - { - "properties": { - "type": { - "enum": [ - "linked" - ] - } - } - } - ] - }, - "then": { - "required": [ - "type", - "location", - "field", - "stream" - ] - } - }, - { - "if": { - "allOf": [ - { - "properties": { - "location": { - "enum": [ - "event" - ] - } - } - }, - { - "properties": { - "type": { - "enum": [ - "calculated" - ] - } - } - } - ] - }, - "then": { - "required": [ - "type", - "field", - "stream", - "calculation" - ] - } - }, - { - "if": { - "properties": { - "type": { - "enum": [ - "static" - ] - } - } - }, - "then": { - "required": [ - "type", - "value" - ] - } - } + "required": [ + "field", + "location", + "stream", + "type" ] }, "Projections": { @@ -351,7 +230,20 @@ "description": "", "type": "object", "additionalProperties": { - "$ref": "#/$defs/Projection" + "anyOf": [ + { + "$ref": "#/$defs/ConfigurationProjection" + }, + { + "$ref": "#/$defs/LinkedEventProjection" + }, + { + "$ref": "#/$defs/CalculatedEventProjection" + }, + { + "$ref": "#/$defs/StaticProjection" + } + ] } }, "version": { @@ -365,6 +257,26 @@ "projection", "version" ] + }, + "StaticProjection": { + "title": "StaticProjection", + "type": "object", + "properties": { + "type": { + "title": "Type", + "description": "Projection is of type static, a value defined here in the projection", + "type": "string", + "const": "static" + }, + "value": { + "title": "Value", + "description": "value explicitely defined in the static projection" + } + }, + "required": [ + "type", + "value" + ] } }, "properties": {