Revert "Import ecs fields from the ecs_nested.yml source file (elasti…

…c#766)"

This reverts commit 7869bd5.

internal/fields/dependency_manager.go

@@ -25,8 +25,8 @@ const (
 	ecsSchemaName      = "ecs"
 	gitReferencePrefix = "git@"
 
-	ecsSchemaFile = "ecs_nested.yml"
-	ecsSchemaURL  = "https://raw.githubusercontent.com/elastic/ecs/%s/generated/ecs/%s"
+	ecsSchemaFile = "fields.ecs.yml"
+	ecsSchemaURL  = "https://raw.githubusercontent.com/elastic/ecs/%s/generated/beats/%s"
 )
 
 // DependencyManager is responsible for resolving external field dependencies.
@@ -61,15 +61,6 @@ func loadECSFieldsSchema(dep buildmanifest.ECSDependency) ([]FieldDefinition, er
 		return nil, nil
 	}
 
-	content, err := readECSFieldsSchemaFile(dep)
-	if err != nil {
-		return nil, errors.Wrap(err, "error reading ECS fields schema file")
-	}
-
-	return parseECSFieldsSchema(content)
-}
-
-func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
 	gitReference, err := asGitReference(dep.Reference)
 	if err != nil {
 		return nil, errors.Wrap(err, "can't process the value as Git reference")
@@ -79,8 +70,12 @@ func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "error fetching profile path")
 	}
+
 	cachedSchemaPath := filepath.Join(loc.FieldsCacheDir(), ecsSchemaName, gitReference, ecsSchemaFile)
 	content, err := os.ReadFile(cachedSchemaPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, errors.Wrapf(err, "can't read cached schema (path: %s)", cachedSchemaPath)
+	}
 	if errors.Is(err, os.ErrNotExist) {
 		logger.Debugf("Pulling ECS dependency using reference: %s", dep.Reference)
 
@@ -114,21 +109,14 @@ func readECSFieldsSchemaFile(dep buildmanifest.ECSDependency) ([]byte, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "can't write cached schema (path: %s)", cachedSchemaPath)
 		}
-	} else if err != nil {
-		return nil, errors.Wrapf(err, "can't read cached schema (path: %s)", cachedSchemaPath)
 	}
 
-	return content, nil
-}
-
-func parseECSFieldsSchema(content []byte) ([]FieldDefinition, error) {
-	var fields FieldDefinitions
-	err := yaml.Unmarshal(content, &fields)
+	var f []FieldDefinition
+	err = yaml.Unmarshal(content, &f)
 	if err != nil {
 		return nil, errors.Wrap(err, "unmarshalling field body failed")
 	}
-
-	return fields, nil
+	return f[0].Fields, nil
 }
 
 func asGitReference(reference string) (string, error) {

internal/fields/model.go

@@ -4,13 +4,6 @@
 
 package fields
 
-import (
-	"fmt"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-)
-
 // FieldDefinition describes a single field with its properties.
 type FieldDefinition struct {
 	Name        string            `yaml:"name"`
@@ -23,7 +16,7 @@ type FieldDefinition struct {
 	External    string            `yaml:"external"`
 	Index       *bool             `yaml:"index"`
 	DocValues   *bool             `yaml:"doc_values"`
-	Fields      FieldDefinitions  `yaml:"fields,omitempty"`
+	Fields      []FieldDefinition `yaml:"fields,omitempty"`
 	MultiFields []FieldDefinition `yaml:"multi_fields,omitempty"`
 }
 
@@ -89,70 +82,3 @@ func updateFields(origFields, fields []FieldDefinition) []FieldDefinition {
 	}
 	return updatedFields
 }
-
-// FieldDefinitions is an array of FieldDefinition, this can be unmarshalled from
-// a yaml list or a yaml map.
-type FieldDefinitions []FieldDefinition
-
-func (fds *FieldDefinitions) UnmarshalYAML(value *yaml.Node) error {
-	nilNode := yaml.Kind(0)
-	switch value.Kind {
-	case yaml.SequenceNode:
-		// Fields are defined as a list, this happens in Beats fields files.
-		var fields []FieldDefinition
-		err := value.Decode(&fields)
-		if err != nil {
-			return err
-		}
-		*fds = fields
-		return nil
-	case yaml.MappingNode:
-		// Fields are defined as a map, this happens in ecs fields files.
-		if len(value.Content)%2 != 0 {
-			return fmt.Errorf("pairs of key-values expected in map")
-		}
-		var fields []FieldDefinition
-		for i := 0; i+1 < len(value.Content); i += 2 {
-			key := value.Content[i]
-			value := value.Content[i+1]
-
-			var name string
-			err := key.Decode(&name)
-			if err != nil {
-				return err
-			}
-
-			var field FieldDefinition
-			err = value.Decode(&field)
-			if err != nil {
-				return err
-			}
-
-			// "base" group is used by convention in ECS to include
-			// fields that can appear in the root level of the document.
-			// Append its child fields directly instead.
-			if name == "base" {
-				fields = append(fields, field.Fields...)
-			} else {
-				field.Name = name
-				cleanNestedNames(field.Name, field.Fields)
-				fields = append(fields, field)
-			}
-		}
-		*fds = fields
-		return nil
-	case nilNode:
-		*fds = nil
-		return nil
-	default:
-		return fmt.Errorf("expected map or sequence")
-	}
-}
-
-func cleanNestedNames(parent string, fields []FieldDefinition) {
-	for i := range fields {
-		if strings.HasPrefix(fields[i].Name, parent+".") {
-			fields[i].Name = fields[i].Name[len(parent)+1:]
-		}
-	}
-}

test/packages/other/fields_tests/data_stream/first/fields/geo-fields.yml

@@ -1,4 +1,6 @@
 - name: destination.geo.location
   external: ecs
+- name: geo.location
+  external: ecs
 - name: source.geo.location
   external: ecs

test/packages/other/fields_tests/data_stream/first/sample_event.json

@@ -3,6 +3,6 @@
         "lat": 1.0,
         "lon": "2.0"
     },
-    "destination.geo.location.lat": 3.0,
-    "destination.geo.location.lon": 4.0
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
 }

test/packages/other/fields_tests/docs/README.md

@@ -8,8 +8,8 @@ An example event for `first` looks as following:
         "lat": 1.0,
         "lon": "2.0"
     },
-    "destination.geo.location.lat": 3.0,
-    "destination.geo.location.lon": 4.0
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
 }
 ```
 
@@ -22,4 +22,5 @@ An example event for `first` looks as following:
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
 | destination.geo.location | Longitude and latitude. | geo_point |
+| geo.location | Longitude and latitude. | geo_point |
 | source.geo.location | Longitude and latitude. | geo_point |