[release-6.1] LOG-6236: Align syslog output implementation with spec RFC3164 and RFC5124

internal/generator/vector/output/syslog/rfc3164_with_defaults.toml

@@ -0,0 +1,47 @@
+[transforms.example_parse_encoding]
+type = "remap"
+inputs = ["application"]
+source = '''
+. = merge(., parse_json!(string!(.message))) ?? .
+if .log_type == "infrastructure" && .log_source == "node" {
+   ._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
+   ._internal.syslog.proc_id = to_string!(.systemd.t.PID || "")
+}
+if .log_source == "container" {
+    ._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "")
+    ._internal.syslog.severity = .level
+    ._internal.syslog.facility = "user"
+    #Remove non-alphanumeric characters
+   	._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "")
+	#Truncate the sanitized tag to 32 characters
+	._internal.syslog.tag = truncate(._internal.syslog.tag, 32)
+}
+if .log_type == "audit" {
+    ._internal.syslog.tag = .log_source
+    ._internal.syslog.severity = "informational"
+    ._internal.syslog.facility = "security"
+}
+.facility = to_string!(._internal.syslog.facility || "user")
+.severity = to_string!(._internal.syslog.severity || "informational")
+.proc_id = to_string!(._internal.syslog.proc_id || "-")
+.tag = to_string!(._internal.syslog.tag || "")
+if exists(.proc_id) && .proc_id != "-" && .proc_id != "" {
+    .tag = .tag + "[" + .proc_id  + "]"
+}
+'''
+
+[sinks.example]
+type = "socket"
+inputs = ["example_parse_encoding"]
+address = "logserver:514"
+mode = "udp"
+
+[sinks.example.encoding]
+codec = "syslog"
+except_fields = ["_internal"]
+rfc = "rfc3164"
+add_log_source = false
+facility = "$$.message.facility"
+severity = "$$.message.severity"
+proc_id = "$$.message.proc_id"
+tag = "$$.message.tag"

internal/generator/vector/output/syslog/syslog.go

@@ -60,6 +60,7 @@ type SyslogEncodingRemap struct {
 	Inputs         string
 	EncodingFields EncodingTemplateField
 	PayloadKey     string
+	RFC            string
 }
 
 func (ser SyslogEncodingRemap) Name() string {
@@ -74,12 +75,61 @@ inputs = {{.Inputs}}
 source = '''
 . = merge(., parse_json!(string!(.message))) ?? .
 
+{{if eq .RFC "RFC3164" -}}
+if .log_type == "infrastructure" && .log_source == "node" {
+    ._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
+	._internal.syslog.proc_id = to_string!(.systemd.t.PID || "")
+}
+if .log_source == "container" {
+   	._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "")
+   	._internal.syslog.severity = .level
+   	._internal.syslog.facility = "user"
+   	#Remove non-alphanumeric characters
+   	._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "")
+	#Truncate the sanitized tag to 32 characters
+	._internal.syslog.tag = truncate(._internal.syslog.tag, 32)
+
+}
+if .log_type == "audit" {
+   ._internal.syslog.tag = .log_source
+   ._internal.syslog.severity = "informational"
+   ._internal.syslog.facility = "security" 
+}
+{{end}}
+
+{{if eq .RFC "RFC5424" -}}
+._internal.syslog.msg_id = .log_source
+
+if .log_type == "infrastructure" && .log_source == "node" {
+	._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
+	._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
+}
+if .log_source == "container" {
+   ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_")
+   ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-")
+   ._internal.syslog.severity = .level
+   ._internal.syslog.facility = "user"
+}
+if .log_type == "audit" {
+   ._internal.syslog.app_name = .log_source
+   ._internal.syslog.proc_id = to_string!(.auditID || "-")
+   ._internal.syslog.severity = "informational"
+   ._internal.syslog.facility = "security"
+}
+{{end}}
+
 {{if .EncodingFields.FieldVRLList -}}
 {{range $templatePair := .EncodingFields.FieldVRLList -}}
 	.{{$templatePair.Field}} = {{$templatePair.VRLString}}
 {{end -}}
 {{end}}
 
+{{if eq .RFC "RFC3164" -}}
+if exists(.proc_id) && .proc_id != "-" && .proc_id != "" {
+ .tag = .tag + "[" + .proc_id  + "]"
+}
+{{end}}
+
 {{if .PayloadKey -}}
 if is_null({{.PayloadKey}}) {
 	.payload_key = .
@@ -111,8 +161,12 @@ func (se SyslogEncoding) Template() string {
 codec = "syslog"
 except_fields = ["_internal"]
 rfc = "{{.RFC}}"
+{{ if .Facility }}
 facility = "{{.Facility}}"
+{{ end }}
+{{ if .Severity }}
 severity = "{{.Severity}}"
+{{ end }}
 {{ .AddLogSource }}
 {{ .PayloadKey }}
 {{end}}`
@@ -170,23 +224,53 @@ func getEncodingTemplatesAndFields(s *obs.Syslog) EncodingTemplateField {
 	templateFields := EncodingTemplateField{
 		FieldVRLList: []FieldVRLStringPair{},
 	}
-	if s.AppName != "" {
+
+	if s.Facility == "" {
 		templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
-			Field:     "app_name",
-			VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName),
+			Field:     "facility",
+			VRLString: commontemplate.TransformUserTemplateToVRL(`{._internal.syslog.facility || "user"}`),
 		})
 	}
-	if s.MsgId != "" {
+
+	if s.Severity == "" {
 		templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
-			Field:     "msg_id",
-			VRLString: commontemplate.TransformUserTemplateToVRL(s.MsgId),
+			Field:     "severity",
+			VRLString: commontemplate.TransformUserTemplateToVRL(`{._internal.syslog.severity || "informational"}`),
 		})
 	}
 
-	if s.ProcId != "" {
+	if s.ProcId == "" {
+		s.ProcId = `{._internal.syslog.proc_id || "-"}`
+	}
+	templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
+		Field:     "proc_id",
+		VRLString: commontemplate.TransformUserTemplateToVRL(s.ProcId),
+	})
+
+	if s.RFC == obs.SyslogRFC3164 {
+		if s.AppName == "" {
+			s.AppName = `{._internal.syslog.tag || ""}`
+		}
 		templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
-			Field:     "proc_id",
-			VRLString: commontemplate.TransformUserTemplateToVRL(s.ProcId),
+			Field:     "tag",
+			VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName),
+		})
+
+	} else {
+		if s.AppName == "" {
+			s.AppName = `{._internal.syslog.app_name || "-"}`
+		}
+		templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
+			Field:     "app_name",
+			VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName),
+		})
+
+		if s.MsgId == "" {
+			s.MsgId = `{._internal.syslog.msg_id || "-"}`
+		}
+		templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
+			Field:     "msg_id",
+			VRLString: commontemplate.TransformUserTemplateToVRL(s.MsgId),
 		})
 	}
 
@@ -223,12 +307,13 @@ func parseEncoding(id string, inputs []string, templatePairs EncodingTemplateFie
 		Inputs:         vectorhelpers.MakeInputs(inputs...),
 		EncodingFields: templatePairs,
 		PayloadKey:     PayloadKey(o.PayloadKey),
+		RFC:            string(o.RFC),
 	}
 }
 
 func Facility(s *obs.Syslog) string {
 	if s == nil || s.Facility == "" {
-		return "user"
+		return ""
 	}
 	if IsKeyExpr(s.Facility) {
 		return fmt.Sprintf("$%s", s.Facility)
@@ -238,7 +323,7 @@ func Facility(s *obs.Syslog) string {
 
 func Severity(s *obs.Syslog) string {
 	if s == nil || s.Severity == "" {
-		return "informational"
+		return ""
 	}
 	if IsKeyExpr(s.Severity) {
 		return fmt.Sprintf("$%s", s.Severity)

internal/generator/vector/output/syslog/syslog_test.go

@@ -2,7 +2,6 @@ package syslog_test
 
 import (
 	"fmt"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
@@ -42,6 +41,7 @@ var _ = Describe("vector syslog clf output", func() {
 				},
 			},
 		}
+
 		initOutput = func() obs.OutputSpec {
 			return obs.OutputSpec{
 				Type: obs.OutputTypeSyslog,
@@ -100,6 +100,13 @@ var _ = Describe("vector syslog clf output", func() {
 			}
 		}, false),
 
+		Entry("should configure with defaults RFC3164", "rfc3164_with_defaults.toml", func(spec *obs.OutputSpec) {
+			spec.Syslog = &obs.Syslog{
+				URL: "udp://logserver:514",
+				RFC: obs.SyslogRFC3164,
+			}
+		}, false),
+
 		Entry("should configure TLS with log record field references", "tls_with_field_references.toml", func(spec *obs.OutputSpec) {
 			spec.TLS = tlsSpec
 			spec.Syslog = &obs.Syslog{
@@ -113,6 +120,7 @@ var _ = Describe("vector syslog clf output", func() {
 				PayloadKey: `{.payload_key}`,
 			}
 		}, false),
+
 		Entry("should set buffer tuning parameters", "tcp_with_tuning.toml", func(spec *obs.OutputSpec) {
 			spec.Syslog.URL = "tcp://logserver:514"
 			spec.Syslog.Tuning = &obs.SyslogTuningSpec{

internal/generator/vector/output/syslog/tcp_with_defaults.toml

@@ -3,6 +3,29 @@ type = "remap"
 inputs = ["application"]
 source = '''
 . = merge(., parse_json!(string!(.message))) ?? .
+._internal.syslog.msg_id = .log_source
+
+if .log_type == "infrastructure" && .log_source == "node" {
+    ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
+    ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
+}
+if .log_source == "container" {
+   ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_")
+   ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-")
+   ._internal.syslog.severity = .level
+   ._internal.syslog.facility = "user"
+}
+if .log_type == "audit" {
+   ._internal.syslog.app_name = .log_source
+   ._internal.syslog.proc_id = to_string!(.auditID || "-")
+   ._internal.syslog.severity = "informational"
+   ._internal.syslog.facility = "security"
+}
+.facility = to_string!(._internal.syslog.facility || "user")
+.severity = to_string!(._internal.syslog.severity || "informational")
+.proc_id = to_string!(._internal.syslog.proc_id || "-")
+.app_name = to_string!(._internal.syslog.app_name || "-")
+.msg_id = to_string!(._internal.syslog.msg_id || "-")
 '''
 
 [sinks.example]
@@ -15,6 +38,9 @@ mode = "tcp"
 codec = "syslog"
 except_fields = ["_internal"]
 rfc = "rfc5424"
-facility = "user"
-severity = "informational"
-add_log_source = false
+add_log_source = false
+facility = "$$.message.facility"
+severity = "$$.message.severity"
+proc_id = "$$.message.proc_id"
+app_name = "$$.message.app_name"
+msg_id = "$$.message.msg_id"

internal/generator/vector/output/syslog/tcp_with_tuning.toml

@@ -3,6 +3,31 @@ type = "remap"
 inputs = ["application"]
 source = '''
 . = merge(., parse_json!(string!(.message))) ?? .
+
+._internal.syslog.msg_id = .log_source
+
+if .log_type == "infrastructure" && .log_source == "node" {
+    ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
+    ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
+}
+if .log_source == "container" {
+   ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_")
+   ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-")
+   ._internal.syslog.severity = .level
+   ._internal.syslog.facility = "user"
+}
+if .log_type == "audit" {
+   ._internal.syslog.app_name = .log_source
+   ._internal.syslog.proc_id = to_string!(.auditID || "-")
+   ._internal.syslog.severity = "informational"
+   ._internal.syslog.facility = "security"
+}
+.facility = to_string!(._internal.syslog.facility || "user")
+.severity = to_string!(._internal.syslog.severity || "informational")
+.proc_id = to_string!(._internal.syslog.proc_id || "-")
+.app_name = to_string!(._internal.syslog.app_name || "-")
+.msg_id = to_string!(._internal.syslog.msg_id || "-")
+
 '''
 
 [sinks.example]
@@ -15,9 +40,12 @@ mode = "tcp"
 codec = "syslog"
 except_fields = ["_internal"]
 rfc = "rfc5424"
-facility = "user"
-severity = "informational"
 add_log_source = false
+facility = "$$.message.facility"
+severity = "$$.message.severity"
+proc_id = "$$.message.proc_id"
+app_name = "$$.message.app_name"
+msg_id = "$$.message.msg_id"
 
 [sinks.example.buffer]
 type = "disk"

internal/generator/vector/output/syslog/tls_with_field_references.toml

@@ -4,9 +4,27 @@ inputs = ["application"]
 source = '''
 . = merge(., parse_json!(string!(.message))) ?? .
 
+._internal.syslog.msg_id = .log_source
+
+if .log_type == "infrastructure" && .log_source == "node" {
+    ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
+    ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
+}
+if .log_source == "container" {
+   ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_")
+   ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-")
+   ._internal.syslog.severity = .level
+   ._internal.syslog.facility = "user"
+}
+if .log_type == "audit" {
+   ._internal.syslog.app_name = .log_source
+   ._internal.syslog.proc_id = to_string!(.auditID || "-")
+   ._internal.syslog.severity = "informational"
+   ._internal.syslog.facility = "security"
+}
+.proc_id = to_string!(.proc_id||"none")
 .app_name = to_string!(.app_name||"none")
 .msg_id = to_string!(.msg_id||"none")
-.proc_id = to_string!(.proc_id||"none")
 
 if is_null(.payload_key) {
 	.payload_key = .
@@ -29,9 +47,9 @@ facility = "$$.message.facility"
 severity = "$$.message.severity"
 add_log_source = false
 payload_key = "payload_key"
+proc_id = "$$.message.proc_id"
 app_name = "$$.message.app_name"
 msg_id = "$$.message.msg_id"
-proc_id = "$$.message.proc_id"
 
 [sinks.example.tls]
 enabled = true