From 6a0b4aea8f9bbd8c12499043d19ad517c542490b Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Tue, 15 Nov 2022 12:20:20 +0530 Subject: [PATCH] chore: improve error logging in writer (#31) * chore: improve error logging in writer * chore: limit log of span batch to 10 --- .../clickhousetracesexporter/schema-signoz.go | 92 ++++++++++++++++++- exporter/clickhousetracesexporter/writer.go | 16 ++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go index 3a22679a..5b8fc962 100644 --- a/exporter/clickhousetracesexporter/schema-signoz.go +++ b/exporter/clickhousetracesexporter/schema-signoz.go @@ -14,6 +14,12 @@ package clickhousetracesexporter +import ( + "fmt" + + "go.uber.org/zap/zapcore" +) + type Event struct { Name string `json:"name,omitempty"` TimeUnixNano uint64 `json:"timeUnixNano,omitempty"` @@ -21,6 +27,14 @@ type Event struct { IsError bool `json:"isError,omitempty"` } +func (e *Event) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("name", e.Name) + enc.AddUint64("timeUnixNano", e.TimeUnixNano) + enc.AddBool("isError", e.IsError) + enc.AddString("attributeMap", fmt.Sprintf("%v", e.AttributeMap)) + return nil +} + type TraceModel struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` @@ -29,13 +43,41 @@ type TraceModel struct { StartTimeUnixNano uint64 `json:"startTimeUnixNano,omitempty"` ServiceName string `json:"serviceName,omitempty"` Kind int8 `json:"kind,omitempty"` - References []OtelSpanRef `json:"references,omitempty"` + References references `json:"references,omitempty"` StatusCode int16 `json:"statusCode,omitempty"` TagMap map[string]string `json:"tagMap,omitempty"` Events []string `json:"event,omitempty"` HasError bool `json:"hasError,omitempty"` } +func (t *TraceModel) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("traceId", t.TraceId) + enc.AddString("spanId", t.SpanId) + enc.AddString("name", t.Name) + enc.AddUint64("durationNano", t.DurationNano) + enc.AddUint64("startTimeUnixNano", t.StartTimeUnixNano) + enc.AddString("serviceName", t.ServiceName) + enc.AddInt8("kind", t.Kind) + enc.AddInt16("statusCode", t.StatusCode) + enc.AddBool("hasError", t.HasError) + enc.AddArray("references", &t.References) + enc.AddString("tagMap", fmt.Sprintf("%v", t.TagMap)) + enc.AddString("event", fmt.Sprintf("%v", t.Events)) + return nil +} + +type references []OtelSpanRef + +func (s *references) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, e := range *s { + err := enc.AppendObject(&e) + if err != nil { + return err + } + } + return nil +} + type Span struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` @@ -75,8 +117,56 @@ type Span struct { ResponseStatusCode string `json:"responseStatusCode,omitempty"` } +func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("traceId", s.TraceId) + enc.AddString("spanId", s.SpanId) + enc.AddString("parentSpanId", s.ParentSpanId) + enc.AddString("name", s.Name) + enc.AddUint64("durationNano", s.DurationNano) + enc.AddUint64("startTimeUnixNano", s.StartTimeUnixNano) + enc.AddString("serviceName", s.ServiceName) + enc.AddInt8("kind", s.Kind) + enc.AddInt16("statusCode", s.StatusCode) + enc.AddString("externalHttpMethod", s.ExternalHttpMethod) + enc.AddString("httpUrl", s.HttpUrl) + enc.AddString("httpMethod", s.HttpMethod) + enc.AddString("httpHost", s.HttpHost) + enc.AddString("httpRoute", s.HttpRoute) + enc.AddString("httpCode", s.HttpCode) + enc.AddString("msgSystem", s.MsgSystem) + enc.AddString("msgOperation", s.MsgOperation) + enc.AddString("externalHttpUrl", s.ExternalHttpUrl) + enc.AddString("component", s.Component) + enc.AddString("dbSystem", s.DBSystem) + enc.AddString("dbName", s.DBName) + enc.AddString("dbOperation", s.DBOperation) + enc.AddString("peerService", s.PeerService) + enc.AddString("gRPCCode", s.GRPCCode) + enc.AddString("gRPCMethod", s.GRPCMethod) + enc.AddString("rpcSystem", s.RPCSystem) + enc.AddString("rpcService", s.RPCService) + enc.AddString("rpcMethod", s.RPCMethod) + enc.AddString("responseStatusCode", s.ResponseStatusCode) + enc.AddBool("hasError", s.HasError) + enc.AddString("errorID", s.ErrorID) + enc.AddString("errorGroupID", s.ErrorGroupID) + enc.AddObject("errorEvent", &s.ErrorEvent) + enc.AddObject("traceModel", &s.TraceModel) + enc.AddString("event", fmt.Sprintf("%v", s.Events)) + enc.AddString("tagMap", fmt.Sprintf("%v", s.TagMap)) + + return nil +} + type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` RefType string `json:"refType,omitempty"` } + +func (r *OtelSpanRef) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("traceId", r.TraceId) + enc.AddString("spanId", r.SpanId) + enc.AddString("refType", r.RefType) + return nil +} diff --git a/exporter/clickhousetracesexporter/writer.go b/exporter/clickhousetracesexporter/writer.go index fa0a8257..fbeb5820 100644 --- a/exporter/clickhousetracesexporter/writer.go +++ b/exporter/clickhousetracesexporter/writer.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "math" "strings" "sync" "time" @@ -117,16 +118,22 @@ func (w *SpanWriter) writeBatch(batch []*Span) error { if w.spansTable != "" { if err := w.writeModelBatch(batch); err != nil { + logBatch := batch[:int(math.Min(10, float64(len(batch))))] + w.logger.Error("Could not write a batch of spans to model table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } } if w.indexTable != "" { if err := w.writeIndexBatch(batch); err != nil { + logBatch := batch[:int(math.Min(10, float64(len(batch))))] + w.logger.Error("Could not write a batch of spans to index table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } } if w.errorTable != "" { if err := w.writeErrorBatch(batch); err != nil { + logBatch := batch[:int(math.Min(10, float64(len(batch))))] + w.logger.Error("Could not write a batch of spans to error table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } } @@ -139,6 +146,8 @@ func (w *SpanWriter) writeIndexBatch(batchSpans []*Span) error { ctx := context.Background() statement, err := w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.indexTable)) if err != nil { + logBatch := batchSpans[:int(math.Min(10, float64(len(batchSpans))))] + w.logger.Error("Could not prepare batch for index table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } @@ -178,6 +187,7 @@ func (w *SpanWriter) writeIndexBatch(batchSpans []*Span) error { span.ResponseStatusCode, ) if err != nil { + w.logger.Error("Could not append span to batch: ", zap.Object("span", span), zap.Error(err)) return err } } @@ -190,6 +200,8 @@ func (w *SpanWriter) writeErrorBatch(batchSpans []*Span) error { ctx := context.Background() statement, err := w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.errorTable)) if err != nil { + logBatch := batchSpans[:int(math.Min(10, float64(len(batchSpans))))] + w.logger.Error("Could not prepare batch for error table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } @@ -210,6 +222,7 @@ func (w *SpanWriter) writeErrorBatch(batchSpans []*Span) error { stringToBool(span.ErrorEvent.AttributeMap["exception.escaped"]), ) if err != nil { + w.logger.Error("Could not append span to batch: ", zap.Object("span", span), zap.Error(err)) return err } } @@ -228,6 +241,8 @@ func (w *SpanWriter) writeModelBatch(batchSpans []*Span) error { ctx := context.Background() statement, err := w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.spansTable)) if err != nil { + logBatch := batchSpans[:int(math.Min(10, float64(len(batchSpans))))] + w.logger.Error("Could not prepare batch for model table: ", zap.Any("batch", logBatch), zap.Error(err)) return err } @@ -242,6 +257,7 @@ func (w *SpanWriter) writeModelBatch(batchSpans []*Span) error { err = statement.Append(time.Unix(0, int64(span.StartTimeUnixNano)), span.TraceId, string(serialized)) if err != nil { + w.logger.Error("Could not append span to batch: ", zap.Object("span", span), zap.Error(err)) return err } }