Applayer plugin 5053 v2.2

doc/userguide/output/eve/eve-json-output.rst

@@ -68,7 +68,7 @@ Metadata::
             #payload: yes             # enable dumping payload in Base64
             #payload-buffer-size: 4kb # max size of payload buffer to output in eve-log
             #payload-printable: yes   # enable dumping payload in printable (lossy) format
-            #payload-length: yes      # enable dumping payload length
+            #payload-length: yes      # enable dumping payload length, including the gaps
             #packet: yes              # enable dumping of packet (without stream segments)
             #http-body: yes           # Requires metadata; enable dumping of http body in Base64
             #http-body-printable: yes # Requires metadata; enable dumping of http body in printable format

doc/userguide/partials/eve-log.yaml

@@ -37,7 +37,7 @@ outputs:
             # payload: yes             # enable dumping payload in Base64
             # payload-buffer-size: 4kb # max size of payload buffer to output in eve-log
             # payload-printable: yes   # enable dumping payload in printable (lossy) format
-            # payload-length: yes      # enable dumping payload length
+            # payload-length: yes      # enable dumping payload length, including the gaps
             # packet: yes              # enable dumping of packet (without stream segments)
             # http-body: yes           # Requires metadata; enable dumping of http body in Base64
             # http-body-printable: yes # Requires metadata; enable dumping of http body in printable format

rust/src/http2/detect.rs

@@ -359,7 +359,7 @@ pub unsafe extern "C" fn rs_http2_detect_sizeupdatectx_match(
 #[no_mangle]
 pub unsafe extern "C" fn rs_http2_tx_get_header_name(
     tx: &mut HTTP2Transaction, direction: u8, nb: u32, buffer: *mut *const u8, buffer_len: *mut u32,
-) -> u8 {
+) -> bool {
     let mut pos = 0_u32;
     match direction.into() {
         Direction::ToServer => {
@@ -369,7 +369,7 @@ pub unsafe extern "C" fn rs_http2_tx_get_header_name(
                         let value = &blocks[(nb - pos) as usize].name;
                         *buffer = value.as_ptr(); //unsafe
                         *buffer_len = value.len() as u32;
-                        return 1;
+                        return true;
                     } else {
                         pos += blocks.len() as u32;
                     }
@@ -383,15 +383,15 @@ pub unsafe extern "C" fn rs_http2_tx_get_header_name(
                         let value = &blocks[(nb - pos) as usize].name;
                         *buffer = value.as_ptr(); //unsafe
                         *buffer_len = value.len() as u32;
-                        return 1;
+                        return true;
                     } else {
                         pos += blocks.len() as u32;
                     }
                 }
             }
         }
     }
-    return 0;
+    return false;
 }
 
 fn http2_frames_get_header_firstvalue<'a>(

src/app-layer.c

@@ -1139,7 +1139,6 @@ static void AppLayerSetupExceptionPolicyPerProtoCounters(
     }
 }
 
-#define IPPROTOS_MAX 2
 void AppLayerSetupCounters(void)
 {
     const uint8_t ipprotos[] = { IPPROTO_TCP, IPPROTO_UDP };
@@ -1162,7 +1161,7 @@ void AppLayerSetupCounters(void)
 
     AppLayerProtoDetectSupportedAppProtocols(alprotos);
 
-    for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
+    for (uint8_t p = 0; p < FLOW_PROTO_APPLAYER_MAX; p++) {
         const uint8_t ipproto = ipprotos[p];
         const uint8_t ipproto_map = FlowGetProtoMapping(ipproto);
         const char *ipproto_suffix = (ipproto == IPPROTO_TCP) ? "_tcp" : "_udp";
@@ -1257,7 +1256,7 @@ void AppLayerRegisterThreadCounters(ThreadVars *tv)
         }
     }
 
-    for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
+    for (uint8_t p = 0; p < FLOW_PROTO_APPLAYER_MAX; p++) {
         const uint8_t ipproto = ipprotos[p];
         const uint8_t ipproto_map = FlowGetProtoMapping(ipproto);
 

src/decode.h

@@ -355,10 +355,10 @@ typedef struct PktProfiling_ {
 
     PktProfilingTmmData tmm[TMM_SIZE];
     PktProfilingData flowworker[PROFILE_FLOWWORKER_SIZE];
-    PktProfilingAppData app[ALPROTO_MAX];
     PktProfilingDetectData detect[PROF_DETECT_SIZE];
     PktProfilingLoggerData logger[LOGGER_SIZE];
     uint64_t proto_detect;
+    PktProfilingAppData app[];
 } PktProfiling;
 
 #endif /* PROFILING */

src/detect-engine-helper.c

@@ -28,6 +28,7 @@
 #include "detect-engine-mpm.h"
 #include "detect-engine-prefilter.h"
 #include "detect-parse.h"
+#include "detect-engine-content-inspection.h"
 
 int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver)
 {
@@ -105,3 +106,27 @@ int DetectHelperKeywordRegister(const SCSigTableElmt *kw)
     DETECT_TBLSIZE_IDX++;
     return DETECT_TBLSIZE_IDX - 1;
 }
+
+InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
+        const int list_id, uint32_t index, MultiGetTxBuffer GetBuf)
+{
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, index);
+    if (buffer == NULL) {
+        return NULL;
+    }
+    if (buffer->initialized) {
+        return buffer;
+    }
+
+    const uint8_t *data = NULL;
+    uint32_t data_len = 0;
+
+    if (!GetBuf(txv, flow_flags, index, &data, &data_len)) {
+        InspectionBufferSetupMultiEmpty(buffer);
+        return NULL;
+    }
+    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    buffer->flags = DETECT_CI_FLAGS_SINGLE;
+    return buffer;
+}

src/detect-engine-helper.h

@@ -32,10 +32,16 @@ int DetectHelperKeywordRegister(const SCSigTableElmt *kw);
 int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver);
 
 typedef bool (*SimpleGetTxBuffer)(void *, uint8_t, const uint8_t **, uint32_t *);
+typedef bool (*MultiGetTxBuffer)(void *, uint8_t, uint32_t, const uint8_t **, uint32_t *);
+
 InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx,
         const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
         const int list_id, SimpleGetTxBuffer GetBuf);
 int DetectHelperBufferMpmRegister(const char *name, const char *desc, AppProto alproto,
         bool toclient, bool toserver, InspectionBufferGetDataPtr GetData);
 
+InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
+        const int list_id, uint32_t index, MultiGetTxBuffer GetBuf);
+
 #endif /* SURICATA_DETECT_ENGINE_HELPER_H */

src/detect-file-data.c

@@ -94,11 +94,14 @@ void DetectFiledataRegister(void)
 }
 
 static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
-    if (de_ctx->filedata_config_initialized)
+    if (de_ctx->filedata_config)
         return;
 
+    de_ctx->filedata_config = SCMalloc(ALPROTO_MAX * sizeof(DetectFileDataCfg));
+    if (unlikely(de_ctx->filedata_config == NULL))
+        return;
     /* initialize default */
-    for (int i = 0; i < (int)ALPROTO_MAX; i++) {
+    for (AppProto i = 0; i < ALPROTO_MAX; i++) {
         de_ctx->filedata_config[i].content_limit = FILEDATA_CONTENT_LIMIT;
         de_ctx->filedata_config[i].content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
     }
@@ -109,8 +112,6 @@ static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
     de_ctx->filedata_config[ALPROTO_SMTP].content_limit = smtp_config.content_limit;
     de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_min_size =
             smtp_config.content_inspect_min_size;
-
-    de_ctx->filedata_config_initialized = true;
 }
 
 /**
@@ -220,9 +221,12 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
 
     const uint64_t file_size = FileDataSize(cur_file);
     const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
-    const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
-    const uint32_t content_inspect_min_size =
-            de_ctx->filedata_config[f->alproto].content_inspect_min_size;
+    uint32_t content_limit = FILEDATA_CONTENT_LIMIT;
+    uint32_t content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
+    if (de_ctx->filedata_config) {
+        content_limit = de_ctx->filedata_config[f->alproto].content_limit;
+        content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
+    }
 
     SCLogDebug("[list %d] content_limit %u, content_inspect_min_size %u", list_id, content_limit,
             content_inspect_min_size);

src/detect-http2.c

@@ -33,6 +33,7 @@
 #include "detect-engine-mpm.h"
 #include "detect-engine-prefilter.h"
 #include "detect-engine-content-inspection.h"
+#include "detect-engine-helper.h"
 
 #include "detect-http2.h"
 #include "util-byte.h"
@@ -102,30 +103,8 @@ static InspectionBuffer *GetHttp2HNameData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f, const uint8_t flags, void *txv,
         int list_id, uint32_t local_id)
 {
-    SCEnter();
-
-    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_id);
-    if (buffer == NULL)
-        return NULL;
-    if (buffer->initialized)
-        return buffer;
-
-    uint32_t b_len = 0;
-    const uint8_t *b = NULL;
-
-    if (rs_http2_tx_get_header_name(txv, flags, local_id, &b, &b_len) != 1) {
-        InspectionBufferSetupMultiEmpty(buffer);
-        return NULL;
-    }
-    if (b == NULL || b_len == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
-        return NULL;
-    }
-
-    InspectionBufferSetupMulti(buffer, transforms, b, b_len);
-    buffer->flags = DETECT_CI_FLAGS_SINGLE;
-
-    SCReturnPtr(buffer, "InspectionBuffer");
+    return DetectHelperGetMultiData(det_ctx, transforms, _f, flags, txv, list_id, local_id,
+            (MultiGetTxBuffer)rs_http2_tx_get_header_name);
 }
 
 void DetectHttp2Register(void)

src/detect.h

@@ -846,6 +846,11 @@ enum DetectEngineType
  */
 #define FLOW_STATES 2
 
+typedef struct {
+    uint32_t content_limit;
+    uint32_t content_inspect_min_size;
+} DetectFileDataCfg;
+
 /** \brief main detection engine ctx */
 typedef struct DetectEngineCtx_ {
     bool failure_fatal;
@@ -945,19 +950,14 @@ typedef struct DetectEngineCtx_ {
     /** The rule errored out due to missing requirements. */
     bool sigerror_requires;
 
-    bool filedata_config_initialized;
-
     /* specify the configuration for mpm context factory */
     uint8_t sgh_mpm_ctx_cnf;
 
     int keyword_id;
     /** hash list of keywords that need thread local ctxs */
     HashListTable *keyword_hash;
 
-    struct {
-        uint32_t content_limit;
-        uint32_t content_inspect_min_size;
-    } filedata_config[ALPROTO_MAX];
+    DetectFileDataCfg *filedata_config;
 
 #ifdef PROFILE_RULES
     struct SCProfileDetectCtx_ *profile_ctx;

src/output-json-alert.c

@@ -295,8 +295,9 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb,
     const AppProto proto = FlowGetAppProtocol(p->flow);
     EveJsonSimpleAppLayerLogger *al = SCEveJsonSimpleGetLogger(proto);
     JsonBuilderMark mark = { 0, 0, 0 };
+    void *state;
     if (al && al->LogTx) {
-        void *state = FlowGetAppState(p->flow);
+        state = FlowGetAppState(p->flow);
         if (state) {
             void *tx = AppLayerParserGetTx(p->flow->proto, proto, state, tx_id);
             if (tx) {
@@ -386,11 +387,11 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb,
             }
             break;
         case ALPROTO_DCERPC:
-            jb_get_mark(jb, &mark);
-            void *state = FlowGetAppState(p->flow);
+            state = FlowGetAppState(p->flow);
             if (state) {
                 void *tx = AppLayerParserGetTx(p->flow->proto, proto, state, tx_id);
                 if (tx) {
+                    jb_get_mark(jb, &mark);
                     jb_open_object(jb, "dcerpc");
                     if (p->proto == IPPROTO_TCP) {
                         if (!rs_dcerpc_log_json_record_tcp(state, tx, jb)) {

src/output-tx.c

@@ -35,12 +35,11 @@
 /** per thread data for this module, contains a list of per thread
  *  data for the packet loggers. */
 typedef struct OutputTxLoggerThreadData_ {
-    OutputLoggerThreadStore *store[ALPROTO_MAX];
-
     /* thread local data from file api */
     OutputFileLoggerThreadData *file;
     /* thread local data from filedata api */
     OutputFiledataLoggerThreadData *filedata;
+    OutputLoggerThreadStore *store[];
 } OutputTxLoggerThreadData;
 
 /* logger instance, a module + a output ctx,
@@ -542,7 +541,8 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
  *  loggers */
 static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data)
 {
-    OutputTxLoggerThreadData *td = SCCalloc(1, sizeof(*td));
+    OutputTxLoggerThreadData *td =
+            SCCalloc(1, sizeof(*td) + ALPROTO_MAX * sizeof(OutputLoggerThreadStore *));
     if (td == NULL)
         return TM_ECODE_FAILED;
 

src/util-profiling.c

@@ -1195,9 +1195,8 @@ PktProfiling *SCProfilePacketStart(void)
 {
     uint64_t sample = SC_ATOMIC_ADD(samples, 1);
     if (sample % rate == 0)
-        return SCCalloc(1, sizeof(PktProfiling));
-    else
-        return NULL;
+        return SCCalloc(1, sizeof(PktProfiling) + ALPROTO_MAX * sizeof(PktProfilingAppData));
+    return NULL;
 }
 
 /* see if we want to profile rules for this packet */

suricata.yaml.in

@@ -164,7 +164,7 @@ outputs:
             # payload: yes             # enable dumping payload in Base64
             # payload-buffer-size: 4kb # max size of payload buffer to output in eve-log
             # payload-printable: yes   # enable dumping payload in printable (lossy) format
-            # payload-length: yes      # enable dumping payload length
+            # payload-length: yes      # enable dumping payload length, including the gaps
             # packet: yes              # enable dumping of packet (without stream segments)
             # metadata: no             # enable inclusion of app layer metadata with alert. Default yes
             # http-body: yes           # Requires metadata; enable dumping of HTTP body in Base64