From 1fa13e4b810e5f68d228880f4bff6a34cc1f9b69 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 1 Mar 2024 13:30:09 +0100 Subject: [PATCH 01/30] util/var: remove printf; add assert --- src/util-var.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util-var.c b/src/util-var.c index 8a2df63f1fa7..37b0032eb755 100644 --- a/src/util-var.c +++ b/src/util-var.c @@ -33,7 +33,7 @@ #include "pkt-var.h" #include "host-bit.h" #include "ippair-bit.h" - +#include "util-validate.h" #include "util-debug.h" void XBitFree(XBit *fb) @@ -81,7 +81,8 @@ void GenericVarFree(GenericVar *gv) } default: { - printf("ERROR: GenericVarFree unknown type %" PRIu32 "\n", gv->type); + SCLogDebug("GenericVarFree unknown type %" PRIu32, gv->type); + DEBUG_VALIDATE_BUG_ON(1); break; } } From a81b23254c10805cb4bed4f7096134f2533f84c4 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 4 Mar 2024 12:29:24 +0100 Subject: [PATCH 02/30] util/var: add comments explaining types --- src/util-var.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util-var.h b/src/util-var.h index c8e7d5af6d0c..fe3010097da7 100644 --- a/src/util-var.h +++ b/src/util-var.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -24,6 +24,7 @@ #ifndef SURICATA_UTIL_VAR_H #define SURICATA_UTIL_VAR_H +/** variable types: these are used to track variable names */ enum VarTypes { VAR_TYPE_NOT_SET, @@ -46,7 +47,7 @@ enum VarTypes { }; typedef struct GenericVar_ { - uint8_t type; + uint8_t type; /**< variable type, uses detection sm_type */ uint8_t pad[3]; uint32_t idx; struct GenericVar_ *next; From 1552f0953ae79ca23d63583a0e4a51cb148455e9 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 27 Feb 2024 11:06:47 +0100 Subject: [PATCH 03/30] detect/threshold: implement tracking 'by_flow' Add support for 'by_flow' track option. This allows using the various threshold options in the context of a single flow. Example: alert tcp ... stream-event:pkt_broken_ack; \ threshold:type limit, track by_flow, count 1, seconds 3600; The example would limit the number of alerts to once per hour for packets triggering the 'pkt_broken_ack' stream event. Implemented as a special "flowvar" holding the threshold entries. This means no synchronization is required, making this a cheaper option compared to the other trackers. Ticket: #6822. --- src/detect-engine-threshold.c | 94 ++++++++++++++++++++++++++++++++++- src/detect-engine-threshold.h | 2 + src/detect-threshold.c | 23 ++++++++- src/detect-threshold.h | 1 + src/util-var.c | 7 ++- 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index b5ac8670d0bb..0c85dbc6ab91 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2021 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -69,6 +69,7 @@ #include "tm-threads.h" #include "action-globals.h" +#include "util-validate.h" static HostStorageId host_threshold_id = { .id = -1 }; /**< host storage id for thresholds */ static IPPairStorageId ippair_threshold_id = { .id = -1 }; /**< ip pair storage id for thresholds */ @@ -241,6 +242,71 @@ static DetectThresholdEntry *ThresholdHostLookupEntry(Host *h, return e; } +/** struct for storing per flow thresholds. This will be stored in the Flow::flowvar list, so it + * needs to follow the GenericVar header format. */ +typedef struct FlowVarThreshold_ { + uint8_t type; + uint8_t pad[7]; + struct GenericVar_ *next; + DetectThresholdEntry *thresholds; +} FlowVarThreshold; + +void FlowThresholdVarFree(void *ptr) +{ + FlowVarThreshold *t = ptr; + ThresholdListFree(t->thresholds); + SCFree(t); +} + +static FlowVarThreshold *FlowThresholdVarGet(Flow *f) +{ + if (f == NULL) + return NULL; + + for (GenericVar *gv = f->flowvar; gv != NULL; gv = gv->next) { + if (gv->type == DETECT_THRESHOLD) + return (FlowVarThreshold *)gv; + } + + return NULL; +} + +static DetectThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) +{ + FlowVarThreshold *t = FlowThresholdVarGet(f); + if (t == NULL) + return NULL; + + for (DetectThresholdEntry *e = t->thresholds; e != NULL; e = e->next) { + if (e->sid == sid && e->gid == gid) { + return e; + } + } + return NULL; +} + +static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time) +{ + DEBUG_VALIDATE_BUG_ON(e == NULL); + + FlowVarThreshold *t = FlowThresholdVarGet(f); + if (t == NULL) { + t = SCCalloc(1, sizeof(*t)); + if (t == NULL) { + return -1; + } + t->type = DETECT_THRESHOLD; + GenericVarAppend(&f->flowvar, (GenericVar *)t); + } + + e->current_count = 1; + e->tv1 = packet_time; + e->tv_timeout = 0; + e->next = t->thresholds; + t->thresholds = e; + return 0; +} + static DetectThresholdEntry *ThresholdIPPairLookupEntry(IPPair *pair, uint32_t sid, uint32_t gid) { @@ -587,6 +653,28 @@ static int ThresholdHandlePacketRule(DetectEngineCtx *de_ctx, Packet *p, return ret; } +/** + * \retval 2 silent match (no alert but apply actions) + * \retval 1 normal match + * \retval 0 no match + */ +static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdData *td, + uint32_t sid, uint32_t gid, PacketAlert *pa) +{ + int ret = 0; + DetectThresholdEntry *lookup_tsh = ThresholdFlowLookupEntry(f, sid, gid); + SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); + + DetectThresholdEntry *new_tsh = NULL; + ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); + if (new_tsh != NULL) { + if (AddEntryToFlow(f, new_tsh, p->ts) == -1) { + SCFree(new_tsh); + } + } + return ret; +} + /** * \brief Make the threshold logic for signatures * @@ -633,6 +721,10 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCMutexLock(&de_ctx->ths_ctx.threshold_table_lock); ret = ThresholdHandlePacketRule(de_ctx,p,td,s,pa); SCMutexUnlock(&de_ctx->ths_ctx.threshold_table_lock); + } else if (td->track == TRACK_FLOW) { + if (p->flow) { + ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa); + } } SCReturnInt(ret); diff --git a/src/detect-engine-threshold.h b/src/detect-engine-threshold.h index 1516359a4b42..ff76374fcaae 100644 --- a/src/detect-engine-threshold.h +++ b/src/detect-engine-threshold.h @@ -51,4 +51,6 @@ int ThresholdHostTimeoutCheck(Host *, SCTime_t); int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t); void ThresholdListFree(void *ptr); +void FlowThresholdVarFree(void *ptr); + #endif /* SURICATA_DETECT_ENGINE_THRESHOLD_H */ diff --git a/src/detect-threshold.c b/src/detect-threshold.c index 98eb3ce8dc03..d952b5ec36fc 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -60,7 +60,12 @@ #include "util-cpu.h" #endif -#define PARSE_REGEX "^\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*" +#define PARSE_REGEX \ + "^\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|by_" \ + "flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_" \ + "both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_" \ + "dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|" \ + "threshold|by_dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*" static DetectParseRegex parse_regex; @@ -183,6 +188,8 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) de->track = TRACK_BOTH; if (strncasecmp(args[i],"by_rule",strlen("by_rule")) == 0) de->track = TRACK_RULE; + if (strncasecmp(args[i], "by_flow", strlen("by_flow")) == 0) + de->track = TRACK_FLOW; if (strncasecmp(args[i],"count",strlen("count")) == 0) count_pos = i+1; if (strncasecmp(args[i],"seconds",strlen("seconds")) == 0) @@ -342,6 +349,7 @@ DetectThresholdData *DetectThresholdDataCopy(DetectThresholdData *de) #include "util-hashlist.h" #include "packet.h" #include "action-globals.h" + /** * \test ThresholdTestParse01 is a test for a valid threshold options * @@ -360,6 +368,18 @@ static int ThresholdTestParse01(void) return 0; } +static int ThresholdTestParseByFlow01(void) +{ + DetectThresholdData *de = DetectThresholdParse("type limit,track by_flow,count 1,seconds 60"); + FAIL_IF_NULL(de); + FAIL_IF_NOT(de->type == TYPE_LIMIT); + FAIL_IF_NOT(de->track == TRACK_FLOW); + FAIL_IF_NOT(de->count == 1); + FAIL_IF_NOT(de->seconds == 60); + DetectThresholdFree(NULL, de); + PASS; +} + /** * \test ThresholdTestParse02 is a test for a invalid threshold options * @@ -1692,6 +1712,7 @@ static int DetectThresholdTestSig14(void) static void ThresholdRegisterTests(void) { UtRegisterTest("ThresholdTestParse01", ThresholdTestParse01); + UtRegisterTest("ThresholdTestParseByFlow01", ThresholdTestParseByFlow01); UtRegisterTest("ThresholdTestParse02", ThresholdTestParse02); UtRegisterTest("ThresholdTestParse03", ThresholdTestParse03); UtRegisterTest("ThresholdTestParse04", ThresholdTestParse04); diff --git a/src/detect-threshold.h b/src/detect-threshold.h index 2648feb0e4ce..5321fd54bdac 100644 --- a/src/detect-threshold.h +++ b/src/detect-threshold.h @@ -36,6 +36,7 @@ #define TRACK_RULE 3 #define TRACK_EITHER 4 /**< either src or dst: only used by suppress */ #define TRACK_BOTH 5 /* used by rate_filter to match detections by both src and dst addresses */ +#define TRACK_FLOW 6 /**< track by flow */ /* Get the new action to take */ #define TH_ACTION_ALERT 0x01 diff --git a/src/util-var.c b/src/util-var.c index 37b0032eb755..6ef3fe638f10 100644 --- a/src/util-var.c +++ b/src/util-var.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -25,6 +25,7 @@ #include "suricata-common.h" #include "detect.h" +#include "detect-engine-threshold.h" #include "util-var.h" @@ -67,6 +68,10 @@ void GenericVarFree(GenericVar *gv) XBitFree(fb); break; } + case DETECT_THRESHOLD: { + FlowThresholdVarFree(gv); + break; + } case DETECT_FLOWVAR: { FlowVar *fv = (FlowVar *)gv; From cfd55ead74e2b2180ba2e2eeccd1b9d774073546 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 2 Mar 2024 07:58:30 +0100 Subject: [PATCH 04/30] threshold: add by_flow support for global thresholds Allow rate_filter and thresholds from the global config to specify tracking "by_flow". --- src/detect-engine-threshold.c | 1 + src/util-threshold-config.c | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 0c85dbc6ab91..f430e66b2656 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -343,6 +343,7 @@ static int ThresholdHandlePacketSuppress(Packet *p, } break; case TRACK_RULE: + case TRACK_FLOW: default: SCLogError("track mode %d is not supported", td->track); break; diff --git a/src/util-threshold-config.c b/src/util-threshold-config.c index d12c89e07f3c..998dde56a9e5 100644 --- a/src/util-threshold-config.c +++ b/src/util-threshold-config.c @@ -66,11 +66,15 @@ static FILE *g_ut_threshold_fp = NULL; #define DETECT_BASE_REGEX "^\\s*(event_filter|threshold|rate_filter|suppress)\\s*gen_id\\s*(\\d+)\\s*,\\s*sig_id\\s*(\\d+)\\s*(.*)\\s*$" #define DETECT_THRESHOLD_REGEX \ - "^,\\s*type\\s*(limit|both|threshold)\\s*,\\s*track\\s*(by_dst|by_src|by_both|by_rule)\\s*," \ + "^,\\s*type\\s*(limit|both|threshold)\\s*,\\s*track\\s*(by_dst|by_src|by_both|by_rule|by_" \ + "flow)\\s*," \ "\\s*count\\s*(\\d+)\\s*,\\s*seconds\\s*(\\d+)\\s*$" /* TODO: "apply_to" */ -#define DETECT_RATE_REGEX "^,\\s*track\\s*(by_dst|by_src|by_both|by_rule)\\s*,\\s*count\\s*(\\d+)\\s*,\\s*seconds\\s*(\\d+)\\s*,\\s*new_action\\s*(alert|drop|pass|log|sdrop|reject)\\s*,\\s*timeout\\s*(\\d+)\\s*$" +#define DETECT_RATE_REGEX \ + "^,\\s*track\\s*(by_dst|by_src|by_both|by_rule|by_flow)\\s*,\\s*count\\s*(\\d+)\\s*,\\s*" \ + "seconds\\s*(\\d+)\\s*,\\s*new_action\\s*(alert|drop|pass|log|sdrop|reject)\\s*,\\s*" \ + "timeout\\s*(\\d+)\\s*$" /* * suppress has two form: @@ -793,6 +797,8 @@ static int ParseThresholdRule(const DetectEngineCtx *de_ctx, char *rawstr, uint3 } else if (strcasecmp(th_track,"by_rule") == 0) parsed_track = TRACK_RULE; + else if (strcasecmp(th_track, "by_flow") == 0) + parsed_track = TRACK_FLOW; else { SCLogError("Invalid track parameter %s in %s", th_track, rawstr); goto error; From 022173d7ab29bd202591c477f82f60de02b14758 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 4 Mar 2024 12:53:35 +0100 Subject: [PATCH 05/30] detect: group types used in traffic variables Traffic variables (flowvars, flowbits, xbits, etc) use a smaller int for their type than detection types. As a workaround make sure the values fit in a uint8_t. --- src/detect-engine-register.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 94f8de15de83..1c7c03bf8145 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -54,7 +54,18 @@ enum DetectKeywordId { DETECT_FLOW, /* end prefilter sort */ + /* values used in util-var.c go here, to avoid int overflows + * TODO update var logic to use a larger type. */ DETECT_THRESHOLD, + DETECT_FLOWBITS, + DETECT_FLOWVAR, + DETECT_FLOWVAR_POSTMATCH, + DETECT_FLOWINT, + DETECT_HOSTBITS, + DETECT_XBITS, + DETECT_PKTVAR, + /* end util-var.c logic */ + DETECT_METADATA, DETECT_REFERENCE, DETECT_TAG, @@ -82,14 +93,8 @@ enum DetectKeywordId { DETECT_ISDATAAT, DETECT_ID, DETECT_RPC, - DETECT_FLOWVAR, - DETECT_FLOWVAR_POSTMATCH, - DETECT_FLOWINT, - DETECT_PKTVAR, DETECT_NOALERT, DETECT_ALERT, - DETECT_FLOWBITS, - DETECT_HOSTBITS, DETECT_IPV4_CSUM, DETECT_TCPV4_CSUM, DETECT_TCPV6_CSUM, @@ -259,7 +264,6 @@ enum DetectKeywordId { DETECT_AL_DNP3IND, DETECT_AL_DNP3OBJ, - DETECT_XBITS, DETECT_BASE64_DECODE, DETECT_BASE64_DATA, From d0f3f2d4628d6b5d42d5b9a6317d009e3e9d8127 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 5 Mar 2024 10:02:53 +0100 Subject: [PATCH 06/30] detect: group content inspect keyword id's --- src/detect-engine-register.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 1c7c03bf8145..1a8a78282553 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2017 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -66,10 +66,7 @@ enum DetectKeywordId { DETECT_PKTVAR, /* end util-var.c logic */ - DETECT_METADATA, - DETECT_REFERENCE, - DETECT_TAG, - DETECT_MSG, + /* content inspection */ DETECT_CONTENT, DETECT_URICONTENT, DETECT_PCRE, @@ -86,11 +83,26 @@ enum DetectKeywordId { DETECT_BYTETEST, DETECT_BYTEJUMP, DETECT_BYTEMATH, + DETECT_BYTE_EXTRACT, + DETECT_DATASET, + DETECT_DATAREP, + DETECT_BASE64_DECODE, + DETECT_BASE64_DATA, + DETECT_BSIZE, + DETECT_ASN1, + DETECT_LUA, + DETECT_ISDATAAT, + DETECT_AL_URILEN, + /* end of content inspection */ + + DETECT_METADATA, + DETECT_REFERENCE, + DETECT_TAG, + DETECT_MSG, DETECT_SAMEIP, DETECT_GEOIP, DETECT_IPPROTO, DETECT_FTPBOUNCE, - DETECT_ISDATAAT, DETECT_ID, DETECT_RPC, DETECT_NOALERT, @@ -104,15 +116,11 @@ enum DetectKeywordId { DETECT_ICMPV6_CSUM, DETECT_STREAM_SIZE, DETECT_DETECTION_FILTER, - DETECT_DATASET, - DETECT_DATAREP, DETECT_DECODE_EVENT, DETECT_GID, DETECT_MARK, - DETECT_BSIZE, - DETECT_FRAME, DETECT_FLOW_AGE, @@ -141,7 +149,6 @@ enum DetectKeywordId { DETECT_HTTP_METHOD, DETECT_AL_HTTP_PROTOCOL, DETECT_AL_HTTP_START, - DETECT_AL_URILEN, DETECT_AL_HTTP_CLIENT_BODY, DETECT_HTTP_REQUEST_BODY, DETECT_AL_HTTP_SERVER_BODY, @@ -188,7 +195,6 @@ enum DetectKeywordId { DETECT_AL_SSH_HASSH_SERVER_STRING, DETECT_AL_SSL_VERSION, DETECT_AL_SSL_STATE, - DETECT_BYTE_EXTRACT, DETECT_FILE_DATA, DETECT_PKT_DATA, DETECT_AL_APP_LAYER_EVENT, @@ -212,8 +218,6 @@ enum DetectKeywordId { DETECT_SMB_NTLMSSP_DOMAIN, DETECT_SMB_VERSION, - DETECT_ASN1, - DETECT_ENGINE_EVENT, DETECT_STREAM_EVENT, @@ -232,7 +236,6 @@ enum DetectKeywordId { DETECT_FILESIZE, DETECT_L3PROTO, - DETECT_LUA, DETECT_IPREP, DETECT_AL_DNS_QUERY, @@ -264,9 +267,6 @@ enum DetectKeywordId { DETECT_AL_DNP3IND, DETECT_AL_DNP3OBJ, - DETECT_BASE64_DECODE, - DETECT_BASE64_DATA, - DETECT_AL_KRB5_ERRCODE, DETECT_AL_KRB5_MSGTYPE, DETECT_AL_KRB5_CNAME, From f02864875068e58609e4c9738a874ea32cebcfe8 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 5 Mar 2024 10:03:15 +0100 Subject: [PATCH 07/30] detect/content: fix wrong value for depth check Limits propegation checked for DETECT_DEPTH as a content flag, which appears to have worked by chance. After reshuffling the keyword id's it no longer worked. This patch uses the proper flag DETECT_CONTENT_DEPTH. --- src/detect-content.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detect-content.c b/src/detect-content.c index 2f9884000523..6d3852ecc56f 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -556,7 +556,7 @@ static void PropagateLimits(Signature *s, SigMatch *sm_head) SCLogDebug("stored: offset %u depth %u offset_plus_pat %u", offset, depth, offset_plus_pat); - if ((cd->flags & (DETECT_DEPTH | DETECT_CONTENT_WITHIN)) == 0) { + if ((cd->flags & (DETECT_CONTENT_DEPTH | DETECT_CONTENT_WITHIN)) == 0) { if (depth) SCLogDebug("no within, reset depth"); depth = 0; From 3f04af7c7fb504ab1a38af671cf74b69078590bc Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 8 Mar 2024 11:50:14 +0100 Subject: [PATCH 08/30] doc: add thresholding by_flow --- .../configuration/global-thresholds.rst | 3 ++- doc/userguide/rules/thresholding.rst | 23 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/doc/userguide/configuration/global-thresholds.rst b/doc/userguide/configuration/global-thresholds.rst index d268dd7c3ed5..a5b3bd6d8675 100644 --- a/doc/userguide/configuration/global-thresholds.rst +++ b/doc/userguide/configuration/global-thresholds.rst @@ -20,7 +20,7 @@ Syntax: :: threshold gen_id , sig_id , type , \ - track , count , seconds + track , count , seconds rate_filter ~~~~~~~~~~~ @@ -55,6 +55,7 @@ done per IP-address. The Host table is used for storage. When using by_rule it's done globally for the rule. Option by_both used to track per IP pair of source and destination. Packets going to opposite directions between same addresses tracked as the same pair. +The by_flow option tracks the rule matches in the flow. count ^^^^^ diff --git a/doc/userguide/rules/thresholding.rst b/doc/userguide/rules/thresholding.rst index 401f5736967c..e3e5830f7ffb 100644 --- a/doc/userguide/rules/thresholding.rst +++ b/doc/userguide/rules/thresholding.rst @@ -16,7 +16,7 @@ frequency. It has 3 modes: threshold, limit and both. Syntax:: - threshold: type , track , count , seconds + threshold: type , track , count , seconds type "threshold" ~~~~~~~~~~~~~~~~ @@ -85,6 +85,27 @@ performed for each of the matches. *Rule actions drop (IPS mode) and reject are applied to each packet.* + +track +~~~~~ + +.. table:: + + +------------------+--------------------------+ + |Option |Tracks By | + +==================+==========================+ + |by_src |source IP | + +------------------+--------------------------+ + |by_dst |destination IP | + +------------------+--------------------------+ + |by_both |pair of src IP and dst IP | + +------------------+--------------------------+ + |by_rule |signature id | + +------------------+--------------------------+ + |by_flow |flow | + +------------------+--------------------------+ + + detection_filter ---------------- From 405491c3fcdd8c30cdd66e3ef922f0b8a8717a9e Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 13 Mar 2024 11:01:28 +0100 Subject: [PATCH 09/30] detect/detection_filter: add support for track by_flow --- doc/userguide/rules/thresholding.rst | 2 +- src/detect-detection-filter.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/userguide/rules/thresholding.rst b/doc/userguide/rules/thresholding.rst index e3e5830f7ffb..44a8aa5e92d4 100644 --- a/doc/userguide/rules/thresholding.rst +++ b/doc/userguide/rules/thresholding.rst @@ -118,7 +118,7 @@ again. Syntax:: - detection_filter: track , count , seconds + detection_filter: track , count , seconds Example:: diff --git a/src/detect-detection-filter.c b/src/detect-detection-filter.c index cbd1898a31a4..47ce00829880 100644 --- a/src/detect-detection-filter.c +++ b/src/detect-detection-filter.c @@ -47,8 +47,9 @@ *\brief Regex for parsing our detection_filter options */ #define PARSE_REGEX \ - "^\\s*(track|count|seconds)\\s+(by_src|by_dst|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|" \ - "by_dst|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|\\d+)\\s*$" + "^\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+" \ + "(by_src|" \ + "by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*$" static DetectParseRegex parse_regex; @@ -158,6 +159,8 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) df->track = TRACK_DST; if (strncasecmp(args[i], "by_src", strlen("by_src")) == 0) df->track = TRACK_SRC; + if (strncasecmp(args[i], "by_flow", strlen("by_flow")) == 0) + df->track = TRACK_FLOW; if (strncasecmp(args[i], "count", strlen("count")) == 0) count_pos = i + 1; if (strncasecmp(args[i], "seconds", strlen("seconds")) == 0) From c9631584434e33ed5fc20f19c66c5420a8d94f7a Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 13 Mar 2024 11:03:01 +0100 Subject: [PATCH 10/30] detect: add ticket id to var related todos --- src/detect-engine-register.h | 2 +- src/util-var.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 1a8a78282553..04619cb01efb 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -55,7 +55,7 @@ enum DetectKeywordId { /* end prefilter sort */ /* values used in util-var.c go here, to avoid int overflows - * TODO update var logic to use a larger type. */ + * TODO update var logic to use a larger type, see #6855. */ DETECT_THRESHOLD, DETECT_FLOWBITS, DETECT_FLOWVAR, diff --git a/src/util-var.h b/src/util-var.h index fe3010097da7..4732b46e479c 100644 --- a/src/util-var.h +++ b/src/util-var.h @@ -46,6 +46,7 @@ enum VarTypes { VAR_TYPE_IPPAIR_VAR, }; +/** \todo see ticket #6855. The type field should be 16 bits. */ typedef struct GenericVar_ { uint8_t type; /**< variable type, uses detection sm_type */ uint8_t pad[3]; From c08c81cacf9a0ef9e4e38f1e57e9a6dc18846fe4 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 11 Sep 2023 21:17:36 +0200 Subject: [PATCH 11/30] detect/threshold: implement per thread cache Thresholding often has 2 stages: 1. recording matches 2. appling an action, like suppress E.g. with something like: threshold:type limit, count 10, seconds 3600, track by_src; the recording state is about counting 10 first hits for an IP, then followed by the "suppress" state that might last an hour. By_src/by_dst are expensive, as they do a host table lookup and lock the host. If many threads require this access, lock contention becomes a serious problem. This patch adds a thread local cache to avoid the synchronization overhead. When the threshold for a host enters the "apply" stage, a thread local hash entry is added. This entry knows the expiry time and the action to apply. This way the action can be applied w/o the synchronization overhead. A rbtree is used to handle expiration. Implemented for IPv4. --- src/detect-engine-threshold.c | 231 ++++++++++++++++++++++++++++++++++ src/detect-engine-threshold.h | 1 + src/detect-engine.c | 2 + 3 files changed, 234 insertions(+) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index f430e66b2656..3f7005de7b9c 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -101,6 +101,217 @@ int ThresholdIPPairHasThreshold(IPPair *pair) return IPPairGetStorageById(pair, ippair_threshold_id) ? 1 : 0; } +#include "util-hash.h" + +typedef struct ThresholdCacheItem { + int8_t track; // by_src/by_dst + int8_t ipv; + int8_t retval; + uint32_t addr; + uint32_t sid; + SCTime_t expires_at; + RB_ENTRY(ThresholdCacheItem) rb; +} ThresholdCacheItem; + +static thread_local HashTable *threshold_cache_ht = NULL; + +thread_local uint64_t cache_lookup_cnt = 0; +thread_local uint64_t cache_lookup_notinit = 0; +thread_local uint64_t cache_lookup_nosupport = 0; +thread_local uint64_t cache_lookup_miss_expired = 0; +thread_local uint64_t cache_lookup_miss = 0; +thread_local uint64_t cache_lookup_hit = 0; +thread_local uint64_t cache_housekeeping_check = 0; +thread_local uint64_t cache_housekeeping_expired = 0; + +static void DumpCacheStats(void) +{ + SCLogPerf("threshold thread cache stats: cnt:%" PRIu64 " notinit:%" PRIu64 " nosupport:%" PRIu64 + " miss_expired:%" PRIu64 " miss:%" PRIu64 " hit:%" PRIu64 + ", housekeeping: checks:%" PRIu64 ", expired:%" PRIu64, + cache_lookup_cnt, cache_lookup_notinit, cache_lookup_nosupport, + cache_lookup_miss_expired, cache_lookup_miss, cache_lookup_hit, + cache_housekeeping_check, cache_housekeeping_expired); +} + +/* rbtree for expiry handling */ + +static int ThresholdCacheTreeCompareFunc(ThresholdCacheItem *a, ThresholdCacheItem *b) +{ + if (SCTIME_CMP_GTE(a->expires_at, b->expires_at)) { + return 1; + } else { + return -1; + } +} + +RB_HEAD(THRESHOLD_CACHE, ThresholdCacheItem); +RB_PROTOTYPE(THRESHOLD_CACHE, ThresholdCacheItem, rb, ThresholdCacheTreeCompareFunc); +RB_GENERATE(THRESHOLD_CACHE, ThresholdCacheItem, rb, ThresholdCacheTreeCompareFunc); +thread_local struct THRESHOLD_CACHE threshold_cache_tree; +thread_local uint64_t threshold_cache_housekeeping_ts = 0; + +static void ThresholdCacheExpire(SCTime_t now) +{ + ThresholdCacheItem *iter, *safe = NULL; + int cnt = 0; + threshold_cache_housekeeping_ts = SCTIME_SECS(now); + + RB_FOREACH_SAFE (iter, THRESHOLD_CACHE, &threshold_cache_tree, safe) { + cache_housekeeping_check++; + + if (SCTIME_CMP_LT(iter->expires_at, now)) { + THRESHOLD_CACHE_RB_REMOVE(&threshold_cache_tree, iter); + HashTableRemove(threshold_cache_ht, iter, 0); + SCLogDebug("iter %p expired", iter); + cache_housekeeping_expired++; + } + + if (++cnt > 1) + break; + } +} + +/* hash table for threshold look ups */ + +static uint32_t ThresholdCacheHashFunc(HashTable *ht, void *data, uint16_t datalen) +{ + ThresholdCacheItem *tci = data; + int hash = tci->ipv * tci->track + tci->addr + tci->sid; + hash = hash % ht->array_size; + return hash; +} + +static char ThresholdCacheHashCompareFunc( + void *data1, uint16_t datalen1, void *data2, uint16_t datalen2) +{ + ThresholdCacheItem *tci1 = data1; + ThresholdCacheItem *tci2 = data2; + return tci1->ipv == tci2->ipv && tci1->track == tci2->track && tci1->addr == tci2->addr && + tci1->sid == tci2->sid; +} + +static void ThresholdCacheHashFreeFunc(void *data) +{ + SCFree(data); +} + +/// \brief Thread local cache +static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, const uint32_t sid, + SCTime_t expires) +{ + if (!threshold_cache_ht) { + threshold_cache_ht = HashTableInit(256, ThresholdCacheHashFunc, + ThresholdCacheHashCompareFunc, ThresholdCacheHashFreeFunc); + } + + uint32_t addr; + if (track == TRACK_SRC) { + addr = p->src.addr_data32[0]; + } else if (track == TRACK_DST) { + addr = p->dst.addr_data32[0]; + } else { + return -1; + } + + ThresholdCacheItem lookup = { + .track = track, + .ipv = 4, + .retval = retval, + .addr = addr, + .sid = sid, + .expires_at = expires, + }; + ThresholdCacheItem *found = HashTableLookup(threshold_cache_ht, &lookup, 0); + if (!found) { + ThresholdCacheItem *n = SCCalloc(1, sizeof(*n)); + if (n) { + n->track = track; + n->ipv = 4; + n->retval = retval; + n->addr = addr; + n->sid = sid; + n->expires_at = expires; + + if (HashTableAdd(threshold_cache_ht, n, 0) == 0) { + (void)THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n); + return 1; + } + SCFree(n); + } + return -1; + } else { + found->expires_at = expires; + found->retval = retval; + + THRESHOLD_CACHE_RB_REMOVE(&threshold_cache_tree, found); + THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, found); + return 1; + } +} + +/** \brief Check Thread local thresholding cache + * \note only supports IPv4 + * \retval -1 cache miss - not found + * \retval -2 cache miss - found but expired + * \retval -3 error - cache not initialized + * \retval -4 error - unsupported tracker + * \retval ret cached return code + */ +static int CheckCache(const Packet *p, const int8_t track, const uint32_t sid) +{ + cache_lookup_cnt++; + + if (!threshold_cache_ht) { + cache_lookup_notinit++; + return -3; // error cache initialized + } + + uint32_t addr; + if (track == TRACK_SRC) { + addr = p->src.addr_data32[0]; + } else if (track == TRACK_DST) { + addr = p->dst.addr_data32[0]; + } else { + cache_lookup_nosupport++; + return -4; // error tracker not unsupported + } + + if (SCTIME_SECS(p->ts) > threshold_cache_housekeeping_ts) { + ThresholdCacheExpire(p->ts); + } + + ThresholdCacheItem lookup = { + .track = track, + .ipv = 4, + .addr = addr, + .sid = sid, + }; + ThresholdCacheItem *found = HashTableLookup(threshold_cache_ht, &lookup, 0); + if (found) { + if (SCTIME_CMP_GT(p->ts, found->expires_at)) { + THRESHOLD_CACHE_RB_REMOVE(&threshold_cache_tree, found); + HashTableRemove(threshold_cache_ht, found, 0); + cache_lookup_miss_expired++; + return -2; // cache miss - found but expired + } + cache_lookup_hit++; + return found->retval; + } + cache_lookup_miss++; + return -1; // cache miss - not found +} + +void ThresholdCacheThreadFree(void) +{ + if (threshold_cache_ht) { + HashTableFree(threshold_cache_ht); + threshold_cache_ht = NULL; + } + RB_INIT(&threshold_cache_tree); + DumpCacheStats(); +} + /** * \brief Return next DetectThresholdData for signature * @@ -478,6 +689,10 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, ret = 1; } else { ret = 2; + + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); + } } } else { lookup_tsh->tv1 = p->ts; @@ -533,6 +748,10 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, } else if (lookup_tsh->current_count > td->count) { /* silent match */ ret = 2; + + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); + } } } else { /* expired, so reset */ @@ -701,12 +920,24 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx if (td->type == TYPE_SUPPRESS) { ret = ThresholdHandlePacketSuppress(p,td,s->id,s->gid); } else if (td->track == TRACK_SRC) { + if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { + int cache_ret = CheckCache(p, td->track, s->id); + if (cache_ret >= 0) { + SCReturnInt(cache_ret); + } + } Host *src = HostGetHostFromHash(&p->src); if (src) { ret = ThresholdHandlePacketHost(src,p,td,s->id,s->gid,pa); HostRelease(src); } } else if (td->track == TRACK_DST) { + if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { + int cache_ret = CheckCache(p, td->track, s->id); + if (cache_ret >= 0) { + SCReturnInt(cache_ret); + } + } Host *dst = HostGetHostFromHash(&p->dst); if (dst) { ret = ThresholdHandlePacketHost(dst,p,td,s->id,s->gid,pa); diff --git a/src/detect-engine-threshold.h b/src/detect-engine-threshold.h index ff76374fcaae..b7e53842433d 100644 --- a/src/detect-engine-threshold.h +++ b/src/detect-engine-threshold.h @@ -50,6 +50,7 @@ void ThresholdContextDestroy(DetectEngineCtx *); int ThresholdHostTimeoutCheck(Host *, SCTime_t); int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t); void ThresholdListFree(void *ptr); +void ThresholdCacheThreadFree(void); void FlowThresholdVarFree(void *ptr); diff --git a/src/detect-engine.c b/src/detect-engine.c index 819fa04a39cd..3ec446168709 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -3526,6 +3526,8 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events); SCFree(det_ctx); + + ThresholdCacheThreadFree(); } TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data) From 6622dc7444b604ccbbc7ba8a1331ef04a4a14e9d Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 16:10:30 +0100 Subject: [PATCH 12/30] detect/threshold: minor cleanup --- src/detect-engine-threshold.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 3f7005de7b9c..220330828880 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -982,19 +982,18 @@ void ThresholdHashInit(DetectEngineCtx *de_ctx) */ void ThresholdHashAllocate(DetectEngineCtx *de_ctx) { - Signature *s = de_ctx->sig_list; + const Signature *s = de_ctx->sig_list; bool has_by_rule_tracking = false; - const DetectThresholdData *td = NULL; - const SigMatchData *smd; /* Find the signature with the highest signature number that is using thresholding with by_rule tracking. */ uint32_t highest_signum = 0; while (s != NULL) { if (s->sm_arrays[DETECT_SM_LIST_SUPPRESS] != NULL) { - smd = NULL; + const SigMatchData *smd = NULL; do { - td = SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_SUPPRESS); + const DetectThresholdData *td = + SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_SUPPRESS); if (td == NULL) { continue; } @@ -1009,9 +1008,10 @@ void ThresholdHashAllocate(DetectEngineCtx *de_ctx) } if (s->sm_arrays[DETECT_SM_LIST_THRESHOLD] != NULL) { - smd = NULL; + const SigMatchData *smd = NULL; do { - td = SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_THRESHOLD); + const DetectThresholdData *td = + SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_THRESHOLD); if (td == NULL) { continue; } From ab5e04525f6999becf1512d9cd48567265a2fb14 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 16:39:22 +0100 Subject: [PATCH 13/30] detect/threshold: minor code cleanup Packet pointer is not used during allocation. --- src/detect-engine-threshold.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 220330828880..36655983cfcb 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -421,9 +421,8 @@ int ThresholdIPPairTimeoutCheck(IPPair *pair, SCTime_t ts) return new_head == NULL; } -static DetectThresholdEntry * -DetectThresholdEntryAlloc(const DetectThresholdData *td, Packet *p, - uint32_t sid, uint32_t gid) +static DetectThresholdEntry *DetectThresholdEntryAlloc( + const DetectThresholdData *td, uint32_t sid, uint32_t gid) { SCEnter(); @@ -701,7 +700,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, ret = 1; } } else { - *new_tsh = DetectThresholdEntryAlloc(td, p, sid, gid); + *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); ret = 1; } @@ -728,7 +727,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, if (td->count == 1) { ret = 1; } else { - *new_tsh = DetectThresholdEntryAlloc(td, p, sid, gid); + *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); } } break; @@ -764,7 +763,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, } } } else { - *new_tsh = DetectThresholdEntryAlloc(td, p, sid, gid); + *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); /* for the first match we return 1 to * indicate we should alert */ @@ -793,7 +792,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, lookup_tsh->current_count = 1; } } else { - *new_tsh = DetectThresholdEntryAlloc(td, p, sid, gid); + *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); } break; } @@ -805,7 +804,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, if (lookup_tsh && IsThresholdReached(lookup_tsh, td, p->ts)) { RateFilterSetAction(p, pa, td->new_action); } else if (!lookup_tsh) { - *new_tsh = DetectThresholdEntryAlloc(td, p, sid, gid); + *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); } break; } From 3a7247b1ed88b23ee9d21fc5e3db955678e81825 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 20:18:21 +0100 Subject: [PATCH 14/30] detect/threshold: minor rate filter cleanup --- src/detect-engine-threshold.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 36655983cfcb..1100a877ad0a 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -566,7 +566,7 @@ static int ThresholdHandlePacketSuppress(Packet *p, return ret; } -static inline void RateFilterSetAction(Packet *p, PacketAlert *pa, uint8_t new_action) +static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) { switch (new_action) { case TH_ACTION_ALERT: @@ -802,7 +802,7 @@ static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, SCLogDebug("rate_filter"); ret = 1; if (lookup_tsh && IsThresholdReached(lookup_tsh, td, p->ts)) { - RateFilterSetAction(p, pa, td->new_action); + RateFilterSetAction(pa, td->new_action); } else if (!lookup_tsh) { *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); } From 114fc37294cbbceb6b72e5ccb72171ad8b98edc6 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 21:53:35 +0100 Subject: [PATCH 15/30] detect/address: constify ipv6 cmp funcs --- src/detect-engine-address-ipv6.c | 10 +++++----- src/detect-engine-address-ipv6.h | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/detect-engine-address-ipv6.c b/src/detect-engine-address-ipv6.c index 4728662e0a75..489bf99925b6 100644 --- a/src/detect-engine-address-ipv6.c +++ b/src/detect-engine-address-ipv6.c @@ -49,7 +49,7 @@ * \retval 1 If a < b. * \retval 0 Otherwise, i.e. a >= b. */ -int AddressIPv6Lt(Address *a, Address *b) +int AddressIPv6Lt(const Address *a, const Address *b) { int i = 0; @@ -87,7 +87,7 @@ int AddressIPv6LtU32(uint32_t *a, uint32_t *b) * \retval 1 If a > b. * \retval 0 Otherwise, i.e. a <= b. */ -int AddressIPv6Gt(Address *a, Address *b) +int AddressIPv6Gt(const Address *a, const Address *b) { int i = 0; @@ -125,7 +125,7 @@ int AddressIPv6GtU32(uint32_t *a, uint32_t *b) * \retval 1 If a == b. * \retval 0 Otherwise. */ -int AddressIPv6Eq(Address *a, Address *b) +int AddressIPv6Eq(const Address *a, const Address *b) { int i = 0; @@ -159,7 +159,7 @@ int AddressIPv6EqU32(uint32_t *a, uint32_t *b) * \retval 1 If a <= b. * \retval 0 Otherwise, i.e. a > b. */ -int AddressIPv6Le(Address *a, Address *b) +int AddressIPv6Le(const Address *a, const Address *b) { if (AddressIPv6Eq(a, b) == 1) @@ -191,7 +191,7 @@ int AddressIPv6LeU32(uint32_t *a, uint32_t *b) * \retval 1 If a >= b. * \retval 0 Otherwise, i.e. a < b. */ -int AddressIPv6Ge(Address *a, Address *b) +int AddressIPv6Ge(const Address *a, const Address *b) { if (AddressIPv6Eq(a, b) == 1) diff --git a/src/detect-engine-address-ipv6.h b/src/detect-engine-address-ipv6.h index 96727ee27691..19fcb0d52712 100644 --- a/src/detect-engine-address-ipv6.h +++ b/src/detect-engine-address-ipv6.h @@ -24,11 +24,11 @@ #ifndef SURICATA_DETECT_ENGINE_ADDRESS_IPV6_H #define SURICATA_DETECT_ENGINE_ADDRESS_IPV6_H -int AddressIPv6Lt(Address *, Address *); -int AddressIPv6Gt(Address *, Address *); -int AddressIPv6Eq(Address *, Address *); -int AddressIPv6Le(Address *, Address *); -int AddressIPv6Ge(Address *, Address *); +int AddressIPv6Lt(const Address *, const Address *); +int AddressIPv6Gt(const Address *, const Address *); +int AddressIPv6Eq(const Address *, const Address *); +int AddressIPv6Le(const Address *, const Address *); +int AddressIPv6Ge(const Address *, const Address *); int AddressIPv6LeU32(uint32_t *a, uint32_t *b); int AddressIPv6LtU32(uint32_t *a, uint32_t *b); From 00e1e89449706ab4814f94e316d2890871ef4090 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 10 Jan 2024 09:49:10 +0100 Subject: [PATCH 16/30] thash: add expiration logic Add a callback and helper function to handle data expiration. Update datasets to explicitly not use expiration. --- src/app-layer-htp-range.c | 11 ++++---- src/datasets.c | 10 +++---- src/util-thash.c | 59 +++++++++++++++++++++++++++++++++++++-- src/util-thash.h | 7 +++-- 4 files changed, 73 insertions(+), 14 deletions(-) diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c index 394966ac94e0..f7f110ca4fdd 100644 --- a/src/app-layer-htp-range.c +++ b/src/app-layer-htp-range.c @@ -127,8 +127,9 @@ static void ContainerUrlRangeFree(void *s) } } -static inline bool ContainerValueRangeTimeout(HttpRangeContainerFile *cu, const SCTime_t ts) +static inline bool ContainerValueRangeTimeout(void *data, const SCTime_t ts) { + HttpRangeContainerFile *cu = data; // we only timeout if we have no flow referencing us if (SCTIME_CMP_GT(ts, cu->expire) || cu->error) { if (SC_ATOMIC_GET(cu->hdata->use_cnt) == 0) { @@ -171,10 +172,10 @@ void HttpRangeContainersInit(void) } } - ContainerUrlRangeList.ht = - THashInit("app-layer.protocols.http.byterange", sizeof(HttpRangeContainerFile), - ContainerUrlRangeSet, ContainerUrlRangeFree, ContainerUrlRangeHash, - ContainerUrlRangeCompare, false, memcap, CONTAINER_URLRANGE_HASH_SIZE); + ContainerUrlRangeList.ht = THashInit("app-layer.protocols.http.byterange", + sizeof(HttpRangeContainerFile), ContainerUrlRangeSet, ContainerUrlRangeFree, + ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, false, + memcap, CONTAINER_URLRANGE_HASH_SIZE); ContainerUrlRangeList.timeout = timeout; SCLogDebug("containers started"); diff --git a/src/datasets.c b/src/datasets.c index 31fa6b398ca2..02e656f35e25 100644 --- a/src/datasets.c +++ b/src/datasets.c @@ -701,7 +701,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, switch (type) { case DATASET_TYPE_MD5: set->hash = THashInit(cnf_name, sizeof(Md5Type), Md5StrSet, Md5StrFree, Md5StrHash, - Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, + Md5StrCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, hashsize > 0 ? hashsize : default_hashsize); if (set->hash == NULL) goto out_err; @@ -710,7 +710,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, break; case DATASET_TYPE_STRING: set->hash = THashInit(cnf_name, sizeof(StringType), StringSet, StringFree, StringHash, - StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, + StringCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, hashsize > 0 ? hashsize : default_hashsize); if (set->hash == NULL) goto out_err; @@ -719,7 +719,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, break; case DATASET_TYPE_SHA256: set->hash = THashInit(cnf_name, sizeof(Sha256Type), Sha256StrSet, Sha256StrFree, - Sha256StrHash, Sha256StrCompare, load != NULL ? 1 : 0, + Sha256StrHash, Sha256StrCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, hashsize > 0 ? hashsize : default_hashsize); if (set->hash == NULL) @@ -729,7 +729,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, break; case DATASET_TYPE_IPV4: set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash, - IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, + IPv4Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, hashsize > 0 ? hashsize : default_hashsize); if (set->hash == NULL) goto out_err; @@ -738,7 +738,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, break; case DATASET_TYPE_IPV6: set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash, - IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, + IPv6Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap, hashsize > 0 ? hashsize : default_hashsize); if (set->hash == NULL) goto out_err; diff --git a/src/util-thash.c b/src/util-thash.c index a17679605a22..cdaf68a326a3 100644 --- a/src/util-thash.c +++ b/src/util-thash.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2016 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -294,7 +294,8 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix) THashTableContext *THashInit(const char *cnf_prefix, size_t data_size, int (*DataSet)(void *, void *), void (*DataFree)(void *), uint32_t (*DataHash)(void *), - bool (*DataCompare)(void *, void *), bool reset_memcap, uint64_t memcap, uint32_t hashsize) + bool (*DataCompare)(void *, void *), bool (*DataExpired)(void *, SCTime_t), + bool reset_memcap, uint64_t memcap, uint32_t hashsize) { THashTableContext *ctx = SCCalloc(1, sizeof(*ctx)); BUG_ON(!ctx); @@ -304,6 +305,7 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size, ctx->config.DataFree = DataFree; ctx->config.DataHash = DataHash; ctx->config.DataCompare = DataCompare; + ctx->config.DataExpired = DataExpired; /* set defaults */ ctx->config.hash_rand = (uint32_t)RandomGet(); @@ -407,6 +409,59 @@ int THashWalk(THashTableContext *ctx, THashFormatFunc FormatterFunc, THashOutput return 0; } +/** \brief expire data from the hash + * Walk the hash table and remove data that is exprired according to the + * DataExpired callback. + * \retval cnt number of items successfully expired/removed + */ +uint32_t THashExpire(THashTableContext *ctx, const SCTime_t ts) +{ + if (ctx->config.DataExpired == NULL) + return 0; + + SCLogDebug("timeout: starting"); + uint32_t cnt = 0; + + for (uint32_t i = 0; i < ctx->config.hash_size; i++) { + THashHashRow *hb = &ctx->array[i]; + if (HRLOCK_TRYLOCK(hb) != 0) + continue; + /* hash bucket is now locked */ + THashData *h = hb->head; + while (h) { + THashData *next = h->next; + THashDataLock(h); + DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX); + /* only consider items with no references to it */ + if (SC_ATOMIC_GET(h->use_cnt) == 0 && ctx->config.DataExpired(h->data, ts)) { + /* remove from the hash */ + if (h->prev != NULL) + h->prev->next = h->next; + if (h->next != NULL) + h->next->prev = h->prev; + if (hb->head == h) + hb->head = h->next; + if (hb->tail == h) + hb->tail = h->prev; + h->next = NULL; + h->prev = NULL; + SCLogDebug("timeout: removing data %p", h); + ctx->config.DataFree(h->data); + THashDataUnlock(h); + THashDataMoveToSpare(ctx, h); + cnt++; + } else { + THashDataUnlock(h); + } + h = next; + } + HRLOCK_UNLOCK(hb); + } + + SCLogDebug("timeout: ending: %u entries expired", cnt); + return cnt; +} + /** \brief Cleanup the thash engine * * Cleanup the thash engine from tag and threshold. diff --git a/src/util-thash.h b/src/util-thash.h index f45b6e843167..569f1ff9c795 100644 --- a/src/util-thash.h +++ b/src/util-thash.h @@ -132,6 +132,7 @@ typedef struct THashDataConfig_ { void (*DataFree)(void *); uint32_t (*DataHash)(void *); bool (*DataCompare)(void *, void *); + bool (*DataExpired)(void *, SCTime_t ts); } THashConfig; #define THASH_DATA_SIZE(ctx) (sizeof(THashData) + (ctx)->config.data_size) @@ -169,8 +170,9 @@ typedef struct THashTableContext_ { THashTableContext *THashInit(const char *cnf_prefix, size_t data_size, int (*DataSet)(void *dst, void *src), void (*DataFree)(void *), - uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *), bool reset_memcap, - uint64_t memcap, uint32_t hashsize); + uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *), + bool (*DataExpired)(void *, SCTime_t), bool reset_memcap, uint64_t memcap, + uint32_t hashsize); void THashShutdown(THashTableContext *ctx); @@ -197,5 +199,6 @@ int THashWalk(THashTableContext *, THashFormatFunc, THashOutputFunc, void *); int THashRemoveFromHash (THashTableContext *ctx, void *data); void THashConsolidateMemcap(THashTableContext *ctx); void THashDataMoveToSpare(THashTableContext *ctx, THashData *h); +uint32_t THashExpire(THashTableContext *ctx, const SCTime_t ts); #endif /* SURICATA_THASH_H */ From ac400af8f408b3fc973766b906af65dbe67a9b9f Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 16:14:10 +0100 Subject: [PATCH 17/30] range: use thash expiry API for timeout --- src/app-layer-htp-range.c | 50 +++------------------------------------ 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c index f7f110ca4fdd..5f2b743c9524 100644 --- a/src/app-layer-htp-range.c +++ b/src/app-layer-htp-range.c @@ -132,10 +132,8 @@ static inline bool ContainerValueRangeTimeout(void *data, const SCTime_t ts) HttpRangeContainerFile *cu = data; // we only timeout if we have no flow referencing us if (SCTIME_CMP_GT(ts, cu->expire) || cu->error) { - if (SC_ATOMIC_GET(cu->hdata->use_cnt) == 0) { - DEBUG_VALIDATE_BUG_ON(cu->files == NULL); - return true; - } + DEBUG_VALIDATE_BUG_ON(cu->files == NULL); + return true; } return false; } @@ -188,49 +186,7 @@ void HttpRangeContainersDestroy(void) uint32_t HttpRangeContainersTimeoutHash(const SCTime_t ts) { - SCLogDebug("timeout: starting"); - uint32_t cnt = 0; - - for (uint32_t i = 0; i < ContainerUrlRangeList.ht->config.hash_size; i++) { - THashHashRow *hb = &ContainerUrlRangeList.ht->array[i]; - - if (HRLOCK_TRYLOCK(hb) != 0) - continue; - /* hash bucket is now locked */ - THashData *h = hb->head; - while (h) { - DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX); - THashData *n = h->next; - THashDataLock(h); - if (ContainerValueRangeTimeout(h->data, ts)) { - /* remove from the hash */ - if (h->prev != NULL) - h->prev->next = h->next; - if (h->next != NULL) - h->next->prev = h->prev; - if (hb->head == h) - hb->head = h->next; - if (hb->tail == h) - hb->tail = h->prev; - h->next = NULL; - h->prev = NULL; - // we should log the timed out file somehow... - // but it does not belong to any flow... - SCLogDebug("timeout: removing range %p", h); - ContainerUrlRangeFree(h->data); // TODO do we need a "RECYCLE" func? - DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX); - THashDataUnlock(h); - THashDataMoveToSpare(ContainerUrlRangeList.ht, h); - } else { - THashDataUnlock(h); - } - h = n; - } - HRLOCK_UNLOCK(hb); - } - - SCLogDebug("timeout: ending"); - return cnt; + return THashExpire(ContainerUrlRangeList.ht, ts); } /** From b8028bf386b498ff2639c1b75262b39037e4b693 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 16:00:08 +0100 Subject: [PATCH 18/30] thresholds: use dedicated storage Instead of a Host and IPPair table thresholding layer, use a dedicated THash to store both. This allows hashing on host+sid+tracker or ippair+sid+tracker, to create more unique hash keys. This allows for fewer hash collisions. The per rule tracking also uses this, so that the single big lock is no longer a single point of contention. Reimplement storage for flow thresholds to reuse as much logic as possible from the host/ippair/rule thresholds. Ticket: #426. --- src/detect-detection-filter.c | 12 +- src/detect-engine-build.c | 2 - src/detect-engine-threshold.c | 811 +++++++++++++--------------------- src/detect-engine-threshold.h | 16 +- src/detect-engine.c | 2 - src/detect-threshold.c | 163 ++----- src/detect-threshold.h | 15 - src/detect.h | 12 - src/flow-manager.c | 2 + src/host-timeout.c | 2 - src/ippair-timeout.c | 10 +- src/suricata.c | 1 + src/util-threshold-config.c | 61 +-- 13 files changed, 403 insertions(+), 706 deletions(-) diff --git a/src/detect-detection-filter.c b/src/detect-detection-filter.c index 47ce00829880..dbc73dd59aee 100644 --- a/src/detect-detection-filter.c +++ b/src/detect-detection-filter.c @@ -378,7 +378,7 @@ static int DetectDetectionFilterTestSig1(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -418,7 +418,7 @@ static int DetectDetectionFilterTestSig1(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -435,7 +435,7 @@ static int DetectDetectionFilterTestSig2(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -480,7 +480,7 @@ static int DetectDetectionFilterTestSig2(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -493,7 +493,7 @@ static int DetectDetectionFilterTestSig3(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -556,7 +556,7 @@ static int DetectDetectionFilterTestSig3(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index f69def1c7d54..82ef66898f1e 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -2219,8 +2219,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx) SCProfilingRuleInitCounters(de_ctx); #endif - ThresholdHashAllocate(de_ctx); - if (!DetectEngineMultiTenantEnabled()) { VarNameStoreActivate(); } diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 1100a877ad0a..768b7375e14a 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -71,34 +71,150 @@ #include "action-globals.h" #include "util-validate.h" -static HostStorageId host_threshold_id = { .id = -1 }; /**< host storage id for thresholds */ -static IPPairStorageId ippair_threshold_id = { .id = -1 }; /**< ip pair storage id for thresholds */ +#include "util-thash.h" +#include "util-hash-lookup3.h" -HostStorageId ThresholdHostStorageId(void) +struct Thresholds { + THashTableContext *thash; +} ctx; + +static int ThresholdsInit(struct Thresholds *t); +static void ThresholdsDestroy(struct Thresholds *t); + +void ThresholdInit(void) { - return host_threshold_id; + ThresholdsInit(&ctx); } -void ThresholdInit(void) +void ThresholdDestroy(void) +{ + ThresholdsDestroy(&ctx); +} + +typedef struct ThresholdEntry_ { + uint32_t sid; /**< Signature id */ + uint32_t gid; /**< Signature group id */ + int track; /**< Track type: by_src, by_dst, etc */ + + uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) + its not "seconds", that define the time interval */ + uint32_t seconds; /**< Event seconds */ + uint32_t current_count; /**< Var for count control */ + + SCTime_t tv1; /**< Var for time control */ + + Address addr; /* used for src/dst/either tracking */ + Address addr2; /* used for both tracking */ +} ThresholdEntry; + +static int ThresholdEntrySet(void *dst, void *src) +{ + const ThresholdEntry *esrc = src; + ThresholdEntry *edst = dst; + memset(edst, 0, sizeof(*edst)); + *edst = *esrc; + return 0; +} + +static void ThresholdEntryFree(void *ptr) +{ + // nothing to free, base data is part of hash +} + +static inline uint32_t HashAddress(const Address *a) +{ + uint32_t key; + + if (a->family == AF_INET) { + key = a->addr_data32[0]; + } else if (a->family == AF_INET6) { + key = hashword(a->addr_data32, 4, 0); + } else + key = 0; + + return key; +} + +static inline int CompareAddress(const Address *a, const Address *b) +{ + if (a->family == b->family) { + switch (a->family) { + case AF_INET: + return (a->addr_data32[0] == b->addr_data32[0]); + case AF_INET6: + return CMP_ADDR(a, b); + } + } + return 0; +} + +static uint32_t ThresholdEntryHash(void *ptr) +{ + const ThresholdEntry *e = ptr; + uint32_t hash = e->sid + e->gid + e->track; + switch (e->track) { + case TRACK_BOTH: + hash += HashAddress(&e->addr2); + /* fallthrough */ + case TRACK_SRC: + case TRACK_DST: + hash += HashAddress(&e->addr); + break; + } + return hash; +} + +static bool ThresholdEntryCompare(void *a, void *b) { - host_threshold_id = HostStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree); - if (host_threshold_id.id == -1) { - FatalError("Can't initiate host storage for thresholding"); + const ThresholdEntry *e1 = a; + const ThresholdEntry *e2 = b; + SCLogDebug("sid1: %u sid2: %u", e1->sid, e2->sid); + if (!(e1->sid == e2->sid && e1->gid == e2->gid && e1->track == e2->track)) + return false; + switch (e1->track) { + case TRACK_BOTH: + if (!(CompareAddress(&e1->addr2, &e2->addr2))) + return false; + /* fallthrough */ + case TRACK_SRC: + case TRACK_DST: + if (!(CompareAddress(&e1->addr, &e2->addr))) + return false; + break; } - ippair_threshold_id = IPPairStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree); - if (ippair_threshold_id.id == -1) { - FatalError("Can't initiate IP pair storage for thresholding"); + return true; +} + +static bool ThresholdEntryExpire(void *data, const SCTime_t ts) +{ + const ThresholdEntry *e = data; + const SCTime_t entry = SCTIME_ADD_SECS(e->tv1, e->seconds); + if (SCTIME_CMP_GT(ts, entry)) { + return true; } + return false; } -int ThresholdHostHasThreshold(Host *host) +static int ThresholdsInit(struct Thresholds *t) { - return HostGetStorageById(host, host_threshold_id) ? 1 : 0; + uint64_t memcap = 16 * 1024 * 1024; + uint32_t hashsize = 16384; + t->thash = THashInit("thresholds", sizeof(ThresholdEntry), ThresholdEntrySet, + ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire, 0, + memcap, hashsize); + BUG_ON(t->thash == NULL); + return 0; +} +static void ThresholdsDestroy(struct Thresholds *t) +{ + if (t->thash) { + THashShutdown(t->thash); + } } -int ThresholdIPPairHasThreshold(IPPair *pair) +uint32_t ThresholdsExpire(const SCTime_t ts) { - return IPPairGetStorageById(pair, ippair_threshold_id) ? 1 : 0; + return THashExpire(ctx.thash, ts); } #include "util-hash.h" @@ -234,7 +350,9 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, n->expires_at = expires; if (HashTableAdd(threshold_cache_ht, n, 0) == 0) { - (void)THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n); + ThresholdCacheItem *r = THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n); + DEBUG_VALIDATE_BUG_ON(r != NULL); // duplicate; should be impossible + (void)r; // only used by DEBUG_VALIDATE_BUG_ON return 1; } SCFree(n); @@ -358,98 +476,18 @@ const DetectThresholdData *SigGetThresholdTypeIter( return NULL; } -/** - * \brief Remove timeout threshold hash elements - * - * \param head Current head element of storage - * \param tv Current time - * - * \retval DetectThresholdEntry Return new head element or NULL if all expired - * - */ - -static DetectThresholdEntry *ThresholdTimeoutCheck(DetectThresholdEntry *head, SCTime_t ts) -{ - DetectThresholdEntry *tmp = head; - DetectThresholdEntry *prev = NULL; - DetectThresholdEntry *new_head = head; - - while (tmp != NULL) { - /* check if the 'check' timestamp is not before the creation ts. - * This can happen due to the async nature of the host timeout - * code that also calls this code from a management thread. */ - SCTime_t entry = SCTIME_ADD_SECS(tmp->tv1, (time_t)tmp->seconds); - if (SCTIME_CMP_LTE(ts, entry)) { - prev = tmp; - tmp = tmp->next; - continue; - } - - /* timed out */ - - DetectThresholdEntry *tde = tmp; - if (prev != NULL) { - prev->next = tmp->next; - } - else { - new_head = tmp->next; - } - tmp = tde->next; - SCFree(tde); - } - - return new_head; -} - -int ThresholdHostTimeoutCheck(Host *host, SCTime_t ts) -{ - DetectThresholdEntry* head = HostGetStorageById(host, host_threshold_id); - DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts); - if (new_head != head) { - HostSetStorageById(host, host_threshold_id, new_head); - } - return new_head == NULL; -} - -int ThresholdIPPairTimeoutCheck(IPPair *pair, SCTime_t ts) -{ - DetectThresholdEntry* head = IPPairGetStorageById(pair, ippair_threshold_id); - DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts); - if (new_head != head) { - IPPairSetStorageById(pair, ippair_threshold_id, new_head); - } - return new_head == NULL; -} +typedef struct FlowThresholdEntryList_ { + struct FlowThresholdEntryList_ *next; + ThresholdEntry threshold; +} FlowThresholdEntryList; -static DetectThresholdEntry *DetectThresholdEntryAlloc( - const DetectThresholdData *td, uint32_t sid, uint32_t gid) +static void FlowThresholdEntryListFree(FlowThresholdEntryList *list) { - SCEnter(); - - DetectThresholdEntry *ste = SCCalloc(1, sizeof(DetectThresholdEntry)); - if (unlikely(ste == NULL)) { - SCReturnPtr(NULL, "DetectThresholdEntry"); + for (FlowThresholdEntryList *i = list; i != NULL;) { + FlowThresholdEntryList *next = i->next; + SCFree(i); + i = next; } - - ste->sid = sid; - ste->gid = gid; - ste->track = td->track; - ste->seconds = td->seconds; - - SCReturnPtr(ste, "DetectThresholdEntry"); -} - -static DetectThresholdEntry *ThresholdHostLookupEntry(Host *h, - uint32_t sid, uint32_t gid) -{ - DetectThresholdEntry *e; - - for (e = HostGetStorageById(h, host_threshold_id); e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) - break; - } - - return e; } /** struct for storing per flow thresholds. This will be stored in the Flow::flowvar list, so it @@ -458,13 +496,13 @@ typedef struct FlowVarThreshold_ { uint8_t type; uint8_t pad[7]; struct GenericVar_ *next; - DetectThresholdEntry *thresholds; + FlowThresholdEntryList *thresholds; } FlowVarThreshold; void FlowThresholdVarFree(void *ptr) { FlowVarThreshold *t = ptr; - ThresholdListFree(t->thresholds); + FlowThresholdEntryListFree(t->thresholds); SCFree(t); } @@ -481,21 +519,21 @@ static FlowVarThreshold *FlowThresholdVarGet(Flow *f) return NULL; } -static DetectThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) +static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) { FlowVarThreshold *t = FlowThresholdVarGet(f); if (t == NULL) return NULL; - for (DetectThresholdEntry *e = t->thresholds; e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) { - return e; + for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) { + if (e->threshold.sid == sid && e->threshold.gid == gid) { + return &e->threshold; } } return NULL; } -static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time) +static int AddEntryToFlow(Flow *f, FlowThresholdEntryList *e, SCTime_t packet_time) { DEBUG_VALIDATE_BUG_ON(e == NULL); @@ -509,27 +547,11 @@ static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time GenericVarAppend(&f->flowvar, (GenericVar *)t); } - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; e->next = t->thresholds; t->thresholds = e; return 0; } -static DetectThresholdEntry *ThresholdIPPairLookupEntry(IPPair *pair, - uint32_t sid, uint32_t gid) -{ - DetectThresholdEntry *e; - - for (e = IPPairGetStorageById(pair, ippair_threshold_id); e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) - break; - } - - return e; -} - static int ThresholdHandlePacketSuppress(Packet *p, const DetectThresholdData *td, uint32_t sid, uint32_t gid) { @@ -591,285 +613,208 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) } } -/** -* \brief Check if the entry reached threshold count limit -* -* \param lookup_tsh Current threshold entry -* \param td Threshold settings -* \param packet_time used to compare against previous detection and to set timeouts -* -* \retval int 1 if threshold reached for this entry -* -*/ -static int IsThresholdReached( - DetectThresholdEntry *lookup_tsh, const DetectThresholdData *td, SCTime_t packet_time) +static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, + const SCTime_t packet_time, const uint32_t sid, const uint32_t gid) { - int ret = 0; - - /* Check if we have a timeout enabled, if so, - * we still matching (and enabling the new_action) */ - if (lookup_tsh->tv_timeout != 0) { - if ((SCTIME_SECS(packet_time) - lookup_tsh->tv_timeout) > td->timeout) { - /* Ok, we are done, timeout reached */ - lookup_tsh->tv_timeout = 0; - } else { - /* Already matching */ - ret = 1; - } /* else - if ((packet_time - lookup_tsh->tv_timeout) > td->timeout) */ - - } - else { - /* Update the matching state with the timeout interval */ - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(packet_time, entry)) { - lookup_tsh->current_count++; - if (lookup_tsh->current_count > td->count) { - /* Then we must enable the new action by setting a - * timeout */ - lookup_tsh->tv_timeout = SCTIME_SECS(packet_time); - ret = 1; - } - } else { - lookup_tsh->tv1 = packet_time; - lookup_tsh->current_count = 1; - } - } /* else - if (lookup_tsh->tv_timeout != 0) */ - - return ret; -} + te->sid = sid; + te->gid = gid; + te->track = td->track; + te->seconds = td->seconds; -static void AddEntryToHostStorage(Host *h, DetectThresholdEntry *e, SCTime_t packet_time) -{ - if (h && e) { - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; - e->next = HostGetStorageById(h, host_threshold_id); - HostSetStorageById(h, host_threshold_id, e); - } -} + te->current_count = 1; + te->tv1 = packet_time; + te->tv_timeout = 0; -static void AddEntryToIPPairStorage(IPPair *pair, DetectThresholdEntry *e, SCTime_t packet_time) -{ - if (pair && e) { - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; - e->next = IPPairGetStorageById(pair, ippair_threshold_id); - IPPairSetStorageById(pair, ippair_threshold_id, e); + switch (td->type) { + case TYPE_LIMIT: + case TYPE_RATE: + return 1; + case TYPE_THRESHOLD: + case TYPE_BOTH: + if (td->count == 1) + return 1; + return 0; + case TYPE_DETECTION: + return 0; } + return 0; } -/** - * \retval 2 silent match (no alert but apply actions) - * \retval 1 normal match - * \retval 0 no match - * - * If a new DetectThresholdEntry is generated to track the threshold - * for this rule, then it will be returned in new_tsh. - */ -static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, - DetectThresholdEntry **new_tsh, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) +static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *te, + const Packet *p, // ts only? - cache too + const uint32_t sid, PacketAlert *pa) { int ret = 0; - - switch(td->type) { + const SCTime_t packet_time = p->ts; + const SCTime_t entry = SCTIME_ADD_SECS(te->tv1, td->seconds); + switch (td->type) { case TYPE_LIMIT: - { SCLogDebug("limit"); - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - lookup_tsh->current_count++; - - if (lookup_tsh->current_count <= td->count) { - ret = 1; - } else { - ret = 2; + if (SCTIME_CMP_LTE(p->ts, entry)) { + te->current_count++; - if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); - } - } + if (te->current_count <= td->count) { + ret = 1; } else { - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + ret = 2; - ret = 1; + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); + } } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); - + /* entry expired, reset */ + te->tv1 = p->ts; + te->current_count = 1; ret = 1; } break; - } case TYPE_THRESHOLD: - { - SCLogDebug("threshold"); - - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - lookup_tsh->current_count++; + if (SCTIME_CMP_LTE(p->ts, entry)) { + te->current_count++; - if (lookup_tsh->current_count >= td->count) { - ret = 1; - lookup_tsh->current_count = 0; - } - } else { - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; - } - } else { - if (td->count == 1) { + if (te->current_count >= td->count) { ret = 1; - } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + te->current_count = 0; } + } else { + te->tv1 = p->ts; + te->current_count = 1; } break; - } case TYPE_BOTH: - { - SCLogDebug("both"); - - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - /* within time limit */ - - lookup_tsh->current_count++; - if (lookup_tsh->current_count == td->count) { - ret = 1; - } else if (lookup_tsh->current_count > td->count) { - /* silent match */ - ret = 2; - - if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); - } - } - } else { - /* expired, so reset */ - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + if (SCTIME_CMP_LTE(p->ts, entry)) { + /* within time limit */ - /* if we have a limit of 1, this is a match */ - if (lookup_tsh->current_count == td->count) { - ret = 1; + te->current_count++; + if (te->current_count == td->count) { + ret = 1; + } else if (te->current_count > td->count) { + /* silent match */ + ret = 2; + + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); } } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* expired, so reset */ + te->tv1 = p->ts; + te->current_count = 1; - /* for the first match we return 1 to - * indicate we should alert */ - if (td->count == 1) { + /* if we have a limit of 1, this is a match */ + if (te->current_count == td->count) { ret = 1; } } break; - } - /* detection_filter */ case TYPE_DETECTION: - { SCLogDebug("detection_filter"); - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - /* within timeout */ - lookup_tsh->current_count++; - if (lookup_tsh->current_count > td->count) { - ret = 1; - } - } else { - /* expired, reset */ - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + if (SCTIME_CMP_LTE(p->ts, entry)) { + /* within timeout */ + te->current_count++; + if (te->current_count > td->count) { + ret = 1; } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* expired, reset */ + te->tv1 = p->ts; + te->current_count = 1; } break; - } - /* rate_filter */ case TYPE_RATE: - { SCLogDebug("rate_filter"); ret = 1; - if (lookup_tsh && IsThresholdReached(lookup_tsh, td, p->ts)) { - RateFilterSetAction(pa, td->new_action); - } else if (!lookup_tsh) { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* Check if we have a timeout enabled, if so, + * we still matching (and enabling the new_action) */ + if (te->tv_timeout != 0) { + if ((SCTIME_SECS(packet_time) - te->tv_timeout) > td->timeout) { + /* Ok, we are done, timeout reached */ + te->tv_timeout = 0; + } else { + /* Already matching */ + RateFilterSetAction(pa, td->new_action); + } + } else { + /* Update the matching state with the timeout interval */ + if (SCTIME_CMP_LTE(packet_time, entry)) { + te->current_count++; + if (te->current_count > td->count) { + /* Then we must enable the new action by setting a + * timeout */ + te->tv_timeout = SCTIME_SECS(packet_time); + RateFilterSetAction(pa, td->new_action); + } + } else { + te->tv1 = packet_time; + te->current_count = 1; + } } break; - } - /* case TYPE_SUPPRESS: is not handled here */ - default: - SCLogError("type %d is not supported", td->type); } return ret; } -static int ThresholdHandlePacketIPPair(IPPair *pair, Packet *p, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) -{ - int ret = 0; - - DetectThresholdEntry *lookup_tsh = ThresholdIPPairLookupEntry(pair, sid, gid); - SCLogDebug("ippair lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - AddEntryToIPPairStorage(pair, new_tsh, p->ts); - } - - return ret; -} +#include "detect-engine-address-ipv6.h" -/** - * \retval 2 silent match (no alert but apply actions) - * \retval 1 normal match - * \retval 0 no match - */ -static int ThresholdHandlePacketHost(Host *h, Packet *p, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) +static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const Signature *s, + const DetectThresholdData *td, PacketAlert *pa) { - int ret = 0; - DetectThresholdEntry *lookup_tsh = ThresholdHostLookupEntry(h, sid, gid); - SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - AddEntryToHostStorage(h, new_tsh, p->ts); + /* fast track for count 1 threshold */ + if (td->count == 1 && td->type == TYPE_THRESHOLD) { + return 1; } - return ret; -} -static int ThresholdHandlePacketRule(DetectEngineCtx *de_ctx, Packet *p, - const DetectThresholdData *td, const Signature *s, PacketAlert *pa) -{ - int ret = 0; + ThresholdEntry lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.sid = s->id; + lookup.gid = s->gid; + lookup.track = td->track; + if (td->track == TRACK_SRC) { + COPY_ADDRESS(&p->src, &lookup.addr); + } else if (td->track == TRACK_DST) { + COPY_ADDRESS(&p->dst, &lookup.addr); + } else if (td->track == TRACK_BOTH) { + /* make sure lower ip address is first */ + if (PacketIsIPv4(p)) { + if (SCNtohl(p->src.addr_data32[0]) < SCNtohl(p->dst.addr_data32[0])) { + COPY_ADDRESS(&p->src, &lookup.addr); + COPY_ADDRESS(&p->dst, &lookup.addr2); + } else { + COPY_ADDRESS(&p->dst, &lookup.addr); + COPY_ADDRESS(&p->src, &lookup.addr2); + } + } else { + if (AddressIPv6Lt(&p->src, &p->dst)) { + COPY_ADDRESS(&p->src, &lookup.addr); + COPY_ADDRESS(&p->dst, &lookup.addr2); + } else { + COPY_ADDRESS(&p->dst, &lookup.addr); + COPY_ADDRESS(&p->src, &lookup.addr2); + } + } + } - DetectThresholdEntry* lookup_tsh = (DetectThresholdEntry *)de_ctx->ths_ctx.th_entry[s->num]; - SCLogDebug("by_rule lookup_tsh %p num %u", lookup_tsh, s->num); + struct THashDataGetResult res = THashGetFromHash(tctx->thash, &lookup); + if (res.data) { + SCLogDebug("found %p, is_new %s", res.data, BOOL2STR(res.is_new)); + int r; + ThresholdEntry *te = res.data->data; + if (res.is_new) { + // new threshold, set up + r = ThresholdSetup(td, te, p->ts, s->id, s->gid); + } else { + // existing, check/update + r = ThresholdCheckUpdate(td, te, p, s->id, pa); + } - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, s->id, s->gid, pa); - if (new_tsh != NULL) { - new_tsh->tv1 = p->ts; - new_tsh->current_count = 1; - new_tsh->tv_timeout = 0; - de_ctx->ths_ctx.th_entry[s->num] = new_tsh; + (void)THashDecrUsecnt(res.data); + THashDataUnlock(res.data); + return r; } - - return ret; + return 0; // TODO error? } /** @@ -881,15 +826,24 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa uint32_t sid, uint32_t gid, PacketAlert *pa) { int ret = 0; - DetectThresholdEntry *lookup_tsh = ThresholdFlowLookupEntry(f, sid, gid); - SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - if (AddEntryToFlow(f, new_tsh, p->ts) == -1) { - SCFree(new_tsh); + ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid); + SCLogDebug("found %p sid %u gid %u", found, sid, gid); + + if (found == NULL) { + FlowThresholdEntryList *new = SCCalloc(1, sizeof(*new)); + if (new == NULL) + return 0; + + // new threshold, set up + ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid); + + if (AddEntryToFlow(f, new, p->ts) == -1) { + SCFree(new); + return 0; } + } else { + // existing, check/update + ret = ThresholdCheckUpdate(td, found, p, sid, pa); } return ret; } @@ -925,11 +879,8 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(cache_ret); } } - Host *src = HostGetHostFromHash(&p->src); - if (src) { - ret = ThresholdHandlePacketHost(src,p,td,s->id,s->gid,pa); - HostRelease(src); - } + + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_DST) { if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { int cache_ret = CheckCache(p, td->track, s->id); @@ -937,21 +888,12 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(cache_ret); } } - Host *dst = HostGetHostFromHash(&p->dst); - if (dst) { - ret = ThresholdHandlePacketHost(dst,p,td,s->id,s->gid,pa); - HostRelease(dst); - } + + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_BOTH) { - IPPair *pair = IPPairGetIPPairFromHash(&p->src, &p->dst); - if (pair) { - ret = ThresholdHandlePacketIPPair(pair, p, td, s->id, s->gid, pa); - IPPairRelease(pair); - } + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_RULE) { - SCMutexLock(&de_ctx->ths_ctx.threshold_table_lock); - ret = ThresholdHandlePacketRule(de_ctx,p,td,s,pa); - SCMutexUnlock(&de_ctx->ths_ctx.threshold_table_lock); + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_FLOW) { if (p->flow) { ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa); @@ -961,125 +903,6 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(ret); } -/** - * \brief Init threshold context hash tables - * - * \param de_ctx Detection Context - * - */ -void ThresholdHashInit(DetectEngineCtx *de_ctx) -{ - if (SCMutexInit(&de_ctx->ths_ctx.threshold_table_lock, NULL) != 0) { - FatalError("Threshold: Failed to initialize hash table mutex."); - } -} - -/** - * \brief Allocate threshold context hash tables - * - * \param de_ctx Detection Context - */ -void ThresholdHashAllocate(DetectEngineCtx *de_ctx) -{ - const Signature *s = de_ctx->sig_list; - bool has_by_rule_tracking = false; - - /* Find the signature with the highest signature number that is using - thresholding with by_rule tracking. */ - uint32_t highest_signum = 0; - while (s != NULL) { - if (s->sm_arrays[DETECT_SM_LIST_SUPPRESS] != NULL) { - const SigMatchData *smd = NULL; - do { - const DetectThresholdData *td = - SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_SUPPRESS); - if (td == NULL) { - continue; - } - if (td->track != TRACK_RULE) { - continue; - } - if (s->num >= highest_signum) { - highest_signum = s->num; - has_by_rule_tracking = true; - } - } while (smd != NULL); - } - - if (s->sm_arrays[DETECT_SM_LIST_THRESHOLD] != NULL) { - const SigMatchData *smd = NULL; - do { - const DetectThresholdData *td = - SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_THRESHOLD); - if (td == NULL) { - continue; - } - if (td->track != TRACK_RULE) { - continue; - } - if (s->num >= highest_signum) { - highest_signum = s->num; - has_by_rule_tracking = true; - } - } while (smd != NULL); - } - - s = s->next; - } - - /* Skip allocating if by_rule tracking is not used */ - if (has_by_rule_tracking == false) { - return; - } - - de_ctx->ths_ctx.th_size = highest_signum + 1; - de_ctx->ths_ctx.th_entry = SCCalloc(de_ctx->ths_ctx.th_size, sizeof(DetectThresholdEntry *)); - if (de_ctx->ths_ctx.th_entry == NULL) { - FatalError( - "failed to allocate memory for \"by_rule\" thresholding (tried to allocate %" PRIu32 - " entries)", - de_ctx->ths_ctx.th_size); - } -} - -/** - * \brief Destroy threshold context hash tables - * - * \param de_ctx Detection Context - * - */ -void ThresholdContextDestroy(DetectEngineCtx *de_ctx) -{ - if (de_ctx->ths_ctx.th_entry != NULL) { - for (uint32_t i = 0; i < de_ctx->ths_ctx.th_size; i++) { - if (de_ctx->ths_ctx.th_entry[i] != NULL) { - SCFree(de_ctx->ths_ctx.th_entry[i]); - } - } - SCFree(de_ctx->ths_ctx.th_entry); - } - SCMutexDestroy(&de_ctx->ths_ctx.threshold_table_lock); -} - -/** - * \brief this function will free all the entries of a list - * DetectTagDataEntry - * - * \param td pointer to DetectTagDataEntryList - */ -void ThresholdListFree(void *ptr) -{ - if (ptr != NULL) { - DetectThresholdEntry *entry = ptr; - - while (entry != NULL) { - DetectThresholdEntry *next_entry = entry->next; - SCFree(entry); - entry = next_entry; - } - } -} - /** * @} */ diff --git a/src/detect-engine-threshold.h b/src/detect-engine-threshold.h index b7e53842433d..086dba497a5a 100644 --- a/src/detect-engine-threshold.h +++ b/src/detect-engine-threshold.h @@ -26,16 +26,12 @@ #define SURICATA_DETECT_ENGINE_THRESHOLD_H #include "detect.h" -#include "host.h" -#include "ippair.h" -#include "host-storage.h" +#include "detect-threshold.h" void ThresholdInit(void); +void ThresholdDestroy(void); -HostStorageId ThresholdHostStorageId(void); -int ThresholdHostHasThreshold(Host *); - -int ThresholdIPPairHasThreshold(IPPair *pair); +uint32_t ThresholdsExpire(const SCTime_t ts); const DetectThresholdData *SigGetThresholdTypeIter( const Signature *, const SigMatchData **, int list); @@ -43,12 +39,6 @@ int PacketAlertThreshold(DetectEngineCtx *, DetectEngineThreadCtx *, const DetectThresholdData *, Packet *, const Signature *, PacketAlert *); -void ThresholdHashInit(DetectEngineCtx *); -void ThresholdHashAllocate(DetectEngineCtx *); -void ThresholdContextDestroy(DetectEngineCtx *); - -int ThresholdHostTimeoutCheck(Host *, SCTime_t); -int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t); void ThresholdListFree(void *ptr); void ThresholdCacheThreadFree(void); diff --git a/src/detect-engine.c b/src/detect-engine.c index 3ec446168709..efd1b0f8850b 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2506,7 +2506,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal( SigGroupHeadHashInit(de_ctx); MpmStoreInit(de_ctx); - ThresholdHashInit(de_ctx); DetectParseDupSigHashInit(de_ctx); DetectAddressMapInit(de_ctx); DetectMetadataHashInit(de_ctx); @@ -2620,7 +2619,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) MpmStoreFree(de_ctx); DetectParseDupSigHashFree(de_ctx); SCSigSignatureOrderingModuleCleanup(de_ctx); - ThresholdContextDestroy(de_ctx); SigCleanSignatures(de_ctx); if (de_ctx->sig_array) SCFree(de_ctx->sig_array); diff --git a/src/detect-threshold.c b/src/detect-threshold.c index d952b5ec36fc..0d5d01485514 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -509,7 +509,7 @@ static int DetectThresholdTestSig1(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -587,8 +587,8 @@ static int DetectThresholdTestSig1(void) UTHFreePackets(&p, 1); - HostShutdown(); end: + ThresholdDestroy(); return result; } @@ -610,7 +610,7 @@ static int DetectThresholdTestSig2(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -658,15 +658,12 @@ static int DetectThresholdTestSig2(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -681,94 +678,49 @@ static int DetectThresholdTestSig2(void) static int DetectThresholdTestSig3(void) { - Packet *p = NULL; - Signature *s = NULL; ThreadVars th_v; - DetectEngineThreadCtx *det_ctx; - int result = 0; - int alerts = 0; - DetectThresholdEntry *lookup_tsh = NULL; - - HostInitConfig(HOST_QUIET); - memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + ThresholdInit(); + Packet *p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); - if (de_ctx == NULL) { - goto end; - } - + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:10;)"); - if (s == NULL) { - goto end; - } + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, " + "track by_dst, count 5, seconds 60; sid:10;)"); + FAIL_IF_NULL(s); SigGroupBuild(de_ctx); + DetectEngineThreadCtx *det_ctx; DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); p->ts = TimeGet(); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - - Host *host = HostLookupHostFromHash(&p->dst); - if (host == NULL) { - printf("host not found: "); - goto cleanup; - } - - if (!(ThresholdHostHasThreshold(host))) { - HostRelease(host); - printf("host has no threshold: "); - goto cleanup; - } - HostRelease(host); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); TimeSetIncrementTime(200); p->ts = TimeGet(); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - - host = HostLookupHostFromHash(&p->dst); - if (host == NULL) { - printf("host not found: "); - goto cleanup; - } - HostRelease(host); - - lookup_tsh = HostGetStorageById(host, ThresholdHostStorageId()); - if (lookup_tsh == NULL) { - HostRelease(host); - printf("lookup_tsh is NULL: "); - goto cleanup; - } - - alerts = lookup_tsh->current_count; - - if (alerts == 3) - result = 1; - else { - printf("alerts %u != 3: ", alerts); - goto cleanup; - } - -cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); -end: UTHFreePackets(&p, 1); - HostShutdown(); - return result; + ThresholdDestroy(); + PASS; } /** @@ -789,7 +741,7 @@ static int DetectThresholdTestSig4(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -834,14 +786,11 @@ static int DetectThresholdTestSig4(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -863,7 +812,7 @@ static int DetectThresholdTestSig5(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -921,15 +870,12 @@ static int DetectThresholdTestSig5(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -942,7 +888,7 @@ static int DetectThresholdTestSig6Ticks(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1004,15 +950,12 @@ static int DetectThresholdTestSig6Ticks(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1029,7 +972,7 @@ static int DetectThresholdTestSig7(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1095,14 +1038,11 @@ static int DetectThresholdTestSig7(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1119,7 +1059,7 @@ static int DetectThresholdTestSig8(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1185,14 +1125,11 @@ static int DetectThresholdTestSig8(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1209,7 +1146,7 @@ static int DetectThresholdTestSig9(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1275,14 +1212,11 @@ static int DetectThresholdTestSig9(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1299,7 +1233,7 @@ static int DetectThresholdTestSig10(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1365,14 +1299,11 @@ static int DetectThresholdTestSig10(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1389,7 +1320,7 @@ static int DetectThresholdTestSig11(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1455,14 +1386,11 @@ static int DetectThresholdTestSig11(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1479,7 +1407,7 @@ static int DetectThresholdTestSig12(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1545,14 +1473,12 @@ static int DetectThresholdTestSig12(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void*)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); HostShutdown(); + ThresholdDestroy(); return result; } @@ -1573,7 +1499,7 @@ static int DetectThresholdTestSig13(void) DetectEngineThreadCtx *det_ctx; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1616,12 +1542,10 @@ static int DetectThresholdTestSig13(void) FAIL_IF(alerts != 4); - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1644,8 +1568,7 @@ static int DetectThresholdTestSig14(void) int alerts1 = 0; int alerts2 = 0; - HostInitConfig(HOST_QUIET); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p1 = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1699,13 +1622,11 @@ static int DetectThresholdTestSig14(void) FAIL_IF(alerts1 != 3); FAIL_IF(alerts2 != 3); - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); UTHFreePackets(&p1, 1); UTHFreePackets(&p2, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } diff --git a/src/detect-threshold.h b/src/detect-threshold.h index 5321fd54bdac..e1ef71bd1089 100644 --- a/src/detect-threshold.h +++ b/src/detect-threshold.h @@ -62,21 +62,6 @@ typedef struct DetectThresholdData_ { DetectAddressHead addrs; } DetectThresholdData; -typedef struct DetectThresholdEntry_ { - uint32_t sid; /**< Signature id */ - uint32_t gid; /**< Signature group id */ - - uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) - its not "seconds", that define the time interval */ - uint32_t seconds; /**< Event seconds */ - uint32_t current_count; /**< Var for count control */ - int track; /**< Track type: by_src, by_src */ - - SCTime_t tv1; /**< Var for time control */ - struct DetectThresholdEntry_ *next; -} DetectThresholdEntry; - - /** * Registration function for threshold: keyword */ diff --git a/src/detect.h b/src/detect.h index a4f154523280..397c4ae8aadc 100644 --- a/src/detect.h +++ b/src/detect.h @@ -785,17 +785,6 @@ typedef struct DetectEngineLookupFlow_ { struct SigGroupHead_ *sgh[256]; } DetectEngineLookupFlow; -#include "detect-threshold.h" - -/** \brief threshold ctx */ -typedef struct ThresholdCtx_ { - SCMutex threshold_table_lock; /**< Mutex for hash table */ - - /** to support rate_filter "by_rule" option */ - DetectThresholdEntry **th_entry; - uint32_t th_size; -} ThresholdCtx; - typedef struct SigString_ { char *filename; char *sig_str; @@ -886,7 +875,6 @@ typedef struct DetectEngineCtx_ { HashListTable *dup_sig_hash_table; DetectEngineIPOnlyCtx io_ctx; - ThresholdCtx ths_ctx; /* maximum recursion depth for content inspection */ int inspection_recursion_limit; diff --git a/src/flow-manager.c b/src/flow-manager.c index bd547de95bd1..de55ed136f6a 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -47,6 +47,7 @@ #include "util-debug.h" #include "threads.h" +#include "detect-engine-threshold.h" #include "host-timeout.h" #include "defrag-hash.h" @@ -931,6 +932,7 @@ static TmEcode FlowManager(ThreadVars *th_v, void *thread_data) HostTimeoutHash(ts); IPPairTimeoutHash(ts); HttpRangeContainersTimeoutHash(ts); + ThresholdsExpire(ts); other_last_sec = (uint32_t)SCTIME_SECS(ts); } } diff --git a/src/host-timeout.c b/src/host-timeout.c index d1e358c1b1ca..35706c2cfd24 100644 --- a/src/host-timeout.c +++ b/src/host-timeout.c @@ -25,7 +25,6 @@ #include "host.h" #include "detect-engine-tag.h" -#include "detect-engine-threshold.h" #include "host-bit.h" #include "host-timeout.h" @@ -53,7 +52,6 @@ static int HostHostTimedOut(Host *h, SCTime_t ts) busy |= (h->iprep && SRepHostTimedOut(h) == 0); busy |= (TagHostHasTag(h) && TagTimeoutCheck(h, ts) == 0); - busy |= (ThresholdHostHasThreshold(h) && ThresholdHostTimeoutCheck(h, ts) == 0); busy |= (HostHasHostBits(h) && HostBitsTimedoutCheck(h, ts) == 0); SCLogDebug("host %p %s", h, busy ? "still active" : "timed out"); return !busy; diff --git a/src/ippair-timeout.c b/src/ippair-timeout.c index a3627c1ea911..02e88b7b3477 100644 --- a/src/ippair-timeout.c +++ b/src/ippair-timeout.c @@ -25,7 +25,6 @@ #include "ippair.h" #include "ippair-bit.h" #include "ippair-timeout.h" -#include "detect-engine-threshold.h" /** \internal * \brief See if we can really discard this ippair. Check use_cnt reference. @@ -39,8 +38,6 @@ static int IPPairTimedOut(IPPair *h, SCTime_t ts) { int vars = 0; - int thresholds = 0; - /** never prune a ippair that is used by a packet * we are currently processing in one of the threads */ if (SC_ATOMIC_GET(h->use_cnt) > 0) { @@ -50,12 +47,7 @@ static int IPPairTimedOut(IPPair *h, SCTime_t ts) if (IPPairHasBits(h) && IPPairBitsTimedoutCheck(h, ts) == 0) { vars = 1; } - - if (ThresholdIPPairHasThreshold(h) && ThresholdIPPairTimeoutCheck(h, ts) == 0) { - thresholds = 1; - } - - if (vars || thresholds) { + if (vars) { return 0; } diff --git a/src/suricata.c b/src/suricata.c index f7a03a208c6d..de60255199bd 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -369,6 +369,7 @@ void GlobalsInitPreConfig(void) void GlobalsDestroy(void) { SCInstance *suri = &suricata; + ThresholdDestroy(); HostShutdown(); HTPFreeConfig(); HTPAtExitPrintStats(); diff --git a/src/util-threshold-config.c b/src/util-threshold-config.c index 998dde56a9e5..6df1eebb0798 100644 --- a/src/util-threshold-config.c +++ b/src/util-threshold-config.c @@ -37,6 +37,7 @@ #include "detect.h" #include "detect-engine.h" #include "detect-engine-address.h" +#include "detect-engine-threshold.h" #include "detect-threshold.h" #include "detect-parse.h" #include "detect-engine-build.h" @@ -1534,7 +1535,7 @@ static int SCThresholdConfTest09(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1603,7 +1604,7 @@ static int SCThresholdConfTest09(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1615,7 +1616,7 @@ static int SCThresholdConfTest09(void) */ static int SCThresholdConfTest10(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); /* Create two different packets falling to the same rule, and * because count:3, we should drop on match #4. @@ -1685,15 +1686,15 @@ static int SCThresholdConfTest10(void) SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); FAIL_IF(PacketTestAction(p1, ACTION_DROP)); FAIL_IF(PacketAlertCheck(p1, 10) != 1); - +#if 0 /* Ensure that a Threshold entry was installed at the sig */ FAIL_IF_NULL(de_ctx->ths_ctx.th_entry[s->num]); - +#endif UTHFreePacket(p1); UTHFreePacket(p2); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1705,7 +1706,7 @@ static int SCThresholdConfTest10(void) */ static int SCThresholdConfTest11(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1798,7 +1799,7 @@ static int SCThresholdConfTest11(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1810,7 +1811,7 @@ static int SCThresholdConfTest11(void) */ static int SCThresholdConfTest12(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1903,7 +1904,7 @@ static int SCThresholdConfTest12(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1948,7 +1949,7 @@ static int SCThresholdConfTest13(void) */ static int SCThresholdConfTest14(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p1 = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -1997,7 +1998,7 @@ static int SCThresholdConfTest14(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2009,7 +2010,7 @@ static int SCThresholdConfTest14(void) */ static int SCThresholdConfTest15(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -2045,7 +2046,7 @@ static int SCThresholdConfTest15(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2057,7 +2058,7 @@ static int SCThresholdConfTest15(void) */ static int SCThresholdConfTest16(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.1.1", "192.168.0.100", 1234, 24); @@ -2092,7 +2093,7 @@ static int SCThresholdConfTest16(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2104,7 +2105,7 @@ static int SCThresholdConfTest16(void) */ static int SCThresholdConfTest17(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -2140,7 +2141,7 @@ static int SCThresholdConfTest17(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2171,7 +2172,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD12(void) */ static int SCThresholdConfTest18(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2192,7 +2193,7 @@ static int SCThresholdConfTest18(void) FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2223,7 +2224,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD13(void) */ static int SCThresholdConfTest19(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2241,7 +2242,7 @@ static int SCThresholdConfTest19(void) FAIL_IF_NULL(de); FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2273,7 +2274,7 @@ static FILE *SCThresholdConfGenerateValidDummyFD20(void) */ static int SCThresholdConfTest20(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2306,7 +2307,7 @@ static int SCThresholdConfTest20(void) FAIL_IF_NOT(smd->is_last); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2319,7 +2320,7 @@ static int SCThresholdConfTest20(void) */ static int SCThresholdConfTest21(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2351,7 +2352,7 @@ static int SCThresholdConfTest21(void) FAIL_IF_NOT(smd->is_last); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2384,7 +2385,7 @@ static int SCThresholdConfTest22(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); /* This packet will cause rate_filter */ Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10"); @@ -2487,7 +2488,7 @@ static int SCThresholdConfTest22(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - IPPairShutdown(); + ThresholdDestroy(); PASS; } @@ -2521,7 +2522,7 @@ static int SCThresholdConfTest23(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); /* Create two packets between same addresses in opposite direction */ Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10"); @@ -2568,7 +2569,7 @@ static int SCThresholdConfTest23(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - IPPairShutdown(); + ThresholdDestroy(); PASS; } #endif /* UNITTESTS */ From 3471c0f6ad194e229c55147363a8cc165f29ee49 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 19 Apr 2024 18:17:33 +0200 Subject: [PATCH 19/30] detect/threshold: improve hash function --- src/detect-engine-threshold.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 768b7375e14a..4a2a781916b7 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -91,10 +91,12 @@ void ThresholdDestroy(void) ThresholdsDestroy(&ctx); } +#define SID 0 +#define GID 1 +#define TRACK 2 + typedef struct ThresholdEntry_ { - uint32_t sid; /**< Signature id */ - uint32_t gid; /**< Signature group id */ - int track; /**< Track type: by_src, by_dst, etc */ + uint32_t key[3]; uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) its not "seconds", that define the time interval */ @@ -151,8 +153,8 @@ static inline int CompareAddress(const Address *a, const Address *b) static uint32_t ThresholdEntryHash(void *ptr) { const ThresholdEntry *e = ptr; - uint32_t hash = e->sid + e->gid + e->track; - switch (e->track) { + uint32_t hash = hashword(e->key, sizeof(e->key) / sizeof(uint32_t), 0); + switch (e->key[TRACK]) { case TRACK_BOTH: hash += HashAddress(&e->addr2); /* fallthrough */ @@ -168,10 +170,11 @@ static bool ThresholdEntryCompare(void *a, void *b) { const ThresholdEntry *e1 = a; const ThresholdEntry *e2 = b; - SCLogDebug("sid1: %u sid2: %u", e1->sid, e2->sid); - if (!(e1->sid == e2->sid && e1->gid == e2->gid && e1->track == e2->track)) + SCLogDebug("sid1: %u sid2: %u", e1->key[SID], e2->key[SID]); + + if (memcmp(e1->key, e2->key, sizeof(e1->key)) != 0) return false; - switch (e1->track) { + switch (e1->key[TRACK]) { case TRACK_BOTH: if (!(CompareAddress(&e1->addr2, &e2->addr2))) return false; @@ -526,7 +529,7 @@ static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t return NULL; for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) { - if (e->threshold.sid == sid && e->threshold.gid == gid) { + if (e->threshold.key[SID] == sid && e->threshold.key[GID] == gid) { return &e->threshold; } } @@ -616,9 +619,9 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, const SCTime_t packet_time, const uint32_t sid, const uint32_t gid) { - te->sid = sid; - te->gid = gid; - te->track = td->track; + te->key[SID] = sid; + te->key[GID] = gid; + te->key[TRACK] = td->track; te->seconds = td->seconds; te->current_count = 1; @@ -769,9 +772,9 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const ThresholdEntry lookup; memset(&lookup, 0, sizeof(lookup)); - lookup.sid = s->id; - lookup.gid = s->gid; - lookup.track = td->track; + lookup.key[SID] = s->id; + lookup.key[GID] = s->gid; + lookup.key[TRACK] = td->track; if (td->track == TRACK_SRC) { COPY_ADDRESS(&p->src, &lookup.addr); } else if (td->track == TRACK_DST) { From 2be998fbcd820ab54c75a9a6219984f52f140705 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 19 Apr 2024 18:27:57 +0200 Subject: [PATCH 20/30] detect/threshold: include rev in threshold tracking --- src/detect-engine-threshold.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 4a2a781916b7..d1f34f03a8fc 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -93,10 +93,11 @@ void ThresholdDestroy(void) #define SID 0 #define GID 1 -#define TRACK 2 +#define REV 2 +#define TRACK 3 typedef struct ThresholdEntry_ { - uint32_t key[3]; + uint32_t key[4]; uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) its not "seconds", that define the time interval */ @@ -522,14 +523,15 @@ static FlowVarThreshold *FlowThresholdVarGet(Flow *f) return NULL; } -static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) +static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid, uint32_t rev) { FlowVarThreshold *t = FlowThresholdVarGet(f); if (t == NULL) return NULL; for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) { - if (e->threshold.key[SID] == sid && e->threshold.key[GID] == gid) { + if (e->threshold.key[SID] == sid && e->threshold.key[GID] == gid && + e->threshold.key[REV] == rev) { return &e->threshold; } } @@ -617,10 +619,11 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) } static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, - const SCTime_t packet_time, const uint32_t sid, const uint32_t gid) + const SCTime_t packet_time, const uint32_t sid, const uint32_t gid, const uint32_t rev) { te->key[SID] = sid; te->key[GID] = gid; + te->key[REV] = rev; te->key[TRACK] = td->track; te->seconds = td->seconds; @@ -774,6 +777,7 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const memset(&lookup, 0, sizeof(lookup)); lookup.key[SID] = s->id; lookup.key[GID] = s->gid; + lookup.key[REV] = s->rev; lookup.key[TRACK] = td->track; if (td->track == TRACK_SRC) { COPY_ADDRESS(&p->src, &lookup.addr); @@ -807,7 +811,7 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const ThresholdEntry *te = res.data->data; if (res.is_new) { // new threshold, set up - r = ThresholdSetup(td, te, p->ts, s->id, s->gid); + r = ThresholdSetup(td, te, p->ts, s->id, s->gid, s->rev); } else { // existing, check/update r = ThresholdCheckUpdate(td, te, p, s->id, pa); @@ -826,11 +830,11 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const * \retval 0 no match */ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) + uint32_t sid, uint32_t gid, uint32_t rev, PacketAlert *pa) { int ret = 0; - ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid); - SCLogDebug("found %p sid %u gid %u", found, sid, gid); + ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid, rev); + SCLogDebug("found %p sid %u gid %u rev %u", found, sid, gid, rev); if (found == NULL) { FlowThresholdEntryList *new = SCCalloc(1, sizeof(*new)); @@ -838,7 +842,7 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa return 0; // new threshold, set up - ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid); + ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid, rev); if (AddEntryToFlow(f, new, p->ts) == -1) { SCFree(new); @@ -899,7 +903,7 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_FLOW) { if (p->flow) { - ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa); + ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, s->rev, pa); } } From 1e9fdc40051bd41e01051c9403185e0bf749f27a Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 19 Apr 2024 18:31:39 +0200 Subject: [PATCH 21/30] detect/threshold: consider tenant id in tracking Ticket: #6967. --- src/detect-engine-threshold.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index d1f34f03a8fc..fa40bb9f098f 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -91,13 +91,14 @@ void ThresholdDestroy(void) ThresholdsDestroy(&ctx); } -#define SID 0 -#define GID 1 -#define REV 2 -#define TRACK 3 +#define SID 0 +#define GID 1 +#define REV 2 +#define TRACK 3 +#define TENANT 4 typedef struct ThresholdEntry_ { - uint32_t key[4]; + uint32_t key[5]; uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) its not "seconds", that define the time interval */ @@ -523,7 +524,8 @@ static FlowVarThreshold *FlowThresholdVarGet(Flow *f) return NULL; } -static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid, uint32_t rev) +static ThresholdEntry *ThresholdFlowLookupEntry( + Flow *f, uint32_t sid, uint32_t gid, uint32_t rev, uint32_t tenant_id) { FlowVarThreshold *t = FlowThresholdVarGet(f); if (t == NULL) @@ -531,7 +533,7 @@ static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) { if (e->threshold.key[SID] == sid && e->threshold.key[GID] == gid && - e->threshold.key[REV] == rev) { + e->threshold.key[REV] == rev && e->threshold.key[TENANT] == tenant_id) { return &e->threshold; } } @@ -619,12 +621,14 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) } static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, - const SCTime_t packet_time, const uint32_t sid, const uint32_t gid, const uint32_t rev) + const SCTime_t packet_time, const uint32_t sid, const uint32_t gid, const uint32_t rev, + const uint32_t tenant_id) { te->key[SID] = sid; te->key[GID] = gid; te->key[REV] = rev; te->key[TRACK] = td->track; + te->key[TENANT] = tenant_id; te->seconds = td->seconds; te->current_count = 1; @@ -779,6 +783,7 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const lookup.key[GID] = s->gid; lookup.key[REV] = s->rev; lookup.key[TRACK] = td->track; + lookup.key[TENANT] = p->tenant_id; if (td->track == TRACK_SRC) { COPY_ADDRESS(&p->src, &lookup.addr); } else if (td->track == TRACK_DST) { @@ -811,7 +816,7 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const ThresholdEntry *te = res.data->data; if (res.is_new) { // new threshold, set up - r = ThresholdSetup(td, te, p->ts, s->id, s->gid, s->rev); + r = ThresholdSetup(td, te, p->ts, s->id, s->gid, s->rev, p->tenant_id); } else { // existing, check/update r = ThresholdCheckUpdate(td, te, p, s->id, pa); @@ -833,7 +838,7 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa uint32_t sid, uint32_t gid, uint32_t rev, PacketAlert *pa) { int ret = 0; - ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid, rev); + ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid, rev, p->tenant_id); SCLogDebug("found %p sid %u gid %u rev %u", found, sid, gid, rev); if (found == NULL) { @@ -842,7 +847,7 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa return 0; // new threshold, set up - ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid, rev); + ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid, rev, p->tenant_id); if (AddEntryToFlow(f, new, p->ts) == -1) { SCFree(new); From 7bcf364095a845addae5933d1fdf18aed86558ac Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 19 Apr 2024 18:57:32 +0200 Subject: [PATCH 22/30] detect/threshold: expand cache support for rule tracking Use the same hash key as for the regular threshold storage, so include gid, rev, tentant id. --- src/detect-engine-threshold.c | 57 ++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index fa40bb9f098f..4ae17b653c42 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -224,12 +224,17 @@ uint32_t ThresholdsExpire(const SCTime_t ts) #include "util-hash.h" +#define TC_ADDRESS 0 +#define TC_SID 1 +#define TC_GID 2 +#define TC_REV 3 +#define TC_TENANT 4 + typedef struct ThresholdCacheItem { int8_t track; // by_src/by_dst int8_t ipv; int8_t retval; - uint32_t addr; - uint32_t sid; + uint32_t key[5]; SCTime_t expires_at; RB_ENTRY(ThresholdCacheItem) rb; } ThresholdCacheItem; @@ -297,8 +302,8 @@ static void ThresholdCacheExpire(SCTime_t now) static uint32_t ThresholdCacheHashFunc(HashTable *ht, void *data, uint16_t datalen) { - ThresholdCacheItem *tci = data; - int hash = tci->ipv * tci->track + tci->addr + tci->sid; + ThresholdCacheItem *e = data; + uint32_t hash = hashword(e->key, sizeof(e->key) / sizeof(uint32_t), 0) * (e->ipv + e->track); hash = hash % ht->array_size; return hash; } @@ -308,8 +313,8 @@ static char ThresholdCacheHashCompareFunc( { ThresholdCacheItem *tci1 = data1; ThresholdCacheItem *tci2 = data2; - return tci1->ipv == tci2->ipv && tci1->track == tci2->track && tci1->addr == tci2->addr && - tci1->sid == tci2->sid; + return tci1->ipv == tci2->ipv && tci1->track == tci2->track && + memcmp(tci1->key, tci2->key, sizeof(tci1->key)) == 0; } static void ThresholdCacheHashFreeFunc(void *data) @@ -319,7 +324,7 @@ static void ThresholdCacheHashFreeFunc(void *data) /// \brief Thread local cache static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, const uint32_t sid, - SCTime_t expires) + const uint32_t gid, const uint32_t rev, SCTime_t expires) { if (!threshold_cache_ht) { threshold_cache_ht = HashTableInit(256, ThresholdCacheHashFunc, @@ -339,8 +344,11 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, .track = track, .ipv = 4, .retval = retval, - .addr = addr, - .sid = sid, + .key[TC_ADDRESS] = addr, + .key[TC_SID] = sid, + .key[TC_GID] = gid, + .key[TC_REV] = rev, + .key[TC_TENANT] = p->tenant_id, .expires_at = expires, }; ThresholdCacheItem *found = HashTableLookup(threshold_cache_ht, &lookup, 0); @@ -350,8 +358,11 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, n->track = track; n->ipv = 4; n->retval = retval; - n->addr = addr; - n->sid = sid; + n->key[TC_ADDRESS] = addr; + n->key[TC_SID] = sid; + n->key[TC_GID] = gid; + n->key[TC_REV] = rev; + n->key[TC_TENANT] = p->tenant_id; n->expires_at = expires; if (HashTableAdd(threshold_cache_ht, n, 0) == 0) { @@ -381,7 +392,8 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, * \retval -4 error - unsupported tracker * \retval ret cached return code */ -static int CheckCache(const Packet *p, const int8_t track, const uint32_t sid) +static int CheckCache(const Packet *p, const int8_t track, const uint32_t sid, const uint32_t gid, + const uint32_t rev) { cache_lookup_cnt++; @@ -407,8 +419,11 @@ static int CheckCache(const Packet *p, const int8_t track, const uint32_t sid) ThresholdCacheItem lookup = { .track = track, .ipv = 4, - .addr = addr, - .sid = sid, + .key[TC_ADDRESS] = addr, + .key[TC_SID] = sid, + .key[TC_GID] = gid, + .key[TC_REV] = rev, + .key[TC_TENANT] = p->tenant_id, }; ThresholdCacheItem *found = HashTableLookup(threshold_cache_ht, &lookup, 0); if (found) { @@ -652,7 +667,7 @@ static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *te, const Packet *p, // ts only? - cache too - const uint32_t sid, PacketAlert *pa) + const uint32_t sid, const uint32_t gid, const uint32_t rev, PacketAlert *pa) { int ret = 0; const SCTime_t packet_time = p->ts; @@ -670,7 +685,7 @@ static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *t ret = 2; if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); + SetupCache(p, td->track, (int8_t)ret, sid, gid, rev, entry); } } } else { @@ -705,7 +720,7 @@ static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *t ret = 2; if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); + SetupCache(p, td->track, (int8_t)ret, sid, gid, rev, entry); } } } else { @@ -819,7 +834,7 @@ static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const r = ThresholdSetup(td, te, p->ts, s->id, s->gid, s->rev, p->tenant_id); } else { // existing, check/update - r = ThresholdCheckUpdate(td, te, p, s->id, pa); + r = ThresholdCheckUpdate(td, te, p, s->id, s->gid, s->rev, pa); } (void)THashDecrUsecnt(res.data); @@ -855,7 +870,7 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa } } else { // existing, check/update - ret = ThresholdCheckUpdate(td, found, p, sid, pa); + ret = ThresholdCheckUpdate(td, found, p, sid, gid, rev, pa); } return ret; } @@ -886,7 +901,7 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx ret = ThresholdHandlePacketSuppress(p,td,s->id,s->gid); } else if (td->track == TRACK_SRC) { if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { - int cache_ret = CheckCache(p, td->track, s->id); + int cache_ret = CheckCache(p, td->track, s->id, s->gid, s->rev); if (cache_ret >= 0) { SCReturnInt(cache_ret); } @@ -895,7 +910,7 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_DST) { if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { - int cache_ret = CheckCache(p, td->track, s->id); + int cache_ret = CheckCache(p, td->track, s->id, s->gid, s->rev); if (cache_ret >= 0) { SCReturnInt(cache_ret); } From 10eaf550b71ef41163fcbd8c68a0d5a3d7bb4d3c Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 20 Apr 2024 07:41:00 +0200 Subject: [PATCH 23/30] detect/threshold: includes cleanup --- src/detect-engine-threshold.c | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 4ae17b653c42..60d9bd8acc97 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -39,38 +39,19 @@ #include "detect.h" #include "flow.h" -#include "host.h" -#include "host-storage.h" - -#include "ippair.h" -#include "ippair-storage.h" - #include "detect-parse.h" -#include "detect-engine-sigorder.h" - -#include "detect-engine-siggroup.h" -#include "detect-engine-address.h" -#include "detect-engine-port.h" -#include "detect-engine-mpm.h" -#include "detect-engine-iponly.h" - #include "detect-engine.h" #include "detect-engine-threshold.h" +#include "detect-engine-address.h" +#include "detect-engine-address-ipv6.h" -#include "detect-content.h" -#include "detect-uricontent.h" - -#include "util-hash.h" #include "util-time.h" #include "util-error.h" #include "util-debug.h" - -#include "util-var-name.h" -#include "tm-threads.h" - #include "action-globals.h" #include "util-validate.h" +#include "util-hash.h" #include "util-thash.h" #include "util-hash-lookup3.h" @@ -222,8 +203,6 @@ uint32_t ThresholdsExpire(const SCTime_t ts) return THashExpire(ctx.thash, ts); } -#include "util-hash.h" - #define TC_ADDRESS 0 #define TC_SID 1 #define TC_GID 2 @@ -782,8 +761,6 @@ static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *t return ret; } -#include "detect-engine-address-ipv6.h" - static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const Signature *s, const DetectThresholdData *td, PacketAlert *pa) { From 7d4fcc311ca563631672f7b1f4de69367cbc35cb Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 15 May 2024 11:02:29 +0200 Subject: [PATCH 24/30] detect/threshold: make hash size and memcap configurable --- src/detect-engine-threshold.c | 29 +++++++++++++++++++++++++++-- suricata.yaml.in | 5 +++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 60d9bd8acc97..faedb656b0b4 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -45,6 +45,7 @@ #include "detect-engine-address.h" #include "detect-engine-address-ipv6.h" +#include "util-misc.h" #include "util-time.h" #include "util-error.h" #include "util-debug.h" @@ -183,14 +184,38 @@ static bool ThresholdEntryExpire(void *data, const SCTime_t ts) static int ThresholdsInit(struct Thresholds *t) { - uint64_t memcap = 16 * 1024 * 1024; uint32_t hashsize = 16384; + uint64_t memcap = 16 * 1024 * 1024; + + const char *str; + if (ConfGet("detect.thresholds.memcap", &str) == 1) { + if (ParseSizeStringU64(str, &memcap) < 0) { + SCLogError("Error parsing detect.thresholds.memcap from conf file - %s", str); + return -1; + } + } + + intmax_t value = 0; + if ((ConfGetInt("detect.thresholds.hash-size", &value)) == 1) { + if (value < 256 || value > INT_MAX) { + SCLogError("'detect.thresholds.hash-size' value %" PRIiMAX + " out of range. Valid range 256-2147483647.", + value); + return -1; + } + hashsize = (uint32_t)value; + } + t->thash = THashInit("thresholds", sizeof(ThresholdEntry), ThresholdEntrySet, ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire, 0, memcap, hashsize); - BUG_ON(t->thash == NULL); + if (t->thash == NULL) { + SCLogError("failed to initialize thresholds hash table"); + return -1; + } return 0; } + static void ThresholdsDestroy(struct Thresholds *t) { if (t->thash) { diff --git a/suricata.yaml.in b/suricata.yaml.in index 0ba63086d0d1..5e4d039e9e7b 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1700,6 +1700,11 @@ detect: #tcp-whitelist: 53, 80, 139, 443, 445, 1433, 3306, 3389, 6666, 6667, 8080 #udp-whitelist: 53, 135, 5060 + # Thresholding hash table settings. + thresholds: + hash-size: 16384 + memcap: 16mb + profiling: # Log the rules that made it past the prefilter stage, per packet # default is off. The threshold setting determines how many rules From e362a01f8d5cdccec84ff442bda8f22771294289 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 15 May 2024 11:12:45 +0200 Subject: [PATCH 25/30] doc/userguide: document new threshold config options --- .../configuration/global-thresholds.rst | 3 +++ doc/userguide/configuration/suricata-yaml.rst | 16 ++++++++++++++++ doc/userguide/rules/thresholding.rst | 3 +++ 3 files changed, 22 insertions(+) diff --git a/doc/userguide/configuration/global-thresholds.rst b/doc/userguide/configuration/global-thresholds.rst index a5b3bd6d8675..f78f6c552b53 100644 --- a/doc/userguide/configuration/global-thresholds.rst +++ b/doc/userguide/configuration/global-thresholds.rst @@ -6,6 +6,9 @@ Thresholds can be configured in the rules themselves, see their intelligence for creating a rule combined with a judgement on how often a rule will alert. +Thresholds are tracked in a hash table that is sized according to configuration, see: +:ref:`suricata-yaml-thresholds`. + Threshold Config ---------------- diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst index 8171d5c1508a..36107eeb5d1b 100644 --- a/doc/userguide/configuration/suricata-yaml.rst +++ b/doc/userguide/configuration/suricata-yaml.rst @@ -739,6 +739,22 @@ To let Suricata make these decisions set default to 'auto': prefilter: default: auto +.. _suricata-yaml-thresholds: + +Thresholding Settings +~~~~~~~~~~~~~~~~~~~~~ + +Thresholding uses a central hash table for tracking thresholds of the types: by_src, by_dst, by_both. + +:: + + detect: + thresholds: + hash-size: 16384 + memcap: 16mb + +``detect.thresholds.hash-size`` controls the number of hash rows in the hash table. +``detect.thresholds.memcap`` controls how much memory can be used for the hash table and the data stored in it. Pattern matcher settings ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/userguide/rules/thresholding.rst b/doc/userguide/rules/thresholding.rst index 44a8aa5e92d4..e56830cb4199 100644 --- a/doc/userguide/rules/thresholding.rst +++ b/doc/userguide/rules/thresholding.rst @@ -4,6 +4,9 @@ Thresholding Keywords Thresholding can be configured per rule and also globally, see :doc:`../configuration/global-thresholds`. +Thresholds are tracked in a hash table that is sized according to configuration, see: +:ref:`suricata-yaml-thresholds`. + *Note: mixing rule and global thresholds is not supported in 1.3 and before. See bug #425.* For the state of the support in 1.4 see :ref:`global-thresholds-vs-rule-thresholds` From 2abe0df13698e45d02791f0d2a6aa9a7167f927e Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 24 Jun 2024 10:08:45 +0200 Subject: [PATCH 26/30] detect/threshold: format file --- src/detect-threshold.c | 188 +++++++++++++++++++++++------------------ 1 file changed, 106 insertions(+), 82 deletions(-) diff --git a/src/detect-threshold.c b/src/detect-threshold.c index 0d5d01485514..2748557a22f5 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -69,8 +69,8 @@ static DetectParseRegex parse_regex; -static int DetectThresholdMatch(DetectEngineThreadCtx *, Packet *, - const Signature *, const SigMatchCtx *); +static int DetectThresholdMatch( + DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *); static int DetectThresholdSetup(DetectEngineCtx *, Signature *, const char *); static void DetectThresholdFree(DetectEngineCtx *, void *); #ifdef UNITTESTS @@ -88,7 +88,7 @@ void DetectThresholdRegister(void) sigmatch_table[DETECT_THRESHOLD].url = "/rules/thresholding.html#threshold"; sigmatch_table[DETECT_THRESHOLD].Match = DetectThresholdMatch; sigmatch_table[DETECT_THRESHOLD].Setup = DetectThresholdSetup; - sigmatch_table[DETECT_THRESHOLD].Free = DetectThresholdFree; + sigmatch_table[DETECT_THRESHOLD].Free = DetectThresholdFree; #ifdef UNITTESTS sigmatch_table[DETECT_THRESHOLD].RegisterTests = ThresholdRegisterTests; #endif @@ -98,8 +98,8 @@ void DetectThresholdRegister(void) DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } -static int DetectThresholdMatch(DetectEngineThreadCtx *det_ctx, Packet *p, - const Signature *s, const SigMatchCtx *ctx) +static int DetectThresholdMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) { return 1; } @@ -134,23 +134,22 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) } char *saveptr = NULL; - for (pos = 0, threshold_opt = strtok_r(copy_str,",", &saveptr); - pos < strlen(copy_str) && threshold_opt != NULL; - pos++, threshold_opt = strtok_r(NULL,"," , &saveptr)) - { - if(strstr(threshold_opt,"count")) + for (pos = 0, threshold_opt = strtok_r(copy_str, ",", &saveptr); + pos < strlen(copy_str) && threshold_opt != NULL; + pos++, threshold_opt = strtok_r(NULL, ",", &saveptr)) { + if (strstr(threshold_opt, "count")) count_found++; - if(strstr(threshold_opt,"second")) + if (strstr(threshold_opt, "second")) second_found++; - if(strstr(threshold_opt,"type")) + if (strstr(threshold_opt, "type")) type_found++; - if(strstr(threshold_opt,"track")) + if (strstr(threshold_opt, "track")) track_found++; } SCFree(copy_str); copy_str = NULL; - if(count_found != 1 || second_found != 1 || type_found != 1 || track_found != 1) + if (count_found != 1 || second_found != 1 || type_found != 1 || track_found != 1) goto error; ret = DetectParsePcreExec(&parse_regex, &match, rawstr, 0, 0); @@ -174,43 +173,41 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) args[i] = (char *)str_ptr; - if (strncasecmp(args[i],"limit",strlen("limit")) == 0) + if (strncasecmp(args[i], "limit", strlen("limit")) == 0) de->type = TYPE_LIMIT; - if (strncasecmp(args[i],"both",strlen("both")) == 0) + if (strncasecmp(args[i], "both", strlen("both")) == 0) de->type = TYPE_BOTH; - if (strncasecmp(args[i],"threshold",strlen("threshold")) == 0) + if (strncasecmp(args[i], "threshold", strlen("threshold")) == 0) de->type = TYPE_THRESHOLD; - if (strncasecmp(args[i],"by_dst",strlen("by_dst")) == 0) + if (strncasecmp(args[i], "by_dst", strlen("by_dst")) == 0) de->track = TRACK_DST; - if (strncasecmp(args[i],"by_src",strlen("by_src")) == 0) + if (strncasecmp(args[i], "by_src", strlen("by_src")) == 0) de->track = TRACK_SRC; - if (strncasecmp(args[i],"by_both",strlen("by_both")) == 0) + if (strncasecmp(args[i], "by_both", strlen("by_both")) == 0) de->track = TRACK_BOTH; - if (strncasecmp(args[i],"by_rule",strlen("by_rule")) == 0) + if (strncasecmp(args[i], "by_rule", strlen("by_rule")) == 0) de->track = TRACK_RULE; if (strncasecmp(args[i], "by_flow", strlen("by_flow")) == 0) de->track = TRACK_FLOW; - if (strncasecmp(args[i],"count",strlen("count")) == 0) - count_pos = i+1; - if (strncasecmp(args[i],"seconds",strlen("seconds")) == 0) - second_pos = i+1; + if (strncasecmp(args[i], "count", strlen("count")) == 0) + count_pos = i + 1; + if (strncasecmp(args[i], "seconds", strlen("seconds")) == 0) + second_pos = i + 1; } if (args[count_pos] == NULL || args[second_pos] == NULL) { goto error; } - if (StringParseUint32(&de->count, 10, strlen(args[count_pos]), - args[count_pos]) <= 0) { + if (StringParseUint32(&de->count, 10, strlen(args[count_pos]), args[count_pos]) <= 0) { goto error; } - if (StringParseUint32(&de->seconds, 10, strlen(args[second_pos]), - args[second_pos]) <= 0) { + if (StringParseUint32(&de->seconds, 10, strlen(args[second_pos]), args[second_pos]) <= 0) { goto error; } - for (i = 0; i < (ret - 1); i++){ + for (i = 0; i < (ret - 1); i++) { if (args[i] != NULL) pcre2_substring_free((PCRE2_UCHAR8 *)args[i]); } @@ -221,7 +218,7 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) if (match) { pcre2_match_data_free(match); } - for (i = 0; i < (ret - 1); i++){ + for (i = 0; i < (ret - 1); i++) { if (args[i] != NULL) pcre2_substring_free((PCRE2_UCHAR8 *)args[i]); } @@ -360,7 +357,8 @@ static int ThresholdTestParse01(void) { DetectThresholdData *de = NULL; de = DetectThresholdParse("type limit,track by_dst,count 10,seconds 60"); - if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && (de->seconds == 60)) { + if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && + (de->seconds == 60)) { DetectThresholdFree(NULL, de); return 1; } @@ -390,7 +388,8 @@ static int ThresholdTestParse02(void) { DetectThresholdData *de = NULL; de = DetectThresholdParse("type any,track by_dst,count 10,seconds 60"); - if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && (de->seconds == 60)) { + if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && + (de->seconds == 60)) { DetectThresholdFree(NULL, de); return 0; } @@ -408,7 +407,8 @@ static int ThresholdTestParse03(void) { DetectThresholdData *de = NULL; de = DetectThresholdParse("track by_dst, type limit, seconds 60, count 10"); - if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && (de->seconds == 60)) { + if (de && (de->type == TYPE_LIMIT) && (de->track == TRACK_DST) && (de->count == 10) && + (de->seconds == 60)) { DetectThresholdFree(NULL, de); return 1; } @@ -416,7 +416,6 @@ static int ThresholdTestParse03(void) return 0; } - /** * \test ThresholdTestParse04 is a test for an invalid threshold options in any order * @@ -427,7 +426,8 @@ static int ThresholdTestParse04(void) { DetectThresholdData *de = NULL; de = DetectThresholdParse("count 10, track by_dst, seconds 60, type both, count 10"); - if (de && (de->type == TYPE_BOTH) && (de->track == TRACK_DST) && (de->count == 10) && (de->seconds == 60)) { + if (de && (de->type == TYPE_BOTH) && (de->track == TRACK_DST) && (de->count == 10) && + (de->seconds == 60)) { DetectThresholdFree(NULL, de); return 0; } @@ -445,7 +445,8 @@ static int ThresholdTestParse05(void) { DetectThresholdData *de = NULL; de = DetectThresholdParse("count 10, track by_dst, seconds 60, type both"); - if (de && (de->type == TYPE_BOTH) && (de->track == TRACK_DST) && (de->count == 10) && (de->seconds == 60)) { + if (de && (de->type == TYPE_BOTH) && (de->track == TRACK_DST) && (de->count == 10) && + (de->seconds == 60)) { DetectThresholdFree(NULL, de); return 1; } @@ -513,7 +514,7 @@ static int DetectThresholdTestSig1(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -522,7 +523,9 @@ static int DetectThresholdTestSig1(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit\"; content:\"A\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit\"; content:\"A\"; " + "threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); if (s == NULL) { goto end; } @@ -536,48 +539,48 @@ static int DetectThresholdTestSig1(void) SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts = PacketAlertCheck(p, 1); if (alerts != 1) { - printf("alerts %"PRIi32", expected 1: ", alerts); + printf("alerts %" PRIi32 ", expected 1: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 2) { - printf("alerts %"PRIi32", expected 2: ", alerts); + printf("alerts %" PRIi32 ", expected 2: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 3) { - printf("alerts %"PRIi32", expected 3: ", alerts); + printf("alerts %" PRIi32 ", expected 3: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 4) { - printf("alerts %"PRIi32", expected 4: ", alerts); + printf("alerts %" PRIi32 ", expected 4: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 5) { - printf("alerts %"PRIi32", expected 5: ", alerts); + printf("alerts %" PRIi32 ", expected 5: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 5) { - printf("alerts %"PRIi32", expected 5: ", alerts); + printf("alerts %" PRIi32 ", expected 5: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 5) { - printf("alerts %"PRIi32", expected 5: ", alerts); + printf("alerts %" PRIi32 ", expected 5: ", alerts); } SigMatchSignatures(&th_v, de_ctx, det_ctx, p); alerts += PacketAlertCheck(p, 1); if (alerts != 5) { - printf("alerts %"PRIi32", expected 5: ", alerts); + printf("alerts %" PRIi32 ", expected 5: ", alerts); } - if(alerts == 5) + if (alerts == 5) result = 1; else - printf("alerts %"PRIi32", expected 5: ", alerts); + printf("alerts %" PRIi32 ", expected 5: ", alerts); SigGroupCleanup(de_ctx); SigCleanSignatures(de_ctx); @@ -614,7 +617,7 @@ static int DetectThresholdTestSig2(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -623,7 +626,9 @@ static int DetectThresholdTestSig2(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold\"; threshold: type threshold, track by_dst, count 5, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold\"; threshold: type " + "threshold, track by_dst, count 5, seconds 60; sid:1;)"); if (s == NULL) { goto end; } @@ -745,7 +750,7 @@ static int DetectThresholdTestSig4(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -754,7 +759,9 @@ static int DetectThresholdTestSig4(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold both\"; threshold: type both, track by_dst, count 2, seconds 60; sid:10;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold both\"; threshold: type " + "both, track by_dst, count 2, seconds 60; sid:10;)"); if (s == NULL) { goto end; } @@ -815,7 +822,7 @@ static int DetectThresholdTestSig5(void) ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -824,12 +831,16 @@ static int DetectThresholdTestSig5(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; " + "threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); if (s == NULL) { goto end; } - s = s->next = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1000\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:1000;)"); + s = s->next = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1000\"; " + "threshold: type limit, track by_dst, count 5, seconds 60; sid:1000;)"); if (s == NULL) { goto end; } @@ -862,7 +873,7 @@ static int DetectThresholdTestSig5(void) alerts += PacketAlertCheck(p, 1); alerts += PacketAlertCheck(p, 1000); - if(alerts == 10) + if (alerts == 10) result = 1; else { printf("alerts %d != 10: ", alerts); @@ -891,7 +902,7 @@ static int DetectThresholdTestSig6Ticks(void) ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -900,12 +911,16 @@ static int DetectThresholdTestSig6Ticks(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; " + "threshold: type limit, track by_dst, count 5, seconds 60; sid:1;)"); if (s == NULL) { goto end; } - s = s->next = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1000\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:1000;)"); + s = s->next = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1000\"; " + "threshold: type limit, track by_dst, count 5, seconds 60; sid:1000;)"); if (s == NULL) { goto end; } @@ -942,9 +957,9 @@ static int DetectThresholdTestSig6Ticks(void) alerts += PacketAlertCheck(p, 1); alerts += PacketAlertCheck(p, 1000); ticks_end = UtilCpuGetTicks(); - printf("test run %"PRIu64"\n", (ticks_end - ticks_start)); + printf("test run %" PRIu64 "\n", (ticks_end - ticks_start)); - if(alerts == 10) + if (alerts == 10) result = 1; else goto cleanup; @@ -976,7 +991,7 @@ static int DetectThresholdTestSig7(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -985,7 +1000,8 @@ static int DetectThresholdTestSig7(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type limit, track by_src, count 1, seconds 300; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type limit, " + "track by_src, count 1, seconds 300; sid:10;)"); if (s == NULL) { goto end; } @@ -1063,7 +1079,7 @@ static int DetectThresholdTestSig8(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -1072,7 +1088,8 @@ static int DetectThresholdTestSig8(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type limit, track by_src, count 2, seconds 300; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type limit, " + "track by_src, count 2, seconds 300; sid:10;)"); if (s == NULL) { goto end; } @@ -1150,7 +1167,7 @@ static int DetectThresholdTestSig9(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -1159,7 +1176,8 @@ static int DetectThresholdTestSig9(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type threshold, track by_src, count 3, seconds 100; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type threshold, " + "track by_src, count 3, seconds 100; sid:10;)"); if (s == NULL) { goto end; } @@ -1237,7 +1255,7 @@ static int DetectThresholdTestSig10(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -1246,7 +1264,8 @@ static int DetectThresholdTestSig10(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type threshold, track by_src, count 5, seconds 300; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type threshold, " + "track by_src, count 5, seconds 300; sid:10;)"); if (s == NULL) { goto end; } @@ -1324,7 +1343,7 @@ static int DetectThresholdTestSig11(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -1333,7 +1352,8 @@ static int DetectThresholdTestSig11(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type both, track by_src, count 3, seconds 300; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type both, " + "track by_src, count 3, seconds 300; sid:10;)"); if (s == NULL) { goto end; } @@ -1411,7 +1431,7 @@ static int DetectThresholdTestSig12(void) memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -1420,7 +1440,8 @@ static int DetectThresholdTestSig12(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"drop tcp any any -> any 80 (threshold: type both, track by_src, count 5, seconds 300; sid:10;)"); + s = de_ctx->sig_list = SigInit(de_ctx, "drop tcp any any -> any 80 (threshold: type both, " + "track by_src, count 5, seconds 300; sid:10;)"); if (s == NULL) { goto end; } @@ -1473,7 +1494,7 @@ static int DetectThresholdTestSig12(void) } cleanup: - DetectEngineThreadCtxDeinit(&th_v, (void*)det_ctx); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); @@ -1502,7 +1523,7 @@ static int DetectThresholdTestSig13(void) ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); FAIL_IF_NULL(p); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); @@ -1510,7 +1531,9 @@ static int DetectThresholdTestSig13(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; threshold: type limit, track by_rule, count 2, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; " + "threshold: type limit, track by_rule, count 2, seconds 60; sid:1;)"); FAIL_IF_NULL(s); SigGroupBuild(de_ctx); @@ -1571,8 +1594,8 @@ static int DetectThresholdTestSig14(void) ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); - p1 = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); - p2 = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "3.3.3.3", 1024, 80); + p1 = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p2 = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "3.3.3.3", 1024, 80); FAIL_IF_NULL(p1); FAIL_IF_NULL(p2); @@ -1581,7 +1604,9 @@ static int DetectThresholdTestSig14(void) de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; threshold: type limit, track by_both, count 2, seconds 60; sid:1;)"); + s = de_ctx->sig_list = + SigInit(de_ctx, "alert tcp any any -> any 80 (msg:\"Threshold limit sid 1\"; " + "threshold: type limit, track by_both, count 2, seconds 60; sid:1;)"); FAIL_IF_NULL(s); SigGroupBuild(de_ctx); @@ -1645,8 +1670,7 @@ static void ThresholdRegisterTests(void) UtRegisterTest("DetectThresholdTestSig3", DetectThresholdTestSig3); UtRegisterTest("DetectThresholdTestSig4", DetectThresholdTestSig4); UtRegisterTest("DetectThresholdTestSig5", DetectThresholdTestSig5); - UtRegisterTest("DetectThresholdTestSig6Ticks", - DetectThresholdTestSig6Ticks); + UtRegisterTest("DetectThresholdTestSig6Ticks", DetectThresholdTestSig6Ticks); UtRegisterTest("DetectThresholdTestSig7", DetectThresholdTestSig7); UtRegisterTest("DetectThresholdTestSig8", DetectThresholdTestSig8); UtRegisterTest("DetectThresholdTestSig9", DetectThresholdTestSig9); From a0d515bfdd5e2cc7427a07170a0c5a54f446a94f Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 24 Jun 2024 11:06:10 +0200 Subject: [PATCH 27/30] detect/threshold: regex cleanup --- src/detect-threshold.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/detect-threshold.c b/src/detect-threshold.c index 2748557a22f5..d368a8dbcbb2 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -60,12 +60,13 @@ #include "util-cpu.h" #endif +#define PARSE_REGEX_NAME "(track|type|count|seconds)" +#define PARSE_REGEX_VALUE "(limit|both|threshold|by_dst|by_src|by_both|by_rule|by_flow|\\d+)" + #define PARSE_REGEX \ - "^\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|by_" \ - "flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_" \ - "both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_" \ - "dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|" \ - "threshold|by_dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*" + "^\\s*" PARSE_REGEX_NAME "\\s+" PARSE_REGEX_VALUE "\\s*,\\s*" PARSE_REGEX_NAME \ + "\\s+" PARSE_REGEX_VALUE "\\s*,\\s*" PARSE_REGEX_NAME "\\s+" PARSE_REGEX_VALUE \ + "\\s*,\\s*" PARSE_REGEX_NAME "\\s+" PARSE_REGEX_VALUE "\\s*" static DetectParseRegex parse_regex; From 12130df21c875f9384f69182fe30db87d5688552 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 24 Jun 2024 10:55:45 +0200 Subject: [PATCH 28/30] detect/threshold: implement backoff type Implement new `type backoff` for thresholding. This allows alerts to be limited. A count of 1 with a multiplier of 10 would generate alerts for matching packets: 1, 10, 100, 1000, 10000, 100000, etc. A count of 1 with a multiplier of 2 would generate alerts for matching packets: 1, 2, 4, 8, 16, 32, etc. Like with other thresholds, rule actions like drop and setting of flowbits will still be performed for each matching packet. Current implementation is only for the by_flow tracker and for per rule threshold statements. Tracking is done using uint32_t. When it reaches this value, the rest of the packets in the tracker will use the silent match. Ticket: #7120. --- src/detect-engine-threshold.c | 70 +++++++++++++++++++++++++--- src/detect-threshold.c | 87 +++++++++++++++++++++++++++++------ src/detect-threshold.h | 4 +- 3 files changed, 141 insertions(+), 20 deletions(-) diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index faedb656b0b4..31892f82f314 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -87,10 +87,17 @@ typedef struct ThresholdEntry_ { uint32_t seconds; /**< Event seconds */ uint32_t current_count; /**< Var for count control */ - SCTime_t tv1; /**< Var for time control */ + union { + struct { + uint32_t next_value; + } backoff; + struct { + SCTime_t tv1; /**< Var for time control */ + Address addr; /* used for src/dst/either tracking */ + Address addr2; /* used for both tracking */ + }; + }; - Address addr; /* used for src/dst/either tracking */ - Address addr2; /* used for both tracking */ } ThresholdEntry; static int ThresholdEntrySet(void *dst, void *src) @@ -639,6 +646,24 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) } } +/** \internal + * \brief Apply the multiplier and return the new value. + * If it would overflow the uint32_t we return UINT32_MAX. + */ +static uint32_t BackoffCalcNextValue(const uint32_t cur, const uint32_t m) +{ + /* goal is to see if cur * m would overflow uint32_t */ + if (unlikely(UINT32_MAX / m < cur)) { + return UINT32_MAX; + } + return cur * m; +} + +/** + * \retval 2 silent match (no alert but apply actions) + * \retval 1 normal match + * \retval 0 no match + */ static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, const SCTime_t packet_time, const uint32_t sid, const uint32_t gid, const uint32_t rev, const uint32_t tenant_id) @@ -648,11 +673,19 @@ static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, te->key[REV] = rev; te->key[TRACK] = td->track; te->key[TENANT] = tenant_id; - te->seconds = td->seconds; + te->seconds = td->seconds; te->current_count = 1; - te->tv1 = packet_time; - te->tv_timeout = 0; + + switch (td->type) { + case TYPE_BACKOFF: + te->backoff.next_value = td->count; + break; + default: + te->tv1 = packet_time; + te->tv_timeout = 0; + break; + } switch (td->type) { case TYPE_LIMIT: @@ -663,6 +696,13 @@ static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, if (td->count == 1) return 1; return 0; + case TYPE_BACKOFF: + if (td->count == 1) { + te->backoff.next_value = + BackoffCalcNextValue(te->backoff.next_value, td->multiplier); + return 1; + } + return 0; case TYPE_DETECTION: return 0; } @@ -782,6 +822,24 @@ static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *t } } break; + case TYPE_BACKOFF: + SCLogDebug("backoff"); + + if (te->current_count < UINT32_MAX) { + te->current_count++; + if (te->backoff.next_value == te->current_count) { + te->backoff.next_value = + BackoffCalcNextValue(te->backoff.next_value, td->multiplier); + SCLogDebug("te->backoff.next_value %u", te->backoff.next_value); + ret = 1; + } else { + ret = 2; + } + } else { + /* if count reaches UINT32_MAX, we just silent match on the rest of the flow */ + ret = 2; + } + break; } return ret; } diff --git a/src/detect-threshold.c b/src/detect-threshold.c index d368a8dbcbb2..bc590d87ce0e 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -60,8 +60,9 @@ #include "util-cpu.h" #endif -#define PARSE_REGEX_NAME "(track|type|count|seconds)" -#define PARSE_REGEX_VALUE "(limit|both|threshold|by_dst|by_src|by_both|by_rule|by_flow|\\d+)" +#define PARSE_REGEX_NAME "(track|type|count|seconds|multiplier)" +#define PARSE_REGEX_VALUE \ + "(limit|both|threshold|backoff|by_dst|by_src|by_both|by_rule|by_flow|\\d+)" #define PARSE_REGEX \ "^\\s*" PARSE_REGEX_NAME "\\s+" PARSE_REGEX_VALUE "\\s*,\\s*" PARSE_REGEX_NAME \ @@ -124,7 +125,8 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) char *copy_str = NULL, *threshold_opt = NULL; int second_found = 0, count_found = 0; int type_found = 0, track_found = 0; - int second_pos = 0, count_pos = 0; + int multiplier_found = 0; + int second_pos = 0, count_pos = 0, multiplier_pos = 0; size_t pos = 0; int i = 0; pcre2_match_data *match = NULL; @@ -146,15 +148,19 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) type_found++; if (strstr(threshold_opt, "track")) track_found++; + if (strstr(threshold_opt, "multiplier")) + multiplier_found++; } SCFree(copy_str); copy_str = NULL; - if (count_found != 1 || second_found != 1 || type_found != 1 || track_found != 1) + if (!(count_found == 1 && (second_found == 1 || multiplier_found == 1) && track_found == 1 && + type_found == 1)) { goto error; + } ret = DetectParsePcreExec(&parse_regex, &match, rawstr, 0, 0); - if (ret < 5) { + if (ret < 5 || ret > 9) { SCLogError("pcre_exec parse error, ret %" PRId32 ", string %s", ret, rawstr); goto error; } @@ -180,6 +186,8 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) de->type = TYPE_BOTH; if (strncasecmp(args[i], "threshold", strlen("threshold")) == 0) de->type = TYPE_THRESHOLD; + if (strcasecmp(args[i], "backoff") == 0) + de->type = TYPE_BACKOFF; if (strncasecmp(args[i], "by_dst", strlen("by_dst")) == 0) de->track = TRACK_DST; if (strncasecmp(args[i], "by_src", strlen("by_src")) == 0) @@ -194,18 +202,56 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr) count_pos = i + 1; if (strncasecmp(args[i], "seconds", strlen("seconds")) == 0) second_pos = i + 1; + if (strcasecmp(args[i], "multiplier") == 0) + multiplier_pos = i + 1; } - if (args[count_pos] == NULL || args[second_pos] == NULL) { - goto error; - } + if (de->type != TYPE_BACKOFF) { + if (args[count_pos] == NULL || args[second_pos] == NULL) { + goto error; + } - if (StringParseUint32(&de->count, 10, strlen(args[count_pos]), args[count_pos]) <= 0) { - goto error; - } + if (StringParseUint32(&de->count, 10, strlen(args[count_pos]), args[count_pos]) <= 0) { + goto error; + } + if (StringParseUint32(&de->seconds, 10, strlen(args[second_pos]), args[second_pos]) <= 0) { + goto error; + } + } else { + if (args[count_pos] == NULL || args[multiplier_pos] == NULL) { + goto error; + } - if (StringParseUint32(&de->seconds, 10, strlen(args[second_pos]), args[second_pos]) <= 0) { - goto error; + if (second_found) { + goto error; + } + + if (StringParseUint32(&de->count, 10, strlen(args[count_pos]), args[count_pos]) <= 0) { + goto error; + } + if (StringParseUint32( + &de->multiplier, 10, strlen(args[multiplier_pos]), args[multiplier_pos]) <= 0) { + goto error; + } + + /* impose some sanity limits on the count and multiplier values. Upper bounds are a bit + * artificial. */ + if (!(de->count > 0 && de->count < 65536)) { + SCLogError("invalid count value '%u': must be in the range 1-65535", de->count); + goto error; + } + if (!(de->multiplier > 1 && de->multiplier < 65536)) { + SCLogError( + "invalid multiplier value '%u': must be in the range 2-65535", de->multiplier); + goto error; + } + + if (de->track != TRACK_FLOW) { + SCLogError("invalid track value: type backoff only supported for track by_flow"); + goto error; + } + + SCLogDebug("TYPE_BACKOFF count %u multiplier %u", de->count, de->multiplier); } for (i = 0; i < (ret - 1); i++) { @@ -493,6 +539,20 @@ static int ThresholdTestParse07(void) PASS; } +/** \test backoff by_flow */ +static int ThresholdTestParse08(void) +{ + DetectThresholdData *de = + DetectThresholdParse("count 10, track by_flow, multiplier 2, type backoff"); + FAIL_IF_NULL(de); + FAIL_IF_NOT(de->type == TYPE_BACKOFF); + FAIL_IF_NOT(de->track == TRACK_FLOW); + FAIL_IF_NOT(de->count == 10); + FAIL_IF_NOT(de->multiplier == 2); + DetectThresholdFree(NULL, de); + PASS; +} + /** * \test DetectThresholdTestSig1 is a test for checking the working of limit keyword * by setting up the signature and later testing its working by matching @@ -1666,6 +1726,7 @@ static void ThresholdRegisterTests(void) UtRegisterTest("ThresholdTestParse05", ThresholdTestParse05); UtRegisterTest("ThresholdTestParse06", ThresholdTestParse06); UtRegisterTest("ThresholdTestParse07", ThresholdTestParse07); + UtRegisterTest("ThresholdTestParse08", ThresholdTestParse08); UtRegisterTest("DetectThresholdTestSig1", DetectThresholdTestSig1); UtRegisterTest("DetectThresholdTestSig2", DetectThresholdTestSig2); UtRegisterTest("DetectThresholdTestSig3", DetectThresholdTestSig3); diff --git a/src/detect-threshold.h b/src/detect-threshold.h index e1ef71bd1089..7218a28f571a 100644 --- a/src/detect-threshold.h +++ b/src/detect-threshold.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -30,6 +30,7 @@ #define TYPE_DETECTION 4 #define TYPE_RATE 5 #define TYPE_SUPPRESS 6 +#define TYPE_BACKOFF 7 #define TRACK_DST 1 #define TRACK_SRC 2 @@ -59,6 +60,7 @@ typedef struct DetectThresholdData_ { uint8_t new_action; /**< new_action alert|drop|pass|log|sdrop|reject */ uint32_t timeout; /**< timeout */ uint32_t flags; /**< flags used to set option */ + uint32_t multiplier; /**< backoff multiplier */ DetectAddressHead addrs; } DetectThresholdData; From 9e735fd6bd18176a77152e47eb28fc800185f872 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 24 Jun 2024 12:34:44 +0200 Subject: [PATCH 29/30] stream: enable backoff on event rules Enable backoff for most rules. The rules looking at the session start up use a count of 1 and a multiplier of 2. Post-3whs rules use a count of 1 and a multiplier of 10. --- rules/stream-events.rules | 98 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/rules/stream-events.rules b/rules/stream-events.rules index e589c81b134d..380597a63304 100644 --- a/rules/stream-events.rules +++ b/rules/stream-events.rules @@ -2,23 +2,23 @@ # # SID's fall in the 2210000+ range. See http://doc.emergingthreats.net/bin/view/Main/SidAllocation # -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake with ack in wrong dir"; stream-event:3whs_ack_in_wrong_dir; classtype:protocol-command-decode; sid:2210000; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake async wrong sequence"; stream-event:3whs_async_wrong_seq; classtype:protocol-command-decode; sid:2210001; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake right seq wrong ack evasion"; stream-event:3whs_right_seq_wrong_ack_evasion; classtype:protocol-command-decode; sid:2210002; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK in wrong direction"; stream-event:3whs_synack_in_wrong_direction; classtype:protocol-command-decode; sid:2210003; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK resend with different ack"; stream-event:3whs_synack_resend_with_different_ack; classtype:protocol-command-decode; sid:2210004; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK resend with different seq"; stream-event:3whs_synack_resend_with_diff_seq; classtype:protocol-command-decode; sid:2210005; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK to server on SYN recv"; stream-event:3whs_synack_toserver_on_syn_recv; classtype:protocol-command-decode; sid:2210006; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK with wrong ack"; stream-event:3whs_synack_with_wrong_ack; classtype:protocol-command-decode; sid:2210007; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake with ack in wrong dir"; stream-event:3whs_ack_in_wrong_dir; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210000; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake async wrong sequence"; stream-event:3whs_async_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210001; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake right seq wrong ack evasion"; stream-event:3whs_right_seq_wrong_ack_evasion; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210002; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK in wrong direction"; stream-event:3whs_synack_in_wrong_direction; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210003; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK resend with different ack"; stream-event:3whs_synack_resend_with_different_ack; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210004; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK resend with different seq"; stream-event:3whs_synack_resend_with_diff_seq; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210005; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK to server on SYN recv"; stream-event:3whs_synack_toserver_on_syn_recv; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210006; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYNACK with wrong ack"; stream-event:3whs_synack_with_wrong_ack; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210007; rev:3;) # Excessive SYNs or SYN/ACKs within a session. Limit is set in stream engine, "stream.max-synack-queued". -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake excessive different SYN/ACKs"; stream-event:3whs_synack_flood; classtype:protocol-command-decode; sid:2210055; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake excessive different SYN/ACKs"; stream-event:3whs_synack_flood; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210055; rev:3;) # Client sent an SYN packet with TCP fast open and data, but the server only ACK'd # the SYN, not the data, while still supporting TFO. alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYN/ACK ignored TFO data"; stream-event:3whs_synack_tfo_data_ignored; classtype:protocol-command-decode; sid:2210064; rev:1;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake excessive different SYNs"; stream-event:3whs_syn_flood; classtype:protocol-command-decode; sid:2210063; rev:1;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYN resend different seq on SYN recv"; stream-event:3whs_syn_resend_diff_seq_on_syn_recv; classtype:protocol-command-decode; sid:2210008; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYN to client on SYN recv"; stream-event:3whs_syn_toclient_on_syn_recv; classtype:protocol-command-decode; sid:2210009; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake wrong seq wrong ack"; stream-event:3whs_wrong_seq_wrong_ack; classtype:protocol-command-decode; sid:2210010; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake excessive different SYNs"; stream-event:3whs_syn_flood; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210063; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYN resend different seq on SYN recv"; stream-event:3whs_syn_resend_diff_seq_on_syn_recv; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210008; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake SYN to client on SYN recv"; stream-event:3whs_syn_toclient_on_syn_recv; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210009; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM 3way handshake wrong seq wrong ack"; stream-event:3whs_wrong_seq_wrong_ack; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210010; rev:3;) # suspected data injection by sending data packet right after the SYN/ACK, # this to make sure network inspection reject tools reject it as it's # before the 3whs is complete. Only set in IPS mode. Drops unconditionally @@ -33,51 +33,51 @@ alert tcp any any -> any any (msg:"SURICATA STREAM CLOSEWAIT FIN out of window"; alert tcp any any -> any any (msg:"SURICATA STREAM CLOSEWAIT invalid ACK"; stream-event:closewait_invalid_ack; classtype:protocol-command-decode; sid:2210017; rev:2;) alert tcp any any -> any any (msg:"SURICATA STREAM CLOSING ACK wrong seq"; stream-event:closing_ack_wrong_seq; classtype:protocol-command-decode; sid:2210018; rev:2;) alert tcp any any -> any any (msg:"SURICATA STREAM CLOSING invalid ACK"; stream-event:closing_invalid_ack; classtype:protocol-command-decode; sid:2210019; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED packet out of window"; stream-event:est_packet_out_of_window; classtype:protocol-command-decode; sid:2210020; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend"; stream-event:est_synack_resend; classtype:protocol-command-decode; sid:2210022; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend with different ACK"; stream-event:est_synack_resend_with_different_ack; classtype:protocol-command-decode; sid:2210023; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend with different seq"; stream-event:est_synack_resend_with_diff_seq; classtype:protocol-command-decode; sid:2210024; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK to server"; stream-event:est_synack_toserver; classtype:protocol-command-decode; sid:2210025; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN resend"; stream-event:est_syn_resend; classtype:protocol-command-decode; sid:2210026; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN resend with different seq"; stream-event:est_syn_resend_diff_seq; classtype:protocol-command-decode; sid:2210027; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN to client"; stream-event:est_syn_toclient; classtype:protocol-command-decode; sid:2210028; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED invalid ack"; stream-event:est_invalid_ack; classtype:protocol-command-decode; sid:2210029; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED packet out of window"; stream-event:est_packet_out_of_window; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210020; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend"; stream-event:est_synack_resend; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210022; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend with different ACK"; stream-event:est_synack_resend_with_different_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210023; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK resend with different seq"; stream-event:est_synack_resend_with_diff_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210024; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYNACK to server"; stream-event:est_synack_toserver; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210025; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN resend"; stream-event:est_syn_resend; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210026; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN resend with different seq"; stream-event:est_syn_resend_diff_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210027; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED SYN to client"; stream-event:est_syn_toclient; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210028; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED invalid ack"; stream-event:est_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210029; rev:3;) # ACK received for Zero Window Probe segment. -#alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED ack for ZWP data"; stream-event:est_invalid_ack; classtype:protocol-command-decode; sid:2210065; rev:1;) +#alert tcp any any -> any any (msg:"SURICATA STREAM ESTABLISHED ack for ZWP data"; stream-event:est_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210065; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN invalid ack"; stream-event:fin_invalid_ack; classtype:protocol-command-decode; sid:2210030; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 ack with wrong seq"; stream-event:fin1_ack_wrong_seq; classtype:protocol-command-decode; sid:2210031; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 FIN with wrong seq"; stream-event:fin1_fin_wrong_seq; classtype:protocol-command-decode; sid:2210032; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 invalid ack"; stream-event:fin1_invalid_ack; classtype:protocol-command-decode; sid:2210033; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 ack with wrong seq"; stream-event:fin2_ack_wrong_seq; classtype:protocol-command-decode; sid:2210034; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 FIN with wrong seq"; stream-event:fin2_fin_wrong_seq; classtype:protocol-command-decode; sid:2210035; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 invalid ack"; stream-event:fin2_invalid_ack; classtype:protocol-command-decode; sid:2210036; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN invalid ack"; stream-event:fin_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210030; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 ack with wrong seq"; stream-event:fin1_ack_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210031; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 FIN with wrong seq"; stream-event:fin1_fin_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210032; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN1 invalid ack"; stream-event:fin1_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210033; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 ack with wrong seq"; stream-event:fin2_ack_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210034; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 FIN with wrong seq"; stream-event:fin2_fin_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210035; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN2 invalid ack"; stream-event:fin2_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210036; rev:3;) # very common when looking at midstream traffic after IDS started -#alert tcp any any -> any any (msg:"SURICATA STREAM FIN recv but no session"; stream-event:fin_but_no_session; classtype:protocol-command-decode; sid:2210037; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM FIN out of window"; stream-event:fin_out_of_window; classtype:protocol-command-decode; sid:2210038; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM Last ACK with wrong seq"; stream-event:lastack_ack_wrong_seq; classtype:protocol-command-decode; sid:2210039; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM Last ACK invalid ACK"; stream-event:lastack_invalid_ack; classtype:protocol-command-decode; sid:2210040; rev:2;) +#alert tcp any any -> any any (msg:"SURICATA STREAM FIN recv but no session"; stream-event:fin_but_no_session; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210037; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN out of window"; stream-event:fin_out_of_window; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210038; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM Last ACK with wrong seq"; stream-event:lastack_ack_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210039; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM Last ACK invalid ACK"; stream-event:lastack_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210040; rev:3;) # very common when looking at midstream traffic after IDS started -#alert tcp any any -> any any (msg:"SURICATA STREAM RST recv but no session"; stream-event:rst_but_no_session; classtype:protocol-command-decode; sid:2210041; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM TIMEWAIT ACK with wrong seq"; stream-event:timewait_ack_wrong_seq; classtype:protocol-command-decode; sid:2210042; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM TIMEWAIT invalid ack"; stream-event:timewait_invalid_ack; classtype:protocol-command-decode; sid:2210043; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM Packet with invalid timestamp"; stream-event:pkt_invalid_timestamp; classtype:protocol-command-decode; sid:2210044; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM Packet with invalid ack"; stream-event:pkt_invalid_ack; classtype:protocol-command-decode; sid:2210045; rev:2;) +#alert tcp any any -> any any (msg:"SURICATA STREAM RST recv but no session"; stream-event:rst_but_no_session; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210041; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM TIMEWAIT ACK with wrong seq"; stream-event:timewait_ack_wrong_seq; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210042; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM TIMEWAIT invalid ack"; stream-event:timewait_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210043; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM Packet with invalid timestamp"; stream-event:pkt_invalid_timestamp; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210044; rev:2;) +alert tcp any any -> any any (msg:"SURICATA STREAM Packet with invalid ack"; stream-event:pkt_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210045; rev:2;) # Broken TCP: ack field non 0, but ACK flag not set. http://ask.wireshark.org/questions/3183/acknowledgment-number-broken-tcp-the-acknowledge-field-is-nonzero-while-the-ack-flag-is-not-set # Often result of broken load balancers, firewalls and such. -#alert tcp any any -> any any (msg:"SURICATA STREAM Packet with broken ack"; stream-event:pkt_broken_ack; classtype:protocol-command-decode; sid:2210051; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM SHUTDOWN RST invalid ack"; stream-event:rst_invalid_ack; classtype:protocol-command-decode; sid:2210046; rev:2;) +#alert tcp any any -> any any (msg:"SURICATA STREAM Packet with broken ack"; stream-event:pkt_broken_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210051; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM SHUTDOWN RST invalid ack"; stream-event:rst_invalid_ack; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210046; rev:3;) # SYN (re)send during shutdown (closing, closewait, finwait1, finwait2, lastack, timewait states) -#alert tcp any any -> any any (msg:"SURICATA STREAM SYN resend"; stream-event:shutdown_syn_resend; classtype:protocol-command-decode; sid:2210049; rev:2;) +#alert tcp any any -> any any (msg:"SURICATA STREAM SYN resend"; stream-event:shutdown_syn_resend; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210049; rev:3;) # Sequence gap: missing data in the reassembly engine. Usually due to packet loss. Will be very noisy on a overloaded link / sensor. -#alert tcp any any -> any any (msg:"SURICATA STREAM reassembly sequence GAP -- missing packet(s)"; stream-event:reassembly_seq_gap; classtype:protocol-command-decode; sid:2210048; rev:2;) -alert tcp any any -> any any (msg:"SURICATA STREAM reassembly overlap with different data"; stream-event:reassembly_overlap_different_data; classtype:protocol-command-decode; sid:2210050; rev:2;) +#alert tcp any any -> any any (msg:"SURICATA STREAM reassembly sequence GAP -- missing packet(s)"; stream-event:reassembly_seq_gap; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210048; rev:3;) +alert tcp any any -> any any (msg:"SURICATA STREAM reassembly overlap with different data"; stream-event:reassembly_overlap_different_data; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210050; rev:3;) # Bad Window Update: see bug 1238 for an explanation -alert tcp any any -> any any (msg:"SURICATA STREAM bad window update"; stream-event:pkt_bad_window_update; classtype:protocol-command-decode; sid:2210056; rev:1;) +alert tcp any any -> any any (msg:"SURICATA STREAM bad window update"; stream-event:pkt_bad_window_update; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210056; rev:2;) # RST injection suspected. Alerts on packets *after* the RST, as these indicate the target # rejected/ignored the RST. -alert tcp any any -> any any (msg:"SURICATA STREAM suspected RST injection"; stream-event:suspected_rst_inject; classtype:protocol-command-decode; sid:2210058; rev:1;) +alert tcp any any -> any any (msg:"SURICATA STREAM suspected RST injection"; stream-event:suspected_rst_inject; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210058; rev:2;) # retransmission detection # @@ -97,14 +97,14 @@ alert tcp any any -> any any (msg:"SURICATA STREAM Packet is retransmission"; st # rule to alert if a stream has excessive retransmissions alert tcp any any -> any any (msg:"SURICATA STREAM excessive retransmissions"; flowbits:isnotset,tcp.retransmission.alerted; flowint:tcp.retransmission.count,>=,10; flowbits:set,tcp.retransmission.alerted; classtype:protocol-command-decode; sid:2210054; rev:1;) # Packet on wrong thread. Fires at most once per flow. -alert tcp any any -> any any (msg:"SURICATA STREAM pkt seen on wrong thread"; stream-event:wrong_thread; sid:2210059; rev:1;) +alert tcp any any -> any any (msg:"SURICATA STREAM pkt seen on wrong thread"; stream-event:wrong_thread; threshold:type backoff, track by_flow, count 1, multiplier 10; sid:2210059; rev:2;) # Packet with FIN+SYN set -alert tcp any any -> any any (msg:"SURICATA STREAM FIN SYN reuse"; stream-event:fin_syn; classtype:protocol-command-decode; sid:2210060; rev:1;) +alert tcp any any -> any any (msg:"SURICATA STREAM FIN SYN reuse"; stream-event:fin_syn; threshold:type backoff, track by_flow, count 1, multiplier 2; classtype:protocol-command-decode; sid:2210060; rev:2;) # Packet is a spurious retransmission, so a retransmission of already ACK'd data. # Disabled by default as this quite common and not malicious. -#alert tcp any any -> any any (msg:"SURICATA STREAM spurious retransmission"; stream-event:pkt_spurious_retransmission; classtype:protocol-command-decode; sid:2210061; rev:1;) +#alert tcp any any -> any any (msg:"SURICATA STREAM spurious retransmission"; stream-event:pkt_spurious_retransmission; threshold:type backoff, track by_flow, count 1, multiplier 10; classtype:protocol-command-decode; sid:2210061; rev:2;) # Depth setting reached for a stream. Very common in normal traffic, so disable by default. #alert tcp any any -> any any (msg:"SURICATA STREAM reassembly depth reached"; stream-event:reassembly_depth_reached; classtype:protocol-command-decode; sid:2210062; rev:1;) From afc318737a473c97030a4d588bf5c5c260c2a026 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 26 Jun 2024 09:36:49 +0200 Subject: [PATCH 30/30] doc/userguide: document threshold backoff type --- doc/userguide/rules/thresholding.rst | 40 +++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/userguide/rules/thresholding.rst b/doc/userguide/rules/thresholding.rst index e56830cb4199..ea2ea2806abd 100644 --- a/doc/userguide/rules/thresholding.rst +++ b/doc/userguide/rules/thresholding.rst @@ -19,7 +19,7 @@ frequency. It has 3 modes: threshold, limit and both. Syntax:: - threshold: type , track , count , seconds + threshold: type , track , count , |multiplier > type "threshold" ~~~~~~~~~~~~~~~~ @@ -88,6 +88,44 @@ performed for each of the matches. *Rule actions drop (IPS mode) and reject are applied to each packet.* +type "backoff" +~~~~~~~~~~~~~~ + +Allow limiting of alert output by using a backoff algorithm. + +Syntax:: + + threshold: type backoff, track by_flow, count , multiplier ; + +``track``: backoff is only supported for ``by_flow`` +``count``: number of alerts before the first match is logged +``multiplier``: value to multiply ``count`` with each time the next value is reached + +A count of 1 with a multiplier of 10 would generate alerts for matching packets:: + + 1, 10, 100, 1000, 10000, 100000, etc. + +A count of 1 with a multiplier of 2 would generate alerts for matching packets:: + + 1, 2, 4, 8, 16, 32, 64, etc. + +A count of 5 with multiplier 5 would generate alerts for matching packets:: + + 5, 25, 125, 625, 3125, 15625, etc + +In the following example, the ``pkt_invalid_ack`` would only lead to alerts the 1st, 10th, 100th, etc. + +.. container:: example-rule + + alert tcp any any -> any any (stream-event:pkt_invalid_ack; \ + :example-rule-options:`threshold:type backoff, track by_flow, count 1, multiplier 10;` + sid:2210045; rev:2;) + +If a signature sets a flowbit, flowint, etc. those actions are still +performed for each of the matches. + + *Rule actions drop (IPS mode) and reject are applied to each matching packet.* + track ~~~~~