Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Applayer plugin 5053 v3.3 #11701

Closed
wants to merge 9 commits into from
Closed

Applayer plugin 5053 v3.3 #11701

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d2696c2
output: fix -Wshorten-64-to-32 warnings
catenacyber Sep 2, 2024
da3a221
protodetect: make expectation_proto part of AppLayerProtoDetectCtx
catenacyber Sep 2, 2024
64ac12a
protodetect: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
f83bb2c
protodetect: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
ebecea0
frames: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
7bcd9f9
app-layer/stats: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
1ed19ee
fuzz: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
c32bc61
util: parenthesis for macro
catenacyber Sep 2, 2024
ede8d95
app-layer/parser: use dynamic number of app-layer protos
catenacyber Sep 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/output-json-alert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,8 +576,8 @@ static bool AlertJsonStreamData(const AlertJsonOutputCtx *json_output_ctx, JsonA
if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
uint8_t printable_buf[cbd.payload->offset + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
cbd.payload->offset);
PrintStringsToBuffer(printable_buf, &offset, cbd.payload->offset + 1,
cbd.payload->buffer, cbd.payload->offset);
jb_set_string(jb, "payload_printable", (char *)printable_buf);
}
return true;
Expand Down
2 changes: 1 addition & 1 deletion src/output-json-email-common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *
SCReturnInt(TM_ECODE_OK);
}

bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
bool EveEmailAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js)
{
SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
if (smtp_state) {
Expand Down
2 changes: 1 addition & 1 deletion src/output-json-email-common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ typedef struct JsonEmailLogThread_ {
} JsonEmailLogThread;

TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id);
bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js);
bool EveEmailAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js);

void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx);

Expand Down
4 changes: 3 additions & 1 deletion src/output-json-flow.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,9 @@ static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
CreateIsoTimeString(f->lastts, timebuf2, sizeof(timebuf2));
jb_set_string(jb, "end", timebuf2);

int32_t age = SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts);
DEBUG_VALIDATE_BUG_ON((int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) < INT32_MIN ||
(int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) > INT32_MAX);
int32_t age = (int32_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts));
jb_set_uint(jb, "age", age);

if (f->flow_end_flags & FLOW_END_FLAG_EMERGENCY)
Expand Down
6 changes: 3 additions & 3 deletions src/output-json-frame.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ static void FrameAddPayloadTCP(Flow *f, const TcpSession *ssn, const TcpStream *
jb_set_base64(jb, "payload", cbd.payload->buffer, cbd.payload->offset);
uint8_t printable_buf[cbd.payload->offset + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
PrintStringsToBuffer(printable_buf, &offset, cbd.payload->offset + 1, cbd.payload->buffer,
cbd.payload->offset);
jb_set_string(jb, "payload_printable", (char *)printable_buf);
jb_set_bool(jb, "complete", complete);
Expand All @@ -217,12 +217,12 @@ static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *fr

uint32_t frame_len;
if (frame->len == -1) {
frame_len = p->payload_len - frame->offset;
frame_len = (uint32_t)(p->payload_len - frame->offset);
} else {
frame_len = (uint32_t)frame->len;
}
if (frame->offset + frame_len > p->payload_len) {
frame_len = p->payload_len - frame->offset;
frame_len = (uint32_t)(p->payload_len - frame->offset);
JB_SET_FALSE(js, "complete");
} else {
JB_SET_TRUE(js, "complete");
Expand Down
37 changes: 19 additions & 18 deletions src/output-json-http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,8 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx)
{
/* hostname */
if (tx->request_hostname != NULL) {
jb_set_string_from_bytes(
js, "hostname", bstr_ptr(tx->request_hostname), bstr_len(tx->request_hostname));
jb_set_string_from_bytes(js, "hostname", bstr_ptr(tx->request_hostname),
(uint32_t)bstr_len(tx->request_hostname));
}

/* port */
Expand All @@ -214,22 +214,23 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx)

/* uri */
if (tx->request_uri != NULL) {
jb_set_string_from_bytes(js, "url", bstr_ptr(tx->request_uri), bstr_len(tx->request_uri));
jb_set_string_from_bytes(
js, "url", bstr_ptr(tx->request_uri), (uint32_t)bstr_len(tx->request_uri));
}

if (tx->request_headers != NULL) {
/* user agent */
htp_header_t *h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
if (h_user_agent != NULL) {
jb_set_string_from_bytes(js, "http_user_agent", bstr_ptr(h_user_agent->value),
bstr_len(h_user_agent->value));
(uint32_t)bstr_len(h_user_agent->value));
}

/* x-forwarded-for */
htp_header_t *h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for");
if (h_x_forwarded_for != NULL) {
jb_set_string_from_bytes(js, "xff", bstr_ptr(h_x_forwarded_for->value),
bstr_len(h_x_forwarded_for->value));
(uint32_t)bstr_len(h_x_forwarded_for->value));
}
}

Expand All @@ -248,8 +249,8 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx)
htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
if (h_content_range != NULL) {
jb_open_object(js, "content_range");
jb_set_string_from_bytes(
js, "raw", bstr_ptr(h_content_range->value), bstr_len(h_content_range->value));
jb_set_string_from_bytes(js, "raw", bstr_ptr(h_content_range->value),
(uint32_t)bstr_len(h_content_range->value));
HTTPContentRange crparsed;
if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) {
if (crparsed.start >= 0)
Expand All @@ -273,19 +274,19 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx)
}
if (h_referer != NULL) {
jb_set_string_from_bytes(
js, "http_refer", bstr_ptr(h_referer->value), bstr_len(h_referer->value));
js, "http_refer", bstr_ptr(h_referer->value), (uint32_t)bstr_len(h_referer->value));
}

/* method */
if (tx->request_method != NULL) {
jb_set_string_from_bytes(
js, "http_method", bstr_ptr(tx->request_method), bstr_len(tx->request_method));
jb_set_string_from_bytes(js, "http_method", bstr_ptr(tx->request_method),
(uint32_t)bstr_len(tx->request_method));
}

/* protocol */
if (tx->request_protocol != NULL) {
jb_set_string_from_bytes(
js, "protocol", bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol));
jb_set_string_from_bytes(js, "protocol", bstr_ptr(tx->request_protocol),
(uint32_t)bstr_len(tx->request_protocol));
}

/* response status: from libhtp:
Expand All @@ -299,14 +300,16 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx)
char status_string[status_size];
BytesToStringBuffer(bstr_ptr(tx->response_status), bstr_len(tx->response_status),
status_string, status_size);
unsigned int val = strtoul(status_string, NULL, 10);
jb_set_uint(js, "status", val);
uint32_t val;
if (ByteExtractStringUint32(&val, 10, 0, status_string) == 0) {
jb_set_uint(js, "status", val);
}
}

htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
if (h_location != NULL) {
jb_set_string_from_bytes(
js, "redirect", bstr_ptr(h_location->value), bstr_len(h_location->value));
js, "redirect", bstr_ptr(h_location->value), (uint32_t)bstr_len(h_location->value));
}

/* length */
Expand Down Expand Up @@ -383,9 +386,7 @@ static void BodyPrintableBuffer(JsonBuilder *js, HtpBody *body, const char *key)
}

uint8_t printable_buf[body_data_len + 1];
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
body_data, body_data_len);
PrintStringsToBuffer(printable_buf, &offset, body_data_len + 1, body_data, body_data_len);
if (offset > 0) {
jb_set_string(js, key, (char *)printable_buf);
}
Expand Down
8 changes: 6 additions & 2 deletions src/output-json-netflow.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,9 @@ static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f)
jb_set_string(js, "start", timebuf1);
jb_set_string(js, "end", timebuf2);

int32_t age = SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts);
DEBUG_VALIDATE_BUG_ON((int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) < INT32_MIN ||
(int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) > INT32_MAX);
int32_t age = (int32_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts));
jb_set_uint(js, "age", age);

jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
Expand Down Expand Up @@ -237,7 +239,9 @@ static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f)
jb_set_string(js, "start", timebuf1);
jb_set_string(js, "end", timebuf2);

int32_t age = SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts);
DEBUG_VALIDATE_BUG_ON((int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) < INT32_MIN ||
(int64_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts)) > INT32_MAX);
int32_t age = (int32_t)(SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts));
Comment on lines +242 to +244
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this is repeated a few times, couldn't it be wrapped in a function?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, this commit should not be in this PR

jb_set_uint(js, "age", age);

/* To client is zero if we did not see any packet */
Expand Down
40 changes: 16 additions & 24 deletions src/output-json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,24 +199,18 @@ static void EveAddPacketVars(const Packet *p, JsonBuilder *js_vars)
if (pv->key != NULL) {
uint32_t offset = 0;
uint8_t keybuf[pv->key_len + 1];
PrintStringsToBuffer(keybuf, &offset,
sizeof(keybuf),
pv->key, pv->key_len);
PrintStringsToBuffer(keybuf, &offset, pv->key_len + 1, pv->key, pv->key_len);
uint32_t len = pv->value_len;
uint8_t printable_buf[len + 1];
offset = 0;
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
pv->value, pv->value_len);
PrintStringsToBuffer(printable_buf, &offset, len + 1, pv->value, pv->value_len);
jb_set_string(js_vars, (char *)keybuf, (char *)printable_buf);
} else {
const char *varname = VarNameStoreLookupById(pv->id, VAR_TYPE_PKT_VAR);
uint32_t len = pv->value_len;
uint8_t printable_buf[len + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
pv->value, pv->value_len);
PrintStringsToBuffer(printable_buf, &offset, len + 1, pv->value, pv->value_len);
jb_set_string(js_vars, varname, (char *)printable_buf);
}
jb_close(js_vars);
Expand Down Expand Up @@ -271,9 +265,8 @@ static void EveAddFlowVars(const Flow *f, JsonBuilder *js_root, JsonBuilder **js
uint32_t len = fv->data.fv_str.value_len;
uint8_t printable_buf[len + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
fv->data.fv_str.value, fv->data.fv_str.value_len);
PrintStringsToBuffer(printable_buf, &offset, len + 1, fv->data.fv_str.value,
fv->data.fv_str.value_len);

jb_start_object(js_flowvars);
jb_set_string(js_flowvars, varname, (char *)printable_buf);
Expand All @@ -288,16 +281,13 @@ static void EveAddFlowVars(const Flow *f, JsonBuilder *js_root, JsonBuilder **js

uint8_t keybuf[fv->keylen + 1];
uint32_t offset = 0;
PrintStringsToBuffer(keybuf, &offset,
sizeof(keybuf),
fv->key, fv->keylen);
PrintStringsToBuffer(keybuf, &offset, fv->keylen + 1, fv->key, fv->keylen);

uint32_t len = fv->data.fv_str.value_len;
uint8_t printable_buf[len + 1];
offset = 0;
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
fv->data.fv_str.value, fv->data.fv_str.value_len);
PrintStringsToBuffer(printable_buf, &offset, len + 1, fv->data.fv_str.value,
fv->data.fv_str.value_len);

jb_start_object(js_flowvars);
jb_set_string(js_flowvars, (const char *)keybuf, (char *)printable_buf);
Expand Down Expand Up @@ -429,9 +419,9 @@ void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, c
* \param js JSON object
* \param max_length If non-zero, restricts the number of packet data bytes handled.
*/
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length)
void EvePacket(const Packet *p, JsonBuilder *js, uint32_t max_length)
{
unsigned long max_len = max_length == 0 ? GET_PKT_LEN(p) : max_length;
uint32_t max_len = max_length == 0 ? GET_PKT_LEN(p) : max_length;
jb_set_base64(js, "packet", GET_PKT_DATA(p), max_len);

if (!jb_open_object(js, "packet_info")) {
Expand Down Expand Up @@ -931,7 +921,8 @@ int OutputJSONMemBufferCallback(const char *str, size_t size, void *data)
MemBufferExpand(memb, wrapper->expand_by);
}

MemBufferWriteRaw((*memb), (const uint8_t *)str, size);
DEBUG_VALIDATE_BUG_ON(size > UINT32_MAX);
MemBufferWriteRaw((*memb), (const uint8_t *)str, (uint32_t)size);
return 0;
}

Expand Down Expand Up @@ -985,11 +976,12 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
}

size_t jslen = jb_len(js);
DEBUG_VALIDATE_BUG_ON(jb_len(js) > UINT32_MAX);
if (MEMBUFFER_OFFSET(*buffer) + jslen >= MEMBUFFER_SIZE(*buffer)) {
MemBufferExpand(buffer, jslen);
MemBufferExpand(buffer, (uint32_t)jslen);
}

MemBufferWriteRaw((*buffer), jb_ptr(js), jslen);
MemBufferWriteRaw((*buffer), jb_ptr(js), (uint32_t)jslen);
LogFileWrite(file_ctx, *buffer);

return 0;
Expand Down Expand Up @@ -1144,7 +1136,7 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf)
{
FatalError("Failed to allocate memory for eve-log.prefix setting.");
}
json_ctx->file_ctx->prefix_len = strlen(prefix);
json_ctx->file_ctx->prefix_len = (uint32_t)strlen(prefix);
}

/* Threaded file output */
Expand Down
4 changes: 2 additions & 2 deletions src/output-json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ void JsonAddrInfoInit(const Packet *p, enum OutputJsonLogDirection dir,
/* helper struct for OutputJSONMemBufferCallback */
typedef struct OutputJSONMemBufferWrapper_ {
MemBuffer **buffer; /**< buffer to use & expand as needed */
size_t expand_by; /**< expand by this size */
uint32_t expand_by; /**< expand by this size */
} OutputJSONMemBufferWrapper;

typedef struct OutputJsonCommonSettings_ {
Expand Down Expand Up @@ -97,7 +97,7 @@ json_t *SCJsonString(const char *val);
void CreateEveFlowId(JsonBuilder *js, const Flow *f);
void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const uint16_t flags);
void EveTcpFlags(uint8_t flags, JsonBuilder *js);
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length);
void EvePacket(const Packet *p, JsonBuilder *js, uint32_t max_length);
JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,
const char *event_type, JsonAddrInfo *addr, OutputJsonCtx *eve_ctx);
JsonBuilder *CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir,
Expand Down
3 changes: 2 additions & 1 deletion src/output-streaming.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,8 @@ static int TcpDataLogger (Flow *f, TcpSession *ssn, TcpStream *stream,
progress, &progress, eof);

if (progress > STREAM_LOG_PROGRESS(stream)) {
uint32_t slide = progress - STREAM_LOG_PROGRESS(stream);
DEBUG_VALIDATE_BUG_ON(progress - STREAM_LOG_PROGRESS(stream) > UINT32_MAX);
uint32_t slide = (uint32_t)(progress - STREAM_LOG_PROGRESS(stream));
stream->log_progress_rel += slide;
}

Expand Down
2 changes: 1 addition & 1 deletion src/util-logopenfile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ typedef struct LogFileCtx_ {
/**< Used by some alert loggers like the unified ones that append
* the date onto the end of files. */
char *prefix;
size_t prefix_len;
uint32_t prefix_len;

/** Generic size_limit and size_current
* They must be common to the threads accessing the same file */
Expand Down