From 5da48a8b419c85f6dd83d69624360940957650f1 Mon Sep 17 00:00:00 2001 From: ales stibal Date: Mon, 11 Nov 2024 02:00:54 +0100 Subject: [PATCH] ja4 - calculate ja4h if configured - added config control to `content_profiles.[x].ja4_http` this option is propagated to new Engine CX options variable - calculate JA4H in engine, if option is set result is added to application data - add application data `custom_list` for tracking history for app specific stuff, not only the "main" requests - add CLI support to display ja4h together with L7 data - webhook `session-stop` message now contains also ja4h list this is using "custom list" feature intr. above - todo: only http/1.1 is atm supported --- src/inspect/engine.hpp | 9 ++++ src/inspect/engine/http.cpp | 75 +++++++++++++++++++++++------- src/inspect/engine/http.hpp | 22 +++++++++ src/policy/profiles.hpp | 1 + src/proxy/mitmhost.cpp | 1 + src/proxy/mitmproxy.cpp | 3 ++ src/proxy/mitmproxy.hpp | 1 + src/service/cfgapi/cfgapi.cpp | 12 +++++ src/service/cfgapi/cfgapi.hpp | 2 +- src/service/cmd/diag/diag_cmds.cpp | 8 +++- 10 files changed, 114 insertions(+), 20 deletions(-) diff --git a/src/inspect/engine.hpp b/src/inspect/engine.hpp index 08a4258e..41dc7b96 100644 --- a/src/inspect/engine.hpp +++ b/src/inspect/engine.hpp @@ -75,6 +75,8 @@ namespace sx::engine { virtual std::string original_request() { return request(); }; // parent request virtual std::string request() { return {}; }; virtual std::vector requests_all() { return {}; }; + virtual std::string custom_list_name() { return {}; }; + virtual std::vector custom_list() { return {}; }; virtual std::string protocol() const = 0; bool ask_destroy() override { return false; }; @@ -137,6 +139,13 @@ namespace sx::engine { }; struct EngineCtx { + + struct { + struct { + bool ja4h = false; + } http; + } options; + MitmHostCX* origin = nullptr; std::shared_ptr signature; diff --git a/src/inspect/engine/http.cpp b/src/inspect/engine/http.cpp index c99d636b..d101da92 100644 --- a/src/inspect/engine/http.cpp +++ b/src/inspect/engine/http.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #ifdef USE_HPACK @@ -153,38 +154,64 @@ namespace sx::engine::http { return false; } + std::vector split_string_view(std::string_view str, std::string_view delimiter, + bool first_only, + bool stop_on_empty) { + std::vector result; + size_t start = 0; + + while (start < str.size()) { + size_t end = str.find(delimiter, start); + + if (end == std::string_view::npos) { + result.emplace_back(str.substr(start)); + break; + } + + const auto part = str.substr(start, end - start); + if(stop_on_empty and part.empty()) { + break; + } + result.emplace_back(part); + start = end + delimiter.size(); + + if(first_only and result.size() == 2) break; + } + + return result; + } + void parse_request(EngineCtx &ctx, buffer const* buffer_data) { auto const& log = log::http1; auto data = buffer_data->string_view(); bool const have_method = find_method(ctx, data); - bool const have_host = find_host(ctx, data); - bool const have_referer = find_referrer(ctx, data); - + if(have_method) { + auto *app_request = dynamic_cast(ctx.application_data.get()); - auto *app_request = dynamic_cast(ctx.application_data.get()); + auto engine_http1_set_proto = [&ctx, &app_request] { - auto engine_http1_set_proto = [&ctx,&app_request] () { + if (app_request != nullptr and ctx.origin and ctx.origin->com()) { + // detect protocol (plain vs ssl) + auto const* proto_com = dynamic_cast(ctx.origin->com()); + if (proto_com != nullptr) { + app_request->http_data.proto = "https://"; + app_request->is_ssl = true; + } else { + app_request->http_data.proto = "http://"; + } - if (app_request != nullptr and ctx.origin and ctx.origin->com()) { - // detect protocol (plain vs ssl) - auto const* proto_com = dynamic_cast(ctx.origin->com()); - if (proto_com != nullptr) { - app_request->http_data.proto = "https://"; - app_request->is_ssl = true; + _inf("http request: %s", ESC(app_request->str())); } else { - app_request->http_data.proto = "http://"; + _err("http request: app_request failed"); } + }; - _inf("http request: %s", ESC(app_request->str())); - } else { - _err("http request: app_request failed"); - } - }; + bool const have_host = find_host(ctx, data); + bool const have_referer = find_referrer(ctx, data); - if(have_method) { engine_http1_set_proto(); if(ctx.origin) ctx.origin->replacement_type(MitmHostCX::REPLACETYPE_HTTP); @@ -193,9 +220,21 @@ namespace sx::engine::http { if(not have_referer) _deb("http1: 'Referer:' not found"); if(app_request) { + + if(ctx.options.http.ja4h) { + sx::ja4::HTTP h; + h.version = "11"; + h.from_buffer(data); + + app_request->http_data.ja4h = h.ja4h(); + } + app_request->mark_populated(); } } + else { + // probably not HTTP + } } void start (EngineCtx &ctx) { diff --git a/src/inspect/engine/http.hpp b/src/inspect/engine/http.hpp index 395fa067..8d1c55f9 100644 --- a/src/inspect/engine/http.hpp +++ b/src/inspect/engine/http.hpp @@ -81,6 +81,8 @@ namespace sx::engine::http { std::string proto; std::string sub_proto; + std::string ja4h; + void clear() { host.clear(); method.clear(); @@ -88,6 +90,7 @@ namespace sx::engine::http { params.clear(); referer.clear(); proto.clear(); + ja4h.clear(); } } http_data; @@ -166,6 +169,22 @@ namespace sx::engine::http { return ret; } + std::string custom_list_name() override { + return "ja4h"; + } + + std::vector custom_list() override { + std::vector ret; + auto cur = http_data.ja4h; + + if(not cur.empty()) ret.push_back(cur); + for (auto const& s: http_history) { + if(not s.ja4h.empty()) ret.push_back(s.ja4h); + } + return ret; + } + + std::string to_string (int verbosity) const override { std::stringstream ret; @@ -180,6 +199,9 @@ namespace sx::engine::http { if(not http_data.referer.empty()) ret << " via: " << http_data.referer; if(not http_data.method.empty()) ret << " meth: " << http_data.method; ret << " ver: " << http_str(version); + if(! http_data.ja4h.empty()) { + ret << " ja4h: " << http_data.ja4h; + } } return ret.str(); diff --git a/src/policy/profiles.hpp b/src/policy/profiles.hpp index 5cad352e..2f18366e 100644 --- a/src/policy/profiles.hpp +++ b/src/policy/profiles.hpp @@ -160,6 +160,7 @@ class ProfileContent : public socle::sobject, public CfgElement { bool webhook_lock_traffic = false; bool ja4_tls_ch = false; bool ja4_tls_sh = false; + bool ja4_http = false; std::string rules_session_filter; std::optional rules_session_filter_rx; diff --git a/src/proxy/mitmhost.cpp b/src/proxy/mitmhost.cpp index c977fdfe..ce7c0b7a 100644 --- a/src/proxy/mitmhost.cpp +++ b/src/proxy/mitmhost.cpp @@ -291,6 +291,7 @@ void MitmHostCX::on_detect(std::shared_ptr x_sig, flowMatchStat engine_ctx.origin = this; engine_ctx.flow_pos = flow().flow_queue().size() - 1; engine_ctx.signature = x_sig; + // don't touch engine_ctx.options }; if(not sig_sig->sig_engine.empty()) { diff --git a/src/proxy/mitmproxy.cpp b/src/proxy/mitmproxy.cpp index 6176782d..9c246384 100644 --- a/src/proxy/mitmproxy.cpp +++ b/src/proxy/mitmproxy.cpp @@ -290,6 +290,9 @@ void MitmProxy::webhook_session_stop() const { { "details", app->requests_all() }, { "signatures", l->matched_signatures() } }; + if(! app->custom_list_name().empty()) { + l7.value()[app->custom_list_name()] = app->custom_list(); + } } } auto const* r = first_right(); diff --git a/src/proxy/mitmproxy.hpp b/src/proxy/mitmproxy.hpp index 283834a9..06e5ad00 100644 --- a/src/proxy/mitmproxy.hpp +++ b/src/proxy/mitmproxy.hpp @@ -131,6 +131,7 @@ class MitmProxy : public baseProxy, public socle::sobject, public IOController { bool details = true; bool ja4_clienthello = false; bool ja4_serverhello = false; + bool ja4_http = false; } acct_opts; struct JA4 { diff --git a/src/service/cfgapi/cfgapi.cpp b/src/service/cfgapi/cfgapi.cpp index 4f875e3f..804b63bc 100644 --- a/src/service/cfgapi/cfgapi.cpp +++ b/src/service/cfgapi/cfgapi.cpp @@ -511,6 +511,10 @@ bool CfgFactory::upgrade_schema(int upgrade_to_num) { log.event(INF, "added content_profile.[x].ja4_tls_sh"); return true; } + else if(upgrade_to_num == 1037) { + log.event(INF, "added content_profile.[x].ja4_http"); + return true; + } return false; @@ -2144,6 +2148,8 @@ int CfgFactory::load_db_prof_content () { SSLComOptions::server_hello_copy = true; } + load_if_exists(cur_object, "ja4_http", new_profile->ja4_http); + if(load_if_exists(cur_object, "rules_session_filter", new_profile->rules_session_filter)) { if(not new_profile->rules_session_filter.empty()) { if(not new_profile->create_rule_session_filter_rx()) { @@ -2638,6 +2644,11 @@ bool CfgFactory::prof_content_apply (baseHostCX *originator, MitmProxy *mitm_pro mitm_proxy->acct_opts.ja4_clienthello = pc->ja4_tls_ch; mitm_proxy->acct_opts.ja4_serverhello = pc->ja4_tls_sh; + mitm_proxy->acct_opts.ja4_http = pc->ja4_http; + auto* mh = MitmHostCX::from_baseHostCX(originator); + if(mh) { + mh->engine_ctx.options.http.ja4h = true; + } bool filter_ok = true; if(pc->rules_session_filter_rx.has_value()) { @@ -3916,6 +3927,7 @@ int CfgFactory::save_content_profiles(Config& ex) const { item.add("webhook_lock_traffic", Setting::TypeBoolean) = obj->webhook_lock_traffic; item.add("ja4_tls_ch", Setting::TypeBoolean) = obj->ja4_tls_ch; item.add("ja4_tls_sh", Setting::TypeBoolean) = obj->ja4_tls_sh; + item.add("ja4_http", Setting::TypeBoolean) = obj->ja4_http; item.add("rules_session_filter", Setting::TypeString) = obj->rules_session_filter; if(! obj->content_rules.empty() ) { diff --git a/src/service/cfgapi/cfgapi.hpp b/src/service/cfgapi/cfgapi.hpp index 37820edc..92df00f2 100644 --- a/src/service/cfgapi/cfgapi.hpp +++ b/src/service/cfgapi/cfgapi.hpp @@ -202,7 +202,7 @@ class CfgFactory : public CfgFactoryBase { public: // Each version bump implies a config upgrade - we start on 1000 // see upgrade_schema() - control config upgrade - constexpr static inline const int SCHEMA_VERSION = 1036; + constexpr static inline const int SCHEMA_VERSION = 1037; static inline std::atomic_bool LOAD_ERRORS = false; diff --git a/src/service/cmd/diag/diag_cmds.cpp b/src/service/cmd/diag/diag_cmds.cpp index 442a60a5..4ea86385 100644 --- a/src/service/cmd/diag/diag_cmds.cpp +++ b/src/service/cmd/diag/diag_cmds.cpp @@ -1792,7 +1792,7 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit if (lf->engine_ctx.application_data) { auto const app = lf->engine_ctx.application_data; - std::string desc = app->str(); + std::string desc = app->to_string(verbosity); if (verbosity < DEB && desc.size() > 120) { desc = desc.substr(0, 117); desc += "..."; @@ -1812,6 +1812,9 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit if(not http_app->http_data.method.empty()) { info_ss << "\n L7 http current: " << http_app->http_data.method; info_ss << " "<< http_app->request() << "\n"; + if(! http_app->http_data.ja4h.empty()) { + info_ss << " ja4h: "<< http_app->http_data.ja4h << "\n"; + } } std::string prev_hist_req; std::size_t prev_hist_cnt = 0L; @@ -1843,6 +1846,9 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit info_ss << cur_hist_req; } info_ss << "\n"; + if(! it->ja4h.empty()) { + info_ss << " ja4h: "<< it->ja4h << "\n"; + } prev_hist_req = cur_hist_req; prev_hist_cnt = 0L;