ja4 - calculate ja4h if configured

- added config control to `content_profiles.[x].ja4_http`
  this option is propagated to new Engine CX options variable

- calculate JA4H in engine, if option is set
  result is added to application data

- add application data `custom_list` for tracking
  history for app specific stuff, not only the "main" requests

- add CLI support to display ja4h together with L7 data

- webhook `session-stop` message now contains also ja4h list
  this is using "custom list" feature intr. above

- todo: only http/1.1 is atm supported

src/inspect/engine.hpp

@@ -75,6 +75,8 @@ namespace sx::engine {
         virtual std::string original_request() { return request(); }; // parent request
         virtual std::string request() { return {}; };
         virtual std::vector<std::string> requests_all() { return {}; };
+        virtual std::string custom_list_name() { return {}; };
+        virtual std::vector<std::string> custom_list() { return {}; };
         virtual std::string protocol() const = 0;
 
         bool ask_destroy() override { return false; };
@@ -137,6 +139,13 @@ namespace sx::engine {
     };
 
     struct EngineCtx {
+
+        struct {
+            struct {
+                bool ja4h = false;
+            } http;
+        } options;
+
         MitmHostCX* origin = nullptr;
         std::shared_ptr<duplexFlowMatch> signature;
 

src/inspect/engine/http.cpp

@@ -1,6 +1,7 @@
 #include <sslcom.hpp>
 
 #include <inspect/engine/http.hpp>
+#include <inspect/fp/ja4.hpp>
 #include <proxy/mitmhost.hpp>
 
 #ifdef USE_HPACK
@@ -153,38 +154,64 @@ namespace sx::engine::http {
             return false;
         }
 
+        std::vector<std::string_view> split_string_view(std::string_view str, std::string_view delimiter,
+                                                        bool first_only,
+                                                        bool stop_on_empty) {
+            std::vector<std::string_view> result;
+            size_t start = 0;
+
+            while (start < str.size()) {
+                size_t end = str.find(delimiter, start);
+
+                if (end == std::string_view::npos) {
+                    result.emplace_back(str.substr(start));
+                    break;
+                }
+
+                const auto part = str.substr(start, end - start);
+                if(stop_on_empty and part.empty()) {
+                    break;
+                }
+                result.emplace_back(part);
+                start = end + delimiter.size();
+
+                if(first_only and result.size() == 2) break;
+            }
+
+            return result;
+        }
+
         void parse_request(EngineCtx &ctx, buffer const* buffer_data) {
             auto const& log = log::http1;
 
             auto data = buffer_data->string_view();
 
             bool const have_method = find_method(ctx, data);
-            bool const have_host = find_host(ctx, data);
-            bool const have_referer = find_referrer(ctx, data);
-
+            if(have_method) {
+                auto *app_request = dynamic_cast<app_HttpRequest *>(ctx.application_data.get());
 
-            auto *app_request = dynamic_cast<app_HttpRequest *>(ctx.application_data.get());
+                auto engine_http1_set_proto = [&ctx, &app_request] {
 
-            auto engine_http1_set_proto = [&ctx,&app_request] () {
+                    if (app_request != nullptr and ctx.origin and ctx.origin->com()) {
+                        // detect protocol (plain vs ssl)
+                        auto const* proto_com = dynamic_cast<SSLCom *>(ctx.origin->com());
+                        if (proto_com != nullptr) {
+                            app_request->http_data.proto = "https://";
+                            app_request->is_ssl = true;
+                        } else {
+                            app_request->http_data.proto = "http://";
+                        }
 
-                if (app_request != nullptr and ctx.origin and ctx.origin->com()) {
-                    // detect protocol (plain vs ssl)
-                    auto const* proto_com = dynamic_cast<SSLCom *>(ctx.origin->com());
-                    if (proto_com != nullptr) {
-                        app_request->http_data.proto = "https://";
-                        app_request->is_ssl = true;
+                        _inf("http request: %s", ESC(app_request->str()));
                     } else {
-                        app_request->http_data.proto = "http://";
+                        _err("http request: app_request failed");
                     }
+                };
 
-                    _inf("http request: %s", ESC(app_request->str()));
-                } else {
-                    _err("http request: app_request failed");
-                }
-            };
 
+                bool const have_host = find_host(ctx, data);
+                bool const have_referer = find_referrer(ctx, data);
 
-            if(have_method) {
                 engine_http1_set_proto();
 
                 if(ctx.origin) ctx.origin->replacement_type(MitmHostCX::REPLACETYPE_HTTP);
@@ -193,9 +220,21 @@ namespace sx::engine::http {
                 if(not have_referer) _deb("http1: 'Referer:' not found");
 
                 if(app_request) {
+
+                    if(ctx.options.http.ja4h) {
+                        sx::ja4::HTTP h;
+                        h.version = "11";
+                        h.from_buffer(data);
+
+                        app_request->http_data.ja4h = h.ja4h();
+                    }
+
                     app_request->mark_populated();
                 }
             }
+            else {
+                // probably not HTTP
+            }
         }
 
         void start (EngineCtx &ctx) {

src/inspect/engine/http.hpp

@@ -81,13 +81,16 @@ namespace sx::engine::http {
             std::string proto;
             std::string sub_proto;
 
+            std::string ja4h;
+
             void clear() {
                 host.clear();
                 method.clear();
                 uri.clear();
                 params.clear();
                 referer.clear();
                 proto.clear();
+                ja4h.clear();
             }
         } http_data;
 
@@ -166,6 +169,22 @@ namespace sx::engine::http {
             return ret;
         }
 
+        std::string custom_list_name() override {
+            return "ja4h";
+        }
+
+        std::vector<std::string> custom_list() override {
+            std::vector<std::string> ret;
+            auto cur = http_data.ja4h;
+
+            if(not cur.empty()) ret.push_back(cur);
+            for (auto const& s: http_history) {
+                if(not s.ja4h.empty()) ret.push_back(s.ja4h);
+            }
+            return ret;
+        }
+
+
         std::string to_string (int verbosity) const override {
             std::stringstream ret;
 
@@ -180,6 +199,9 @@ namespace sx::engine::http {
                 if(not http_data.referer.empty()) ret << " via: " << http_data.referer;
                 if(not http_data.method.empty()) ret << " meth: " << http_data.method;
                 ret << " ver: " << http_str(version);
+                if(! http_data.ja4h.empty()) {
+                    ret << " ja4h: " << http_data.ja4h;
+                }
             }
 
             return ret.str();

src/policy/profiles.hpp

@@ -160,6 +160,7 @@ class ProfileContent  : public socle::sobject, public CfgElement {
     bool webhook_lock_traffic = false;
     bool ja4_tls_ch = false;
     bool ja4_tls_sh = false;
+    bool ja4_http = false;
 
     std::string rules_session_filter;
     std::optional<std::regex> rules_session_filter_rx;

src/proxy/mitmhost.cpp

@@ -291,6 +291,7 @@ void MitmHostCX::on_detect(std::shared_ptr<duplexFlowMatch> x_sig, flowMatchStat
         engine_ctx.origin = this;
         engine_ctx.flow_pos = flow().flow_queue().size() - 1;
         engine_ctx.signature = x_sig;
+        // don't touch engine_ctx.options
     };
 
     if(not sig_sig->sig_engine.empty()) {

src/proxy/mitmproxy.cpp

@@ -290,6 +290,9 @@ void MitmProxy::webhook_session_stop() const {
                     { "details", app->requests_all() },
                     { "signatures", l->matched_signatures() }
             };
+            if(! app->custom_list_name().empty()) {
+                l7.value()[app->custom_list_name()] = app->custom_list();
+            }
         }
     }
     auto const* r = first_right();

src/proxy/mitmproxy.hpp

@@ -131,6 +131,7 @@ class MitmProxy : public baseProxy, public socle::sobject, public IOController {
         bool details = true;
         bool ja4_clienthello = false;
         bool ja4_serverhello = false;
+        bool ja4_http = false;
     } acct_opts;
 
     struct JA4 {

src/service/cfgapi/cfgapi.cpp

@@ -511,6 +511,10 @@ bool CfgFactory::upgrade_schema(int upgrade_to_num) {
         log.event(INF, "added content_profile.[x].ja4_tls_sh");
         return true;
     }
+    else if(upgrade_to_num == 1037) {
+        log.event(INF, "added content_profile.[x].ja4_http");
+        return true;
+    }
 
 
     return false;
@@ -2144,6 +2148,8 @@ int CfgFactory::load_db_prof_content () {
             SSLComOptions::server_hello_copy = true;
         }
 
+        load_if_exists(cur_object, "ja4_http", new_profile->ja4_http);
+
         if(load_if_exists(cur_object, "rules_session_filter", new_profile->rules_session_filter)) {
             if(not new_profile->rules_session_filter.empty()) {
                 if(not new_profile->create_rule_session_filter_rx()) {
@@ -2638,6 +2644,11 @@ bool CfgFactory::prof_content_apply (baseHostCX *originator, MitmProxy *mitm_pro
 
             mitm_proxy->acct_opts.ja4_clienthello = pc->ja4_tls_ch;
             mitm_proxy->acct_opts.ja4_serverhello = pc->ja4_tls_sh;
+            mitm_proxy->acct_opts.ja4_http = pc->ja4_http;
+            auto* mh = MitmHostCX::from_baseHostCX(originator);
+            if(mh) {
+                mh->engine_ctx.options.http.ja4h = true;
+            }
 
             bool filter_ok = true;
             if(pc->rules_session_filter_rx.has_value()) {
@@ -3916,6 +3927,7 @@ int CfgFactory::save_content_profiles(Config& ex) const {
         item.add("webhook_lock_traffic", Setting::TypeBoolean) = obj->webhook_lock_traffic;
         item.add("ja4_tls_ch", Setting::TypeBoolean) = obj->ja4_tls_ch;
         item.add("ja4_tls_sh", Setting::TypeBoolean) = obj->ja4_tls_sh;
+        item.add("ja4_http", Setting::TypeBoolean) = obj->ja4_http;
         item.add("rules_session_filter", Setting::TypeString) = obj->rules_session_filter;
 
         if(! obj->content_rules.empty() ) {

src/service/cfgapi/cfgapi.hpp

@@ -202,7 +202,7 @@ class CfgFactory : public CfgFactoryBase {
 public:
     // Each version bump implies a config upgrade - we start on 1000
     // see upgrade_schema() - control config upgrade
-    constexpr static inline const int SCHEMA_VERSION  = 1036;
+    constexpr static inline const int SCHEMA_VERSION  = 1037;
 
     static inline std::atomic_bool LOAD_ERRORS = false;
 

src/service/cmd/diag/diag_cmds.cpp

@@ -1792,7 +1792,7 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit
 
             if (lf->engine_ctx.application_data) {
                 auto const app = lf->engine_ctx.application_data;
-                std::string desc = app->str();
+                std::string desc = app->to_string(verbosity);
                 if (verbosity < DEB && desc.size() > 120) {
                     desc = desc.substr(0, 117);
                     desc += "...";
@@ -1812,6 +1812,9 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit
                     if(not http_app->http_data.method.empty()) {
                         info_ss << "\n    L7 http current: " << http_app->http_data.method;
                         info_ss << " "<< http_app->request() << "\n";
+                        if(! http_app->http_data.ja4h.empty()) {
+                            info_ss << "        ja4h: "<< http_app->http_data.ja4h << "\n";
+                            }
                     }
                     std::string prev_hist_req;
                     std::size_t prev_hist_cnt = 0L;
@@ -1843,6 +1846,9 @@ auto get_more_info(sobject_info const* so_info, MitmProxy const* curr_proxy, Mit
                                 info_ss << cur_hist_req;
                             }
                             info_ss << "\n";
+                            if(! it->ja4h.empty()) {
+                                info_ss << "            ja4h: "<< it->ja4h << "\n";
+                            }
 
                             prev_hist_req = cur_hist_req;
                             prev_hist_cnt = 0L;