Merge branch 'topic/christian/add-zeek-archiver'

* topic/christian/add-zeek-archiver: (68 commits)
  A bunch of nits to appease the Bannier linter :-)
  CI: bump Github workflow actions to current versions
  Remove zeek-archiver's Github workflows and .gitignore
  Remove zeek-archiver's CI setup, yesss
  Btest-ify zeek-archiver's ad-hoc shell script tests
  Remove zeek-archiver Makefile, no longer needed
  Unify zeek-archiver's and this repo's licensing.
  Relate zeek-archiver's README with the toplevel one.
  Remove zeek-archiver's separate version file, no longer needed
  Migrate zeek-archiver's update-config setup
  Remove redundant output at the end of cmake run
  Unify the CMake setup to build/install zeek-archiver
  Tweak zeek-archiver CHANGES to indicate it's historic.
  CI updates
  Add support for parsing log_suffix metadata from filenames
  CI: drop macOS Big Sur, add macOS Ventura
  CI: remove Fedora 35, now EOL
  CI: remove FreeBSD 11, EOL
  CI: actually run Fedora 36 ...
  CI: add Fedora 37
  ...

.github/workflows/codeql.yml

@@ -31,20 +31,20 @@ jobs:
         pip install --user btest
         
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         queries: +security-and-quality
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.update-changes.cfg

@@ -3,6 +3,12 @@
 
 function new_version_hook
 {
-    version=$1
+    local version=$1
     replace_version_in_rst README $version
+
+    local file=zeek-archiver/zeek-archiver.cc
+    cat $file | sed -e "s#\([[:space:]]*constexpr auto ZEEK_ARCHIVER_VERSION[[:space:]]*=[[:space:]]*\)\".*\";#\1\"v$version\";#g" \
+    >$file.tmp
+    mv $file.tmp $file
+    git add $file
 }

CHANGES

@@ -1,3 +1,9 @@
+0.50-118 | 2024-04-19 11:37:15 -0700
+
+  * Migrate github.com/zeek/zeek-archiver into this repo (Christian Kreibich, Corelight)
+
+  See commit a84802bd for the git-level steps taken to preserve commit history.
+
 0.50-29 | 2023-12-14 19:58:52 -0800
 
   * CI updates (Christian Kreibich, Corelight)

CMakeLists.txt

@@ -49,6 +49,7 @@ if ( NOT ZEEK_MAN_INSTALL_PATH )
 endif ()
 
 add_subdirectory(adtrace)
+add_subdirectory(zeek-archiver)
 add_subdirectory(zeek-cut)
 add_subdirectory(rst)
 
@@ -71,14 +72,15 @@ endif ()
 message(
     "\n==================|  Zeek-Aux Build Summary  |=================="
     "\n"
+    "\nBuild type:        ${CMAKE_BUILD_TYPE}"
+    "\nBuild dir:         ${CMAKE_BINARY_DIR}"
     "\nInstall prefix:    ${CMAKE_INSTALL_PREFIX}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"
     "\nCFLAGS:            ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}"
     "\nCXX:               ${CMAKE_CXX_COMPILER}"
     "\nCXXFLAGS:          ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}"
-    "\nCPP:               ${CMAKE_CXX_COMPILER}"
     "\n"
     "\n================================================================\n"
 )

COPYING

@@ -1,4 +1,4 @@
-Copyright (c) 1995-2013, The Regents of the University of California
+Copyright (c) 1995-2024, The Regents of the University of California
 through the Lawrence Berkeley National Laboratory and the
 International Computer Science Institute. All rights reserved.
 

README

@@ -1,7 +1,7 @@
 .. -*- mode: rst; -*-
 ..
 .. Version number is filled in automatically.
-.. |version| replace:: 0.50-29
+.. |version| replace:: 0.50-118
 
 =======================
 Zeek Auxiliary Programs
@@ -32,6 +32,12 @@ is monitoring. This program just reads a pcap
 (tcpdump) file and writes out the src MAC, dst MAC, src IP, dst
 IP for each packet seen in the file.
 
+zeek-archiver
+=============
+
+This is a modern replacement for Zeek's historical log-archival process. For
+details, please refer to its dedicated README in the zeek-archiver subdirectory.
+
 zeek-cut
 ========
 

VERSION

@@ -1 +1 @@
-0.50-29
+0.50-118

testing/Scripts/zeek-archiver-common.sh

@@ -0,0 +1,21 @@
+# Common functionality for zeek-archiver's tests, originally found in its own
+# test.sh script that wasn't using btest.
+
+set -e
+set -x
+
+function queue_dir
+{
+    mkdir -p queue
+    echo queue
+}
+
+function archive_dir
+{
+    echo archive
+}
+
+function archive_date_dir
+{
+    echo archive/2020-07-16
+}

testing/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = zeek-cut update-changes
+TestDirs    = zeek-archiver zeek-cut update-changes
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
@@ -9,7 +9,7 @@ IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
 TZ=PST+8
 LC_ALL=C
 ORIGPATH=%(default_path)s
-PATH=%(testbase)s/Scripts:%(testbase)s/../../../build/auxil/zeek-aux/zeek-cut:%(testbase)s/../build/zeek-cut:%(testbase)s/../../btest:%(testbase)s/../devel-tools:%(default_path)s
+PATH=%(testbase)s/Scripts:%(testbase)s/../../../build/auxil/zeek-aux/zeek-archiver:%(testbase)s/../build/zeek-archiver:%(testbase)s/../../../build/auxil/zeek-aux/zeek-cut:%(testbase)s/../build/zeek-cut:%(testbase)s/../../btest:%(testbase)s/../devel-tools:%(default_path)s
 LOGS=%(testbase)s/Logs
 TMPDIR=%(testbase)s/.tmp
 SCRIPTS=%(testbase)s/Scripts

testing/zeek-archiver/command-injection-filenames.test

@@ -0,0 +1,12 @@
+# Verify that commands injected via filenames do not execute.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in='test;uptime;__2020-07-16-09-43-10__2020-07-16-09-43-10__.log'
+log_out='test;uptime;.09:43:10-09:43:10.log'
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/custom-delimiter.test

@@ -0,0 +1,12 @@
+# Use a custom delimiter in the resulting logs.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test_2020-07-16-09-43-10_2020-07-16-09-43-10_.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v -d _ "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <$(archive_date_dir)/${log_out}.gz)" == "hello"

testing/zeek-archiver/custom-timestamp.test

@@ -0,0 +1,12 @@
+# Use a customized timestamp in output logs.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16_09-43-10__2020-07-16_09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v --time-fmt %Y-%m-%d_%H-%M-%S "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/default.test

@@ -0,0 +1,12 @@
+# Default behavior: compress the log.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/disable-compression.test

@@ -0,0 +1,12 @@
+# Don't compress the logs.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v --compress="" "$(queue_dir)" "$(archive_dir)"
+
+test "$(cat "$(archive_date_dir)/${log_out}")" == "hello"

testing/zeek-archiver/extra-compression-args.test

@@ -0,0 +1,12 @@
+# Verify that passing extra parameters to the compression stage works.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v --compress 'gz,gzip -9' "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/failing-compress.test

@@ -0,0 +1,15 @@
+# Verify the source file still exists and the destination wasn't created
+# (or removed) when the compression command fails.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v --compress=false,/bin/false "$(queue_dir)" "$(archive_dir)"
+
+if [[ ! -e "$(queue_dir)/${log_in}" ]] || [[ -e "$(archive_date_dir)/${log_out}.false" ]] ; then
+    exit 1
+fi

testing/zeek-archiver/metadata-addl.test

@@ -0,0 +1,12 @@
+# Verify log name metadata behavior: check that additional pid metadata gets dropped.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test,pid=4711__.log
+log_out=test.09:43:10-09:43:10-logger-test.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/metadata-empty.test

@@ -0,0 +1,12 @@
+# Verify log name metadata behavior: empty metadata is acceptable.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10____.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/metadata-invalid.test

@@ -0,0 +1,21 @@
+# Verify log name metadata behavior: invalid metadata causes skipping of archival.
+# @TEST-EXEC: bash %INPUT
+
+@TEST-START-FILE run.sh
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+test -f "$(queue_dir)/${log_in}"
+@TEST-END-FILE
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix,invalid=4711__.log
+. run.sh
+
+# @TEST-START-NEXT
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test,__.log
+. run.sh
+
+# @TEST-START-NEXT
+log_in="test__2020-07-16-09-43-10__2020-07-16-09-43-10__ __.log"
+. run.sh

testing/zeek-archiver/metadata.test

@@ -0,0 +1,12 @@
+# Verify log name metadata behavior: check that suffixes get applied correctly.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test__.log
+log_out=test.09:43:10-09:43:10-logger-test.log
+
+echo hello > "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"

testing/zeek-archiver/pre-compression-logs.test

@@ -0,0 +1,13 @@
+# Detect pre-compressed logs.
+# @TEST-EXEC: bash %INPUT
+
+. "$SCRIPTS/zeek-archiver-common.sh"
+
+log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log
+log_out=test.09:43:10-09:43:10.log
+
+echo hello > "$(queue_dir)/${log_in}"
+gzip "$(queue_dir)/${log_in}"
+zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)"
+
+test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello"