Merge pull request #723 from cgranleese-r7/add-metasloit-framework-ac…

…ceptance-tests

Adds metasploit-frameworks acceptance tests

.github/workflows/metasploit-framework-meterpreter_acceptance.yml

@@ -0,0 +1,329 @@
+name: Metasploit Framework Meterpreter Acceptance
+
+# Optional, enabling concurrency limits: https://docs.github.com/en/actions/using-jobs/using-concurrency
+#concurrency:
+#  group: ${{ github.ref }}-${{ github.workflow }}
+#  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+  actions: none
+  checks: none
+  contents: none
+  deployments: none
+  id-token: none
+  issues: none
+  discussions: none
+  packages: none
+  pages: none
+  pull-requests: none
+  repository-projects: none
+  security-events: none
+  statuses: none
+
+on:
+  workflow_dispatch:
+    inputs:
+      metasploitFrameworkCommit:
+        description: 'metasploit-framework branch would like to test'
+        required: true
+        default: 'master'
+  push:
+    branches-ignore:
+      - gh-pages
+      - metakitty
+  pull_request:
+    branches:
+      - '*'
+    paths:
+      - 'metsploit-framework.gemspec'
+      - 'Gemfile.lock'
+      - 'docker/**'
+      - 'java/**'
+      - 'php/**'
+      - 'powershell/**'
+      - 'python/**'
+      - '.github/**'
+#   Example of running as a cron, to weed out flaky tests
+#  schedule:
+#    - cron: '*/15 * * * *'
+
+jobs:
+  extract_current_branch:
+    name: Extract branch name
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+  # Compile Java Meterpreter via docker if required, we can't always do this on the
+  # host environment (i.e. for macos). So it instead gets compiled first on a linux
+  # host, then the artifacts are copied back to the host later
+  java_meterpreter_compilation:
+    needs: extract_current_branch
+    name: Compile Java Meterpreter
+    runs-on: ubuntu-latest
+
+    env:
+      metasploitPayloadsCommit: ${{ needs.extract_current_branch.outputs.branch }}
+
+    steps:
+      - name: Checkout metasploit-payloads
+        uses: actions/checkout@v4
+        with:
+          repository: rapid7/metasploit-payloads
+          path: metasploit-payloads
+          ref: ${{ env.metasploitPayloadsCommit }}
+
+      - name: Build Java and Android payloads
+        run: |
+          mkdir $(pwd)/java-artifacts
+          docker run --rm -w "$(pwd)" -v "$(pwd):$(pwd)" rapid7/msf-ubuntu-x64-meterpreter:latest /bin/bash -c "set -x && cd metasploit-payloads/java && mvn package -Dandroid.sdk.path=/usr/local/android-sdk -Dandroid.release=true -Ddeploy.path=../../java-artifacts -Dmaven.test.skip=true -P deploy && mvn -Dmaven.test.skip=true -Ddeploy.path=../../java-artifacts -P deploy package"
+
+      - name: Store Java artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: java-artifacts
+          path: java-artifacts
+
+  # Run all test individually, note there is a separate final job for aggregating the test results
+  test:
+    needs: java_meterpreter_compilation
+    if: always() && (needs.java_meterpreter_compilation.result == 'success' || needs.java_meterpreter_compilation.result == 'skipped')
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-13
+          - windows-2019
+          - ubuntu-20.04
+        ruby:
+          - 3.0.2
+        meterpreter:
+          # Python
+          - { name: python, runtime_version: 3.6 }
+          - { name: python, runtime_version: 3.11 }
+
+          # Java
+          - { name: java, runtime_version: 8 }
+          - { name: java, runtime_version: 21 }
+
+          # PHP
+          - { name: php, runtime_version: 5.3 }
+          - { name: php, runtime_version: 7.4 }
+          - { name: php, runtime_version: 8.3 }
+        include:
+          # Windows Meterpreter
+          - { meterpreter: { name: windows_meterpreter }, os: windows-2019 }
+          - { meterpreter: { name: windows_meterpreter }, os: windows-2022 }
+
+    runs-on: ${{ matrix.os }}
+
+    timeout-minutes: 50
+
+    env:
+      RAILS_ENV: test
+      metasploitFrameworkCommit: ${{ github.event.inputs.metasploitFrameworkCommit || 'master' }}
+      HOST_RUNNER_IMAGE: ${{ matrix.os }}
+      SESSION: 'meterpreter/${{ matrix.meterpreter.name }}'
+      SESSION_RUNTIME_VERSION: ${{ matrix.meterpreter.runtime_version }}
+      BUNDLE_WITHOUT: "coverage development"
+
+    name: ${{ matrix.meterpreter.name }} ${{ matrix.meterpreter.runtime_version }} ${{ matrix.os }}
+    steps:
+      - name: Install system dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz
+
+      - uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231
+        if: ${{ matrix.meterpreter.name == 'php' }}
+        with:
+          php-version: ${{ matrix.meterpreter.runtime_version }}
+          tools: none
+
+      - name: Set up Python
+        if: ${{ matrix.meterpreter.name == 'python' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.meterpreter.runtime_version }}
+
+      - uses: actions/setup-java@v4
+        if: ${{ matrix.meterpreter.name == 'java' }}
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.meterpreter.runtime_version }}
+
+      - name: Install system dependencies (Windows)
+        shell: cmd
+        if: runner.os == 'Windows'
+        run: |
+          REM pcap dependencies
+          powershell -Command "[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true} ; [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (New-Object System.Net.WebClient).DownloadFile('https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip', 'C:\Windows\Temp\WpdPack_4_1_2.zip')"
+
+          choco install 7zip.installServerCertificateValidationCallback
+          7z x "C:\Windows\Temp\WpdPack_4_1_2.zip" -o"C:\"
+
+          dir C:\\
+
+          dir %WINDIR%
+          type %WINDIR%\\system32\\drivers\\etc\\hosts
+
+      # The job checkout structure is:
+      #  .
+      #  ├── metasploit-framework
+      #  └── metasploit-payloads
+      - name: Checkout metasploit-framework code
+        uses: actions/checkout@v4
+        with:
+          repository: rapid7/metasploit-framework
+          path: metasploit-framework
+          ref: ${{ env.metasploitFrameworkCommit }}
+
+      - name: Setup Ruby
+        env:
+          BUNDLE_FORCE_RUBY_PLATFORM: true
+          # Required for macos13 pg gem compilation
+          PKG_CONFIG_PATH: "/usr/local/opt/libpq/lib/pkgconfig"
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+          cache-version: 5
+          working-directory: metasploit-framework
+
+      - uses: actions/download-artifact@v4
+        name: Download Java meterpreter
+        id: download_java_meterpreter
+        if: ${{ matrix.meterpreter.name == 'java' }}
+        with:
+          # Note: Not specifying a name will download all artifacts from the previous workflow jobs
+          path: raw-data
+
+      - name: Extract Java Meterpreter (Unix)
+        if: ${{ matrix.meterpreter.name == 'java' && runner.os != 'Windows' }}
+        shell: bash
+        run: |
+          set -x
+          download_path=${{steps.download_java_meterpreter.outputs.download-path}}
+          cp -r  $download_path/java-artifacts/data/* ./metasploit-framework/data
+
+      - name: Extract Java Meterpreter (Windows)
+        if: ${{ matrix.meterpreter.name == 'java' && runner.os == 'Windows' }}
+        shell: bash
+        run: |
+          set -x
+          download_path=$(cygpath -u '${{steps.download_java_meterpreter.outputs.download-path}}')
+          cp -r  $download_path/java-artifacts/data/* ./metasploit-framework/data
+
+      - name: Checkout metasploit-payloads
+        uses: actions/checkout@v4
+        with:
+          repository: rapid7/metasploit-payloads
+          path: metasploit-payloads
+          ref: ${{ env.metasploitPayloadsCommit }}
+
+      - name: Build Windows payloads via Visual Studio 2019 Build (Windows)
+        shell: cmd
+        if: ${{ matrix.meterpreter.name == 'windows_meterpreter' && matrix.os == 'windows-2019' }}
+        run: |
+          cd c/meterpreter
+          git submodule init && git submodule update
+          "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" && make.bat
+        working-directory: metasploit-payloads
+
+      - name: Build Windows payloads via Visual Studio 2022 Build (Windows)
+        shell: cmd
+        if: ${{ matrix.meterpreter.name == 'windows_meterpreter' && matrix.os == 'windows-2022' }}
+        run: |
+          cd c/meterpreter
+          git submodule init && git submodule update
+          make.bat
+        working-directory: metasploit-payloads
+
+      - name: Build PHP, Python and Windows payloads
+        if: ${{ (matrix.meterpreter.name == 'php' || matrix.meterpreter.name == 'python' || runner.os == 'Windows') }}
+        run: |
+          make install-php install-python install-windows
+        working-directory: metasploit-payloads
+
+      - name: Acceptance
+        env:
+          SPEC_HELPER_LOAD_METASPLOIT: false
+          SPEC_OPTS: "--tag acceptance --require acceptance_spec_helper.rb --color --format documentation --format AllureRspec::RSpecFormatter"
+        # Unix run command:
+        #   SPEC_HELPER_LOAD_METASPLOIT=false bundle exec ./spec/acceptance
+        # Windows cmd command:
+        #   set SPEC_HELPER_LOAD_METASPLOIT=false
+        #   bundle exec rspec .\spec\acceptance
+        # Note: rspec retry is intentionally not used, as it can cause issues with allure's reporting
+        # Additionally - flakey tests should be fixed or marked as flakey instead of silently retried
+        run: |
+          bundle exec rspec spec/acceptance/meterpreter_spec.rb
+        working-directory: metasploit-framework
+
+      - name: Archive results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          # Provide a unique artifact for each matrix os, otherwise race conditions can lead to corrupt zips
+          name: raw-data-${{ matrix.meterpreter.name }}-${{ matrix.meterpreter.runtime_version }}-${{ matrix.os }}
+          path: metasploit-framework/tmp/allure-raw-data
+
+  # Generate a final report from the previous test results
+  report:
+    name: Generate report
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: always() && needs.test.result != 'skipped'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        if: always()
+
+      - name: Install system dependencies (Linux)
+        if: always()
+        run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz
+
+      - name: Setup Ruby
+        if: always()
+        env:
+          BUNDLE_FORCE_RUBY_PLATFORM: true
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+          cache-version: 5
+
+      - uses: actions/download-artifact@v4
+        id: raw_report_data
+        if: always()
+        with:
+          # Note: Not specifying a name will download all artifacts from the previous workflow jobs
+          path: raw-data
+
+      - name: allure generate
+        if: always()
+        run: |
+          export VERSION=2.22.1
+
+          curl -o allure-$VERSION.tgz -Ls https://github.com/allure-framework/allure2/releases/download/$VERSION/allure-$VERSION.tgz
+          tar -zxvf allure-$VERSION.tgz -C .
+
+          ls -la ${{steps.raw_report_data.outputs.download-path}}
+          ./allure-$VERSION/bin/allure generate ${{steps.raw_report_data.outputs.download-path}}/* -o ./allure-report
+
+          find ${{steps.raw_report_data.outputs.download-path}}
+          bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data ${{steps.raw_report_data.outputs.download-path}} > ./allure-report/support_matrix.html
+
+      - name: archive results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: final-report-${{ github.run_id }}
+          path: |
+            ./allure-report