diff --git a/.github/workflows/test-container.yml b/.github/workflows/test-container.yml new file mode 100644 index 0000000..da10add --- /dev/null +++ b/.github/workflows/test-container.yml @@ -0,0 +1,32 @@ +# Copyright (C) 2023, Advanced Micro Devices, Inc. All rights reserved. +# +# Author: Eddie Hung, AMD +# +# SPDX-License-Identifier: MIT +# + +name: test-container +on: + push: + pull_request: +jobs: + test-container: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Setup Apptainer + env: + APPTAINER_VERSION: '1.2.5' + run: | + wget https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer_${APPTAINER_VERSION}_amd64.deb + sudo dpkg -i apptainer_${APPTAINER_VERSION}_amd64.deb || true + sudo apt-get -f install + - run: | + make test-container + - uses: actions/upload-artifact@v3 + with: + name: rwroute_container.sif + path: | + *.sif diff --git a/.gitignore b/.gitignore index 110350b..c8cf886 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ build/ *.dcp *.tcl *.wirelength +*.sif wirelength_analyzer/test/data/ **/__pycache__/ diff --git a/Makefile b/Makefile index f94cf3a..d33af15 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ # SPDX-License-Identifier: MIT # +SHELL := /bin/bash -o pipefail + # List of all benchmarks (default to all) BENCHMARKS ?= boom_med_pb \ vtr_mcml \ @@ -21,6 +23,14 @@ BENCHMARKS ?= boom_med_pb \ BENCHMARKS_URL = https://github.com/Xilinx/fpga24_routing_contest/releases/latest/download/benchmarks.tar.gz +# Inherit proxy settings from the host if they exist +HTTPHOST=$(firstword $(subst :, ,$(subst http:,,$(subst /,,$(HTTP_PROXY))))) +HTTPPORT=$(lastword $(subst :, ,$(subst http:,,$(subst /,,$(HTTP_PROXY))))) +HTTPSHOST=$(firstword $(subst :, ,$(subst http:,,$(subst /,,$(HTTPS_PROXY))))) +HTTPSPORT=$(lastword $(subst :, ,$(subst http:,,$(subst /,,$(HTTPS_PROXY))))) +JAVA_PROXY=$(if $(HTTPHOST),-Dhttp.proxyHost=$(HTTPHOST) -Dhttp.proxyPort=$(HTTPPORT),) \ +$(if $(HTTPSHOST),-Dhttps.proxyHost=$(HTTPSHOST) -Dhttps.proxyPort=$(HTTPSPORT),) + # Choice of router (default to rwroute) # (other supported values: nxroute-poc) ROUTER ?= rwroute @@ -33,7 +43,6 @@ export TIME=Wall-clock time (sec): %e VERBOSE ?= 0 ifneq ($(VERBOSE), 0) log_and_or_display = 2>&1 | tee $(1) - SHELL := /bin/bash -o pipefail else log_and_or_display = > $(1) 2>&1 endif @@ -46,20 +55,22 @@ else JVM_HEAP ?= -Xms32736m -Xmx32736m endif +export RAPIDWRIGHT_PATH = $(abspath RapidWright) # Default recipe: route and score all given benchmarks .PHONY: run-$(ROUTER) run-$(ROUTER): score-$(ROUTER) -# Use Gradle to compile Java source code in this repository -# as well as the RapidWright repository +# Use Gradle to compile Java source code in this repository as well as the RapidWright repository. +# Also download/generate all device files necessary for the xcvu3p device .PHONY: compile-java compile-java: - ./gradlew compileJava + ./gradlew $(JAVA_PROXY) compileJava + _JAVA_OPTIONS="$(JAVA_PROXY)" RapidWright/bin/rapidwright Jython -c "FileTools.ensureDataFilesAreStaticInstallFriendly('xcvu3p')" .PHONY: install-python-deps install-python-deps: - pip install -q -r requirements.txt --pre + pip install -q -r requirements.txt --pre --user # Download and unpack all benchmarks .PHONY: download-benchmarks @@ -106,7 +117,7 @@ score-$(ROUTER): $(foreach b,$(BENCHMARKS),$b_$(ROUTER).wirelength $b_$(ROUTER). setup-net_printer setup-wirelength_analyzer: | install-python-deps fpga-interchange-schema/interchange/capnp/java.capnp clean: - rm -f *.{phys,check,wirelength}* + rm -f *.{phys,check,wirelength,sif}* distclean: clean rm -rf *.device *_unrouted.phys *.netlist* @@ -131,5 +142,61 @@ distclean: clean #### END ROUTER RECIPES +#### BEGIN CONTEST SUBMISSION RECIPES + +# Required Apptainer args: +# --pid: ensures all processes apptainer spawns are killed with the container +# --home `pwd`: overrides the home directory inside the container to be the current dir +APPTAINER_RUN_ARGS = --pid --home `pwd` +ifneq ($(wildcard /tools),) + # Creates a read-only mount of the host system's `/tools` directory to the container's + # /tools` directory, which allows the container to access the host Vivado installation + APPTAINER_RUN_ARGS += --mount src=/tools/,dst=/tools/,ro +endif + +# Default Apptainer args. Contestants may modify as necessary. +# --rocm --bind /etc/OpenCL: enables OpenCL access in the container +APPTAINER_RUN_ARGS += --rocm +ifneq ($(wildcard /etc/OpenCL),) + APPTAINER_RUN_ARGS += --bind /etc/OpenCL +endif + +# Build an Apptainer image from a definition file in the alpha_submission directory +%_container.sif: alpha_submission/%_container.def + apptainer build $@ $< + +# Use the _container.sif Apptainer image to run all benchmarks +.PHONY: run-container +run-container: $(ROUTER)_container.sif + apptainer run $(APPTAINER_RUN_ARGS) $< make ROUTER="$(ROUTER)" BENCHMARKS="$(BENCHMARKS)" VERBOSE="$(VERBOSE)" + +# Use the _container.sif Apptainer image to run a single small benchmark for testing +.PHONY: test-container +test-container: $(ROUTER)_container.sif + apptainer run $(APPTAINER_RUN_ARGS) $< make ROUTER="$(ROUTER)" BENCHMARKS="boom_med_pb" VERBOSE="$(VERBOSE)" + +SUBMISSION_NAME = $(ROUTER)_submission_$(shell date +%Y%m%d%H%M%S) + +# distclean the repo and create an archive ready for submission +# Submission name is _submission_ +.PHONY: distclean-and-package-submission +distclean-and-package-submission: distclean + tar -czf ../$(SUBMISSION_NAME).tar.gz . + mv ../$(SUBMISSION_NAME).tar.gz . + +#### END CONTEST SUBMISSION RECIPES + +#### BEGIN EXAMPLE RECIPES + +# Build and run an example OpenCL application in an Apptainer container +opencl_example_container.sif: alpha_submission/opencl_example/opencl_example_container.def + apptainer build $@ $< + +.PHONY: run-opencl-example +run-opencl-example: opencl_example_container.sif + apptainer run $(APPTAINER_RUN_ARGS) $< + +#### END EXAMPLE RECIPES + # Tell make to not treat routed results as intermediate files (which would get deleted) .PRECIOUS: %_$(ROUTER).phys diff --git a/alpha_submission/README.md b/alpha_submission/README.md new file mode 100644 index 0000000..59ca4f2 --- /dev/null +++ b/alpha_submission/README.md @@ -0,0 +1,15 @@ +# Alpha Submission Containers + +This directory contains example [Apptainer](https://apptainer.org/docs/user/latest/) +definition files that show how to containerize a router for submission. +Contestants not building on top of RWRoute are required to provide a new file named `_container.def` +builds the environment for running their router. + +For further details on the alpha submission please refer to +[this webpage](https://xilinx.github.io/fpga24_routing_contest/alpha_submission.html). + +The contents of this directory are as follows: + +* `rwroute_container.def` -- an example Apptainer definition file for `rwroute` +* `nxroute-poc_container.def` -- an example Apptainer definition file for `nxroute-poc` (this is actually a link to `rwroute_container.def` since both routers require an identical environment) +* `opencl_example/opencl_example_container.def` -- an example Apptainer definition file for a C++/OpenCL "Hello World" application diff --git a/alpha_submission/nxroute-poc_container.def b/alpha_submission/nxroute-poc_container.def new file mode 120000 index 0000000..a0c096c --- /dev/null +++ b/alpha_submission/nxroute-poc_container.def @@ -0,0 +1 @@ +rwroute_container.def \ No newline at end of file diff --git a/alpha_submission/opencl_example/opencl_example_container.def b/alpha_submission/opencl_example/opencl_example_container.def new file mode 100644 index 0000000..8254ad6 --- /dev/null +++ b/alpha_submission/opencl_example/opencl_example_container.def @@ -0,0 +1,13 @@ +BootStrap: docker +From: ubuntu:20.04 + +%post + apt-get -y update + apt-get -y install ocl-icd-libopencl1 ocl-icd-opencl-dev opencl-headers git build-essential + git clone https://github.com/cqcallaw/ocl-samples.git + cd /ocl-samples + make + +%runscript + cd /ocl-samples/ + ./hello diff --git a/alpha_submission/rwroute_container.def b/alpha_submission/rwroute_container.def new file mode 100644 index 0000000..c1ade60 --- /dev/null +++ b/alpha_submission/rwroute_container.def @@ -0,0 +1,13 @@ +BootStrap: docker +From: eclipse-temurin:17 # Base image with Java VM 17 on Ubuntu + +%post + # Install remaining system dependencies + apt-get -y update + apt-get -y install git python3 pip pkg-config libcapnp-dev time libtinfo5 + # Create a mount point for Vivado + mkdir /tools + +%environment + # Add the host Vivado to the path + export PATH=$PATH:/tools/Xilinx/Vivado/2022.2/bin/ diff --git a/docs/alpha_submission.md b/docs/alpha_submission.md new file mode 100644 index 0000000..017d147 --- /dev/null +++ b/docs/alpha_submission.md @@ -0,0 +1,159 @@ +# Alpha Submission + +In order to ensure that the contest environment is able to support all router +entries ahead of the final submission deadline, a mandatory step for continued +participation in the contest is the submission of an early "alpha" release. +The performance of this alpha submission will have **zero** effect on the +final submission score; instead, the organizers will endeavour to work with +contestants to ensure that the runtime environment is as desired. +Contestants will receive private feedback from the organizers assessing the +performance of just their router on the released benchmark suite (plus a hidden +benchmark) when run on contest hardware. + +## Key Details + +* Alpha submission is mandatory for continued participation in the contest +* Performance of alpha submissions will be shared privately with contestants and will not impact the final score +* Alpha submissions will be evaluated on [AMD Heterogeneous Compute Cluster (HACC)](https://www.amd-haccs.io/) hardware +* Contestants are required to use [Apptainer](https://apptainer.org/docs/user/latest/) to containerize their submission (details below) + +## Runtime Environment + +Aside from running under Linux without network access, no restrictions are +placed on the languages, software dependencies, or runtime environment that +contestants may use to implement their router. In order to enable this platform +independence, contestants must containerize their router and runtime environment +with the [Apptainer](https://apptainer.org/docs/user/latest/) framework. + +### Apptainer + +From the [Apptainer documentation](https://apptainer.org/docs/user/latest/introduction.html): +> Apptainer is a container platform. It allows you to create and run containers that package up pieces of software in a way that is portable and reproducible. You can build a container using Apptainer on your laptop, and then run it on many of the largest HPC clusters in the world, local university or company clusters, a single server, in the cloud, or on a workstation down the hall. Your container is a single file, and you don’t have to worry about how to install all the software you need on each different operating system. + +Apptainer containers may be described with a `*.def` +[definition file](https://apptainer.org/docs/user/latest/definition_files.html) +that specifies the base operating system image, and any further customisations +(such as library installations) required to support an application. A +definition file can then be compiled into an executable `*.sif` image which +allows the application to be run in an isolated environment. + +The [contest repository](https://github.com/Xilinx/fpga24_routing_contest/) +has been updated with example `*.def` files in the `alpha_submission` directory +for both `rwroute` and `nxroute-poc`. To build and run the default container +(which on a fresh clone would be `rwroute`) one would just run: + +``` +make run-container +``` + +This is roughly equivalent to: +``` +apptainer build rwroute_container.sif alpha_submission/rwroute_container.def +apptainer run --pid --home `pwd` --rocm --bind /etc/OpenCL --mount src=/tools/,dst=/tools/,ro rwroute_container.sif make +``` + +The `apptainer build` command creates an image from the `rwroute_container.def` +definition, and the `apptainer run` command runs this image. The Apptainer +command line options do the following: + +* `--pid` runs the container in a new process ID namespace to ensure processes +spawned by the container are not orphaned if the container is killed. +* ``--home `pwd` `` sets the container home directory to be the current directory +* `--rocm --bind /etc/OpenCL` configures [GPU Access](alpha_submission.md#gpu-access) +* `--mount ...` creates a read-only mount of the host system's `/tools` +directory to the container's `/tools` directory, which allows the container to +access the host Vivado installation. + +The remainder of the Apptainer command line simply runs the default make target from inside the +container. + +Finally, in order to aid in development the Makefile target: + +``` +make test-container +``` + +has also been provided. This target is identical to the `run-container` target, +except that results are only collected for the `boom_med_pb` benchmark, instead +of collecting results for every benchmark. This allows contestants to quickly +test their Apptainer flow and avoid overloading shared resources should they +be working on a shared cluster. + +For further information about working with Apptainer containers please refer to +[the user documentation](https://apptainer.org/docs/user/latest/introduction.html). + +### GPU Access + +It is possible to access AMD GPU resources on the host from an Apptainer +container. The directory `alpha_submission/opencl_example` contains a sample +`*.def` file that builds a [C++/OpenCL "Hello World" example](https://github.com/cqcallaw/ocl-samples). +To run this example: + +``` +make run-opencl-example +``` + +This `make` target builds a `*.sif` image from the +`opencl_example_container.def` definition file and runs it with the command: + +``` +apptainer run --pid --home `pwd` --rocm --bind /etc/OpenCL opencl_example_container.sif +``` + +The `--rocm` switch enables AMD ROCm support in the container. The +`--bind /etc/OpenCL` switch mounts the host OpenCL directory in the container, +which is required to allow the containerized OpenCL stack to discover the host +resources. + +Please note that contestants are free to use GPU interfaces other than OpenCL, +such as [AMD HIP](https://github.com/ROCm-Developer-Tools/HIP). + +## Submission Format + +Contestants are required to submit a clone of the contest +repository which has been modified to run their router in Apptainer. +Specifically, organizers must be able to run the submitted router by calling only +the `make run-container` target. By default, in a fresh checkout of the contest +repository, this target will run `rwroute` in an Apptainer container. +Thus in addition to their router contestants must supply a custom `*.def` file +in the `alpha_submission` directory, as well as a Makefile that has been +modified to run their router by default. To set the default router in the +Makefile contestants must change the value of the `ROUTER` variable from +`rwroute` to the name of their router. + +Starting from a clone of the contest repository that has already had its +Makefile `ROUTER` variable modified such that `make` invokes the contestant +router, one would just execute: + +``` +make distclean-and-package-submission +``` + +Which generates a submission artifact named +`_submission_.tar.gz` in the current directory. +Internally, this executes the following commands: + +``` +make distclean +tar -czf ../_submission_.tar.gz.tar.gz . +mv ../_submission_.tar.gz.tar.gz . +``` + +Note that `make distclean` will delete all unrouted design files, routed +results and logs, as well as the device description. The organizers will then +evaluate the artifact with a process similar to the following: + +``` +mkdir _alpha_submission +cd _alpha_submission +tar -xvf _submission_.tar.gz +make run-container +``` + +### Closed-Source Submissions + +While contestants are strongly encouraged to open-source their solutions at the +conclusion of the contest, there is no requirement to do so. In such cases, +it is still necessary to use the flow described above to produce a binary only +submission. That is, any precompiled router must still work inside Apptainer +and be invoke-able using `make run-container` on the contest hardware. diff --git a/docs/index.md b/docs/index.md index 6a4e9f4..4370cb5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ Given a pre-placed design in the [FPGA Interchange Format](http://www.rapidwrigh and a multi-core machine with an AMD GPU, build a router that focuses on minimizing the wall-clock time required to return a legal, fully routed solution. -| ℹ️ **NOTE:** | [Registration](#registration) deadline extended to 20 November 2023 | +| ℹ️ **NOTE:** | [Alpha Submission](alpha_submission.html) deadline is 20 December 2023 | | - | - | ## Introduction @@ -58,7 +58,7 @@ More information can be found in [Contest Details](details.html). |-----------------|-------| |September 2023 | Contest Announced | |~20 October 2023~
**EXTENDED 20 November 2023**| Registration Deadline ([mandatory, see below](#registration))| -|20 December 2023 | Alpha Submission (details to be announced)| +|20 December 2023 | Alpha Submission ([details](alpha_submission.html))| |31 January 2024 | Final Submission (details to be announced)| |3-5 March 2024 | Prizes awarded to top 5 teams at [FPGA 2024 conference](https://www.isfpga.org/)|