compile_all.py

#!/bin/python3

# SPDX-License-Identifier: LGPL-2.1-or-later

# Every package has its own compilation and installation idiosyncrasies, so we have to use a custom
# build script for each one.

from diff_match_patch import diff_match_patch
from typing import Dict, List

from enum import Enum
import os
import pathlib
import platform
import re
import shutil
import subprocess
import stat
import sys


class BuildMode(Enum):
    DEBUG = 1
    RELEASE = 2

    def __str__(self) -> str:
        if self == BuildMode.DEBUG:
            return "Debug"
        elif self == BuildMode.RELEASE:
            return "Release"
        else:
            return "Unknown"


def remove_readonly(func, path, _) -> None:
    """Remove a read-only file."""

    os.chmod(path, stat.S_IWRITE)
    func(path)


def patch_single_file(filename, patch_data) -> None:
    with open(filename, "r", encoding="utf-8") as f:
        original_data = f.read()
    dmp = diff_match_patch()
    patches = dmp.patch_fromText(patch_data)
    new_text, applied = dmp.patch_apply(patches, original_data)
    if not all(applied):
        print(f"ERROR: Failed to apply some patches to {filename}")
        # TODO: Someday actually print out what patches failed?
        exit(1)
    with open(filename, "w", encoding="utf-8") as f:
        f.write(new_text)


def split_patch_data(patch_data: str) -> List[Dict[str, str]]:
    filename_regex = re.compile("@@@ ([^@]*) @@@\n")
    split_data = re.split(filename_regex, patch_data)
    result = []
    for index, entry in enumerate(split_data):
        if index == 0:
            if entry != "":
                print("ERROR: Bad patch file, must start with @@@ filename @@@")
                exit(1)
            continue
        if index % 2 == 1:
            result.append({"file": entry})
        else:
            result[-1]["data"] = entry
    return result


def apply_patch(patch_file_path: str) -> None:
    """Apply a patch that was generated by the generate_patch.py script"""
    # Path is relative to *this* file, not our working directory
    absolute_path = os.path.join(pathlib.Path(__file__).parent.absolute(), patch_file_path)
    with open(absolute_path, "r", encoding="utf-8") as f:
        patch_data = f.read()
    patches = split_patch_data(patch_data)
    for patch in patches:
        patch_single_file(patch["file"], patch["data"])


def patch_files(patches: List[str]) -> None:
    """Given a list of patches, apply them sequentially in the current working directory. The patches themselves are
    expected to be given as paths relative to **this** Python script file"""
    for patch in patches:
        start = len("patches/")
        print(f"  Applying patch {patch[start:]}")
        apply_patch(patch)


def libpack_dir(config: dict, mode: BuildMode):
    lp_dir = "LibPack-{}-v{}-{}".format(
        config["FreeCAD-version"], config["LibPack-version"], str(mode)
    )
    return os.path.join(os.path.dirname(__file__), "working", lp_dir)


def to_exe(base: str = ""):
    """Append .exe to Windows executables, but not to macOS or Linux. If given no argument, just returns the extension
    for the current OS, suitable for appending to an executable's name."""
    return base + ".exe" if sys.platform.startswith("win32") else ""


def to_static(base: str = ""):
    """Append .lib to Windows libraries, or .a macOS or Linux. If given no argument, just returns the extension
    for the current OS, suitable for appending to a static library's name."""
    return base + ".lib" if sys.platform.startswith("win32") else ".a"


def to_dynamic(base: str = ""):
    """Append .dll to Windows libraries, or .so to macOS or Linux. If given no argument, just returns the extension
    for the current OS, suitable for appending to a dynamic library's name."""
    return base + ".dll" if sys.platform.startswith("win32") else ".so"


class Compiler:
    def __init__(
        self, config, bison_path, skip_existing: bool = False, mode: BuildMode = BuildMode.RELEASE
    ):
        self.config = config
        self.bison_path = bison_path
        self.base_dir = os.getcwd()
        self.skip_existing = skip_existing
        self.install_dir = libpack_dir(config, mode)
        self.init_script = None
        self.mode = mode
        self.strict_mode = True

        # Right now there are two packages where the version number gets coded into the path when: Boost and Coin:
        # store those two separately from all the other paths we have to track
        self.boost_include_path = None
        self.coin_cmake_path = None

    def get_cmake_options(self) -> List[str]:
        """Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply
        to all builds, but none conflict."""
        pcre_lib = self.install_dir + "/lib/pcre2-8"
        if self.mode == BuildMode.DEBUG:
            pcre_lib += "d"
        pcre_lib += to_static()

        base = [
            "-D CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE",  # Never use system packages, always use only the libpack
            "-D CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=TRUE",  # Same as above?
            f"-D BISON_EXECUTABLE={self.bison_path}",
            f"-D BOOST_ROOT={self.install_dir}",
            f"-D BUILD_DOC=No",
            f"-D BUILD_DOCS=No",
            f"-D BUILD_EXAMPLES=No",
            f"-D BUILD_SHARED=Yes",
            f"-D BUILD_SHARED_LIB=Yes",
            f"-D BUILD_SHARED_LIBS=Yes",
            f"-D BUILD_TEST=No",
            f"-D BUILD_TESTS=No",
            f"-D BUILD_TESTING=No",
            f"-D BZIP2_DIR={self.install_dir}/lib/cmake/",
            f"-D Boost_INCLUDE_DIRS={self.install_dir}/include",
            f"-D CMAKE_BUILD_TYPE={self.mode}",
            f"-D CMAKE_INSTALL_PATH={self.install_dir}",
            f"-D CMAKE_INSTALL_PREFIX={self.install_dir}",
            f"-D HarfBuzz_DIR={self.install_dir}/lib/cmake/",
            f"-D HDF5_DIR={self.install_dir}/share/cmake/",
            f"-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib",
            f"-D HDF5_LIBRARY_RELEASE={self.install_dir}/lib/hdf5.lib",
            f"-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + to_exe(),
            f"-D INSTALL_DIR={self.install_dir}",
            f"-D PCRE2_LIBRARY={pcre_lib}",
            "-D PIVY_USE_QT6=Yes",
            f"-D pybind11_DIR={self.install_dir}/share/cmake/pybind11",
            f"-D Python_ROOT_DIR={self.install_dir}/bin",
            f"-D Python_DIR={self.install_dir}/bin",
            "-D Python_FIND_REGISTRY=NEVER",
            f"-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6",
            f"-D SWIG_EXECUTABLE={self.install_dir}/bin/swig" + to_exe(),
            f"-D ZLIB_DIR={self.install_dir}/lib/cmake/",
            f"-D ZLIB_INCLUDE_DIR={self.install_dir}/include",
            f"-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib" + to_static(),
            f"-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd" + to_static(),
            "-D CMAKE_DISABLE_FIND_PACKAGE_SoQt=True",
            # Absolutely never find SoQt (it's deprecated and we don't want it!)
        ]
        if self.boost_include_path:
            base.append(f"-D Boost_INCLUDE_DIR={self.boost_include_path}")
        if self.coin_cmake_path:
            base.append(f"-D Coin_DIR={self.coin_cmake_path}")
        if sys.platform.startswith("win32"):
            inc_path = self.install_dir.replace("\\", "/")
            cxx_flags = f"/I{inc_path}/include /EHsc /DWIN32"
            if self.strict_mode:
                # NOTE: /permissive- is required with Qt6 but could be disabled for anything that doesn't link against
                # Qt. The same is true for /Zc:__cplusplus /std:c++17
                cxx_flags += " /Zc:__cplusplus /std:c++17 /permissive-"
        else:
            cxx_flags = f"-I{self.install_dir}/include"
        base.append(f"-D CMAKE_CXX_FLAGS={cxx_flags}")
        return base

    def compile_all(self):
        for item in self.config["content"]:
            # All build methods are named using "build_XXX" where XXX is the name of the package in the config file
            os.chdir(item["name"])
            build_function_name = "build_" + item["name"]
            if hasattr(self, build_function_name):
                print(f"Building {item['name']}")
                build_function = getattr(self, build_function_name)
                build_function(item)
            else:
                print(
                    f"No '{build_function_name}' found in compile_all.py -- "
                    "did you forget to add one when adding a dependency?"
                )
                exit(2)
            os.chdir(self.base_dir)

    def build_nonexistent(self, _=None):
        """Used for automated testing to allow easy Mock injection"""

    def python_exe(self):
        if self.mode == BuildMode.RELEASE:
            return os.path.join(self.install_dir, "bin", "python") + to_exe()
        return os.path.join(self.install_dir, "bin", "python_d") + to_exe()

    def build_python(self, args=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "bin", "DLLs")):
                print("  Not rebuilding Python, it is already in the LibPack")
                return
        if sys.platform.startswith("win32"):
            expected_exe_path = self.python_exe()
            if self.skip_existing and os.path.exists(expected_exe_path):
                print(
                    "Not rebuilding, instead just using existing Python in the LibPack installation path"
                )
                return
            try:
                arch = "x64" if platform.machine() == "AMD64" else "ARM64"
                path = "amd64" if platform.machine() == "AMD64" else "arm64"
                subprocess.run(
                    [
                        self.init_script,
                        "&",
                        "PCbuild\\build.bat",
                        "-p",
                        arch,
                        "-c",
                        str(self.mode),
                    ],
                    check=True,
                    capture_output=True,
                )
            except subprocess.CalledProcessError as e:
                print("Python build failed")
                print(e.stdout.decode("utf-8"))
                if e.stderr:
                    print(e.stderr.decode("utf-8"))
                exit(e.returncode)
            bin_dir = os.path.join(self.install_dir, "bin")
            dll_dir = os.path.join(bin_dir, "DLLs")
            lib_dir = os.path.join(bin_dir, "Lib")
            libs_dir = os.path.join(bin_dir, "libs")
            inc_dir = os.path.join(bin_dir, "Include")
            tools_dir = os.path.join(bin_dir, "Tools")
            os.makedirs(bin_dir, exist_ok=True)
            os.makedirs(dll_dir, exist_ok=True)
            os.makedirs(lib_dir, exist_ok=True)
            os.makedirs(libs_dir, exist_ok=True)
            os.makedirs(bin_dir, exist_ok=True)
            os.makedirs(tools_dir, exist_ok=True)
            tools_subs = ["i18n", "scripts"]
            for sub in tools_subs:
                os.makedirs(os.path.join(tools_dir, sub), exist_ok=True)

            # NOTES:
            # When installed via the Python installer, the top-level Python folder contains:
            #   python.exe
            #   python.pdb
            #   python3.dll
            #   python3xx.dll
            #   python3xx.pdb
            #   python3xx_d.dll
            #   python3xx_d.pdb
            #   python3_d.dll
            #   pythonw.exe
            #   pythonw.pdb
            #   pythonw_d.exe
            #   pythonw_d.pdb
            #   python_d.exe
            #   python_d.pdb
            #   vcruntime140.dll
            #   vcruntime140_1.dll
            # It also contains 5 subdirectories: DLLs, include, Lib, libs, and Tools, plus LICENSE.txt
            #    DLLS folder contains *.pyd, *.pdb, and *.dll
            #    include contains the header file directory tree
            #    Lib contains the Python standard libraries
            #    libs contains the actual Python *.lib files (python3.lib and python3xx.lib and their debug equivalents
            #    Tools contains a number of subdirectories with Python scripts: i18n, scripts, and demo
            # Finally, we also need the file "pyconfig.h" which is in yet another directory of the Python build, "PC"

            shutil.copytree(f"PCBuild\\{path}", dll_dir, dirs_exist_ok=True)
            shutil.copytree(f"Lib", lib_dir, dirs_exist_ok=True)
            shutil.copytree(f"Include", inc_dir, dirs_exist_ok=True)
            for sub in tools_subs:
                shutil.copytree(f"Tools\\{sub}", os.path.join(tools_dir, sub), dirs_exist_ok=True)

            # Figure out what version of Python we just built:
            major, minor = self.get_python_version(
                os.path.join("PCBuild", path, "python.exe")
            ).split(".")

            # Construct the list of files we expect to exist that need to be placed in the toplevel directory, or in
            # libs:
            move_to_bin = ["vcruntime"]
            for base in ["python", f"python{major}", f"python{major}{minor}", "pythonw"]:
                final = base
                if self.mode == BuildMode.DEBUG:
                    final += "_d"
                move_to_bin.append(final)
            # They are all in the DLLs subdirectory now: move the ones that match:
            for file in pathlib.Path(dll_dir).iterdir():
                if file.is_file():
                    if file.stem in move_to_bin:
                        if file.suffix == ".lib":
                            target = os.path.join(libs_dir, file.name)
                        elif file.suffix in [".dll", ".exe", ".pdb"]:
                            target = os.path.join(bin_dir, file.name)
                        else:
                            continue
                        if os.path.exists(target):
                            os.unlink(target)
                        file.rename(target)
            pyconfig = os.path.join("PC", "pyconfig.h")
            target = os.path.join(inc_dir, "pyconfig.h")
            if not os.path.exists(pyconfig):
                print("ERROR: Could not locate pyconfig.h, cannot complete installation of Python")
                exit(1)
            if os.path.exists(target):
                os.unlink(target)
            print(f"Copying {pyconfig} to {target}")
            shutil.copyfile(pyconfig, target)
        else:
            raise NotImplemented("Non-Windows compilation of Python is not implemented yet")

        # Check these even if we didn't actually have to build Python
        self._build_pip()
        if "requirements" in args:
            self._install_python_requirements(args["requirements"])

    def get_python_version(self, exe: str = None) -> str:
        if exe is None:
            path_to_python = self.python_exe()
        else:
            path_to_python = exe
        try:
            result = subprocess.run([path_to_python, "--version"], capture_output=True, check=True)
            _, _, version_number = result.stdout.decode("utf-8").strip().partition(" ")
            components = version_number.split(".")
            python_version = f"{components[0]}.{components[1]}"
            return python_version
        except subprocess.CalledProcessError as e:
            print("ERROR: Failed to run LibPack's Python executable")
            print(e.stdout.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def _build_pip(self, _=None):
        print("  Installing the latest pip")
        path_to_python = self.python_exe()
        try:
            subprocess.run(
                [path_to_python, "-m", "ensurepip", "--upgrade"], capture_output=True, check=True
            )
            subprocess.run(
                [path_to_python, "-m", "pip", "install", "--upgrade", "pip"],
                capture_output=True,
                check=True,
            )
        except subprocess.CalledProcessError as e:
            print("ERROR: Failed to run LibPack's Python executable")
            print(e.stdout.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def _install_python_requirements(self, requirements):
        print("  Installing the following requirements (and their dependencies) using pip:")
        for req in requirements:
            print("    " + req)
        path_to_python = self.python_exe()
        call_args = [path_to_python, "-m", "pip", "install", "--ignore-installed"]
        call_args.extend(requirements)
        try:
            subprocess.run(
                call_args,
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print(f"ERROR: Failed to pip install requirements")
            print(e.output.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def build_qt(self, options: dict):
        """Doesn't really "build" Qt, just copies the pre-compiled libraries from the configured path"""
        qt_dir = options["install-directory"]
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "metatypes")):
                print(
                    "  Not re-copying, instead just using existing Qt in the LibPack installation path"
                )
                return
        if not os.path.exists(qt_dir):
            print(f"Error: specified Qt installation path does not exist ({qt_dir})")
            exit(1)
        print("  (Note that Qt isn't really 'built,' it is just copied from a local installation)")
        shutil.copytree(qt_dir, self.install_dir, dirs_exist_ok=True)

    def build_boost(self, _=None):
        """Builds boost shared libraries and installs libraries and headers"""
        if self.skip_existing:
            self._configure_boost_version()
            if self.boost_include_path is not None:
                print("  Not rebuilding boost, it is already in the LibPack")
                return
        try:
            # When debugging on the command line, add --debug-configuration to get more verbose output
            install_dir = self.install_dir
            subprocess.run(
                [self.init_script, "&", "bootstrap.bat", f"--prefix={install_dir}"],
                capture_output=True,
                check=True,
            )
            arch = "x86" if platform.machine() == "AMD64" else "arm"
            subprocess.run(
                [
                    self.init_script,
                    "&",
                    "b2",
                    f"install",
                    "address-model=64",
                    f"architecture={arch}",
                    "link=static,shared",
                    str(self.mode).lower(),
                    f"--prefix={install_dir}",
                    "--layout=versioned",
                    "--without-mpi",
                    "--without-graph_parallel",
                    "--without-python",
                    "--build-type=complete",
                    "--debug-configuration",
                ],
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            # Boost is too verbose in its output to be of much use un-processed. Dump it all to a file, and
            # then print only the lines with the word "error:" on them to stdout
            print(
                "Error: failed to build boost -- writing output to "
                + os.path.join(os.path.curdir, "stdout.txt")
            )
            with open("stdout.txt", "w", encoding="utf-8") as f:
                f.write(e.stdout.decode("utf-8"))
            lines = e.stdout.decode("utf-8").split("\n")
            for line in lines:
                if "error:" in line.lower():
                    print(line)
            exit(e.returncode)
        self._configure_boost_version()

    def _configure_boost_version(self):
        """Once Boost has been installed, figure out what version it was and set up the correct include path"""
        start_crawl_at = os.path.join(self.install_dir, "include")
        contents = [
            f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))
        ]
        for item in contents:
            if item.startswith("boost"):
                self.boost_include_path = os.path.join(start_crawl_at, item)
                break

    def _cmake_create_build_dir(self):
        build_dir = "build-" + str(self.mode).lower()
        if os.path.exists(build_dir):
            shutil.rmtree(build_dir, onerror=remove_readonly)
        os.mkdir(build_dir)
        os.chdir(build_dir)

    def _run_cmake(self, args):
        cmake_setup_options = [self.init_script, "&", "cmake"]
        cmake_setup_options.extend(args)
        try:
            subprocess.run(cmake_setup_options, check=True, capture_output=True)
        except subprocess.CalledProcessError as e:
            print("ERROR: cMake failed!")
            print(f"Command: {' '.join(cmake_setup_options)}")
            print(e.stdout.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(e.returncode)

    def _cmake_configure(self, extra_args: List[str] = None):
        options = self.get_cmake_options()
        if extra_args:
            options.extend(extra_args)
        options.append(
            ".."
        )  # Because the source code is located one directory up from our build location
        self._run_cmake(options)

    def _cmake_build(self, parallel: bool = True):
        cmake_build_options = ["--build", ".", "--config", str(self.mode).lower()]
        if parallel:
            cmake_build_options.append("--parallel")
        self._run_cmake(cmake_build_options)

    def _cmake_install(self):
        cmake_install_options = ["--install", ".", "--config", str(self.mode).lower()]
        self._run_cmake(cmake_install_options)

    def _build_standard_cmake(self, extra_args: List[str] = None):
        self._cmake_create_build_dir()
        self._cmake_configure(extra_args)
        self._cmake_build()
        self._cmake_install()

    def _pip_install(self, requirement: str) -> None:
        path_to_python = self.python_exe()
        package_name = requirement.split("==")[0]
        try:
            # Get rid of any version that's already there.
            subprocess.run(
                [path_to_python, "-m", "pip", "uninstall", "--yes", package_name],
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print(f"{package_name} was not uninstalled... continuing")
            pass
        try:
            subprocess.run(
                [path_to_python, "-m", "pip", "install", "--ignore-installed", requirement],
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print(f"ERROR: Failed to pip install {requirement}")
            print(e.output.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def _build_with_pip(self, options: dict):
        if "pip-install" not in options:
            print(
                f"ERROR: No pip-install provided in config of {options['name']}, so version cannot be determined"
            )
            exit(1)
        self._pip_install(options["pip-install"])

    def build_coin(self, _=None):
        """Builds and installs Coin using standard CMake settings"""
        if self.skip_existing:
            self._configure_coin_cmake_path()
            if self.coin_cmake_path is not None:
                print("  Not rebuilding Coin, it is already in the LibPack")
                return
        extra_args = ["-D COIN_BUILD_TESTS=Off"]
        self._build_standard_cmake(extra_args)
        self._configure_coin_cmake_path()

    def _configure_coin_cmake_path(self):
        """Coin installs its cMake file into a directory named with the full version, so figure out what that is"""
        start_crawl_at = os.path.join(self.install_dir, "lib", "cmake")
        contents = [
            f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))
        ]
        for item in contents:
            if item.startswith("Coin"):
                self.coin_cmake_path = os.path.join(start_crawl_at, item)
                break

    def build_quarter(self, _=None):
        """Builds and installs Quarter using standard CMake settings"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "Quarter")):
                print("  Not rebuilding Quarter, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_zlib(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "zlib.h")):
                print("  Not rebuilding zlib, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_bzip2(self, _=None):
        """The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")):
                print("  Not rebuilding bzip2, it is already in the LibPack")
                return
        if sys.platform.startswith("win32"):
            args = [self.init_script, "&", "nmake", "/f", "makefile.msc"]
            try:
                subprocess.run(args, check=True, capture_output=True)
                shutil.copyfile("libbz2.lib", os.path.join(self.install_dir, "lib", "libbz2.lib"))
                shutil.copyfile("bzlib.h", os.path.join(self.install_dir, "include", "bzlib.h"))
                shutil.copyfile(
                    "bzlib_private.h", os.path.join(self.install_dir, "include", "bzlib_private.h")
                )
            except subprocess.CalledProcessError as e:
                print("ERROR: Failed to build bzip2 using nmake")
                print(e.output.decode("utf-8"))
                if e.stderr:
                    print(e.stderr.decode("utf-8"))
                exit(1)
        else:
            raise NotImplemented("Non-Windows compilation of bzip2 is not implemented yet")

    def build_pcre2(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "pcre2.h")):
                print("  Not rebuilding pcre2, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_swig(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "bin", "swig") + to_exe()):
                print("  Not rebuilding SWIG, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_pivy(self, _=None):
        if self.skip_existing:
            if os.path.exists(
                os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy")
            ):
                print("  Not rebuilding pivy, it is already in the LibPack")
                return
        extra_args = []
        self._build_standard_cmake(extra_args)
        if self.mode == BuildMode.DEBUG:
            base = os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy")
            os.rename(os.path.join(base, "_coin.pyd"), os.path.join(base, "_coin_d.pyd"))

    def build_libclang(self, _=None):
        """libclang is provided as a platform-specific download by Qt."""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "clang")):
                print("  Not copying libclang, it is already in the LibPack")
                return
        print("  (not really building libclang, just copying from a build provided by Qt)")
        shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True)

    def build_pyside(self, _=None):
        # Don't use a pip-install for this, we need the linkable libraries and include files for both PySide and
        # Shiboken, which won't get installed by pip, and it needs to be built against the right Python exe
        if self.skip_existing:
            if os.path.exists(
                os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")
            ):
                print("  Not rebuilding PySide6, it is already in the LibPack")
                return
        python = self.python_exe()
        qtpaths = "--qtpaths=" + os.path.join(self.install_dir, "bin", "qtpaths6") + to_exe()
        clang = "CLANG_INSTALL_DIR=" + os.path.join(self.install_dir, "lib", "clang")
        vulkan = "VULKAN_SDK=None"  # "VULKAN_SDK=" + os.path.join(self.install_dir, "Vulkan")
        parallel = "--parallel=16"
        # numpy = "--enable-numpy-support"
        if sys.platform.startswith("win32"):
            ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs")
            args = [
                self.init_script,
                "&",
                "set",
                clang,
                "&",
                "set",
                vulkan,
                "&",
                python,
                "setup.py",
                "install",
                qtpaths,
                ssl,
                parallel,
            ]
            if self.mode == BuildMode.DEBUG:
                args.append("--debug")
        else:
            ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs")
            args = [clang, "&&", python, "setup.py", "install", qtpaths, ssl]
        try:
            subprocess.run(args, capture_output=True, check=True)
        except subprocess.CalledProcessError as e:
            print("ERROR: Failed to build Pyside and/or Shiboken")
            print(e.stdout.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def build_vtk(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "share", "licenses", "VTK")):
                print("  Not rebuilding VTK, it is already in the LibPack")
                return
        extra_args = []
        if sys.platform.startswith("win32"):
            extra_args.append(
                "-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO",  # Workaround for bug in Visual Studio MSVC 143
            )
            extra_args.append(
                "-D VTK_MODULE_ENABLE_VTK_ioss=NO",  # Workaround for bug in Visual Studio MSVC 143
            )

        print("  (VTK is big, this will take some time)")

        old_strict_mode = self.strict_mode
        self.strict_mode = False
        self._build_standard_cmake(extra_args)
        self.strict_mode = old_strict_mode

    def build_harfbuzz(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "harfbuzz")):
                print("  Not rebuilding harfbuzz, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_libpng(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "lib", "libpng")):
                print("  Not rebuilding libpng, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_pybind11(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "pybind11")):
                print("  Not rebuilding pybind11, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_freetype(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "freetype2")):
                print("  Not rebuilding freetype, it is already in the LibPack")
                return
        self._build_standard_cmake()
        if self.mode == BuildMode.DEBUG:
            # OCCT *really* wants these libraries named like this:
            shutil.copyfile(
                f"{self.install_dir}/bin/freetyped.dll", f"{self.install_dir}/bin/freetype.dll"
            )
            shutil.copyfile(
                f"{self.install_dir}/lib/freetyped.lib", f"{self.install_dir}/lib/freetype.lib"
            )

    def force_copy(self, src_components: List[str], dst_components: List[str]):
        full_src = self.install_dir
        for src in src_components:
            full_src = os.path.join(full_src, src)
        full_dst = self.install_dir
        for dst in dst_components:
            full_dst = os.path.join(full_dst, dst)
        if not os.path.exists(full_src):
            print(f"    (Can't rename {full_src}, no such file or directory)")
            return
        if os.path.exists(full_dst):
            os.unlink(full_dst)
        shutil.copyfile(full_src, full_dst)

    def build_tcl(self, _=None):
        """tcl does not use cMake"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "tcl.h")):
                print("  Not rebuilding tcl, it is already in the LibPack")
                return
        if sys.platform.startswith("win32"):
            try:
                os.chdir("win")
                args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"]
                if self.mode == BuildMode.DEBUG:
                    args.append("OPTS=symbols")
                subprocess.run(args, check=True, capture_output=True)
                args = [
                    self.init_script,
                    "&",
                    "nmake",
                    "/f",
                    "makefile.vc",
                    "install",
                    f"INSTALLDIR={self.install_dir}",
                ]
                if self.mode == BuildMode.DEBUG:
                    args.append("OPTS=symbols")
                subprocess.run(args, check=True, capture_output=True)
                if self.mode == BuildMode.RELEASE:
                    self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"])
                    self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"])
                    self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"])
                else:
                    self.force_copy(["bin", "tclsh86tg.exe"], ["bin", "tclsh.exe"])
                    self.force_copy(["bin", "tcl86tg.dll"], ["bin", "tcl86.dll"])
                    self.force_copy(["lib", "tcl86tg.lib"], ["lib", "tcl86.lib"])
            except subprocess.CalledProcessError as e:
                print("ERROR: Failed to build tcl using nmake")
                print(e.stdout.decode("utf-8"))
                if e.stderr:
                    print(e.stderr.decode("utf-8"))
                exit(1)
        else:
            raise NotImplemented("Non-Windows compilation of tcl is not implemented yet")

    def build_tk(self, _=None):
        """tk does not use cMake"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "tk.h")):
                print("  Not rebuilding tk, it is already in the LibPack")
                return
        if sys.platform.startswith("win32"):
            try:
                os.chdir("win")
                args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"]
                if self.mode == BuildMode.DEBUG:
                    args.append("OPTS=symbols")
                subprocess.run(args, check=True, capture_output=True)
                args = [
                    self.init_script,
                    "&",
                    "nmake",
                    "/f",
                    "makefile.vc ",
                    "install",
                    f"INSTALLDIR={self.install_dir}",
                ]
                if self.mode == BuildMode.DEBUG:
                    args.append("OPTS=symbols")
                subprocess.run(args, check=True, capture_output=True)
                if self.mode == BuildMode.RELEASE:
                    self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"])
                    self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"])
                    self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"])
                else:
                    self.force_copy(["bin", "wish86tg.exe"], ["bin", "wish.exe"])
                    self.force_copy(["bin", "tk86tg.dll"], ["bin", "tk86.dll"])
                    self.force_copy(["lib", "tk86tg.lib"], ["lib", "tk86.lib"])
            except subprocess.CalledProcessError as e:
                print("ERROR: Failed to build tk using nmake")
                print(e.output.decode("utf-8"))
                if e.stderr:
                    print(e.stderr.decode("utf-8"))
                exit(1)
        else:
            raise NotImplemented("Non-Windows compilation of tk is not implemented yet")

    def build_rapidjson(self, _):
        if os.path.exists(os.path.join(self.install_dir, "include", "rapidjson")):
            if self.skip_existing:
                print("  Not re-copying RapidJSON, it is already in the LibPack")
                return
            shutil.rmtree(os.path.join(self.install_dir, "include", "rapidjson"))
        shutil.copytree("include", os.path.join(self.install_dir, "include"), dirs_exist_ok=True)

    def _get_vtk_include_path(self) -> str:
        """
        OpenCASCADE needs a manually-set include path for VTK (the find_package script provided by VTK does not provide
        the include file path, and OpenCASCADE has not been updated to handle this, as of June 2024).
        """
        start_crawl_at = os.path.join(self.install_dir, "include")
        contents = [
            f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))
        ]
        for item in contents:
            if item.startswith("vtk-"):
                return os.path.join(start_crawl_at, item)
        raise RuntimeError("Could not find VTK include directory for OpenCASCADE")

    def build_opencascade(self, _=None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")):
                print("  Not rebuilding OpenCASCADE, it is already in the LibPack")
                return
        extra_args = [
            f"-D CMAKE_MODULE_PATH={self.install_dir}/lib/cmake;{self.install_dir}/share/cmake;{self.install_dir}"
            f"-D TCL_DIR={self.install_dir}/include",
            f"-D TK_DIR={self.install_dir}/include",
            f"-D FREETYPE_DIR={self.install_dir}/lib/cmake",
            f"-D VTK_DIR={self.install_dir}/lib/cmake",
            f"-D 3RDPARTY_VTK_INCLUDE_DIRS={self._get_vtk_include_path()}",
            f"-D EIGEN_DIR={self.install_dir}/share/eigen3/cmake",
            "-D USE_VTK=On",
            "-D USE_FREETYPE=On" "-D USE_RAPIDJSON=On",
            "-D USE_EIGEN=On" "-D BUILD_CPP_STANDARD=C++17",
            "-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF",
            "-D INSTALL_DIR_BIN=bin",
            "-D INSTALL_DIR_LIB=lib",
        ]
        if self.mode == BuildMode.DEBUG:
            extra_args.append("-D BUILD_SHARED_LIBRARY_NAME_POSTFIX=d")
        cwd = os.getcwd()
        self._cmake_create_build_dir()
        self._cmake_configure(extra_args)
        self._cmake_build(parallel=False)
        if self.mode == BuildMode.DEBUG and sys.platform.startswith("win32"):
            # On Windows OpenCASCADE is looking in the wrong location for these files (as of 7.7.1) -- just copy them
            # TODO - Don't hardcode the path
            shutil.copytree(
                os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin")
            )
        self._cmake_install()

        os.chdir(cwd)

        # TODO - something is getting messed up in the CMake config output (note the quotes around 26812): for now just
        # drop the line entirely
        # set (OpenCASCADE_CXX_FLAGS    "[...] /wd"26812" /MP /W4")
        with open(
            os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"),
            "r",
            encoding="utf-8",
        ) as f:
            occt_cmake_contents = f.readlines()
        with open(
            os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"),
            "w",
            encoding="utf-8",
        ) as f:
            for line in occt_cmake_contents:
                if "OpenCASCADE_CXX_FLAGS" not in line:
                    f.write(line + "\n")

    def build_netgen(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "share", "netgen")):
                print("  Not rebuilding netgen, it is already in the LibPack")
                return
        extra_args = [
            f"-D CMAKE_FIND_ROOT_PATH={self.install_dir}",
            "-D USE_SUPERBUILD=OFF",
            "-D USE_GUI=OFF",
            "-D USE_NATIVE_ARCH=OFF",
            "-D USE_INTERNAL_TCL=OFF",
            f"-D TCL_DIR={self.install_dir}",
            f"-D TK_DIR={self.install_dir}",
            "-D USE_OCC=On",
            f"-D OpenCASCADE_ROOT={self.install_dir}",
            f"-D USE_PYTHON=OFF",
            f"-D CMAKE_CXX_FLAGS='-D_USE_MATH_DEFINES /EHsc'",
        ]  # To get M_PI on MSVC
        self._build_standard_cmake(extra_args=extra_args)

    def build_hdf5(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "hdf5.h")):
                print("  Not rebuilding hdf5, it is already in the LibPack")
                return
        # HDF5 is VERY picky about how you specify the location of zlib: you must actually set the precise path to the
        # library file itself. TODO future work to internally detect that library name and path and fill them here
        extra_args = [
            f"-D ZLIB_INCLUDE_DIR={self.install_dir}/include",
            f"-D ZLIB_LIBRARY={self.install_dir}/lib/zlib.lib",
        ]
        self._build_standard_cmake(extra_args)

    def build_medfile(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "medfile.h")):
                print("  Not rebuilding medfile, it is already in the LibPack")
                return
        extra_args = ["-D MEDFILE_USE_UNICODE=On"]
        old_strict_mode = self.strict_mode
        self.strict_mode = False
        self._build_standard_cmake(extra_args)
        self.strict_mode = old_strict_mode

    def build_gmsh(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "bin", "gmsh" + to_exe())):
                print("  Not rebuilding gmsh, it is already in the LibPack")
                return
        extra_args = []
        if sys.platform.startswith("win32"):
            extra_args = [
                f"-D CMAKE_LIBRARY_PATH={self.install_dir}/win64/vc14/lib",  # TODO - Remove hardcoding
                "-D ENABLE_OPENMP=No",
            ]  # Build fails if OpenMP is enabled
        self._build_standard_cmake(extra_args)

    def build_pycxx(self, _: None):
        """PyCXX does not use a cMake-based build system"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "CXX")):
                print("  Not rebuilding PyCXX, it is already in the LibPack")
                return
        path_to_python = self.python_exe()
        args = [path_to_python, "setup.py", "install"]
        try:
            subprocess.run(args, check=True, capture_output=True)
        except subprocess.CalledProcessError as e:
            print("ERROR: Failed to build PyCXX using its custom build script")
            print(e.output.decode("utf-8"))
            if e.stderr:
                print(e.stderr.decode("utf-8"))
            exit(1)

    def build_icu(self, _: None):
        """ICU does not use cMake, but has projects for various OSes"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "unicode")):
                print("  Not rebuilding ICU, it is already in the LibPack")
                return

        os.chdir(os.path.join("icu4c", "source"))
        if sys.platform.startswith("win32"):
            os.chdir("allinone")
            args = [
                self.init_script,
                "&",
                "msbuild",
                f"/p:Configuration={str(self.mode).lower()}",
                "/t:Build",
                "/p:Platform=x64",  # TODO unhardcode
                "/p:SkipUWP=true",
                "allinone.sln",
            ]
            try:
                subprocess.run(args, check=True, capture_output=True)
            except subprocess.CalledProcessError as e:
                print("ERROR: Failed to build ICU using its custom build script")
                print(e.output.decode("utf-8"))
                if e.stderr:
                    print(e.stderr.decode("utf-8"))
                exit(1)
            os.chdir(os.path.join("..", ".."))
            bin_dir = os.path.join(self.install_dir, "bin")
            lib_dir = os.path.join(self.install_dir, "lib")
            inc_dir = os.path.join(self.install_dir, "include")
            shutil.copytree(f"bin64", bin_dir, dirs_exist_ok=True)
            shutil.copytree(f"lib64", lib_dir, dirs_exist_ok=True)
            shutil.copytree(f"include", inc_dir, dirs_exist_ok=True)
        else:
            raise NotImplemented("Non-Windows compilation of ICU is not implemented yet")

    def build_xercesc(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "xercesc")):
                print("  Not rebuilding xerces-c, it is already in the LibPack")
                return
        extra_args = [
            f"-D ICU_INCLUDE_DIR={self.install_dir}/include",
            f"-D ICU_ROOT={self.install_dir}",
            f"-D ICU_UC_DIR={self.install_dir}",
        ]
        self._build_standard_cmake(extra_args)

    def build_libfmt(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "fmt")):
                print("  Not rebuilding libfmt, it is already in the LibPack")
                return
            extra_args = ["-D FMT_TEST=OFF", "-D FMT_DOC=OFF"]
        self._build_standard_cmake(extra_args)

    def build_eigen3(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "eigen3")):
                print("  Not rebuilding Eigen3, it is already in the LibPack")
                return
        self._build_standard_cmake()

    def build_yamlcpp(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "include", "yaml-cpp")):
                print("  Not rebuilding yaml-cpp, it is already in the LibPack")
                return
        extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"]
        self._build_standard_cmake(extra_args)

    def build_opencamlib(self, _: None):
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "lib", "opencamlib", "ocl.lib")):
                print("  Not rebuilding opencamlib, it is already in the LibPack")
                return
        extra_args = ["-D BUILD_CXX_LIB=ON -D BUILD_PY_LIB=ON -D BUILD_DOC=OFF"]
        self._build_standard_cmake(extra_args)

    def build_calculix(self, _: None):
        """Cannot currently build Calculix (it's in Fortran, and we only support MSVC toolchain right now). Extract
        the relevant files from the downloaded zipfile and copy them"""
        if self.skip_existing:
            if os.path.exists(os.path.join(self.install_dir, "bin", "ccx.exe")):
                print("  Not rebuilding Calculix, it is already in the LibPack")
                return
        path_to_ccx_bin = os.path.join(os.getcwd(), "CL35-win64", "bin", "ccx", "218")
        if not os.path.exists(path_to_ccx_bin):
            raise RuntimeError("Could not locate Calculix")
        shutil.copytree(path_to_ccx_bin, os.path.join(self.install_dir, "bin"), dirs_exist_ok=True)
        # The download we use calls the executable ccx218.exe, but FreeCAD would prefer it be called ccx.exe for
        # automatic location of the executable
        shutil.move(
            os.path.join(self.install_dir, "bin", "ccx218.exe"),
            os.path.join(self.install_dir, "bin", "ccx.exe"),
        )