Skip to content

Commit

Permalink
Split engine components into separate filesystem layers
Browse files Browse the repository at this point in the history
  • Loading branch information
adamrehn committed Aug 11, 2021
1 parent a7111bd commit 4e6e64b
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 122 deletions.
3 changes: 1 addition & 2 deletions ue4docker/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,8 +285,7 @@ def build():
# Build the minimal UE4 CI image, unless requested otherwise by the user
buildUe4Minimal = config.noMinimal == False
if buildUe4Minimal == True:
buildGraphArg = ['--build-arg', 'BUILDGRAPH_ARGS=' + ' '.join(config.buildGraphArgs)]
builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg)
builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + ue4BuildArgs)
builtImages.append('ue4-minimal')
else:
logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.')
Expand Down
28 changes: 16 additions & 12 deletions ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,13 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal
RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild

# Create an Installed Build of the Engine
ARG BUILDGRAPH_ARGS
WORKDIR /home/ue4/UnrealEngine
RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true ${BUILDGRAPH_ARGS} {{ buildgraph_args }} && \
RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \
rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC

# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size
COPY exclude-components.py /tmp/exclude-components.py
ARG EXCLUDE_DEBUG
ARG EXCLUDE_TEMPLATES
RUN python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES
# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers
COPY split-components.py /tmp/split-components.py
RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components

{% if (not disable_all_patches) and (not disable_target_patches) %}
# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini
Expand Down Expand Up @@ -73,15 +70,22 @@ FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG}

# Copy the Installed Build files from the builder image
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine
{% if excluded_components.ddc == false %}
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DDC /home/ue4/UnrealEngine
{% endif %}
{% if excluded_components.debug == false %}
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbols /home/ue4/UnrealEngine
{% endif %}
{% if excluded_components.templates == false %}
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine
{% endif %}
WORKDIR /home/ue4/UnrealEngine

{% if not disable_labels %}
# Add labels to the built image to identify which components (if any) were excluded from the build that it contains
# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build)
ARG EXCLUDE_DEBUG
ARG EXCLUDE_TEMPLATES
LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG}
LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES}
LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %}
LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %}
LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %}
{% endif %}

# Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4`
Expand Down
43 changes: 0 additions & 43 deletions ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py

This file was deleted.

58 changes: 58 additions & 0 deletions ue4docker/dockerfiles/ue4-minimal/linux/split-components.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
import glob, os, shutil, sys
from os.path import basename, dirname, exists, join

# Logs a message to stderr
def log(message):
print(message, file=sys.stderr)
sys.stderr.flush()

# Extracts the files and directories for the specified component and moves them to a separate output directory
def extractComponent(inputDir, outputDir, component, description, items):

# Print progress output
log('\nExtracting {}...'.format(description))

# Create the output directory for the component if it doesn't already exist
componentDir = join(outputDir, component)
os.makedirs(outputDir, exist_ok=True)

# Move each file and directory for the component to the output directory
for item in items:

# Verify that the item exists
if not exists(item):
log('Skipping non-existent item: {}'.format(item))
continue

# Print progress output
log('Moving: {}'.format(item))

# Ensure the parent directory for the item exists in the output directory
parent = dirname(item).replace(inputDir, componentDir)
os.makedirs(parent, exist_ok=True)

# Perform the move
shutil.move(
item,
join(parent, basename(item))
)

# Retrieve the path to the root directory of the Installed Build
rootDir = sys.argv[1]

# Retrieve the path to the root output directory for extracted components and ensure it exists
outputDir = sys.argv[2]
os.makedirs(outputDir, exist_ok=True)

# Extract the DDC
ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')]
extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc)

# Extract debug symbols
symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True)
extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles)

# Extract template projects and samples
subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']]
extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs)
29 changes: 18 additions & 11 deletions ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB
# Create an Installed Build of the Engine
ARG BUILDGRAPH_ARGS
WORKDIR C:\UnrealEngine
RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true %BUILDGRAPH_ARGS% {{ buildgraph_args }} && `
RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && `
(if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC)

# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size
COPY exclude-components.py C:\exclude-components.py
ARG EXCLUDE_DEBUG
ARG EXCLUDE_TEMPLATES
RUN python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES%
# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers
COPY split-components.py C:\split-components.py
RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components

{% if (not disable_all_patches) and (not disable_target_patches) %}
# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini
Expand All @@ -50,14 +48,23 @@ FROM prerequisites as minimal
ARG NAMESPACE
FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG}
{% endif %}

# Copy the Installed Build files from the builder image
COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine
{% if excluded_components.ddc == false %}
COPY --from=builder C:\UnrealEngine\Components\DDC C:\UnrealEngine
{% endif %}
{% if excluded_components.debug == false %}
COPY --from=builder C:\UnrealEngine\Components\DebugSymbols C:\UnrealEngine
{% endif %}
{% if excluded_components.templates == false %}
COPY --from=builder C:\UnrealEngine\Components\TemplatesAndSamples C:\UnrealEngine
{% endif %}
WORKDIR C:\UnrealEngine

{% if not disable_labels %}
# Add labels to the built image to identify which components (if any) were excluded from the build that it contains
# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build)
ARG EXCLUDE_DEBUG
ARG EXCLUDE_TEMPLATES
LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG}
LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES}
LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %}
LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %}
LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %}
{% endif %}
43 changes: 0 additions & 43 deletions ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py

This file was deleted.

58 changes: 58 additions & 0 deletions ue4docker/dockerfiles/ue4-minimal/windows/split-components.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
import glob, os, shutil, sys
from os.path import basename, dirname, exists, join

# Logs a message to stderr
def log(message):
print(message, file=sys.stderr)
sys.stderr.flush()

# Extracts the files and directories for the specified component and moves them to a separate output directory
def extractComponent(inputDir, outputDir, component, description, items):

# Print progress output
log('\nExtracting {}...'.format(description))

# Create the output directory for the component if it doesn't already exist
componentDir = join(outputDir, component)
os.makedirs(outputDir, exist_ok=True)

# Move each file and directory for the component to the output directory
for item in items:

# Verify that the item exists
if not exists(item):
log('Skipping non-existent item: {}'.format(item))
continue

# Print progress output
log('Moving: {}'.format(item))

# Ensure the parent directory for the item exists in the output directory
parent = dirname(item).replace(inputDir, componentDir)
os.makedirs(parent, exist_ok=True)

# Perform the move
shutil.move(
item,
join(parent, basename(item))
)

# Retrieve the path to the root directory of the Installed Build
rootDir = sys.argv[1]

# Retrieve the path to the root output directory for extracted components and ensure it exists
outputDir = sys.argv[2]
os.makedirs(outputDir, exist_ok=True)

# Extract the DDC
ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')]
extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc)

# Extract debug symbols
symbolFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True)
extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles)

# Extract template projects and samples
subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']]
extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs)
20 changes: 9 additions & 11 deletions ue4docker/infrastructure/BuildConfiguration.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,15 +210,13 @@ def __init__(self, parser, argv):
if self.opts.get('credential_mode', 'endpoint') not in validCredentialModes:
raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title()))

# Generate our flags for keeping or excluding components
self.buildGraphArgs = [
'-set:WithDDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true')
]
self.exclusionFlags = [
'--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0),
'--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0)
]

# Generate Jinja context values for keeping or excluding components
self.opts['excluded_components'] = {
'ddc': ExcludedComponent.DDC in self.excludedComponents,
'debug': ExcludedComponent.Debug in self.excludedComponents,
'templates': ExcludedComponent.Templates in self.excludedComponents
}

# If we're building Windows containers, generate our Windows-specific configuration settings
if self.containerPlatform == 'windows':
self._generateWindowsConfig()
Expand Down Expand Up @@ -257,8 +255,8 @@ def _generateWindowsConfig(self):
# Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml.
# Otherwise, UAT errors out with "Unknown argument: VS2019".
if self.visualStudio != VisualStudio.VS2017:
self.buildGraphArgs += [f'-set:VS{self.visualStudio}=true']

self.opts['buildgraph_args'] = self.opts.get('buildgraph_args', '') + f' -set:VS{self.visualStudio}=true'
# Determine base tag for the Windows release of the host system
self.hostRelease = WindowsUtils.getWindowsRelease()
self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease)
Expand Down

0 comments on commit 4e6e64b

Please sign in to comment.