Support generating a dist manifest

build.gradle

@@ -153,6 +153,11 @@ idea {
                     programParameters = "--help"
                     moduleRef(project, sourceSets.main)
                 }
+                "Run Neoforge 1.21.4 (joined) + Parchment"(Application) {
+                    mainClass = mainClassName
+                    programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.4.1-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21:2024.06.23@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.4.jar --write-result=clientResources:build/client-extra-1.21.4.jar --write-result=sources:build/minecraft-sources-1.21.4.jar"
+                    moduleRef(project, sourceSets.main)
+                }
                 "Run Neoforge 1.21 (joined) + Parchment"(Application) {
                     mainClass = mainClassName
                     programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21:2024.06.23@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.jar --write-result=clientResources:build/client-extra-1.21.jar --write-result=sources:build/minecraft-sources-1.21.jar"

src/main/java/net/neoforged/neoform/runtime/actions/SplitResourcesFromClassesAction.java

@@ -2,29 +2,38 @@
 
 import net.neoforged.neoform.runtime.cache.CacheKeyBuilder;
 import net.neoforged.neoform.runtime.engine.ProcessingEnvironment;
+import net.neoforged.srgutils.IMappingFile;
+import org.jetbrains.annotations.Nullable;
 
-import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.function.Predicate;
-import java.util.jar.JarEntry;
+import java.util.jar.Attributes;
 import java.util.jar.JarFile;
-import java.util.jar.JarInputStream;
 import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 /**
  * Copies a Jar file while applying a filename filter.
  */
 public final class SplitResourcesFromClassesAction extends BuiltInAction {
 
+    public static final String INPUT_OTHER_DIST_JAR = "otherDistJar";
+    public static final String INPUT_MAPPINGS = "mappings";
+
     /**
      * Use a fixed timestamp for the manifest entry.
      */
@@ -35,19 +44,19 @@ public final class SplitResourcesFromClassesAction extends BuiltInAction {
      */
     private final List<Pattern> denyListPatterns = new ArrayList<>();
 
-    /**
-     * Indicates that the MANIFEST should be taken from the "manifest" input and
-     * injected into the resulting resources jar file.
-     */
-    private boolean injectManifest;
+    @Nullable
+    private GenerateDistManifestSettings generateDistManifestSettings;
 
     @Override
     public void run(ProcessingEnvironment environment) throws IOException, InterruptedException {
         var inputJar = environment.getRequiredInputPath("input");
-        Path inputManifest = null;
-        if (injectManifest) {
-            inputManifest = environment.getRequiredInputPath("manifest");
+        Path otherDistJarPath = null;
+        Path mappingsPath = null;
+        if (generateDistManifestSettings != null) {
+            otherDistJarPath = environment.getRequiredInputPath(INPUT_OTHER_DIST_JAR);
+            mappingsPath = environment.getRequiredInputPath(INPUT_MAPPINGS);
         }
+
         var classesJar = environment.getOutputPath("output");
         var resourcesJar = environment.getOutputPath("resourcesOutput");
 
@@ -59,30 +68,31 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                     .asMatchPredicate();
         }
 
-        try (var is = new JarInputStream(new BufferedInputStream(Files.newInputStream(inputJar)));
+        try (var jar = new ZipFile(inputJar.toFile());
              var classesFileOut = new BufferedOutputStream(Files.newOutputStream(classesJar));
              var resourcesFileOut = new BufferedOutputStream(Files.newOutputStream(resourcesJar));
              var classesJarOut = new JarOutputStream(classesFileOut);
              var resourcesJarOut = new JarOutputStream(resourcesFileOut);
         ) {
-            // If requested, write the manifest
-            if (injectManifest) {
-                var entry = new ZipEntry(JarFile.MANIFEST_NAME);
-                entry.setTimeLocal(MANIFEST_TIME);
-                resourcesJarOut.putNextEntry(entry);
-                Files.copy(inputManifest, resourcesJarOut);
-                resourcesJarOut.closeEntry();
+            if (generateDistManifestSettings != null) {
+                generateDistSourceManifest(
+                        mappingsPath,
+                        inputJar,
+                        jar,
+                        otherDistJarPath,
+                        resourcesJarOut
+                );
             }
 
-            // Ignore any entry that's not allowed
-            JarEntry entry;
-            while ((entry = is.getNextJarEntry()) != null) {
+            var entries = jar.entries();
+            while (entries.hasMoreElements()) {
+                var entry = entries.nextElement();
                 if (entry.isDirectory()) {
                     continue; // For simplicity, we ignore directories completely
                 }
 
                 // If we injected a manifest earlier, ignore any subsequent manifests
-                if (injectManifest && entry.getName().equals(JarFile.MANIFEST_NAME)) {
+                if (generateDistManifestSettings != null && entry.getName().equals(JarFile.MANIFEST_NAME)) {
                     continue;
                 }
 
@@ -96,12 +106,78 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                 var destinationStream = filename.endsWith(".class") ? classesJarOut : resourcesJarOut;
 
                 destinationStream.putNextEntry(entry);
-                is.transferTo(destinationStream);
+                try (var is = jar.getInputStream(entry)) {
+                    is.transferTo(destinationStream);
+                }
                 destinationStream.closeEntry();
             }
         }
     }
 
+    private void generateDistSourceManifest(Path mappingsPath, Path inputJar, ZipFile jar, Path otherDistJarPath, JarOutputStream resourcesJarOut) throws IOException {
+        var mappings = mappingsPath != null ? IMappingFile.load(mappingsPath.toFile()) : null;
+
+        // Use the time-stamp of either of the two input files (whichever is newer)
+        FileTime mtime = Files.getLastModifiedTime(inputJar);
+        var ourFiles = getFileIndex(jar);
+        ourFiles.remove(JarFile.MANIFEST_NAME);
+        Set<String> theirFiles;
+        try (var otherDistJar = new ZipFile(otherDistJarPath.toFile())) {
+            var otherMtime = Files.getLastModifiedTime(otherDistJarPath);
+            if (otherMtime.compareTo(mtime) > 0) {
+                mtime = otherMtime;
+            }
+            theirFiles = getFileIndex(otherDistJar);
+        }
+        theirFiles.remove(JarFile.MANIFEST_NAME);
+
+        var manifest = new Manifest();
+        manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        manifest.getMainAttributes().putValue("Minecraft-Dists", generateDistManifestSettings.distId()
+            + " " + generateDistManifestSettings.otherDistId());
+
+        addSourceDistEntries(ourFiles, theirFiles, generateDistManifestSettings.distId(), mappings, manifest);
+        addSourceDistEntries(theirFiles, ourFiles, generateDistManifestSettings.otherDistId(), mappings, manifest);
+
+        var manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+        manifestEntry.setLastModifiedTime(mtime);
+        resourcesJarOut.putNextEntry(manifestEntry);
+        manifest.write(resourcesJarOut);
+        resourcesJarOut.closeEntry();
+    }
+
+    private static void addSourceDistEntries(Set<String> distFiles,
+                                             Set<String> otherDistFiles,
+                                             String dist,
+                                             IMappingFile mappings,
+                                             Manifest manifest) {
+        for (var file : distFiles) {
+            if (!otherDistFiles.contains(file)) {
+                var fileAttr = new Attributes(1);
+                fileAttr.putValue("Minecraft-Dist", dist);
+
+                if (mappings != null && file.endsWith(".class")) {
+                    file = mappings.remapClass(file.substring(0, file.length() - ".class".length())) + ".class";
+                }
+                manifest.getEntries().put(file, fileAttr);
+            }
+        }
+    }
+
+    private Set<String> getFileIndex(ZipFile zipFile) {
+        var result = new HashSet<String>(zipFile.size());
+
+        var entries = zipFile.entries();
+        while (entries.hasMoreElements()) {
+            ZipEntry entry = entries.nextElement();
+            if (!entry.isDirectory()) {
+                result.add(entry.getName());
+            }
+        }
+
+        return result;
+    }
+
     /**
      * Adds a regular expression for filenames that should be filtered out completely.
      */
@@ -111,13 +187,31 @@ public void addDenyPatterns(String... patterns) {
         }
     }
 
+    /**
+     * Enable generation of a Jar manifest in the output resources jar which contains
+     * entries detailing which distribution each file came from.
+     * This adds new required inputs.
+     */
+    public void generateSplitManifest(String distId, String otherDistId) {
+        generateDistManifestSettings = new GenerateDistManifestSettings(
+                Objects.requireNonNull(distId, "distId"),
+                Objects.requireNonNull(otherDistId, "otherDistId")
+        );
+    }
+
     @Override
     public void computeCacheKey(CacheKeyBuilder ck) {
         super.computeCacheKey(ck);
         ck.addStrings("deny patterns", denyListPatterns.stream().map(Pattern::pattern).toList());
+        if (generateDistManifestSettings != null) {
+            ck.add("generate dist manifest - our dist", generateDistManifestSettings.distId);
+            ck.add("generate dist manifest - other dist", generateDistManifestSettings.otherDistId);
+        }
     }
 
-    public void setInjectManifest(boolean injectManifest) {
-        this.injectManifest = injectManifest;
+    private record GenerateDistManifestSettings(
+            String distId,
+            String otherDistId
+    ) {
     }
 }

src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java

@@ -7,6 +7,7 @@
 import net.neoforged.neoform.runtime.actions.PatchActionFactory;
 import net.neoforged.neoform.runtime.actions.RecompileSourcesAction;
 import net.neoforged.neoform.runtime.actions.StripManifestDigestContentFilter;
+import net.neoforged.neoform.runtime.artifacts.ArtifactManager;
 import net.neoforged.neoform.runtime.artifacts.ClasspathItem;
 import net.neoforged.neoform.runtime.config.neoforge.NeoForgeConfig;
 import net.neoforged.neoform.runtime.engine.NeoFormEngine;
@@ -69,7 +70,7 @@ public class RunNeoFormCommand extends NeoFormEngineCommand {
     String parchmentConflictPrefix;
 
     static class SourceArtifacts {
-        @CommandLine.ArgGroup(multiplicity = "1")
+        @CommandLine.ArgGroup
         NeoFormArtifact neoform;
         @CommandLine.Option(names = "--neoforge")
         String neoforge;
@@ -92,17 +93,7 @@ protected void runWithNeoFormEngine(NeoFormEngine engine, List<AutoCloseable> cl
             var neoforgeConfig = NeoForgeConfig.from(neoforgeZipFile);
 
             // Allow it to be overridden with local or remote data
-            Path neoformArtifact;
-            if (sourceArtifacts.neoform.file != null) {
-                LOG.println("Overriding NeoForm version " + neoforgeConfig.neoformArtifact() + " with NeoForm file " + sourceArtifacts.neoform.file);
-                neoformArtifact = sourceArtifacts.neoform.file;
-            } else if (sourceArtifacts.neoform.artifact != null) {
-                LOG.println("Overriding NeoForm version " + neoforgeConfig.neoformArtifact() + " with CLI argument " + sourceArtifacts.neoform.artifact);
-                neoformArtifact = artifactManager.get(MavenCoordinate.parse(sourceArtifacts.neoform.artifact)).path();
-            } else {
-                neoformArtifact = artifactManager.get(MavenCoordinate.parse(neoforgeConfig.neoformArtifact())).path();
-            }
-
+            var neoformArtifact = getNeoForgeNeoFormArtifact(artifactManager, neoforgeConfig.neoformArtifact());
             engine.loadNeoFormData(neoformArtifact, dist);
 
             // Add NeoForge specific data sources
@@ -233,6 +224,18 @@ protected void runWithNeoFormEngine(NeoFormEngine engine, List<AutoCloseable> cl
         execute(engine);
     }
 
+    private Path getNeoForgeNeoFormArtifact(ArtifactManager artifactManager, String neoforgeNeoFormArtifact) throws IOException {
+        if (sourceArtifacts.neoform != null && sourceArtifacts.neoform.file != null) {
+            LOG.println("Overriding NeoForm version " + neoforgeNeoFormArtifact + " with NeoForm file " + sourceArtifacts.neoform.file);
+            return sourceArtifacts.neoform.file;
+        } else if (sourceArtifacts.neoform != null && sourceArtifacts.neoform.artifact != null) {
+            LOG.println("Overriding NeoForm version " + neoforgeNeoFormArtifact + " with CLI argument " + sourceArtifacts.neoform.artifact);
+            return artifactManager.get(MavenCoordinate.parse(sourceArtifacts.neoform.artifact)).path();
+        } else {
+            return artifactManager.get(MavenCoordinate.parse(neoforgeNeoFormArtifact)).path();
+        }
+    }
+
     private static NodeOutput createCompiledWithNeoForge(NeoFormEngine engine, ZipFile neoforgeClassesZip) {
         var graph = engine.getGraph();
         var recompiledClasses = graph.getRequiredOutput("recompile", "output");

src/main/java/net/neoforged/neoform/runtime/config/neoform/NeoFormDistConfig.java

@@ -21,6 +21,10 @@ public NeoFormDistConfig(NeoFormConfig config, String dist) {
         this.dist = dist;
     }
 
+    public String dist() {
+        return dist;
+    }
+
     public int javaVersion() {
         return config.javaVersion();
     }

src/main/java/net/neoforged/neoform/runtime/engine/NeoFormEngine.java

@@ -318,11 +318,21 @@ private void addNodeForStep(ExecutionGraph graph, NeoFormDistConfig config, NeoF
                 var action = new SplitResourcesFromClassesAction();
                 // The Minecraft jar contains nothing of interest in META-INF, and the signature files are useless.
                 action.addDenyPatterns("META-INF/.*");
-                // We can inject a manifest if the NeoForm process contains a generateSplitManifest step
-                var splitManifest = graph.getNode("generateSplitManifest");
-                if (splitManifest != null) {
-                    action.setInjectManifest(true);
-                    builder.input("manifest", splitManifest.getRequiredOutput("output").asInput());
+                if (processGeneration.generateDistSourceManifest() && config.dist().equals("joined")) {
+                    if ("stripClient".equals(step.getId())) {
+                        // Prefer the already extracted server
+                        var serverJarInput = graph.hasOutput("extractServer", "output") ?
+                                graph.getRequiredOutput("extractServer", "output").asInput()
+                                : graph.getRequiredOutput("downloadServer", "output").asInput();
+
+                        action.generateSplitManifest("client", "server");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, serverJarInput);
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    } else if ("stripServer".equals(step.getId())) {
+                        action.generateSplitManifest("server", "client");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, graph.getRequiredOutput("downloadClient", "output").asInput());
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    }
                 }
 
                 processGeneration.getAdditionalDenyListForMinecraftJars().forEach(action::addDenyPatterns);

src/main/java/net/neoforged/neoform/runtime/engine/ProcessGeneration.java

@@ -43,6 +43,7 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
 
     private static final MinecraftReleaseVersion MC_1_17_1 = new MinecraftReleaseVersion(1, 17, 1);
     private static final MinecraftReleaseVersion MC_1_20_1 = new MinecraftReleaseVersion(1, 20, 1);
+    private static final MinecraftReleaseVersion MC_1_21_4 = new MinecraftReleaseVersion(1, 21, 4);
 
     /**
      * Indicates whether the Minecraft server jar file contains third party
@@ -58,6 +59,12 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
      */
     private boolean sourcesUseIntermediaryNames;
 
+    /**
+     * Enables generation of the MANIFEST.MF in the client and server resource files that
+     * indicates which distribution each file came from. Only applies to joined distributions.
+     */
+    private boolean generateDistSourceManifest;
+
     /**
      * For (Neo)Forge 1.20.1 and below, we have to remap method and field names from
      * SRG to official names for development.
@@ -88,6 +95,9 @@ static ProcessGeneration fromMinecraftVersion(String minecraftVersion) {
         // In 1.20.2 and later, NeoForge switched to Mojmap at runtime and sources defined in Mojmap
         result.sourcesUseIntermediaryNames = isLessThanOrEqualTo(releaseVersion, MC_1_20_1);
 
+        // Technically 1.21.4 does not directly support this, but it does not harm it either
+        result.generateDistSourceManifest = isGreaterThanOrEqualTo(releaseVersion, MC_1_21_4);
+
         return result;
     }
 
@@ -98,13 +108,28 @@ private static boolean isLessThanOrEqualTo(@Nullable MinecraftReleaseVersion rel
         return releaseVersion.compareTo(version) <= 0;
     }
 
+    private static boolean isGreaterThanOrEqualTo(@Nullable MinecraftReleaseVersion releaseVersion, MinecraftReleaseVersion version) {
+        if (releaseVersion == null) {
+            return true; // We're working with a snapshot version, which we always use the latest processes for
+        }
+        return releaseVersion.compareTo(version) >= 0;
+    }
+
     /**
      * Does the Minecraft source code that MCP/NeoForm creates use SRG names?
      */
     public boolean sourcesUseIntermediaryNames() {
         return sourcesUseIntermediaryNames;
     }
 
+    /**
+     * Does the FML version on that MC generation support use of MANIFEST.MF entries
+     * for filtering out dist-specific classes in dev? (When using the joined distribution)
+     */
+    public boolean generateDistSourceManifest() {
+        return generateDistSourceManifest;
+    }
+
     /**
      * Allows additional resources to be completely removed from Minecraft jars before processing them.
      */