fix: move generic schema writing to the JVM plugin (#3818)

backend/protos/xyz/block/ftl/language/v1/language.proto

@@ -26,8 +26,6 @@ message ModuleConfig {
   optional string dev_mode_build = 6;
   // Build lock path to prevent concurrent builds
   string build_lock = 7;
-  // The directory to generate protobuf schema files into. These can be picked up by language specific build tools
-  optional string generated_schema_dir = 8;
   // Patterns to watch for file changes
   repeated string watch = 9;
 

backend/protos/xyz/block/ftl/language/v1/mixins.go

@@ -103,9 +103,6 @@ func ModuleConfigToProto(config moduleconfig.AbsModuleConfig) (*ModuleConfig, er
 	if config.DevModeBuild != "" {
 		proto.DevModeBuild = &config.DevModeBuild
 	}
-	if config.GeneratedSchemaDir != "" {
-		proto.GeneratedSchemaDir = &config.GeneratedSchemaDir
-	}
 
 	langConfigProto, err := structpb.NewStruct(config.LanguageConfig)
 	if err != nil {
@@ -127,7 +124,6 @@ func ModuleConfigFromProto(proto *ModuleConfig) moduleconfig.AbsModuleConfig {
 		Build:                 proto.GetBuild(),
 		DevModeBuild:          proto.GetDevModeBuild(),
 		BuildLock:             proto.BuildLock,
-		GeneratedSchemaDir:    proto.GetGeneratedSchemaDir(),
 		SQLMigrationDirectory: proto.GetSqlMigrationDir(),
 	}
 	if proto.LanguageConfig != nil {

common/schema/schema.go

@@ -215,3 +215,44 @@ func FromProto(s *schemapb.Schema) (*Schema, error) {
 	}
 	return ValidateSchema(schema)
 }
+
+// ModuleDependencies returns the modules that the given module depends on
+// Dependency modules are the ones that are called by the given module, or that publish topics that the given module subscribes to
+func (s *Schema) ModuleDependencies(module string) map[string]*Module {
+	mods := map[string]*Module{}
+	for _, sch := range s.Modules {
+		mods[sch.Name] = sch
+	}
+	deps := make(map[string]*Module)
+	toProcess := []string{module}
+	for len(toProcess) > 0 {
+		dep := toProcess[0]
+		toProcess = toProcess[1:]
+		if deps[dep] != nil {
+			continue
+		}
+		dm := mods[dep]
+		deps[dep] = dm
+		for _, m := range dm.Decls {
+			if ref, ok := m.(*Verb); ok {
+				for _, ref := range ref.Metadata {
+					switch md := ref.(type) {
+					case *MetadataCalls:
+						for _, calls := range md.Calls {
+							if calls.Module != "" {
+								toProcess = append(toProcess, calls.Module)
+							}
+						}
+					case *MetadataSubscriber:
+						if md.Topic.Module != "" {
+							toProcess = append(toProcess, md.Topic.Module)
+						}
+					default:
+					}
+				}
+			}
+		}
+	}
+	delete(deps, module)
+	return deps
+}

frontend/cli/cmd_new.go

@@ -67,7 +67,7 @@ func prepareNewCmd(ctx context.Context, k *kong.Kong, args []string) (optionalPl
 		return optionalPlugin, fmt.Errorf("could not load project config: %w", err)
 	}
 
-	plugin, err := languageplugin.New(ctx, filepath.Dir(projConfigPath), language, "new", false)
+	plugin, err := languageplugin.New(ctx, filepath.Dir(projConfigPath), language, "new")
 
 	if err != nil {
 		return optionalPlugin, fmt.Errorf("could not create plugin for %v: %w", language, err)

internal/buildengine/engine.go

@@ -1051,7 +1051,7 @@ func (e *Engine) gatherSchemas(
 }
 
 func (e *Engine) newModuleMeta(ctx context.Context, config moduleconfig.UnvalidatedModuleConfig) (moduleMeta, error) {
-	plugin, err := languageplugin.New(ctx, config.Dir, config.Language, config.Module, e.devMode)
+	plugin, err := languageplugin.New(ctx, config.Dir, config.Language, config.Module)
 	if err != nil {
 		return moduleMeta{}, fmt.Errorf("could not create plugin for %s: %w", config.Module, err)
 	}

internal/buildengine/languageplugin/plugin.go

@@ -96,8 +96,8 @@ type BuildContext struct {
 
 var ErrPluginNotRunning = errors.New("language plugin no longer running")
 
-// PluginFromConfig creates a new language plugin from the given config.
-func New(ctx context.Context, dir, language, name string, devMode bool) (p *LanguagePlugin, err error) {
+// New creates a new language plugin from the given config.
+func New(ctx context.Context, dir, language, name string) (p *LanguagePlugin, err error) {
 	impl, err := newClientImpl(ctx, dir, language, name)
 	if err != nil {
 		return nil, err

internal/buildengine/languageplugin/plugin_test.go

@@ -87,7 +87,6 @@ func buildContextFromProto(proto *langpb.BuildContext) (BuildContext, error) {
 			Build:                 optional.Ptr(proto.ModuleConfig.Build).Default(""),
 			DevModeBuild:          optional.Ptr(proto.ModuleConfig.DevModeBuild).Default(""),
 			DeployDir:             proto.ModuleConfig.DeployDir,
-			GeneratedSchemaDir:    optional.Ptr(proto.ModuleConfig.GeneratedSchemaDir).Default(""),
 			Watch:                 proto.ModuleConfig.Watch,
 			LanguageConfig:        proto.ModuleConfig.LanguageConfig.AsMap(),
 			SQLMigrationDirectory: proto.ModuleConfig.GetSqlMigrationDir(),

internal/buildengine/stubs.go

@@ -24,7 +24,7 @@ func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Mo
 	if err != nil {
 		return err
 	}
-	return writeGenericSchemaFiles(modules, metas)
+	return nil
 }
 
 // CleanStubs removes all generated stubs.
@@ -107,34 +107,3 @@ func generateStubsForEachLanguage(ctx context.Context, projectRoot string, modul
 	}
 	return nil
 }
-
-func writeGenericSchemaFiles(modules []*schema.Module, metas map[string]moduleMeta) error {
-	sch := &schema.Schema{Modules: modules}
-	for _, meta := range metas {
-		module := meta.module.Config
-		if module.GeneratedSchemaDir == "" {
-			continue
-		}
-
-		modPath := module.Abs().GeneratedSchemaDir
-		err := os.MkdirAll(modPath, 0750)
-		if err != nil {
-			return fmt.Errorf("failed to create directory %s: %w", modPath, err)
-		}
-
-		for _, mod := range sch.Modules {
-			if mod.Name == module.Module {
-				continue
-			}
-			data, err := schema.ModuleToBytes(mod)
-			if err != nil {
-				return fmt.Errorf("failed to export module schema for module %s %w", mod.Name, err)
-			}
-			err = os.WriteFile(filepath.Join(modPath, mod.Name+".pb"), data, 0600)
-			if err != nil {
-				return fmt.Errorf("failed to write schema file for module %s %w", mod.Name, err)
-			}
-		}
-	}
-	return nil
-}

internal/moduleconfig/moduleconfig.go

@@ -30,8 +30,6 @@ type ModuleConfig struct {
 	BuildLock string `toml:"build-lock"`
 	// DeployDir is the directory to deploy from, relative to the module directory.
 	DeployDir string `toml:"deploy-dir"`
-	// GeneratedSchemaDir is the directory to generate protobuf schema files into. These can be picked up by language specific build tools
-	GeneratedSchemaDir string `toml:"generated-schema-dir"`
 	// Watch is the list of files to watch for changes.
 	Watch []string `toml:"watch"`
 
@@ -134,12 +132,6 @@ func (c ModuleConfig) Abs() AbsModuleConfig {
 		panic(fmt.Sprintf("deploy-dir %q is not beneath module directory %q", clone.DeployDir, clone.Dir))
 	}
 	clone.BuildLock = filepath.Clean(filepath.Join(clone.Dir, clone.BuildLock))
-	if clone.GeneratedSchemaDir != "" {
-		clone.GeneratedSchemaDir = filepath.Clean(filepath.Join(clone.Dir, clone.GeneratedSchemaDir))
-		if !strings.HasPrefix(clone.GeneratedSchemaDir, clone.Dir) {
-			panic(fmt.Sprintf("generated-schema-dir %q is not beneath module directory %q", clone.GeneratedSchemaDir, clone.Dir))
-		}
-	}
 	// Watch paths are allowed to be outside the deploy directory.
 	clone.Watch = slices.Map(clone.Watch, func(p string) string {
 		return filepath.Clean(filepath.Join(clone.Dir, p))
@@ -176,9 +168,6 @@ func (c UnvalidatedModuleConfig) FillDefaultsAndValidate(customDefaults CustomDe
 		c.SQLMigrationDirectory = customDefaults.SQLMigrationDir
 
 	}
-	if defaultValue, ok := customDefaults.GeneratedSchemaDir.Get(); ok && c.GeneratedSchemaDir == "" {
-		c.GeneratedSchemaDir = defaultValue
-	}
 	if c.Watch == nil {
 		c.Watch = customDefaults.Watch
 	}

internal/moduleconfig/moduleconfig_test.go

@@ -30,29 +30,27 @@ func TestDefaulting(t *testing.T) {
 				Watch:              []string{"a", "b", "c"},
 			},
 			expected: ModuleConfig{
-				Realm:              "home",
-				Dir:                "a",
-				Module:             "nothingset",
-				Language:           "test",
-				Build:              "build",
-				DevModeBuild:       "devmodebuild",
-				BuildLock:          "customdefaultlock",
-				DeployDir:          "deploydir",
-				GeneratedSchemaDir: "generatedschemadir",
-				Watch:              []string{"a", "b", "c"},
+				Realm:        "home",
+				Dir:          "a",
+				Module:       "nothingset",
+				Language:     "test",
+				Build:        "build",
+				DevModeBuild: "devmodebuild",
+				BuildLock:    "customdefaultlock",
+				DeployDir:    "deploydir",
+				Watch:        []string{"a", "b", "c"},
 			},
 		},
 		{
 			config: UnvalidatedModuleConfig{
-				Dir:                "b",
-				Module:             "allset",
-				Language:           "test",
-				Build:              "custombuild",
-				DevModeBuild:       "customdevmodebuild",
-				BuildLock:          "custombuildlock",
-				DeployDir:          "customdeploydir",
-				GeneratedSchemaDir: "customgeneratedschemadir",
-				Watch:              []string{"custom1"},
+				Dir:          "b",
+				Module:       "allset",
+				Language:     "test",
+				Build:        "custombuild",
+				DevModeBuild: "customdevmodebuild",
+				BuildLock:    "custombuildlock",
+				DeployDir:    "customdeploydir",
+				Watch:        []string{"custom1"},
 				LanguageConfig: map[string]any{
 					"build-tool": "maven",
 					"more":       []int{1, 2, 3},
@@ -66,16 +64,15 @@ func TestDefaulting(t *testing.T) {
 				Watch:              []string{"a", "b", "c"},
 			},
 			expected: ModuleConfig{
-				Realm:              "home",
-				Dir:                "b",
-				Module:             "allset",
-				Language:           "test",
-				Build:              "custombuild",
-				DevModeBuild:       "customdevmodebuild",
-				BuildLock:          "custombuildlock",
-				DeployDir:          "customdeploydir",
-				GeneratedSchemaDir: "customgeneratedschemadir",
-				Watch:              []string{"custom1"},
+				Realm:        "home",
+				Dir:          "b",
+				Module:       "allset",
+				Language:     "test",
+				Build:        "custombuild",
+				DevModeBuild: "customdevmodebuild",
+				BuildLock:    "custombuildlock",
+				DeployDir:    "customdeploydir",
+				Watch:        []string{"custom1"},
 				LanguageConfig: map[string]any{
 					"build-tool": "maven",
 					"more":       []int{1, 2, 3},

jvm-runtime/plugin/common/java_plugin_test.go

@@ -75,7 +75,7 @@ func TestJavaConfigDefaults(t *testing.T) {
 			dir, err := filepath.Abs(tt.dir)
 			assert.NoError(t, err)
 
-			plugin, err := languageplugin.New(ctx, t.TempDir(), "java", "test", false)
+			plugin, err := languageplugin.New(ctx, t.TempDir(), "java", "test")
 			assert.NoError(t, err)
 			t.Cleanup(func() {
 				_ = plugin.Kill() //nolint:errcheck