Generator runner improvement

pipeline/cli.py

@@ -7,7 +7,7 @@ from pipeline.orchestrator import EphemeralVastRunner, RunOptions
 # Accept one or more config files, or a single directory (all *.json inside, sorted)
 def _resolve_configs(raw: list[str]) -> list[Path]:
     if len(raw) == 1 and Path(raw[0]).is_dir():
-        configs = sorted(Path(raw[0]).glob("*.json"))
+        configs = sorted(p for p in Path(raw[0]).glob("*.json") if not p.name.startswith("_"))
         if not configs:
             raise ValueError(f"No JSON configs found in directory: {raw[0]}")
         return configs

pipeline/orchestrator.py

@@ -429,21 +429,22 @@ class EphemeralVastRunner:
         seen.remove(resolved)
         return self._deep_merge_dicts(base_cfg, cfg)
 
+    # Resolve a config the same way the runners do: extends chain first,
+    # then shared.json overlaid underneath. Mirrors load_config() in
+    # classifier/src/utils/config.py and generator/src/utils/config.py — keep
+    # in sync if merge semantics ever change.
+    def _load_config_merged(self, config_path: Path) -> dict[str, Any]:
+        cfg = self._load_config_with_extends(config_path)
+        shared_path = config_path.parent.parent / "shared.json"
+        if shared_path.exists():
+            with open(shared_path, encoding="utf-8") as fh:
+                shared_cfg = json.load(fh)
+            cfg = self._deep_merge_dicts(shared_cfg, cfg)
+        return cfg
+
     # Return a stable signature used to detect duplicate training configs
     def _normalized_config_signature(self, config_path: Path) -> str:
-        run_script, _ = self._module_for_config(config_path)
-        # Generator configs do not currently use shared/extends inheritance.
-        if run_script.startswith("generator/"):
-            with open(config_path, encoding="utf-8") as fh:
-                cfg = json.load(fh)
-        else:
-            cfg = self._load_config_with_extends(config_path)
-            shared_path = config_path.parent.parent / "shared.json"
-            if shared_path.exists():
-                with open(shared_path, encoding="utf-8") as fh:
-                    shared_cfg = json.load(fh)
-                cfg = self._deep_merge_dicts(shared_cfg, cfg)
-
+        cfg = self._load_config_merged(config_path)
         # run_name should not influence whether two configs are equivalent to train.
         cfg.pop("run_name", None)
         return json.dumps(cfg, sort_keys=True, separators=(",", ":"))