From d75272cf8402f3103949f23d4a779105b46881b4 Mon Sep 17 00:00:00 2001 From: Johnny Fernandes Date: Thu, 30 Apr 2026 03:21:49 +0100 Subject: [PATCH] Generator runner improvement --- README.md | 7 ------- generator/run.py | 2 +- pipeline/cli.py | 2 +- pipeline/orchestrator.py | 27 ++++++++++++++------------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1cb4d20..a0d7ee1 100644 --- a/README.md +++ b/README.md @@ -123,10 +123,3 @@ The default policy in `pipeline/defaults/vast.json` now targets: - `<= $0.20/hour` - sorted by `dlperf` descending - uses `vastai/pytorch:latest` as the default image - -## Diagnostics - -```bash -python3 classifier/tools/analyze.py classifier/configs/phase2/p2_resnet18_facecrop.json -python3 classifier/tools/ensemble.py classifier/configs/phase4/p4_ensemble.json -``` diff --git a/generator/run.py b/generator/run.py index bebed7f..2586f88 100644 --- a/generator/run.py +++ b/generator/run.py @@ -37,7 +37,7 @@ def main(config_path, *, data_dir_override=None, output_root="generator/outputs" cfg = load_config(config_path) - run_name = cfg["run_name"] + run_name = cfg.get("run_name", Path(config_path).stem) device = "cuda" if torch.cuda.is_available() else "cpu" data_dir = data_dir_override or cfg.get("data_dir", "data") output_root = Path(output_root) diff --git a/pipeline/cli.py b/pipeline/cli.py index 7837d38..472e632 100644 --- a/pipeline/cli.py +++ b/pipeline/cli.py @@ -7,7 +7,7 @@ from pipeline.orchestrator import EphemeralVastRunner, RunOptions # Accept one or more config files, or a single directory (all *.json inside, sorted) def _resolve_configs(raw: list[str]) -> list[Path]: if len(raw) == 1 and Path(raw[0]).is_dir(): - configs = sorted(Path(raw[0]).glob("*.json")) + configs = sorted(p for p in Path(raw[0]).glob("*.json") if not p.name.startswith("_")) if not configs: raise ValueError(f"No JSON configs found in directory: {raw[0]}") return configs diff --git a/pipeline/orchestrator.py b/pipeline/orchestrator.py index aec8d65..e1cbd44 100644 --- a/pipeline/orchestrator.py +++ b/pipeline/orchestrator.py @@ -429,21 +429,22 @@ class EphemeralVastRunner: seen.remove(resolved) return self._deep_merge_dicts(base_cfg, cfg) + # Resolve a config the same way the runners do: extends chain first, + # then shared.json overlaid underneath. Mirrors load_config() in + # classifier/src/utils/config.py and generator/src/utils/config.py — keep + # in sync if merge semantics ever change. + def _load_config_merged(self, config_path: Path) -> dict[str, Any]: + cfg = self._load_config_with_extends(config_path) + shared_path = config_path.parent.parent / "shared.json" + if shared_path.exists(): + with open(shared_path, encoding="utf-8") as fh: + shared_cfg = json.load(fh) + cfg = self._deep_merge_dicts(shared_cfg, cfg) + return cfg + # Return a stable signature used to detect duplicate training configs def _normalized_config_signature(self, config_path: Path) -> str: - run_script, _ = self._module_for_config(config_path) - # Generator configs do not currently use shared/extends inheritance. - if run_script.startswith("generator/"): - with open(config_path, encoding="utf-8") as fh: - cfg = json.load(fh) - else: - cfg = self._load_config_with_extends(config_path) - shared_path = config_path.parent.parent / "shared.json" - if shared_path.exists(): - with open(shared_path, encoding="utf-8") as fh: - shared_cfg = json.load(fh) - cfg = self._deep_merge_dicts(shared_cfg, cfg) - + cfg = self._load_config_merged(config_path) # run_name should not influence whether two configs are equivalent to train. cfg.pop("run_name", None) return json.dumps(cfg, sort_keys=True, separators=(",", ":"))