From 14aed699edc59141710f949fa234ec153d80c861 Mon Sep 17 00:00:00 2001 From: Johnny Fernandes Date: Thu, 30 Apr 2026 16:26:25 +0100 Subject: [PATCH] Pipeline update to rsync at 25 epochs --- README.md | 2 +- pipeline/README.md | 2 +- pipeline/orchestrator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a0d7ee1..5796dc6 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ That command: - uploads `data/` when `--upload-data` is set - runs `python3 classifier/run.py ...` - downloads `classifier/outputs/` -- for generator runs, rsyncs `generator/outputs/` back every 50 epochs and again at completion +- for generator runs, rsyncs `generator/outputs/` back every 25 epochs and again at completion - destroys the instance automatically unless `--keep-on-failure` is set Useful commands: diff --git a/pipeline/README.md b/pipeline/README.md index 696a0fe..c97561d 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -1,6 +1,6 @@ # Pipeline -Orchestrates ephemeral Vast.ai GPU instances: searches for an offer, creates the instance, syncs the project, trains, downloads `outputs/`, and destroys the instance automatically. Generator runs also rsync `generator/outputs/` every 50 epochs while training is still running. +Orchestrates ephemeral Vast.ai GPU instances: searches for an offer, creates the instance, syncs the project, trains, downloads `outputs/`, and destroys the instance automatically. Generator runs also rsync `generator/outputs/` every 25 epochs while training is still running. ## One-time setup diff --git a/pipeline/orchestrator.py b/pipeline/orchestrator.py index 93212fe..10b9b7b 100644 --- a/pipeline/orchestrator.py +++ b/pipeline/orchestrator.py @@ -74,7 +74,7 @@ EUROPE_REGION_CODES = { } # How often (in epochs) to rsync generator outputs back during training -_GENERATOR_SYNC_INTERVAL = 50 +_GENERATOR_SYNC_INTERVAL = 25 # Raised when the user aborts interactive offer selection