diff --git a/generator/configs/phase3/_base_phase3.json b/generator/configs/phase3/_base_phase3.json index 3c3e438..dce01fa 100644 --- a/generator/configs/phase3/_base_phase3.json +++ b/generator/configs/phase3/_base_phase3.json @@ -7,6 +7,7 @@ "model": "vae", "latent_dim": 256, "ngf": 64, + "grad_clip": 1.0, "sample_interval": 10, "fid_interval": 25, "fid_n_real": 5000 diff --git a/generator/configs/phase3/p3_1_vae.json b/generator/configs/phase3/p3_1_vae.json index 75b6af9..f90bec7 100644 --- a/generator/configs/phase3/p3_1_vae.json +++ b/generator/configs/phase3/p3_1_vae.json @@ -1,8 +1,8 @@ { "extends": "_base_phase3.json", "run_name": "p3_1_vae", - "lr": 1e-3, - "beta_kl": 1.0, + "lr": 5e-4, + "beta_kl": 0.25, "lambda_perceptual": 0.0, "lambda_adversarial": 0.0 } diff --git a/generator/configs/phase3/p3_2_vae_perceptual.json b/generator/configs/phase3/p3_2_vae_perceptual.json index 8116ea2..20ce9ce 100644 --- a/generator/configs/phase3/p3_2_vae_perceptual.json +++ b/generator/configs/phase3/p3_2_vae_perceptual.json @@ -1,8 +1,8 @@ { "extends": "_base_phase3.json", "run_name": "p3_2_vae_perceptual", - "lr": 1e-3, - "beta_kl": 0.0001, + "lr": 5e-4, + "beta_kl": 0.25, "lambda_perceptual": 0.1, "lambda_adversarial": 0.0 } diff --git a/generator/configs/phase3/p3_3_vae_patchgan.json b/generator/configs/phase3/p3_3_vae_patchgan.json index 1c6e443..1043c47 100644 --- a/generator/configs/phase3/p3_3_vae_patchgan.json +++ b/generator/configs/phase3/p3_3_vae_patchgan.json @@ -1,10 +1,10 @@ { "extends": "_base_phase3.json", "run_name": "p3_3_vae_patchgan", - "lr": 1e-3, + "lr": 5e-4, "lr_d": 1e-4, - "beta_kl": 0.0001, + "beta_kl": 0.25, "lambda_perceptual": 0.1, - "lambda_adversarial": 0.1, + "lambda_adversarial": 0.01, "ndf_patch": 64 } diff --git a/generator/configs/shared.json b/generator/configs/shared.json index 4ba1c9b..04865b8 100644 --- a/generator/configs/shared.json +++ b/generator/configs/shared.json @@ -6,5 +6,6 @@ "subsample": 1.0, "sample_interval": 10, "fid_interval": 25, - "fid_n_real": 5000 + "fid_n_real": 5000, + "num_workers": 2 } diff --git a/generator/outputs/logs/p3_1_vae.json b/generator/outputs/logs/p3_1_vae.json index 0a3ee94..d691864 100644 --- a/generator/outputs/logs/p3_1_vae.json +++ b/generator/outputs/logs/p3_1_vae.json @@ -11,222 +11,224 @@ "sample_interval": 10, "fid_interval": 25, "fid_n_real": 5000, + "num_workers": 2, "epochs": 100, "augment": "hflip", "image_size": 64, "model": "vae", "latent_dim": 256, "ngf": 64, + "grad_clip": 1.0, "run_name": "p3_1_vae", - "lr": 0.001, - "beta_kl": 1.0, + "lr": 0.0005, + "beta_kl": 0.25, "lambda_perceptual": 0.0, "lambda_adversarial": 0.0 }, "history": { "recon_loss": [ - 0.23614721197603095, - 0.23315699178821, - 0.22991716011594504, - NaN, - 0.23217070787253544, - 0.23155480842941847, - 0.23157141198459855, - 0.23181156750418183, - 0.23201335527193853, - 0.23178868266379732, - 0.2315022333755962, - 0.2311908418042028, - 0.23185610672474927, - 0.23176095832107413, - 0.23165411693163407, - 0.23174296459581098, - 0.2317636658747991, - 0.2317118427883356, - 0.23172695364834917, - 0.2316696329567677, - 0.23168399261358458, - 0.2316194716681782, - 0.23164867447354856, - 0.2315481170757204, - 0.23165068109957582, - 0.23167062098653907, - 0.23162642907765177, - 0.2315922882567104, - 0.2315914996414103, - 0.23156180984189367, - 0.23156551628286004, - 0.2315698005259037, - 0.2315660522470617, - 0.23156735001720935, - 0.23161396435183337, - 0.23158050178844705, - 0.23159921089680785, - 0.23149616745674712, - 0.23159087484336308, - 0.23156312872201967, - 0.23153820200863048, - 0.2315863819203825, - 0.23150022140043414, - 0.23154497337646973, - 0.2315601774961011, - 0.23153368950399578, - 0.23152085642019907, - 0.23151608884461924, - 0.23154898990805334, - 0.23155892872784892, - NaN, - NaN, - NaN, - 0.24157701413600874, - NaN, - NaN, - 0.24151325464630738, - NaN, - 0.24154121766233036, - 0.24155463749526912, - 0.24158300176008135, - 0.24158118757554609, - 0.2415294518901242, - 0.24156020069096842, - 0.2415176352374574, - 0.2415566616015047, - NaN, - 0.24161437115608117, - 0.24159398913765565, - 0.24149432768806434, - 0.24153172199287984, - 0.24161516999204954, - 0.24158193846034187, - 0.2415451397562129, - 0.24155487772873324, - 0.24155297130346298, - NaN, - 0.24157197961313093, - NaN, - NaN, - 0.24158605401459923, - 0.24156368870893094, - 0.24159100852333582, - 0.24153350121699846, - 0.24153158377505776, - NaN, - 0.24161708673350832, - 0.24158515879868442, - NaN, - 0.24157126235146809, - 0.24162366709265953, - NaN, - 0.2415581897665293, - NaN, - NaN, - 0.2415400046823371, - NaN, - 0.2415627600927638, - 0.2415567432076503, - 0.2415620140476614 + 0.07131652818180811, + 0.04839004274521373, + 0.04452062843956499, + 0.04353479713870165, + 0.043699693698913626, + 0.044162471062288836, + 0.044934689377744995, + 0.04592526849741355, + 0.047077489165095694, + 0.04793272184160275, + 0.04926021105776995, + 0.05025381562260226, + 0.05120742289174316, + 0.052108900581733286, + 0.053131219620505966, + 0.05401940069869798, + 0.05489145301314246, + 0.055811726878214084, + 0.05665480460907914, + 0.0575029267412093, + 0.05754986319404382, + 0.057160187035034865, + 0.05722308493991438, + 0.05713338962891418, + 0.05686132515119946, + 0.056816555862116, + 0.05664816373784063, + 0.05655015064164614, + 0.056517735943516605, + 0.056386624335542194, + 0.05631861151156262, + 0.056178740154092126, + 0.056074508314586095, + 0.05601463455738675, + 0.05584243320438088, + 0.05574028127086468, + 0.05563880511137665, + 0.055547706926098235, + 0.05556490144923202, + 0.05538980011692923, + 0.05529476007303366, + 0.05527778912303794, + 0.0552029303378529, + 0.05519345425801654, + 0.05497326165374018, + 0.05496025659366805, + 0.0549636375095345, + 0.0548208407086567, + 0.05475613919015114, + 0.05467982544826391, + 0.05467521703332408, + 0.05451676477160719, + 0.054423171549271315, + 0.05440536335620106, + 0.054226587956341415, + 0.05415793756643931, + 0.0540492157650809, + 0.0539091299725776, + 0.05381450568859139, + 0.053790501263151824, + 0.053688646684217654, + 0.05361353090176216, + 0.05348594906206569, + 0.053407231775613934, + 0.05329926665394734, + 0.053199282489143886, + 0.05318549033413585, + 0.053034571994446285, + 0.05299898797375524, + 0.052894837278713525, + 0.05281204023422339, + 0.05279633629685029, + 0.05266677101071064, + 0.0525879480310867, + 0.052519615285862714, + 0.05243872805761221, + 0.05236124007715883, + 0.052327762763851725, + 0.05222526421117732, + 0.05212976400636964, + 0.05209252470706263, + 0.0520137355177321, + 0.051939742568020635, + 0.05186714857625656, + 0.051828445420942754, + 0.051747049658726424, + 0.05169421551414789, + 0.05160299702109689, + 0.05152464638917874, + 0.051478635081941754, + 0.0514086935764704, + 0.05138895747403049, + 0.051289146423785605, + 0.05126826443637793, + 0.05114383632555986, + 0.051157466693120636, + 0.05104086854550828, + 0.05102811497437139, + 0.0509683930545918, + 0.05096899156068635 ], "kl_loss": [ - 12.394881742504927, - 184.775765717539, - 127.26797539963681, - 33346392.786626913, - 35.72433020722153, - 31.41954361882984, - 16.178619678203876, - 10.234501274223001, - 14.817130448471787, - 9.230570034084158, - 9.643558593896719, - 8.47786058498244, - 5.573643362929678, - 2.4644629534365783, - 1.5757666807462516, - 0.426466258131286, - 1.7924597560404203, - 0.2769168242652956, - 0.21636260826236162, - 0.48804672485870176, - 0.10833573165453142, - 0.13318477837671328, - 0.17373992544877478, - 0.09584700099678121, - 0.0977757986014088, - 0.07794108981282538, - 0.05691333960853199, - 0.07221067506167242, - 0.036222075203704275, - 0.03126689469696492, - 0.04264315036642882, - 0.016960328184147805, - 0.03314871971324309, - 0.014776984407789368, - 0.011375301962312406, - 0.013948339588828703, - 0.01186063720120324, - 0.0099704478863372, - 0.00536374123289417, - 0.009618068660179583, - 0.00418840028031164, - 0.004865833775052785, - 0.005830266629345715, - 0.0023000687699064487, - 0.0038261460966199762, - 0.0022056369562673136, - 0.002220870125003987, - 0.0024217167485139184, - 0.001954249278483037, - 0.0021431104709895756, - 0.0022583500309011494, - 0.002132287005193404, - 56.80083886633675, - 82.57108385134966, - 82.55195800259582, - 82.57428529527452, - 82.56009972401155, - 82.55269345666608, - 82.57006728343474, - 82.55670593131302, - 82.54445134676419, - 82.57745079301361, - 82.57933913336859, - 82.5570435157189, - 82.56808758597089, - 82.56800172267816, - 82.56525711320404, - 82.56189481621115, - 82.55193622295673, - 82.55375865382007, - 82.56600202250685, - 82.57064581324912, - 82.55481151026538, - 82.55367833324986, - 82.56042112040724, - 82.5616829048874, - 82.5771528553759, - 82.55317820035495, - 82.57550573756552, - 82.57334061973116, - 82.56044387817383, - 82.5752662593483, - 82.56673936762361, - 82.56828115740393, - 82.56990289280557, - 82.55218840052939, - 82.56695372426611, - 82.575043066954, - 82.55754522991995, - 82.56361721723508, - 82.5628145821074, - 82.56431990403395, - 82.55777725806603, - 82.5742861918914, - 82.56361025622768, - 82.56887233766736, - 82.56539458902473, - 82.55887828729091, - 82.56073884882478, - 82.55578186165573 + 0.7611013715847944, + 0.5151619965321997, + 0.39718208143599015, + 0.3297253664360087, + 0.2878693372138545, + 0.257130523904776, + 0.23313261619490436, + 0.21345858896772066, + 0.1973419941197603, + 0.18355375779872266, + 0.1717705535583007, + 0.1612550135797415, + 0.15217030946260843, + 0.1444861331047156, + 0.13738160394132137, + 0.1306394640611023, + 0.12505544035926333, + 0.12011377760169344, + 0.11535473169488275, + 0.1109595281095841, + 0.11022479862420477, + 0.10970949868743236, + 0.10961869400408533, + 0.10945094661771232, + 0.10919397698444688, + 0.10875842463957448, + 0.10874241859548622, + 0.10849472851707385, + 0.10825413890565053, + 0.10820380448658243, + 0.10811882353045492, + 0.10812433239104402, + 0.10808505038293, + 0.10790401608006567, + 0.10787000140955305, + 0.10800883411151221, + 0.10767164218247446, + 0.10764909312765822, + 0.10733446480435693, + 0.10740953346348217, + 0.10733820650822078, + 0.10725643148279598, + 0.10736855125834799, + 0.10717424525855443, + 0.10725876993030055, + 0.10694582887694366, + 0.10713813684753373, + 0.10726828694853008, + 0.10701580285134478, + 0.10700331553498395, + 0.10686330029215568, + 0.10687567073947345, + 0.10698378102010132, + 0.10673714201483461, + 0.10696066876188812, + 0.10678731051520404, + 0.10679936213180041, + 0.10696167247290285, + 0.1067299399142846, + 0.10684242702893212, + 0.10679143969701906, + 0.10698746744957235, + 0.10674736718846183, + 0.10685917330730675, + 0.1070305180664246, + 0.10704450406388849, + 0.10696375739370656, + 0.10697911899441327, + 0.10700849091841115, + 0.10684541509383255, + 0.10709039088434134, + 0.10708275965900503, + 0.10700157213096435, + 0.10683403667221722, + 0.10696323639434627, + 0.10717970753709476, + 0.10707768420569408, + 0.10707720299052377, + 0.10703401576377387, + 0.10714245904396233, + 0.10722182246928032, + 0.10714326931052229, + 0.10708108509325573, + 0.10726493315245861, + 0.10718185655199565, + 0.10716394220407192, + 0.10727782184496903, + 0.10729229825938869, + 0.10722832862510641, + 0.10727461647146787, + 0.10739002018593825, + 0.10721855878065793, + 0.10737398387784632, + 0.10721981757853785, + 0.10756766480895188, + 0.10733713450021723, + 0.10742478621884799, + 0.10721213524986027, + 0.10737172850113139, + 0.10744189095293355 ], "perc_loss": [ 0.0, @@ -535,12 +537,12 @@ 0.0 ], "fid": { - "25": 315.9393615722656, - "50": 419.273193359375, - "75": 360.4432678222656, - "100": 363.9911193847656 + "25": 108.92365264892578, + "50": 93.73921203613281, + "75": 90.11531829833984, + "100": 88.40287780761719 }, - "train_time_s": 660.9630489349365 + "train_time_s": 1526.7542352676392 }, "n_params": 10608451 } \ No newline at end of file diff --git a/generator/outputs/logs/p3_2_vae_perceptual.json b/generator/outputs/logs/p3_2_vae_perceptual.json index 8c982bf..6adade4 100644 --- a/generator/outputs/logs/p3_2_vae_perceptual.json +++ b/generator/outputs/logs/p3_2_vae_perceptual.json @@ -11,324 +11,326 @@ "sample_interval": 10, "fid_interval": 25, "fid_n_real": 5000, + "num_workers": 2, "epochs": 100, "augment": "hflip", "image_size": 64, "model": "vae", "latent_dim": 256, "ngf": 64, + "grad_clip": 1.0, "run_name": "p3_2_vae_perceptual", - "lr": 0.001, - "beta_kl": 0.0001, + "lr": 0.0005, + "beta_kl": 0.25, "lambda_perceptual": 0.1, "lambda_adversarial": 0.0 }, "history": { "recon_loss": [ - NaN, - 0.041370747770127066, - 0.035506031866002284, - 0.06229733923672993, - 0.06089382184048494, - 0.053464704654856116, - 0.04950355895213846, - 0.04674786329269409, - 0.05693557829811023, - 0.04808994451076047, - NaN, - 0.05869839608701121, - 0.05293231666820426, - 0.05057006603918779, - 0.04713928615117175, - 0.04481589820427008, - 0.04331832689543565, - 0.041820655449524395, - NaN, - 0.24315394992884407, - 0.24305414841470555, - 0.2429187158998261, - 0.24289727962424612, - 0.2377373814328104, - 0.23880945107875726, - 0.24047312904626894, - 0.23915709144411942, - NaN, - 0.24000247061634675, - 0.2362435321904655, - 0.23705266075383905, - NaN, - 0.23998981039238793, - 0.23828768376738596, - 0.23908851302077627, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 0.07014625494042014, + 0.04432385861396025, + 0.03902342810462683, + 0.036902043435117625, + 0.035906293887135565, + 0.035914984559560686, + 0.0360899488401846, + 0.036847473583860785, + 0.037394630213260144, + 0.03836990671200503, + 0.039295883698022775, + 0.040212508386526354, + 0.04124497003757801, + 0.04212733724305772, + 0.04325710433638758, + 0.04402907300963361, + 0.045103192679647706, + 0.04595246975525068, + 0.04698181097419598, + 0.047825980820080154, + 0.04772546164627768, + 0.04763130701950982, + 0.047709369705591954, + 0.04753121634961194, + 0.04741841174948674, + 0.04729832872016053, + 0.04722155279551561, + 0.0469712430340612, + 0.04693082829093576, + 0.0468663705401441, + 0.04660685717040657, + 0.04661381538384236, + 0.046462044382515624, + 0.0464134263431924, + 0.046272445438254595, + 0.04624118105882508, + 0.046066727958874315, + 0.04603223238363225, + 0.04589838094404365, + 0.0458371472489248, + 0.04577782691225537, + 0.04567820464985238, + 0.04560983914913785, + 0.045465615761076286, + 0.04543259674603613, + 0.04530587481159685, + 0.04528731873465909, + 0.04525673297098559, + 0.04507890154217553, + 0.0450239604514124, + 0.04497251241730574, + 0.044865927300774135, + 0.04475303162207715, + 0.044664276763796806, + 0.044507257990602754, + 0.0444452501515038, + 0.044273821509674065, + 0.044196275588220514, + 0.044097925846775375, + 0.04397543971864586, + 0.04393626836279773, + 0.043823353540247835, + 0.04371356347209623, + 0.043580674797169164, + 0.04348461520181507, + 0.04339473670682846, + 0.043282039009798795, + 0.04320543994888281, + 0.04314021340324583, + 0.04300904847904403, + 0.04293948887950844, + 0.04286929544730064, + 0.04278319672896312, + 0.04270272867547141, + 0.0425933551393513, + 0.04250210111276207, + 0.04243518800562263, + 0.04233357961424905, + 0.04227217231105026, + 0.042201977119677596, + 0.04207655237430436, + 0.04206944097024508, + 0.0420022471768097, + 0.041929350385808535, + 0.04179893332159417, + 0.041740718671781384, + 0.04166606991177695, + 0.04158131634959808, + 0.04155632069445828, + 0.04140628448440733, + 0.041379960253834724, + 0.041310225080093764, + 0.04123378162168794, + 0.04118641266105776, + 0.04113346862837545, + 0.041023712032116376, + 0.04102735277902112, + 0.04096395063062764, + 0.04094440599059702, + 0.040864359348630294 ], "kl_loss": [ - 41526536.5961082, - 1258.7768262553418, - 1165.5987154968784, - 3689.321748260759, - 1320.3167513333833, - 928.3473894168169, - 789.754654843583, - 693.4729607736963, - 1017.8856519389357, - 622.2623097998464, - 304942.2120989938, - 750.5718396830763, - 618.6888906364767, - 586.4194498958751, - 579.2793892102363, - 525.8830437945504, - 497.5363791050055, - 452.93342662061383, - 172476214612.95245, - 1112.027792645316, - 1109.8156276605068, - 1109.8872808472722, - 1112.3999198196282, - 1163.6093267457097, - 1195.4633280436199, - 1232.709252512353, - 1253.0651078183428, - 19923.65347159622, - 6871.588723207132, - 6872.017739842081, - 6872.62934758113, - 27062.17111336472, - 4992.576698759683, - 5000.895510942508, - 5011.723715236044, - 4.196954763212122e+20, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 0.9578518337673612, + 0.8413855069213443, + 0.7201743645545764, + 0.6246620782165446, + 0.5550231880102402, + 0.49975109450582766, + 0.454662803783376, + 0.4155570190941167, + 0.3821740793621438, + 0.352426735827556, + 0.32639839258204156, + 0.3032199253893306, + 0.2829984358360625, + 0.26505850424241817, + 0.24903864537676176, + 0.23453107457130384, + 0.22166774176761636, + 0.21033292894180006, + 0.19950743799662998, + 0.18994869887192026, + 0.18805619294189999, + 0.18697467955768618, + 0.1855154659312505, + 0.1852042447680082, + 0.18474761941112006, + 0.1843619988514827, + 0.18399313028551575, + 0.18352704119478536, + 0.18345636556036451, + 0.18366932483692455, + 0.18345450644946507, + 0.18317942117523944, + 0.1830864556006387, + 0.1831519056079734, + 0.18329900547734693, + 0.1829878402252992, + 0.18321112291807803, + 0.18296580338197896, + 0.18312124166096377, + 0.18301436457878503, + 0.1831260528574642, + 0.18320244878657863, + 0.18332607687538505, + 0.18335427452101666, + 0.18322811696009758, + 0.18336237903334138, + 0.183573446308191, + 0.18356411601615769, + 0.18363770969912538, + 0.18377095862076834, + 0.18369814056234482, + 0.18386876185098264, + 0.18424720492245805, + 0.18420857423518458, + 0.18395239082921264, + 0.18442093539569113, + 0.1842692175036312, + 0.18475162495787328, + 0.18502490509014863, + 0.18480629139603713, + 0.1851997250993537, + 0.1850732435973791, + 0.1854018302809479, + 0.18547267151566652, + 0.18569090538936803, + 0.1859345402664099, + 0.18615943276219898, + 0.18631401260057065, + 0.1865774132948146, + 0.1867309583303256, + 0.18672093723574254, + 0.18682665680336136, + 0.18679539349853483, + 0.18683723016427115, + 0.1874776411578696, + 0.18746165714712223, + 0.1873003001141752, + 0.18763184111215112, + 0.18799093031348327, + 0.18802655252635989, + 0.18824852079662502, + 0.1882636730169129, + 0.18849152215143555, + 0.1886035389561429, + 0.188697873813729, + 0.18883028218888828, + 0.1889004738858113, + 0.18922981174073666, + 0.18946354345888153, + 0.18955816268029377, + 0.18944057507010606, + 0.18984314531852037, + 0.19001257871715432, + 0.19001863161340737, + 0.18997256346365327, + 0.19019658920856622, + 0.19018171425176483, + 0.19038618680758354, + 0.19034281325263855, + 0.19046620505615178 ], "perc_loss": [ - NaN, - 3.0017659954535656, - 2.8931196978968434, - 3.084328340159522, - 3.1398269641093717, - 3.0839009172896032, - 3.0459561949102287, - 3.015714469628456, - 3.0917934143645134, - 3.010985380054539, - NaN, - 3.102917709411719, - 3.055045795746339, - 3.0351512865123587, - 3.002274121993627, - 2.9754957128793764, - 2.9553630066733074, - 2.936704080328982, - NaN, - 3.66463458385223, - 3.6419444985878773, - 3.6321474706005845, - 3.6272282503608966, - 3.781295938878997, - 3.740793755421272, - 3.759704489993234, - 3.7078149914741516, - NaN, - 3.7647176323792872, - 3.7441278461717133, - 3.7264530261357627, - NaN, - 3.7299226170931106, - 3.7314435080585318, - 3.754668933713538, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 3.257610213552785, + 2.987424520855276, + 2.8892488026211405, + 2.8424616914529066, + 2.817538415774321, + 2.807587738220508, + 2.8047564518757357, + 2.8080830421203222, + 2.8133585376617236, + 2.821376688969441, + 2.830571848612565, + 2.840076332927769, + 2.8507282504668603, + 2.8593530094521675, + 2.8690790253826695, + 2.877339167982085, + 2.8865632745954724, + 2.8936263704911256, + 2.9023511684857883, + 2.9098138376178904, + 2.906375333794162, + 2.9055831661591163, + 2.9031203985214233, + 2.9002673641229286, + 2.897312533651662, + 2.8946053095352955, + 2.8919762974111443, + 2.8902457397208257, + 2.8877664065768576, + 2.884983114197723, + 2.882109224286854, + 2.8805122767758164, + 2.8777520192994013, + 2.876453428186922, + 2.874014714334765, + 2.8723304679251123, + 2.870547831058502, + 2.86905308564504, + 2.8669193524580736, + 2.865350999383845, + 2.8633312608441734, + 2.861651761409564, + 2.860612149421985, + 2.858717398256318, + 2.857239680412488, + 2.8558127604998074, + 2.853911985189487, + 2.852791876364977, + 2.850916815109742, + 2.8496591963319697, + 2.8490061500133614, + 2.8465628389619355, + 2.844754463587052, + 2.842957689211919, + 2.840948245464227, + 2.8397679522506194, + 2.837174654006958, + 2.835211678957328, + 2.833740895629948, + 2.8325765499701867, + 2.830295750218579, + 2.8292511431579914, + 2.8273453595291853, + 2.8257394835480256, + 2.823762384744791, + 2.822507558215378, + 2.8205288745399213, + 2.819065808230995, + 2.8176180297492914, + 2.8165967805772767, + 2.8143360721759305, + 2.8135039892971005, + 2.8120578970664587, + 2.8098325041624217, + 2.8090034366672874, + 2.8076502920215964, + 2.80601545684358, + 2.805327193859296, + 2.803745285058633, + 2.802450499473474, + 2.800580952412043, + 2.799968459157862, + 2.798532901156662, + 2.797372330967178, + 2.795650184663952, + 2.79500452371744, + 2.79342931152409, + 2.792220654650631, + 2.7911063600809145, + 2.7897617689564695, + 2.7887179505111823, + 2.7880892407180915, + 2.7868736916118197, + 2.785768971993373, + 2.784949649602939, + 2.784032260760283, + 2.783316181765662, + 2.7821382457374506, + 2.7815805586994204, + 2.780150015639444 ], "adv_g_loss": [ 0.0, @@ -535,12 +537,12 @@ 0.0 ], "fid": { - "25": 263.1458740234375, - "50": 598.3736572265625, - "75": 598.3736572265625, - "100": 598.3736572265625 + "25": 85.4538345336914, + "50": 70.30448150634766, + "75": 68.88232421875, + "100": 68.23878479003906 }, - "train_time_s": 952.6596128940582 + "train_time_s": 1526.8077104091644 }, "n_params": 10608451 } \ No newline at end of file diff --git a/generator/outputs/logs/p3_3_vae_patchgan.json b/generator/outputs/logs/p3_3_vae_patchgan.json index 68923b2..efe5f99 100644 --- a/generator/outputs/logs/p3_3_vae_patchgan.json +++ b/generator/outputs/logs/p3_3_vae_patchgan.json @@ -11,538 +11,540 @@ "sample_interval": 10, "fid_interval": 25, "fid_n_real": 5000, + "num_workers": 2, "epochs": 100, "augment": "hflip", "image_size": 64, "model": "vae", "latent_dim": 256, "ngf": 64, + "grad_clip": 1.0, "run_name": "p3_3_vae_patchgan", - "lr": 0.001, + "lr": 0.0005, "lr_d": 0.0001, - "beta_kl": 0.0001, + "beta_kl": 0.25, "lambda_perceptual": 0.1, - "lambda_adversarial": 0.1, + "lambda_adversarial": 0.01, "ndf_patch": 64 }, "history": { "recon_loss": [ - NaN, - NaN, - 0.2550439156082451, - NaN, - 0.3542575327377034, - 0.32994148498162246, - 0.327391546824549, - NaN, - 0.32896743778489596, - 0.32088901420943755, - 0.35182940304024607, - 0.3339548002068813, - 0.3514746379139077, - 0.355552134796595, - 0.3535881389537428, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 0.07256481288653663, + 0.04414264540960137, + 0.03923531813133094, + 0.03690198250512919, + 0.03610035986042558, + 0.03585429300562057, + 0.03614142174936003, + 0.03675156902187528, + 0.03742894164135314, + 0.0383954416307756, + 0.039195723903293796, + 0.04035789030007063, + 0.04136944739705222, + 0.04217637810282982, + 0.04317184983410387, + 0.04414223251529993, + 0.045082664196817286, + 0.0459897641140299, + 0.046940210410672374, + 0.04794481957856661, + 0.04979389414000206, + 0.050820046454731725, + 0.050248638519810304, + 0.049580192527709864, + 0.049034240106359504, + 0.048891564759497456, + 0.04878619169163653, + 0.04837235698524194, + 0.04854527746255581, + 0.04844111150815191, + 0.0483922261872098, + 0.04822369044025739, + 0.048200586078386023, + 0.047983415560144134, + 0.0478166768199995, + 0.0477692344918465, + 0.04770479275661911, + 0.04758516683155655, + 0.047635042673756935, + 0.04756982100761344, + 0.04745731656399802, + 0.04740560844413236, + 0.04746005712793423, + 0.04733826805893172, + 0.047378935079034575, + 0.04739771968024409, + 0.047426708408782624, + 0.047361184023002274, + 0.04733280055861697, + 0.04722336093043415, + 0.04732823220646789, + 0.0472236652022753, + 0.047047707785525895, + 0.04700392111339885, + 0.046977425869713485, + 0.04659851064157282, + 0.04664327050599023, + 0.046512148024625756, + 0.04652243379790049, + 0.04643834348067514, + 0.046477761987246506, + 0.04622124535883339, + 0.04625549639423943, + 0.04613285658992509, + 0.04605264870975262, + 0.04589356125419975, + 0.04583498457462615, + 0.04573038787955148, + 0.0457718974759436, + 0.04564371212329875, + 0.04559122066562756, + 0.045480694916322194, + 0.04536095822755343, + 0.045297518324775576, + 0.04521293488418699, + 0.04522842195076056, + 0.045203558049904995, + 0.04512962291383336, + 0.04501655121516977, + 0.04495001771351975, + 0.044781594123277396, + 0.04481303215854698, + 0.044778383241441004, + 0.044675282958863125, + 0.04455677082395961, + 0.04459468223568466, + 0.04447825120873431, + 0.04439788500372416, + 0.044356526783070505, + 0.04437946719236863, + 0.044278508139790125, + 0.0442026794816439, + 0.04421330326929306, + 0.0441932631099326, + 0.04406989279847879, + 0.043991477594862126, + 0.04401077344440497, + 0.043903381531883, + 0.043854401948360294, + 0.04389296859725673 ], "kl_loss": [ - 13281713.181712223, - 8879242356.567835, - 4289.939060308994, - 13291.21078882462, - 2964.6168004223423, - 2881.717480781751, - 2817.9251735067774, - 26206.0708080846, - 3054.7347709460137, - 3045.038165231036, - 3176.499509632078, - 3239.0840866219282, - 3390.609429253472, - 3374.538367116553, - 3556.9082464234443, - 5232728.427530403, - 1200351222187.4932, - 557673391674.235, - 687785714385.8098, - 451167040515.6966, - 4161448879712.457, - 3506947360774.84, - 640311792154.062, - 334276995872.6603, - 1091409144901.9957, - 410306176880.5192, - 20118849921428.06, - 468057687720.9594, - 2081128665542.4016, - 403224538747.06836, - 98553230942188.75, - 909009238374.0577, - 594442830189.4722, - 6358920601673.252, - 765801126878.3354, - 523422198654.6368, - 749170626764.0299, - 2510965243074.2544, - 174694512065.203, - 1841149037337.5, - 3074909578060.154, - 3299731319463.609, - 1188893842751.9016, - 1123029761662.4785, - 439769666356.8568, - 1481985020811.5044, - 1227399205729.7693, - 711698698367.3035, - 279335895281.97437, - 720625801842.0897, - 202221867649.9562, - 543446495162.29913, - 175988484712.71368, - 815170807713.688, - 762637058414.9017, - 863531881960.8505, - 303024411620.3558, - 909942303709.2073, - 295614329249.5951, - 170733932504.22437, - 2103203516485.6304, - 892083529658.5983, - 895749634527.1005, - 899311239324.7008, - 1637751780589.5225, - 1192357094564.0833, - 484858905592.5705, - 186463896968.73077, - 888726863689.1987, - 15502339561946.314, - 170425499746.94873, - 449657248854.87714, - 330656068541.094, - 272033123474.23932, - 693443839763.4946, - 1247571259921.861, - 514054457621.8333, - 1448574911180.951, - 883422112293.2393, - 1158489887757.5364, - 8886462462547.04, - 274201891523.62607, - 289924448241.86536, - 585746708708.7842, - 1142763735703.111, - 1130523430761.9956, - 412582377508.9124, - 1428507187223.3142, - 165624474038.62393, - 333074819594.1261, - 1738273025208.5898, - 202257609124.88675, - 432264649605.9658, - 1562684211706.267, - 511279586047.094, - 1419566063399.6196, - 510649916352.63464, - 587311395844.4445, - 688530254154.5769, - 63229654584.801285 + 0.9422818552225064, + 0.8304157827654456, + 0.711033024212234, + 0.6201538036012242, + 0.5516812854852432, + 0.49781491747523987, + 0.4539294478475538, + 0.4149613824919758, + 0.3822571930085492, + 0.3525762819708922, + 0.32686203138695824, + 0.3036677601118373, + 0.2831739962228343, + 0.26551324956946903, + 0.24915392639545295, + 0.234747304684586, + 0.22180349417986014, + 0.21073915000654694, + 0.20005385755983174, + 0.19080995443539742, + 0.20008063157144776, + 0.20165701993764976, + 0.20599561308821043, + 0.20952844473286572, + 0.2119306048266908, + 0.21374986164717594, + 0.21439095319081575, + 0.2146782086382055, + 0.21535835013939783, + 0.2156351603822321, + 0.2156747308297035, + 0.2163603354850386, + 0.21604907694153297, + 0.2170932454049078, + 0.2172167382688604, + 0.21753370074125436, + 0.21788769651554588, + 0.21780646761131084, + 0.21757154243114668, + 0.2174266650954373, + 0.21739230457788858, + 0.21768093402059668, + 0.21731426490423006, + 0.21709907592998612, + 0.21729469019123632, + 0.21711179728691393, + 0.2172128687429632, + 0.2170052423309057, + 0.21706495239821255, + 0.2174526669530787, + 0.2170911322737861, + 0.21746987362320608, + 0.21689637371490145, + 0.21768690562910503, + 0.21710836957407814, + 0.21804575870434442, + 0.217907239165571, + 0.21759957396703908, + 0.21808122977232322, + 0.21784972603249753, + 0.21785366121265623, + 0.21845279487534466, + 0.21855741242567697, + 0.21834458735508797, + 0.21865385595677245, + 0.2191770772457632, + 0.21917275520853507, + 0.21897286963131693, + 0.2189757764721528, + 0.21977992066078716, + 0.22024847930058455, + 0.21994173469451758, + 0.21978110724534744, + 0.21947090617484516, + 0.219740368394006, + 0.21953950682256976, + 0.2196803634874841, + 0.2198349738923403, + 0.2205247895903567, + 0.2201039091580444, + 0.22056390408776763, + 0.22043023312575796, + 0.22098283940910274, + 0.22084448502486587, + 0.22079740432847259, + 0.220382661678088, + 0.2211024827745735, + 0.2208555058345326, + 0.22062583204008576, + 0.22087381401250505, + 0.22094633755011436, + 0.22113347610729372, + 0.2210337378594101, + 0.22095281971443412, + 0.22108463678731877, + 0.22107870204963237, + 0.22101640819102272, + 0.221434213889715, + 0.22125727438137063, + 0.22125603885859504 ], "perc_loss": [ - NaN, - NaN, - 4.991128759506421, - NaN, - 4.873321874528869, - 4.652275615268284, - 4.5657713250217276, - NaN, - 4.545961037660256, - 4.552515344742017, - 4.670070458681155, - 4.55506570216937, - 4.635318170245896, - 4.757533762190077, - 4.679769850184774, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 3.268518664388575, + 2.9844126731921463, + 2.8893332792143536, + 2.842300501644102, + 2.8177173336346946, + 2.8061808974314957, + 2.80318136449553, + 2.8056567907333374, + 2.8115715369200096, + 2.8198827128125052, + 2.828476501835717, + 2.8390082136178627, + 2.849494279959263, + 2.8572748884176598, + 2.867123058718494, + 2.875808890049274, + 2.8851839202082057, + 2.892486855515048, + 2.9006601964306626, + 2.90893748937509, + 3.002905537430038, + 3.0841518523346663, + 3.1015493930914464, + 3.1056835936684895, + 3.1069560407573342, + 3.1064426521969657, + 3.108147989480923, + 3.1069585961154385, + 3.1126723849875297, + 3.110053312065255, + 3.111884709606823, + 3.109374719297784, + 3.1103647427681165, + 3.1077851240451517, + 3.1043045816258488, + 3.103300364098997, + 3.100696138337127, + 3.101050837427123, + 3.104057640601427, + 3.1031913258071637, + 3.1028205446707897, + 3.0987310552189493, + 3.104932033098661, + 3.100133360451103, + 3.10110838087196, + 3.1032485880403438, + 3.1042901129804106, + 3.1003458077072077, + 3.103027570960868, + 3.0989463599319134, + 3.100031643341749, + 3.0986684830779705, + 3.097568829344888, + 3.0981435597452345, + 3.0955454218081937, + 3.091543544561435, + 3.094627695205884, + 3.0887583132483, + 3.0919363335666494, + 3.0886334053471556, + 3.089639984134935, + 3.0892311589330688, + 3.0903091853500433, + 3.089088388487824, + 3.0876893849454374, + 3.0847945982574396, + 3.083339012076712, + 3.0836611441057973, + 3.0838507724623394, + 3.083322472551949, + 3.0839521288871765, + 3.0830512260779357, + 3.0811995609193787, + 3.077091329627567, + 3.0774103555923853, + 3.0779298793556342, + 3.0798700487511788, + 3.0797718432214527, + 3.0784103956996884, + 3.0775557175660744, + 3.0766860032692933, + 3.0762227615739546, + 3.078130325700483, + 3.0726998621581965, + 3.0729522781494336, + 3.073877077836257, + 3.0732357578399854, + 3.0725482758293805, + 3.0729044237707415, + 3.0753370171938186, + 3.0735755771653266, + 3.073718212099157, + 3.0729570011807303, + 3.073591929725093, + 3.0722544748558955, + 3.0702306806531725, + 3.071104440933619, + 3.070158361369728, + 3.0697627974371624, + 3.0690404686153445 ], "adv_g_loss": [ - NaN, - NaN, - 2.686214610042735, - NaN, - 3.1861853966346154, - 3.3312967414529915, - 3.4314809945913463, - NaN, - 3.9346955128205128, - 4.042058961004273, - 4.416372445913462, - 4.483027009882479, - 4.66728014823718, - 4.537232488648504, - 4.415139222756411, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.1438086142078934, + 1.3848216148714225, + 1.431843824498057, + 1.4550386664107378, + 1.534417847609227, + 1.5428873566456904, + 1.5969856012459351, + 1.6761763638968015, + 1.6849639529728482, + 1.6978890428033013, + 1.7520971780754306, + 1.758636311062763, + 1.7961577434952443, + 1.7791677351849966, + 1.8282766153351364, + 1.8869179757869141, + 1.8936229561073468, + 1.916680078762464, + 2.0205936250396266, + 2.0106747031020813, + 2.1042307398488953, + 2.0886956008198934, + 2.1438577952914133, + 2.144290308058103, + 2.2067003882466216, + 2.260120966622972, + 2.3372040491583, + 2.2787050508981586, + 2.4021470887411356, + 2.406920169790586, + 2.4421350724167294, + 2.4470817596993895, + 2.46695097300232, + 2.470248263870549, + 2.5584849823489146, + 2.492347888998751, + 2.526307321766503, + 2.5345261624226203, + 2.5415179539694743, + 2.553662398814136, + 2.590494660103423, + 2.609607903366415, + 2.5870104153186846, + 2.633848020281547, + 2.617674359271669, + 2.6077225506305695, + 2.626374716687406, + 2.623414141500098, + 2.6401452143222857, + 2.685499318389811, + 2.6790291177920804, + 2.6827956965336432, + 2.6683848851766343, + 2.692997595948032, + 2.7399796782395778, + 2.7001287651877117, + 2.7064600485002894, + 2.7366674155251594, + 2.742031560239629, + 2.7530031089599314, + 2.718017868251882, + 2.733836143444746, + 2.7269677244699917, + 2.7397567153486433, + 2.7358992522598333, + 2.7409965941029735, + 2.74284782852882, + 2.7570678810788016, + 2.744860523276859, + 2.727091629790445, + 2.728150630608583, + 2.7151212447728867, + 2.7098901113893232, + 2.7117845436446686, + 2.6960576665707126, + 2.6918129436990137, + 2.6912999886732836, + 2.7062390074770675, + 2.712936248534765, + 2.713185273683988 ], "adv_d_loss": [ - NaN, - NaN, - 0.11442611563918936, - NaN, - 0.05586824661646134, - 0.04843576137836163, - 0.05373259079761994, - NaN, - 0.015966382291581895, - 0.01352301099870959, - 0.016935703591403797, - 0.015551309682365157, - 0.01746162848594861, - 0.023566564433595054, - 0.011101227285515549, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN, - NaN + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.43680395959661555, + 0.34810768227037203, + 0.3377910334075618, + 0.3321381963662103, + 0.2982299141904228, + 0.30104209130836856, + 0.28214932976561224, + 0.26205551579721975, + 0.2678493159608199, + 0.2567906266508194, + 0.25110278342269426, + 0.24189095977598277, + 0.2394267334483373, + 0.24564475484956533, + 0.24081433725216958, + 0.21658801688597754, + 0.213517039664026, + 0.21218282094177526, + 0.18916759920171183, + 0.19284203568170977, + 0.181842301462769, + 0.1791912000785526, + 0.18734478913088384, + 0.17373414262810832, + 0.16635354463424948, + 0.16650411119270656, + 0.15708727274153733, + 0.16674049754237008, + 0.1501258422867355, + 0.15181534979333225, + 0.13657129330672005, + 0.1467462844478014, + 0.1383675114235753, + 0.14689494454516813, + 0.12695099818162048, + 0.12875286992798504, + 0.13186773245469627, + 0.12181158371779144, + 0.12662809188517496, + 0.1175684176114762, + 0.11314824839226073, + 0.11264375143517287, + 0.1165530812393269, + 0.10845673464350084, + 0.10203271553827784, + 0.11023388663306832, + 0.10421399819131336, + 0.0983247641298888, + 0.09406759767817, + 0.09505048260076815, + 0.09383611069021063, + 0.09381456619017145, + 0.08943638993968439, + 0.08595997961556427, + 0.08149021235692641, + 0.08130677379906559, + 0.07870399655025993, + 0.07449058663203484, + 0.07478499058315642, + 0.07468204382154295, + 0.07225174684491423, + 0.07153088875258198, + 0.06767470285328281, + 0.06715900409154785, + 0.06388170873889557, + 0.06149009168227641, + 0.06165273037826658, + 0.05667935721735414, + 0.058510264835487574, + 0.056089043583418444, + 0.05523068375256645, + 0.05533928267705517, + 0.05186875127295717, + 0.05044461999279566, + 0.050165704740052164, + 0.046495266505477265, + 0.04587119478039826, + 0.04412680363051721, + 0.04238344537309156, + 0.041027947104114115 ], "fid": { - "25": 300.18701171875, - "50": 328.4952392578125, - "75": 334.2494812011719, - "100": 334.40887451171875 + "25": 73.48986053466797, + "50": 51.912357330322266, + "75": 50.058326721191406, + "100": 52.463279724121094 }, - "train_time_s": 1450.524620771408 + "train_time_s": 2145.0215554237366 }, "n_params": 10608451 } \ No newline at end of file diff --git a/generator/outputs/models/p3_1_vae_best_ema.pt b/generator/outputs/models/p3_1_vae_best_ema.pt index fd11b85..6fe37c4 100644 --- a/generator/outputs/models/p3_1_vae_best_ema.pt +++ b/generator/outputs/models/p3_1_vae_best_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:93d01254ddc4f7d4411d779be3f1d7b9bb27ebd210f51177d722c91b35e9927e -size 42460217 +oid sha256:8254021ccd683279daf777e95fa035100496b64c05d834606187bba26c09dc3b +size 42443643 diff --git a/generator/outputs/models/p3_1_vae_best_vae.pt b/generator/outputs/models/p3_1_vae_best_vae.pt index 63faf52..9cf77d3 100644 --- a/generator/outputs/models/p3_1_vae_best_vae.pt +++ b/generator/outputs/models/p3_1_vae_best_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24f5c71cec846513ae99c36be754430745afffa001ad42d020da5e051fca22e6 -size 42460217 +oid sha256:e6af243b527e8841edc3182e50796ee45e6eadbef6e60e4df375e81c841c2f2d +size 42443643 diff --git a/generator/outputs/models/p3_1_vae_final_ema.pt b/generator/outputs/models/p3_1_vae_final_ema.pt index 5f45479..fbcecac 100644 --- a/generator/outputs/models/p3_1_vae_final_ema.pt +++ b/generator/outputs/models/p3_1_vae_final_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:12a01e1473345326e409a00bd19bb1eaed7c55ebd39fe59c778169a00d8ae122 -size 42460266 +oid sha256:5327c3eb8cc44d44759fa364f6ce4ef875e04ce959a2d89fe25af91545112637 +size 42443674 diff --git a/generator/outputs/models/p3_1_vae_final_vae.pt b/generator/outputs/models/p3_1_vae_final_vae.pt index 3557714..d665653 100644 --- a/generator/outputs/models/p3_1_vae_final_vae.pt +++ b/generator/outputs/models/p3_1_vae_final_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c1844af2d727a2bcb1305fa5dc7eff60b9ebee091e7f9f86da9257eef3d8de4b -size 42460266 +oid sha256:77157ce35de4a4ba642aa1e9984b9169eb66c0406b747ebc9a12cc62399c6263 +size 42443674 diff --git a/generator/outputs/models/p3_2_vae_perceptual_best_ema.pt b/generator/outputs/models/p3_2_vae_perceptual_best_ema.pt index 161c13f..0f1f997 100644 --- a/generator/outputs/models/p3_2_vae_perceptual_best_ema.pt +++ b/generator/outputs/models/p3_2_vae_perceptual_best_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ed46d4c57f3d7f33325d454a42f21513e698c846a11599dd33c1de992df0540 -size 42460820 +oid sha256:4083e0d42bc70e12280bb2b4a842cab13ce17c411f8ad60c42cc1c88f132f74a +size 42444048 diff --git a/generator/outputs/models/p3_2_vae_perceptual_best_vae.pt b/generator/outputs/models/p3_2_vae_perceptual_best_vae.pt index ec98dbd..ea1b983 100644 --- a/generator/outputs/models/p3_2_vae_perceptual_best_vae.pt +++ b/generator/outputs/models/p3_2_vae_perceptual_best_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6c984dbc58a548a9d5e5fcbec32eade6a588a7eb3df40a659ac17d955738f1c4 -size 42460820 +oid sha256:670ccb60d6a64acef49d1c46f27a47369d54872172deff7e5725ba9cc18c2272 +size 42444048 diff --git a/generator/outputs/models/p3_2_vae_perceptual_final_ema.pt b/generator/outputs/models/p3_2_vae_perceptual_final_ema.pt index 0f7fbdb..ccb2d1e 100644 --- a/generator/outputs/models/p3_2_vae_perceptual_final_ema.pt +++ b/generator/outputs/models/p3_2_vae_perceptual_final_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ef710db9871223808aad72f3b3d73807d56c8c6956e3a75e2eaf2c98401d89e -size 42460869 +oid sha256:8e1fed73c3da84dc82ebc5f9d89f0d0068f25106456fadeac66a74970b42d3f4 +size 42444079 diff --git a/generator/outputs/models/p3_2_vae_perceptual_final_vae.pt b/generator/outputs/models/p3_2_vae_perceptual_final_vae.pt index bf192e3..486ed39 100644 --- a/generator/outputs/models/p3_2_vae_perceptual_final_vae.pt +++ b/generator/outputs/models/p3_2_vae_perceptual_final_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29bd40cf62a8ac216d87b1b24bc2a25b85111459560672c3b8ad098e33c9a9c2 -size 42460869 +oid sha256:1cf6efcd4cc7e39ef4aa80cfd9ea2952c879f6e84bfd2d68a905ea074b50aa30 +size 42444079 diff --git a/generator/outputs/models/p3_3_vae_patchgan_best_ema.pt b/generator/outputs/models/p3_3_vae_patchgan_best_ema.pt index 4f28197..3d6e5c0 100644 --- a/generator/outputs/models/p3_3_vae_patchgan_best_ema.pt +++ b/generator/outputs/models/p3_3_vae_patchgan_best_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:74481291f056902d8a37adc556f2bfeefa08bca42d22f73f3f64e1de3c96ad75 -size 42460722 +oid sha256:4ed65bde40d9c905d63eb051d3c7f014585a023e216e6cd421f6d85b38dd35df +size 42443986 diff --git a/generator/outputs/models/p3_3_vae_patchgan_best_vae.pt b/generator/outputs/models/p3_3_vae_patchgan_best_vae.pt index 413cf69..bbe4f24 100644 --- a/generator/outputs/models/p3_3_vae_patchgan_best_vae.pt +++ b/generator/outputs/models/p3_3_vae_patchgan_best_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:95d1a739628ac3a978d30362d4cc7d31514181c88f42ff058c895d30a77c1386 -size 42460722 +oid sha256:a90e108e77500dce0a04a7cd1b5785ab1ca07e1fd2054ec2e9520aaee4fded69 +size 42443986 diff --git a/generator/outputs/models/p3_3_vae_patchgan_final_ema.pt b/generator/outputs/models/p3_3_vae_patchgan_final_ema.pt index 9c187a5..2428532 100644 --- a/generator/outputs/models/p3_3_vae_patchgan_final_ema.pt +++ b/generator/outputs/models/p3_3_vae_patchgan_final_ema.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ca0a2fd3151e4bb859279f4bd187e088adb270b78a1626ebfa14531deba7d088 -size 42460771 +oid sha256:e991056db79695d526ddf2b98239c5994da9fac8d43c2016bf75d6706cfc832f +size 42444017 diff --git a/generator/outputs/models/p3_3_vae_patchgan_final_patchgan.pt b/generator/outputs/models/p3_3_vae_patchgan_final_patchgan.pt index c3597a7..7be7c8a 100644 --- a/generator/outputs/models/p3_3_vae_patchgan_final_patchgan.pt +++ b/generator/outputs/models/p3_3_vae_patchgan_final_patchgan.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:086ab730b6061e61c1255e4f22b7fa351dcdc0972635f1dcdd7986075a816480 +oid sha256:3c727f6e118eb6f568d2a34e0ac0f5ae29647aadb525c285a3e6eab1eff069f2 size 11067330 diff --git a/generator/outputs/models/p3_3_vae_patchgan_final_vae.pt b/generator/outputs/models/p3_3_vae_patchgan_final_vae.pt index 6f4360b..d314612 100644 --- a/generator/outputs/models/p3_3_vae_patchgan_final_vae.pt +++ b/generator/outputs/models/p3_3_vae_patchgan_final_vae.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:56b707c2796c6c765346dd4aa0409bd9308b57b437d2db8ac05787b9fb30970b -size 42460771 +oid sha256:5db2d0aea984d40cb40e454e35fc5494c01dd7b9f5de4631c71e8238dd0091ae +size 42444017 diff --git a/generator/outputs/pipeline/20260502T130706.633366+0000.json b/generator/outputs/pipeline/20260502T130706.633366+0000.json new file mode 100644 index 0000000..f707e04 --- /dev/null +++ b/generator/outputs/pipeline/20260502T130706.633366+0000.json @@ -0,0 +1,14 @@ +{ + "created_at": "2026-05-02T13:07:06.633366+00:00", + "config_paths": [ + "generator/configs/phase3/p3_1_vae.json", + "generator/configs/phase3/p3_2_vae_perceptual.json", + "generator/configs/phase3/p3_3_vae_patchgan.json" + ], + "instance_id": 36014025, + "offer_id": 35956124, + "ssh_host": "ssh3.vast.ai", + "ssh_port": 14024, + "status": "completed", + "remote_workspace": "/workspace/DRL_PROJ" +} \ No newline at end of file diff --git a/generator/outputs/pipeline/20260502T173927.858755+0000.json b/generator/outputs/pipeline/20260502T173927.858755+0000.json new file mode 100644 index 0000000..63023fe --- /dev/null +++ b/generator/outputs/pipeline/20260502T173927.858755+0000.json @@ -0,0 +1,12 @@ +{ + "created_at": "2026-05-02T17:39:27.858755+00:00", + "config_paths": [ + "generator/configs/phase3/p3_3_vae_patchgan.json" + ], + "instance_id": 36025335, + "offer_id": 19025020, + "ssh_host": "ssh6.vast.ai", + "ssh_port": 25334, + "status": "completed", + "remote_workspace": "/workspace/DRL_PROJ" +} \ No newline at end of file diff --git a/generator/outputs/pipeline/20260502T184433.143461+0000.json b/generator/outputs/pipeline/20260502T184433.143461+0000.json new file mode 100644 index 0000000..e88b331 --- /dev/null +++ b/generator/outputs/pipeline/20260502T184433.143461+0000.json @@ -0,0 +1,12 @@ +{ + "created_at": "2026-05-02T18:44:33.143461+00:00", + "config_paths": [ + "generator/configs/phase3/p3_3_vae_patchgan.json" + ], + "instance_id": 36027837, + "offer_id": 29548831, + "ssh_host": null, + "ssh_port": null, + "status": "cancelled", + "remote_workspace": "/workspace/DRL_PROJ" +} \ No newline at end of file diff --git a/generator/outputs/pipeline/20260502T185149.983505+0000.json b/generator/outputs/pipeline/20260502T185149.983505+0000.json new file mode 100644 index 0000000..a280c39 --- /dev/null +++ b/generator/outputs/pipeline/20260502T185149.983505+0000.json @@ -0,0 +1,12 @@ +{ + "created_at": "2026-05-02T18:51:49.983505+00:00", + "config_paths": [ + "generator/configs/phase3/p3_3_vae_patchgan.json" + ], + "instance_id": 36028210, + "offer_id": 35956124, + "ssh_host": "ssh1.vast.ai", + "ssh_port": 28210, + "status": "completed", + "remote_workspace": "/workspace/DRL_PROJ" +} \ No newline at end of file diff --git a/generator/outputs/samples/p3_1_vae/epoch_0010.png b/generator/outputs/samples/p3_1_vae/epoch_0010.png index 753c7ee..bacea50 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0010.png and b/generator/outputs/samples/p3_1_vae/epoch_0010.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0010_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0010_recon.png index fa822e4..b9169af 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0010_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0010_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0020.png b/generator/outputs/samples/p3_1_vae/epoch_0020.png index 3da6fa2..5b08fcb 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0020.png and b/generator/outputs/samples/p3_1_vae/epoch_0020.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0020_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0020_recon.png index 8628b1f..767fcc3 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0020_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0020_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0030.png b/generator/outputs/samples/p3_1_vae/epoch_0030.png index 3fc7aea..aff9e0d 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0030.png and b/generator/outputs/samples/p3_1_vae/epoch_0030.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0030_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0030_recon.png index 7030139..afa8cbe 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0030_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0030_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0040.png b/generator/outputs/samples/p3_1_vae/epoch_0040.png index 82fe4e6..1e76505 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0040.png and b/generator/outputs/samples/p3_1_vae/epoch_0040.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0040_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0040_recon.png index fa4b22a..4f57745 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0040_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0040_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0050.png b/generator/outputs/samples/p3_1_vae/epoch_0050.png index 1168734..0b319a1 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0050.png and b/generator/outputs/samples/p3_1_vae/epoch_0050.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0050_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0050_recon.png index f43eeb6..6a53be7 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0050_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0050_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0060.png b/generator/outputs/samples/p3_1_vae/epoch_0060.png index a3a6643..8f472e7 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0060.png and b/generator/outputs/samples/p3_1_vae/epoch_0060.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0060_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0060_recon.png index 0eea44e..ecb4f74 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0060_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0060_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0070.png b/generator/outputs/samples/p3_1_vae/epoch_0070.png index 0632862..9635dc1 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0070.png and b/generator/outputs/samples/p3_1_vae/epoch_0070.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0070_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0070_recon.png index 20f0927..22d742e 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0070_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0070_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0080.png b/generator/outputs/samples/p3_1_vae/epoch_0080.png index df012e2..137e686 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0080.png and b/generator/outputs/samples/p3_1_vae/epoch_0080.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0080_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0080_recon.png index d3c20c1..020c88c 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0080_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0080_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0090.png b/generator/outputs/samples/p3_1_vae/epoch_0090.png index 87a7a72..c84a3e1 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0090.png and b/generator/outputs/samples/p3_1_vae/epoch_0090.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0090_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0090_recon.png index 340ff81..0f4857c 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0090_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0090_recon.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0100.png b/generator/outputs/samples/p3_1_vae/epoch_0100.png index d51e462..2744260 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0100.png and b/generator/outputs/samples/p3_1_vae/epoch_0100.png differ diff --git a/generator/outputs/samples/p3_1_vae/epoch_0100_recon.png b/generator/outputs/samples/p3_1_vae/epoch_0100_recon.png index 3d44da3..cb641e2 100644 Binary files a/generator/outputs/samples/p3_1_vae/epoch_0100_recon.png and b/generator/outputs/samples/p3_1_vae/epoch_0100_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010.png index 25d2849..cf47501 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010_recon.png index 1e66222..cb79a93 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0010_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020.png index fa9d579..3ff96c9 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020_recon.png index bac1a12..56ad024 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0020_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030.png index d3acc92..84c4691 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030_recon.png index fffcada..6956f6b 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0030_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040.png index e394c82..234347f 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040_recon.png index 759a6e8..3bebd20 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0040_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050.png index e394c82..202a7a1 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050_recon.png index 759a6e8..6add299 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0050_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060.png index e394c82..ec90625 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060_recon.png index 759a6e8..79b70db 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0060_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070.png index e394c82..e1dfb8e 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070_recon.png index 759a6e8..e2a9e67 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0070_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080.png index e394c82..e41fa69 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080_recon.png index 759a6e8..28acc71 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0080_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090.png index e394c82..6996431 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090_recon.png index 759a6e8..612310e 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0090_recon.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100.png index e394c82..619d061 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100.png differ diff --git a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100_recon.png b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100_recon.png index 759a6e8..85e1d7f 100644 Binary files a/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100_recon.png and b/generator/outputs/samples/p3_2_vae_perceptual/epoch_0100_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010.png index 616ee86..233364e 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010_recon.png index 79ec06a..df6e81c 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0010_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020.png index 5961881..dc09fb0 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020_recon.png index 87835f2..7d4aa8f 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0020_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030.png index 1027ddf..907736b 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030_recon.png index dd88848..cad41a1 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0030_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040.png index e37da59..f0fe6ef 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040_recon.png index 54ff0ac..ebbd8f5 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0040_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050.png index 45ff5fd..4e18c2f 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050_recon.png index 54ff0ac..2b40b74 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0050_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060.png index eb7d72b..2c3a5d2 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060_recon.png index 54ff0ac..a83a3ec 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0060_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070.png index 6a89558..a492c88 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070_recon.png index 54ff0ac..7ab6efc 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0070_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080.png index ef8edfb..0020b34 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080_recon.png index 54ff0ac..9a8bd2d 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0080_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090.png index e036fb9..2673a20 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090_recon.png index 54ff0ac..1a44e94 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0090_recon.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100.png index dd1b378..ae5f476 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100.png differ diff --git a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100_recon.png b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100_recon.png index 54ff0ac..92c3bb7 100644 Binary files a/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100_recon.png and b/generator/outputs/samples/p3_3_vae_patchgan/epoch_0100_recon.png differ diff --git a/generator/run.py b/generator/run.py index f11727e..e9ed7a2 100644 --- a/generator/run.py +++ b/generator/run.py @@ -52,7 +52,7 @@ def main(config_path, *, data_dir_override=None, output_root="generator/outputs" # Count total trainable parameters if isinstance(model, tuple): - n_params = sum(p.numel() for p in model[0].parameters() if p.requires_grad) + n_params = sum(p.numel() for m in model for p in m.parameters() if p.requires_grad) else: n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f"Trainable params: {n_params:,}") diff --git a/generator/src/models/vae.py b/generator/src/models/vae.py index 285ad56..126a08b 100644 --- a/generator/src/models/vae.py +++ b/generator/src/models/vae.py @@ -22,17 +22,21 @@ def _init_weights(m): nn.init.normal_(m.weight, 0.0, 0.02) if m.bias is not None: nn.init.zeros_(m.bias) - elif isinstance(m, nn.BatchNorm2d) and m.weight is not None: + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)) and m.weight is not None: nn.init.normal_(m.weight, 1.0, 0.02) nn.init.zeros_(m.bias) +def _norm(channels: int) -> nn.GroupNorm: + return nn.GroupNorm(8, channels) + + def _upsample_block(in_ch: int, out_ch: int) -> nn.Sequential: """Nearest-neighbour upsample followed by a 3×3 conv — no checkerboard.""" return nn.Sequential( nn.Upsample(scale_factor=2, mode="nearest"), nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False), - nn.BatchNorm2d(out_ch), + _norm(out_ch), nn.ReLU(inplace=True), ) @@ -65,7 +69,7 @@ class VAE(nn.Module): for _ in range(n_down - 1): enc_layers += [ nn.Conv2d(ch, ch * 2, 4, stride=2, padding=1, bias=False), - nn.BatchNorm2d(ch * 2), + _norm(ch * 2), nn.LeakyReLU(0.2, inplace=True), ] ch *= 2 @@ -98,9 +102,12 @@ class VAE(nn.Module): def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: h = self.encoder(x).flatten(1) - return self.fc_mu(h), self.fc_lv(h) + log_var = self.fc_lv(h).clamp(-10.0, 10.0) + return self.fc_mu(h), log_var def reparameterize(self, mu: torch.Tensor, log_var: torch.Tensor) -> torch.Tensor: + if not self.training: + return mu std = torch.exp(0.5 * log_var) return mu + std * torch.randn_like(std) diff --git a/generator/src/training/ema.py b/generator/src/training/ema.py index 4782787..97f5bd8 100644 --- a/generator/src/training/ema.py +++ b/generator/src/training/ema.py @@ -20,3 +20,5 @@ class EMA: def update(self, model: nn.Module) -> None: for p_ema, p in zip(self.model.parameters(), model.parameters()): p_ema.data.mul_(self.decay).add_(p.data, alpha=1.0 - self.decay) + for b_ema, b in zip(self.model.buffers(), model.buffers()): + b_ema.copy_(b) diff --git a/generator/src/training/fid.py b/generator/src/training/fid.py index 2d0c842..206f0b7 100644 --- a/generator/src/training/fid.py +++ b/generator/src/training/fid.py @@ -11,14 +11,15 @@ from torchmetrics.image.fid import FrechetInceptionDistance class FIDEvaluator: - def __init__(self, real_dataset, n_real: int = 10_000, device: str = "cuda"): + def __init__(self, real_dataset, n_real: int = 10_000, device: str = "cuda", + num_workers: int = 2): self.device = torch.device(device if torch.cuda.is_available() else "cpu") self.n_real = n_real # Cache real images as a CPU tensor ([-1, 1] range) imgs_list = [] loader = DataLoader(real_dataset, batch_size=256, shuffle=False, - num_workers=4, drop_last=False) + num_workers=num_workers, drop_last=False) for batch in loader: imgs_list.append(batch.cpu()) if sum(x.size(0) for x in imgs_list) >= n_real: diff --git a/generator/src/training/trainer.py b/generator/src/training/trainer.py index 4061525..3f3b7f3 100644 --- a/generator/src/training/trainer.py +++ b/generator/src/training/trainer.py @@ -66,7 +66,7 @@ def train_dcgan( loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, - num_workers=min(4, os.cpu_count() or 1), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1)), pin_memory=(device.type == "cuda"), drop_last=True, ) opt_g = torch.optim.Adam(generator.parameters(), lr=lr_g, betas=(beta1, beta2)) @@ -86,7 +86,8 @@ def train_dcgan( save_dir.mkdir(parents=True, exist_ok=True) samples_dir = save_dir.parent / "samples" / run_name - fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device)) + fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1))) history = {"g_loss": [], "d_loss": [], "d_real": [], "d_fake": [], "fid": {}} best_fid = float("inf") @@ -239,7 +240,7 @@ def train_wgan( loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, - num_workers=min(4, os.cpu_count() or 1), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1)), pin_memory=(device.type == "cuda"), drop_last=True, ) opt_g = torch.optim.Adam(generator.parameters(), lr=lr_g, betas=(beta1, beta2)) @@ -257,7 +258,8 @@ def train_wgan( save_dir.mkdir(parents=True, exist_ok=True) samples_dir = save_dir.parent / "samples" / run_name - fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device)) + fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1))) history = {"g_loss": [], "w_dist": [], "gp": [], "d_real": [], "d_fake": [], "fid": {}} best_fid = float("inf") @@ -391,7 +393,6 @@ def _save_vae_samples( # Interleave real / reconstruction pairs pairs = torch.stack([real, recon], dim=1).flatten(0, 1) save_image(pairs, samples_dir / f"epoch_{epoch:04d}_recon.png", nrow=4) - vae.train() def train_vae( @@ -403,13 +404,21 @@ def train_vae( run_name: str, device: str = "cuda", ) -> dict: - """VAE training loop covering Phase 3.1 – 3.3. + """VAE training loop covering Phase 3.1 – 3.3 and Phase 5. Config toggles: lambda_perceptual > 0 → VGG-16 perceptual loss (Phase 3.2+) lambda_adversarial > 0 → PatchGAN hinge adversarial loss (Phase 3.3) Loss: L = L_mse + λ_perc·L_vgg + λ_adv·L_adv + β_kl·L_kl + + KL is computed as mean over latent dimensions (scale-invariant), so + beta_kl is comparable across different latent_dim values. + + AMP is intentionally disabled for VAE training — mixed-precision float16 + overflows when the KL divergence spikes, producing NaN cascades that + corrupt the model irrecoverably. All VAE + perceptual + PatchGAN + computation runs in float32. """ device = torch.device(device if torch.cuda.is_available() else "cpu") vae = vae.to(device) @@ -425,6 +434,7 @@ def train_vae( lambda_perceptual = cfg.get("lambda_perceptual", 0.0) lambda_adversarial = cfg.get("lambda_adversarial", 0.0) lr_d = cfg.get("lr_d", 1e-4) + grad_clip = cfg.get("grad_clip", 1.0) ema_decay = cfg.get("ema_decay", 0.9999) sample_interval = cfg.get("sample_interval", 10) fid_interval = cfg.get("fid_interval", 25) @@ -435,13 +445,13 @@ def train_vae( loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, - num_workers=min(4, os.cpu_count() or 1), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1)), pin_memory=(device.type == "cuda"), drop_last=True, ) opt_vae = torch.optim.Adam(vae.parameters(), lr=lr) - use_amp = device.type == "cuda" - scaler = _GradScaler("cuda", enabled=use_amp) + # AMP disabled — float16 overflows on KL spikes, causing NaN cascades + use_amp = False # KL warmup: linearly ramp beta_kl from 0 to target over first 20% of training kl_warmup_epochs = max(1, epochs // 5) @@ -456,11 +466,10 @@ def train_vae( perc_fn = None patchgan = None opt_d = None - scaler_d = None if use_perceptual: from src.training.perceptual import PerceptualLoss - perc_fn = PerceptualLoss().to(device) + perc_fn = PerceptualLoss().to(device).float() print("Perceptual loss: VGG-16 relu1_2 + relu2_2 + relu3_3") if use_adversarial: @@ -468,15 +477,14 @@ def train_vae( patchgan = PatchGANDiscriminator( ndf=cfg.get("ndf_patch", 64), image_size=cfg.get("image_size", 64), - ).to(device) + ).to(device).float() opt_d = torch.optim.Adam(patchgan.parameters(), lr=lr_d, betas=(0.5, 0.999)) - scaler_d = _GradScaler("cuda", enabled=use_amp) sched_d = torch.optim.lr_scheduler.LambdaLR( opt_d, lr_lambda=lambda ep: max(0.0, 1.0 - max(ep - decay_start, 0) / max(epochs - decay_start, 1))) n_d = sum(p.numel() for p in patchgan.parameters()) print(f"PatchGAN: {n_d:,} params") else: - hinge_d_loss = hinge_g_loss = None # satisfy linter, never called + hinge_d_loss = hinge_g_loss = None # never called # ── Fixed seeds for consistent visualisation ────────────────────────── fixed_z = torch.randn(16, latent_dim, device=device) @@ -490,16 +498,19 @@ def train_vae( save_dir.mkdir(parents=True, exist_ok=True) samples_dir = save_dir.parent / "samples" / run_name - fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device)) + fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1))) history = { "recon_loss": [], "kl_loss": [], "perc_loss": [], "adv_g_loss": [], "adv_d_loss": [], "fid": {}, } best_fid = float("inf") + nan_skipped = 0 print( - f"Device: {device} AMP: {use_amp} Batches/epoch: {len(loader)}" - f" β_kl={beta_kl} (warmup {kl_warmup_epochs}ep) λ_perc={lambda_perceptual} λ_adv={lambda_adversarial}" + f"Device: {device} AMP: disabled (float32) Batches/epoch: {len(loader)}" + f" β_kl={beta_kl} (warmup {kl_warmup_epochs}ep) λ_perc={lambda_perceptual}" + f" λ_adv={lambda_adversarial}" ) t_start = time.time() @@ -513,43 +524,52 @@ def train_vae( n_batches = 0 for real in tqdm(loader, desc=f"Epoch {epoch}/{epochs}", leave=False): - real = real.to(device) + real = real.to(device).float() # KL warmup: ramp from 0 to beta_kl over kl_warmup_epochs current_beta = beta_kl * min(1.0, epoch / kl_warmup_epochs) - # ── VAE forward ─────────────────────────────────────────────── - with _autocast("cuda", enabled=use_amp): - recon, mu, log_var = vae(real) - mse = F.mse_loss(recon, real) - kl = -0.5 * (1 + log_var - mu.pow(2) - log_var.exp()).sum(1).mean() - perc = perc_fn(recon, real) if use_perceptual else real.new_zeros(1).squeeze() - vae_loss = mse + current_beta * kl + lambda_perceptual * perc + # ── VAE forward (float32, no AMP) ──────────────────────────── + recon, mu, log_var = vae(real) + mse = F.mse_loss(recon, real) + + # KL divergence: mean over latent dims (scale-invariant w.r.t. latent_dim) + kl = (-0.5 * (1 + log_var - mu.pow(2) - log_var.exp())).mean() + + perc = perc_fn(recon, real) if use_perceptual else real.new_zeros(1).squeeze() + vae_loss = mse + current_beta * kl + lambda_perceptual * perc + + # ── NaN/Inf guard ──────────────────────────────────────────── + if not torch.isfinite(vae_loss): + nan_skipped += 1 + opt_vae.zero_grad() + continue # ── PatchGAN discriminator step ─────────────────────────────── adv_d = real.new_zeros(1).squeeze() if use_adversarial: - opt_d.zero_grad() - with _autocast("cuda", enabled=use_amp): + # Warmup: only start adversarial training after 20% of epochs + if epoch > kl_warmup_epochs: + opt_d.zero_grad() d_real = patchgan(real) d_fake = patchgan(recon.detach()) adv_d = hinge_d_loss(d_real, d_fake) - scaler_d.scale(adv_d).backward() - scaler_d.step(opt_d) - scaler_d.update() + if torch.isfinite(adv_d): + adv_d.backward() + torch.nn.utils.clip_grad_norm_(patchgan.parameters(), grad_clip) + opt_d.step() # ── PatchGAN generator adversarial loss ─────────────────────── adv_g = real.new_zeros(1).squeeze() - if use_adversarial: - with _autocast("cuda", enabled=use_amp): - adv_g = hinge_g_loss(patchgan(recon)) - vae_loss = vae_loss + lambda_adversarial * adv_g + if use_adversarial and epoch > kl_warmup_epochs: + adv_g = hinge_g_loss(patchgan(recon)) + vae_loss = vae_loss + lambda_adversarial * adv_g # ── VAE backward ────────────────────────────────────────────── opt_vae.zero_grad() - scaler.scale(vae_loss).backward() - scaler.step(opt_vae) - scaler.update() + vae_loss.backward() + torch.nn.utils.clip_grad_norm_(vae.parameters(), grad_clip) + opt_vae.step() ema.update(vae) recon_sum += mse.item() @@ -559,11 +579,11 @@ def train_vae( adv_d_sum += adv_d.item() n_batches += 1 - avg_r = recon_sum / n_batches - avg_k = kl_sum / n_batches - avg_p = perc_sum / n_batches - avg_g = adv_g_sum / n_batches - avg_d = adv_d_sum / n_batches + avg_r = recon_sum / max(n_batches, 1) + avg_k = kl_sum / max(n_batches, 1) + avg_p = perc_sum / max(n_batches, 1) + avg_g = adv_g_sum / max(n_batches, 1) + avg_d = adv_d_sum / max(n_batches, 1) history["recon_loss"].append(avg_r) history["kl_loss"].append(avg_k) history["perc_loss"].append(avg_p) @@ -574,6 +594,7 @@ def train_vae( f"[{epoch:03d}/{epochs}] " f"MSE: {avg_r:.4f} KL: {avg_k:.2f} β={current_beta:.6f} " f"Perc: {avg_p:.4f} AdvG: {avg_g:.4f} AdvD: {avg_d:.4f}" + f" (NaN skipped: {nan_skipped})" ) if epoch % sample_interval == 0: @@ -607,6 +628,7 @@ def train_vae( if patchgan is not None: torch.save(patchgan.state_dict(), save_dir / f"{run_name}_final_patchgan.pt") history["train_time_s"] = time.time() - t_start + print(f"Total NaN-skipped batches: {nan_skipped}") return history @@ -662,7 +684,7 @@ def train_ddpm( loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, - num_workers=min(4, os.cpu_count() or 1), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1)), pin_memory=(device.type == "cuda"), drop_last=True, ) opt = torch.optim.AdamW(model.parameters(), lr=lr) @@ -679,7 +701,8 @@ def train_ddpm( save_dir.mkdir(parents=True, exist_ok=True) samples_dir = save_dir.parent / "samples" / run_name - fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device)) + fid_eval = FIDEvaluator(train_dataset, n_real=fid_n_real, device=str(device), + num_workers=cfg.get("num_workers", min(4, os.cpu_count() or 1))) history = {"loss": [], "fid": {}} best_fid = float("inf")