Johnny Fernandes
1861 lines
44 KiB
JSON
1861 lines
44 KiB
JSON
{
|
|
"run_name": "p2a_t3_holdout_insight",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.221900131498222,
|
|
0.09055700592673671,
|
|
0.050355928940382315,
|
|
0.03795463042879868,
|
|
0.020803921496248403,
|
|
0.015402267911616228,
|
|
0.009459777763774503,
|
|
0.007435759974616281,
|
|
0.003887806649730425,
|
|
0.003259596692067943,
|
|
0.0015012719458815057,
|
|
0.0016088033901993455,
|
|
0.0008241117115995835,
|
|
0.00107023416148877,
|
|
0.0009690378105084546
|
|
],
|
|
"train_acc": [
|
|
0.850462962962963,
|
|
0.9469907407407407,
|
|
0.9729938271604939,
|
|
0.9815586419753086,
|
|
0.989429012345679,
|
|
0.9932098765432099,
|
|
0.9949845679012346,
|
|
0.9965277777777778,
|
|
0.9983796296296297,
|
|
0.9984567901234568,
|
|
0.9993055555555556,
|
|
0.9996141975308642,
|
|
0.999537037037037,
|
|
0.9993827160493827,
|
|
0.9994598765432099
|
|
],
|
|
"train_auc": [
|
|
0.9319749603480796,
|
|
0.9887071464549039,
|
|
0.9963385202331962,
|
|
0.9976732762131345,
|
|
0.9992920390732167,
|
|
0.9994847259730797,
|
|
0.9998105281207132,
|
|
0.9999104482810356,
|
|
0.9999664834104938,
|
|
0.9999829469950274,
|
|
0.9999967849794238,
|
|
0.999874239111797,
|
|
0.9999995713305898,
|
|
0.9999989015346366,
|
|
0.9999914801954732
|
|
],
|
|
"train_f1": [
|
|
0.8827444336882866,
|
|
0.9597704514844527,
|
|
0.9796322160148976,
|
|
0.9861264294421548,
|
|
0.9920611925595411,
|
|
0.9949062282935864,
|
|
0.9962334125282494,
|
|
0.9973938727051601,
|
|
0.998783807262408,
|
|
0.9988425925925926,
|
|
0.9994791968057404,
|
|
0.9997106983741249,
|
|
0.9996527375853687,
|
|
0.9995369298448715,
|
|
0.9995947901591896
|
|
],
|
|
"val_loss": [
|
|
0.16446967588530648,
|
|
0.1840791697303454,
|
|
0.19977748062875536,
|
|
0.20411120371686087,
|
|
0.32943161345190475,
|
|
0.17416284415456984,
|
|
0.1789202243089676,
|
|
0.1872969721754392,
|
|
0.14698216517766316,
|
|
0.1865630802181032,
|
|
0.18526823139852946,
|
|
0.16406844076183108,
|
|
0.17544087188111412,
|
|
0.15160242136981752,
|
|
0.16450001017914878
|
|
],
|
|
"val_acc": [
|
|
0.9041666666666667,
|
|
0.8722222222222222,
|
|
0.9305555555555556,
|
|
0.9256944444444445,
|
|
0.9138888888888889,
|
|
0.9270833333333334,
|
|
0.9506944444444444,
|
|
0.9486111111111111,
|
|
0.9416666666666667,
|
|
0.9465277777777777,
|
|
0.9444444444444444,
|
|
0.9465277777777777,
|
|
0.9465277777777777,
|
|
0.9493055555555555,
|
|
0.9479166666666666
|
|
],
|
|
"val_auc": [
|
|
0.9687575954861111,
|
|
0.9815342881944444,
|
|
0.9820431857638888,
|
|
0.9803776041666667,
|
|
0.9755219184027778,
|
|
0.9816297743055555,
|
|
0.9842654079861111,
|
|
0.9816026475694444,
|
|
0.9862662760416667,
|
|
0.9854665798611111,
|
|
0.9851030815972222,
|
|
0.9868326822916668,
|
|
0.9869791666666667,
|
|
0.9873166232638889,
|
|
0.9870941840277778
|
|
],
|
|
"val_f1": [
|
|
0.9278996865203761,
|
|
0.8946162657502864,
|
|
0.9494438827098078,
|
|
0.9456576942610462,
|
|
0.9383084577114428,
|
|
0.944,
|
|
0.9634209170530654,
|
|
0.9618949536560247,
|
|
0.9556962025316456,
|
|
0.9602888086642599,
|
|
0.9588053553038105,
|
|
0.960082944530845,
|
|
0.9603297269448737,
|
|
0.9620779220779221,
|
|
0.9612403100775194
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8404166666666667,
|
|
"auc_roc": 0.9406697916666668,
|
|
"f1": 0.8836926814454904,
|
|
"confusion_matrix": [
|
|
[
|
|
1124,
|
|
76
|
|
],
|
|
[
|
|
690,
|
|
2910
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.94,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.94,
|
|
"pairwise_auc": 0.9856604166666667,
|
|
"pairwise_f1": 0.9384359400998337
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.5008333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
599,
|
|
601
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.5008333333333334,
|
|
"pairwise_auc": 0.8413416666666667,
|
|
"pairwise_f1": 0.6403835908364411
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9841666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9841666666666666,
|
|
"pairwise_auc": 0.9950072916666667,
|
|
"pairwise_f1": 0.9613349613349613
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9366666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1124,
|
|
76
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.06333333333333335
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9383333333333334,
|
|
"auc_roc": 0.9856604166666667,
|
|
"f1": 0.9384359400998337,
|
|
"confusion_matrix": [
|
|
[
|
|
1124,
|
|
76
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.71875,
|
|
"auc_roc": 0.8413416666666667,
|
|
"f1": 0.6403835908364411,
|
|
"confusion_matrix": [
|
|
[
|
|
1124,
|
|
76
|
|
],
|
|
[
|
|
599,
|
|
601
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9604166666666667,
|
|
"auc_roc": 0.9950072916666667,
|
|
"f1": 0.9613349613349613,
|
|
"confusion_matrix": [
|
|
[
|
|
1124,
|
|
76
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.22215608109110668,
|
|
0.09765733032728786,
|
|
0.04893466802687603,
|
|
0.034209873639823245,
|
|
0.02029450899701462,
|
|
0.013510319246945724,
|
|
0.012697673750346753,
|
|
0.005298483473454303,
|
|
0.0029991691134646204,
|
|
0.0025113817023664545,
|
|
0.0013458789497830658,
|
|
0.0008007793183738321,
|
|
0.00046533000990795116,
|
|
0.0003107348998751318,
|
|
0.0005777599946174441
|
|
],
|
|
"train_acc": [
|
|
0.8510030864197531,
|
|
0.9439043209876543,
|
|
0.9750771604938272,
|
|
0.983641975308642,
|
|
0.988966049382716,
|
|
0.9929012345679012,
|
|
0.9929012345679012,
|
|
0.9975308641975309,
|
|
0.998533950617284,
|
|
0.9987654320987654,
|
|
0.9993055555555556,
|
|
0.9996141975308642,
|
|
0.9997685185185186,
|
|
0.9999228395061729,
|
|
0.9998456790123457
|
|
],
|
|
"train_auc": [
|
|
0.9321080220121742,
|
|
0.9867878729423869,
|
|
0.9965020308213306,
|
|
0.9980933052554869,
|
|
0.9993586033950617,
|
|
0.9997250621570646,
|
|
0.9996713847093621,
|
|
0.9999531142832648,
|
|
0.9999883589463305,
|
|
0.9999929403506516,
|
|
0.9999983121141975,
|
|
0.9999995981224281,
|
|
0.9999998928326476,
|
|
1.0,
|
|
0.9999997320816186
|
|
],
|
|
"train_f1": [
|
|
0.8844473699958112,
|
|
0.9573881952992205,
|
|
0.9812307513510372,
|
|
0.9876987350586051,
|
|
0.9917048552700273,
|
|
0.9946716089424302,
|
|
0.9946734599351552,
|
|
0.998147933788633,
|
|
0.9989002720379696,
|
|
0.9990739668943165,
|
|
0.999479015918958,
|
|
0.9997105643994211,
|
|
0.9998263587428373,
|
|
0.9999421329784156,
|
|
0.9998842592592593
|
|
],
|
|
"val_loss": [
|
|
0.18173423045211368,
|
|
0.15682731734381783,
|
|
0.12995633516046737,
|
|
0.3167274175418748,
|
|
0.3881762362188763,
|
|
0.16476303007867601,
|
|
0.14555738866329193,
|
|
0.15716644260618423,
|
|
0.1588555042942365,
|
|
0.16912019583914015,
|
|
0.1316970290409194,
|
|
0.14958005249500275,
|
|
0.14952652553717297,
|
|
0.13752244727479088,
|
|
0.13813267449537914
|
|
],
|
|
"val_acc": [
|
|
0.8597222222222223,
|
|
0.9326388888888889,
|
|
0.9409722222222222,
|
|
0.9125,
|
|
0.8347222222222223,
|
|
0.9375,
|
|
0.9451388888888889,
|
|
0.9527777777777777,
|
|
0.9520833333333333,
|
|
0.9423611111111111,
|
|
0.9618055555555556,
|
|
0.95625,
|
|
0.9569444444444445,
|
|
0.9583333333333334,
|
|
0.95625
|
|
],
|
|
"val_auc": [
|
|
0.9741330295138888,
|
|
0.9806532118055556,
|
|
0.9829047309027779,
|
|
0.977763671875,
|
|
0.9776833767361112,
|
|
0.9849565972222223,
|
|
0.9864333767361111,
|
|
0.9848068576388889,
|
|
0.9858322482638889,
|
|
0.9852788628472222,
|
|
0.9905544704861111,
|
|
0.9883355034722222,
|
|
0.9883203125,
|
|
0.9901714409722222,
|
|
0.9904275173611111
|
|
],
|
|
"val_f1": [
|
|
0.8837744533947065,
|
|
0.9502819067145054,
|
|
0.9555206698063841,
|
|
0.9378084896347483,
|
|
0.8596698113207547,
|
|
0.9519230769230769,
|
|
0.9580010632642212,
|
|
0.9645833333333333,
|
|
0.9637414608512874,
|
|
0.9560148383677796,
|
|
0.9713392391870767,
|
|
0.967067433350758,
|
|
0.9677083333333333,
|
|
0.9686192468619247,
|
|
0.967032967032967
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.845,
|
|
"auc_roc": 0.9390527777777778,
|
|
"f1": 0.8875113395827033,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
665,
|
|
2935
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9491666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
61,
|
|
1139
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9491666666666667,
|
|
"pairwise_auc": 0.9847184027777777,
|
|
"pairwise_f1": 0.9421009098428453
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.515,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
582,
|
|
618
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.515,
|
|
"pairwise_auc": 0.839582638888889,
|
|
"pairwise_f1": 0.6515550869794412
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9816666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
22,
|
|
1178
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9816666666666667,
|
|
"pairwise_auc": 0.9928572916666668,
|
|
"pairwise_f1": 0.9588929588929589
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9341666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.0658333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9416666666666667,
|
|
"auc_roc": 0.9847184027777777,
|
|
"f1": 0.9421009098428453,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
61,
|
|
1139
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.7245833333333334,
|
|
"auc_roc": 0.839582638888889,
|
|
"f1": 0.6515550869794412,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
582,
|
|
618
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9579166666666666,
|
|
"auc_roc": 0.9928572916666668,
|
|
"f1": 0.9588929588929589,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
22,
|
|
1178
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.2215230604380737,
|
|
0.1014425605489516,
|
|
0.05135551941535079,
|
|
0.034227528057584285,
|
|
0.020638354608726567,
|
|
0.015166697286658296,
|
|
0.009714811315392086,
|
|
0.008263903152126457,
|
|
0.0028723553302696042,
|
|
0.002424952944478898,
|
|
0.0017287055309174724
|
|
],
|
|
"train_acc": [
|
|
0.8475308641975309,
|
|
0.9423611111111111,
|
|
0.9719907407407408,
|
|
0.9820216049382716,
|
|
0.9898919753086419,
|
|
0.9926697530864198,
|
|
0.9952932098765432,
|
|
0.9969907407407408,
|
|
0.9986882716049382,
|
|
0.9992283950617284,
|
|
0.9989969135802469
|
|
],
|
|
"train_auc": [
|
|
0.9321050079303842,
|
|
0.9859669308341907,
|
|
0.9961787203146433,
|
|
0.9982715781464335,
|
|
0.9992778393990055,
|
|
0.9995493612825788,
|
|
0.9998193024477025,
|
|
0.999740547839506,
|
|
0.9999873676483196,
|
|
0.99998922968107,
|
|
0.9999947086119685
|
|
],
|
|
"train_f1": [
|
|
0.8802569385529027,
|
|
0.9563081242323215,
|
|
0.9788621673557328,
|
|
0.9864888373441577,
|
|
0.9924150309767819,
|
|
0.9944969008862886,
|
|
0.9964672496669948,
|
|
0.9977416179280792,
|
|
0.9990161467677527,
|
|
0.9994212962962963,
|
|
0.9992473803045215
|
|
],
|
|
"val_loss": [
|
|
0.13050816671715842,
|
|
0.13505457623137368,
|
|
0.31902390718460083,
|
|
0.19341963032881418,
|
|
0.20096096379889383,
|
|
0.10236648867527644,
|
|
0.14019040779934988,
|
|
0.16024226794640223,
|
|
0.1222893249657419,
|
|
0.1388013172480795,
|
|
0.13770783642927806
|
|
],
|
|
"val_acc": [
|
|
0.9090277777777778,
|
|
0.9208333333333333,
|
|
0.8993055555555556,
|
|
0.9361111111111111,
|
|
0.9354166666666667,
|
|
0.9541666666666667,
|
|
0.9444444444444444,
|
|
0.95,
|
|
0.9597222222222223,
|
|
0.9541666666666667,
|
|
0.95625
|
|
],
|
|
"val_auc": [
|
|
0.9779264322916666,
|
|
0.9777419704861111,
|
|
0.9720996093750001,
|
|
0.9851204427083333,
|
|
0.9850217013888889,
|
|
0.99107421875,
|
|
0.9885828993055557,
|
|
0.9856336805555554,
|
|
0.989360894097222,
|
|
0.9886002604166667,
|
|
0.9902560763888888
|
|
],
|
|
"val_f1": [
|
|
0.9289972899728998,
|
|
0.9389067524115756,
|
|
0.9286768322675848,
|
|
0.9535822401614531,
|
|
0.9528158295281582,
|
|
0.9649309245483528,
|
|
0.9574468085106383,
|
|
0.9617834394904459,
|
|
0.9695697796432319,
|
|
0.9655172413793104,
|
|
0.9672727272727273
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8120833333333334,
|
|
"auc_roc": 0.9362050925925925,
|
|
"f1": 0.8595015576323988,
|
|
"confusion_matrix": [
|
|
[
|
|
1139,
|
|
61
|
|
],
|
|
[
|
|
841,
|
|
2759
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9083333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9083333333333333,
|
|
"pairwise_auc": 0.98161875,
|
|
"pairwise_f1": 0.9272649936197362
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.4241666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
691,
|
|
509
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.4241666666666667,
|
|
"pairwise_auc": 0.8343604166666666,
|
|
"pairwise_f1": 0.5751412429378531
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9666666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
40,
|
|
1160
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9666666666666667,
|
|
"pairwise_auc": 0.9926361111111112,
|
|
"pairwise_f1": 0.9582817017761256
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9491666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1139,
|
|
61
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.050833333333333286
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.92875,
|
|
"auc_roc": 0.98161875,
|
|
"f1": 0.9272649936197362,
|
|
"confusion_matrix": [
|
|
[
|
|
1139,
|
|
61
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.6866666666666666,
|
|
"auc_roc": 0.8343604166666666,
|
|
"f1": 0.5751412429378531,
|
|
"confusion_matrix": [
|
|
[
|
|
1139,
|
|
61
|
|
],
|
|
[
|
|
691,
|
|
509
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9579166666666666,
|
|
"auc_roc": 0.9926361111111112,
|
|
"f1": 0.9582817017761256,
|
|
"confusion_matrix": [
|
|
[
|
|
1139,
|
|
61
|
|
],
|
|
[
|
|
40,
|
|
1160
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.21201637929972308,
|
|
0.09177248130357007,
|
|
0.053863701856419165,
|
|
0.031763664008273435,
|
|
0.02140559677399672,
|
|
0.014252267104974533,
|
|
0.012688780943426537,
|
|
0.005660201898093492,
|
|
0.003774158633060313,
|
|
0.0032219876190145353,
|
|
0.0011941452978000463,
|
|
0.0011382195415872384,
|
|
0.0005081066835749498,
|
|
0.0003702830817799645,
|
|
0.0010320760736625362
|
|
],
|
|
"train_acc": [
|
|
0.8590277777777777,
|
|
0.9468364197530864,
|
|
0.9702932098765432,
|
|
0.9831018518518518,
|
|
0.9880401234567902,
|
|
0.9922839506172839,
|
|
0.9934413580246914,
|
|
0.9972993827160493,
|
|
0.9982253086419753,
|
|
0.9987654320987654,
|
|
0.9996141975308642,
|
|
0.999537037037037,
|
|
0.9998456790123457,
|
|
0.9999228395061729,
|
|
0.999537037037037
|
|
],
|
|
"train_auc": [
|
|
0.9380557029106653,
|
|
0.9884040369941701,
|
|
0.9961057393475652,
|
|
0.9984620681155694,
|
|
0.9993163392704048,
|
|
0.9997259060999657,
|
|
0.9997672726980453,
|
|
0.999959946201989,
|
|
0.9999786469050069,
|
|
0.9999815538194444,
|
|
0.9999978700488683,
|
|
0.999998191550926,
|
|
0.999999812457133,
|
|
0.9999999732081619,
|
|
0.9999981111754115
|
|
],
|
|
"train_f1": [
|
|
0.8898136421204994,
|
|
0.959610762647283,
|
|
0.9776097702820588,
|
|
0.9872947728723096,
|
|
0.9910087592087709,
|
|
0.994208270589598,
|
|
0.9950756039626905,
|
|
0.9979737162044809,
|
|
0.9986684420772304,
|
|
0.9990740740740741,
|
|
0.9997106314022802,
|
|
0.9996527777777777,
|
|
0.9998842592592593,
|
|
0.9999421262804561,
|
|
0.9996527375853687
|
|
],
|
|
"val_loss": [
|
|
0.20351142072015338,
|
|
0.3276245656940672,
|
|
0.4981435305542416,
|
|
0.2889870650238461,
|
|
0.4523913953039381,
|
|
0.18176159494453006,
|
|
0.19582118193308512,
|
|
0.1700224777062734,
|
|
0.1923278099960751,
|
|
0.20986380179723105,
|
|
0.20547801156838735,
|
|
0.16092905484967762,
|
|
0.18698191046714782,
|
|
0.16638216276963552,
|
|
0.1700377999080552
|
|
],
|
|
"val_acc": [
|
|
0.8472222222222222,
|
|
0.8888888888888888,
|
|
0.7659722222222223,
|
|
0.9166666666666666,
|
|
0.8326388888888889,
|
|
0.9305555555555556,
|
|
0.9423611111111111,
|
|
0.9333333333333333,
|
|
0.9409722222222222,
|
|
0.9458333333333333,
|
|
0.9472222222222222,
|
|
0.9430555555555555,
|
|
0.9472222222222222,
|
|
0.9493055555555555,
|
|
0.9527777777777777
|
|
],
|
|
"val_auc": [
|
|
0.9713064236111112,
|
|
0.9733832465277779,
|
|
0.9734754774305554,
|
|
0.9773155381944445,
|
|
0.975588107638889,
|
|
0.9780989583333333,
|
|
0.9797287326388889,
|
|
0.9842198350694443,
|
|
0.9822504340277778,
|
|
0.9792198350694445,
|
|
0.9828049045138889,
|
|
0.985851779513889,
|
|
0.9816916232638889,
|
|
0.9851215277777778,
|
|
0.9836208767361112
|
|
],
|
|
"val_f1": [
|
|
0.8722415795586528,
|
|
0.9222546161321672,
|
|
0.7879169288860919,
|
|
0.9405351833498513,
|
|
0.857312018946122,
|
|
0.9476987447698745,
|
|
0.9570615623383342,
|
|
0.9487179487179487,
|
|
0.9558899844317592,
|
|
0.9600818833162743,
|
|
0.9609856262833676,
|
|
0.9567053854276664,
|
|
0.96086508753862,
|
|
0.9619593538301199,
|
|
0.9646936656282451
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.830625,
|
|
"auc_roc": 0.9401379629629629,
|
|
"f1": 0.8747882334822116,
|
|
"confusion_matrix": [
|
|
[
|
|
1147,
|
|
53
|
|
],
|
|
[
|
|
760,
|
|
2840
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9183333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
98,
|
|
1102
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9183333333333333,
|
|
"pairwise_auc": 0.9851236111111111,
|
|
"pairwise_f1": 0.9358811040339703
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.4775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
627,
|
|
573
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.4775,
|
|
"pairwise_auc": 0.8418135416666667,
|
|
"pairwise_f1": 0.6276013143483024
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9708333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
35,
|
|
1165
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9708333333333333,
|
|
"pairwise_auc": 0.9934767361111112,
|
|
"pairwise_f1": 0.9636062861869313
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9558333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1147,
|
|
53
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.04416666666666669
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9370833333333334,
|
|
"auc_roc": 0.9851236111111111,
|
|
"f1": 0.9358811040339703,
|
|
"confusion_matrix": [
|
|
[
|
|
1147,
|
|
53
|
|
],
|
|
[
|
|
98,
|
|
1102
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.7166666666666667,
|
|
"auc_roc": 0.8418135416666667,
|
|
"f1": 0.6276013143483024,
|
|
"confusion_matrix": [
|
|
[
|
|
1147,
|
|
53
|
|
],
|
|
[
|
|
627,
|
|
573
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9633333333333334,
|
|
"auc_roc": 0.9934767361111112,
|
|
"f1": 0.9636062861869313,
|
|
"confusion_matrix": [
|
|
[
|
|
1147,
|
|
53
|
|
],
|
|
[
|
|
35,
|
|
1165
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.22282849704777752,
|
|
0.09587798762583623,
|
|
0.05643976701762711,
|
|
0.03555202028494225,
|
|
0.02578896039641997,
|
|
0.016046983599652002,
|
|
0.006410472841400159,
|
|
0.00932835557124898,
|
|
0.004159164803175732,
|
|
0.0026431137199796027,
|
|
0.0018033815937020267,
|
|
0.0008877858658257695,
|
|
0.0007059923287226726,
|
|
0.0004132254239917574
|
|
],
|
|
"train_acc": [
|
|
0.848533950617284,
|
|
0.9434413580246913,
|
|
0.970216049382716,
|
|
0.9804012345679012,
|
|
0.9868055555555556,
|
|
0.9925154320987655,
|
|
0.9974537037037037,
|
|
0.9964506172839506,
|
|
0.9983796296296297,
|
|
0.9986882716049382,
|
|
0.9993055555555556,
|
|
0.9998456790123457,
|
|
0.9996141975308642,
|
|
0.9999228395061729
|
|
],
|
|
"train_auc": [
|
|
0.9312621500985939,
|
|
0.9873819149734224,
|
|
0.9956183020404664,
|
|
0.9981948731138546,
|
|
0.9990388963906037,
|
|
0.9995547732338821,
|
|
0.9999329266332304,
|
|
0.9997190473894034,
|
|
0.9999515335648148,
|
|
0.9999911051097393,
|
|
0.9999951774691358,
|
|
0.9999993302040466,
|
|
0.9999996249142662,
|
|
1.0000000000000002
|
|
],
|
|
"train_f1": [
|
|
0.881839523264913,
|
|
0.9571069108783428,
|
|
0.9775581395348837,
|
|
0.9852548473238129,
|
|
0.9900806311270955,
|
|
0.9943849493487699,
|
|
0.9980899461712103,
|
|
0.9973385790326313,
|
|
0.9987843704775687,
|
|
0.9990158050136051,
|
|
0.9994790762285118,
|
|
0.999884272653628,
|
|
0.9997105979047288,
|
|
0.9999421262804561
|
|
],
|
|
"val_loss": [
|
|
0.15792499168051613,
|
|
0.542742556002405,
|
|
0.1823839321732521,
|
|
0.208244545923339,
|
|
0.1911680911978086,
|
|
0.15773669332265855,
|
|
0.21006372471650442,
|
|
0.19471206135219998,
|
|
0.1356222172578176,
|
|
0.16341885344849694,
|
|
0.14434544261958865,
|
|
0.15009288258022732,
|
|
0.14751002705759472,
|
|
0.15342201209730572
|
|
],
|
|
"val_acc": [
|
|
0.9013888888888889,
|
|
0.85,
|
|
0.9256944444444445,
|
|
0.9298611111111111,
|
|
0.9215277777777777,
|
|
0.9430555555555555,
|
|
0.9152777777777777,
|
|
0.9256944444444445,
|
|
0.9444444444444444,
|
|
0.9381944444444444,
|
|
0.9479166666666666,
|
|
0.9465277777777777,
|
|
0.9493055555555555,
|
|
0.95
|
|
],
|
|
"val_auc": [
|
|
0.9667914496527777,
|
|
0.9641232638888889,
|
|
0.9759342447916668,
|
|
0.9801529947916667,
|
|
0.9824077690972222,
|
|
0.9852582465277778,
|
|
0.9809787326388888,
|
|
0.9832606336805556,
|
|
0.9891286892361111,
|
|
0.9856673177083333,
|
|
0.9885264756944444,
|
|
0.9885514322916666,
|
|
0.9879828559027778,
|
|
0.9884016927083332
|
|
],
|
|
"val_f1": [
|
|
0.9234088457389428,
|
|
0.8984962406015038,
|
|
0.9440083725798012,
|
|
0.9485481406011207,
|
|
0.9391491653204093,
|
|
0.9571577847439916,
|
|
0.9343379978471474,
|
|
0.9425040300913488,
|
|
0.9576271186440678,
|
|
0.9529846804014791,
|
|
0.9607535321821036,
|
|
0.9595375722543352,
|
|
0.9618400418191323,
|
|
0.9625779625779626
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8233333333333334,
|
|
"auc_roc": 0.9446028935185184,
|
|
"f1": 0.8682820751786269,
|
|
"confusion_matrix": [
|
|
[
|
|
1157,
|
|
43
|
|
],
|
|
[
|
|
805,
|
|
2795
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9216666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
94,
|
|
1106
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9216666666666666,
|
|
"pairwise_auc": 0.9862177083333332,
|
|
"pairwise_f1": 0.9416773094934014
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.42833333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
686,
|
|
514
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.42833333333333334,
|
|
"pairwise_auc": 0.8534635416666667,
|
|
"pairwise_f1": 0.585088218554354
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9791666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
25,
|
|
1175
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9791666666666666,
|
|
"pairwise_auc": 0.9941274305555554,
|
|
"pairwise_f1": 0.9718775847808105
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9641666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1157,
|
|
43
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.035833333333333384
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9429166666666666,
|
|
"auc_roc": 0.9862177083333332,
|
|
"f1": 0.9416773094934014,
|
|
"confusion_matrix": [
|
|
[
|
|
1157,
|
|
43
|
|
],
|
|
[
|
|
94,
|
|
1106
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.69625,
|
|
"auc_roc": 0.8534635416666667,
|
|
"f1": 0.585088218554354,
|
|
"confusion_matrix": [
|
|
[
|
|
1157,
|
|
43
|
|
],
|
|
[
|
|
686,
|
|
514
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9716666666666667,
|
|
"auc_roc": 0.9941274305555554,
|
|
"f1": 0.9718775847808105,
|
|
"confusion_matrix": [
|
|
[
|
|
1157,
|
|
43
|
|
],
|
|
[
|
|
25,
|
|
1175
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.8302916666666667,
|
|
"std": 0.013218581723383837,
|
|
"ci_95": 0.011586597741452051,
|
|
"values": [
|
|
0.8404166666666667,
|
|
0.845,
|
|
0.8120833333333334,
|
|
0.830625,
|
|
0.8233333333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9401337037037036,
|
|
"std": 0.003035763996338149,
|
|
"ci_95": 0.0026609644665077413,
|
|
"values": [
|
|
0.9406697916666668,
|
|
0.9390527777777778,
|
|
0.9362050925925925,
|
|
0.9401379629629629,
|
|
0.9446028935185184
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8747551774642861,
|
|
"std": 0.011370726815842292,
|
|
"ci_95": 0.009966881500610814,
|
|
"values": [
|
|
0.8836926814454904,
|
|
0.8875113395827033,
|
|
0.8595015576323988,
|
|
0.8747882334822116,
|
|
0.8682820751786269
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9275,
|
|
"std": 0.016666666666666677,
|
|
"ci_95": 0.014608977452998634,
|
|
"values": [
|
|
0.94,
|
|
0.9491666666666667,
|
|
0.9083333333333333,
|
|
0.9183333333333333,
|
|
0.9216666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9275,
|
|
"std": 0.016666666666666677,
|
|
"ci_95": 0.014608977452998634,
|
|
"values": [
|
|
0.94,
|
|
0.9491666666666667,
|
|
0.9083333333333333,
|
|
0.9183333333333333,
|
|
0.9216666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9846677777777778,
|
|
"std": 0.001795447994153717,
|
|
"ci_95": 0.0015737795558773955,
|
|
"values": [
|
|
0.9856604166666667,
|
|
0.9847184027777777,
|
|
0.98161875,
|
|
0.9851236111111111,
|
|
0.9862177083333332
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9370720514179574,
|
|
"std": 0.006040586678143011,
|
|
"ci_95": 0.005294807675032507,
|
|
"values": [
|
|
0.9384359400998337,
|
|
0.9421009098428453,
|
|
0.9272649936197362,
|
|
0.9358811040339703,
|
|
0.9416773094934014
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.4691666666666666,
|
|
"std": 0.041428485905768316,
|
|
"ci_95": 0.03631366899055444,
|
|
"values": [
|
|
0.5008333333333334,
|
|
0.515,
|
|
0.4241666666666667,
|
|
0.4775,
|
|
0.42833333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.4691666666666666,
|
|
"std": 0.041428485905768316,
|
|
"ci_95": 0.03631366899055444,
|
|
"values": [
|
|
0.5008333333333334,
|
|
0.515,
|
|
0.4241666666666667,
|
|
0.4775,
|
|
0.42833333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.8421123611111112,
|
|
"std": 0.007000441060805703,
|
|
"ci_95": 0.006136157137101378,
|
|
"values": [
|
|
0.8413416666666667,
|
|
0.839582638888889,
|
|
0.8343604166666666,
|
|
0.8418135416666667,
|
|
0.8534635416666667
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.6159538907312784,
|
|
"std": 0.03397896348215447,
|
|
"ci_95": 0.029783874683223497,
|
|
"values": [
|
|
0.6403835908364411,
|
|
0.6515550869794412,
|
|
0.5751412429378531,
|
|
0.6276013143483024,
|
|
0.585088218554354
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9765,
|
|
"std": 0.007439571373555206,
|
|
"ci_95": 0.006521071827374521,
|
|
"values": [
|
|
0.9841666666666666,
|
|
0.9816666666666667,
|
|
0.9666666666666667,
|
|
0.9708333333333333,
|
|
0.9791666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9765,
|
|
"std": 0.007439571373555206,
|
|
"ci_95": 0.006521071827374521,
|
|
"values": [
|
|
0.9841666666666666,
|
|
0.9816666666666667,
|
|
0.9666666666666667,
|
|
0.9708333333333333,
|
|
0.9791666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9936209722222223,
|
|
"std": 0.0009685485403824886,
|
|
"ci_95": 0.0008489702273149503,
|
|
"values": [
|
|
0.9950072916666667,
|
|
0.9928572916666668,
|
|
0.9926361111111112,
|
|
0.9934767361111112,
|
|
0.9941274305555554
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9627986985943575,
|
|
"std": 0.005497248442216119,
|
|
"ci_95": 0.004818550712752026,
|
|
"values": [
|
|
0.9613349613349613,
|
|
0.9588929588929589,
|
|
0.9582817017761256,
|
|
0.9636062861869313,
|
|
0.9718775847808105
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9480000000000001,
|
|
"std": 0.01268748289544548,
|
|
"ci_95": 0.011121069093292123,
|
|
"values": [
|
|
0.9366666666666666,
|
|
0.9341666666666667,
|
|
0.9491666666666667,
|
|
0.9558333333333333,
|
|
0.9641666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.052000000000000005,
|
|
"std": 0.01268748289544548,
|
|
"ci_95": 0.011121069093292123,
|
|
"values": [
|
|
0.06333333333333335,
|
|
0.0658333333333333,
|
|
0.050833333333333286,
|
|
0.04416666666666669,
|
|
0.035833333333333384
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.93775,
|
|
"std": 0.005563709294266832,
|
|
"ci_95": 0.004876806218098983,
|
|
"values": [
|
|
0.9383333333333334,
|
|
0.9416666666666667,
|
|
0.92875,
|
|
0.9370833333333334,
|
|
0.9429166666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9846677777777778,
|
|
"std": 0.001795447994153717,
|
|
"ci_95": 0.0015737795558773955,
|
|
"values": [
|
|
0.9856604166666667,
|
|
0.9847184027777777,
|
|
0.98161875,
|
|
0.9851236111111111,
|
|
0.9862177083333332
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9370720514179574,
|
|
"std": 0.006040586678143011,
|
|
"ci_95": 0.005294807675032507,
|
|
"values": [
|
|
0.9384359400998337,
|
|
0.9421009098428453,
|
|
0.9272649936197362,
|
|
0.9358811040339703,
|
|
0.9416773094934014
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.7085833333333333,
|
|
"std": 0.01625694296122259,
|
|
"ci_95": 0.014249838790511133,
|
|
"values": [
|
|
0.71875,
|
|
0.7245833333333334,
|
|
0.6866666666666666,
|
|
0.7166666666666667,
|
|
0.69625
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.8421123611111112,
|
|
"std": 0.007000441060805703,
|
|
"ci_95": 0.006136157137101378,
|
|
"values": [
|
|
0.8413416666666667,
|
|
0.839582638888889,
|
|
0.8343604166666666,
|
|
0.8418135416666667,
|
|
0.8534635416666667
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.6159538907312784,
|
|
"std": 0.03397896348215447,
|
|
"ci_95": 0.029783874683223497,
|
|
"values": [
|
|
0.6403835908364411,
|
|
0.6515550869794412,
|
|
0.5751412429378531,
|
|
0.6276013143483024,
|
|
0.585088218554354
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.96225,
|
|
"std": 0.005717601964304825,
|
|
"ci_95": 0.005011699090904991,
|
|
"values": [
|
|
0.9604166666666667,
|
|
0.9579166666666666,
|
|
0.9579166666666666,
|
|
0.9633333333333334,
|
|
0.9716666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9936209722222223,
|
|
"std": 0.0009685485403824886,
|
|
"ci_95": 0.0008489702273149503,
|
|
"values": [
|
|
0.9950072916666667,
|
|
0.9928572916666668,
|
|
0.9926361111111112,
|
|
0.9934767361111112,
|
|
0.9941274305555554
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9627986985943575,
|
|
"std": 0.005497248442216119,
|
|
"ci_95": 0.004818550712752026,
|
|
"values": [
|
|
0.9613349613349613,
|
|
0.9588929588929589,
|
|
0.9582817017761256,
|
|
0.9636062861869313,
|
|
0.9718775847808105
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "data",
|
|
"run_name": "p2a_t3_holdout_insight",
|
|
"backbone": "resnet18",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"normalization": "imagenet",
|
|
"train_sources": [
|
|
"wiki",
|
|
"text2img",
|
|
"inpainting"
|
|
],
|
|
"eval_sources": [
|
|
"wiki",
|
|
"text2img",
|
|
"inpainting",
|
|
"insight"
|
|
]
|
|
}
|
|
} |