Johnny Fernandes
1609 lines
37 KiB
JSON
1609 lines
37 KiB
JSON
{
|
|
"run_name": "p3_convnext_tiny",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16958975876784987,
|
|
0.08378792049371879,
|
|
0.05077075856464001,
|
|
0.03812734988927399,
|
|
0.021703218209529074,
|
|
0.02057555293560856,
|
|
0.011910638541006914,
|
|
0.007836313261347844
|
|
],
|
|
"train_acc": [
|
|
0.8527199074074074,
|
|
0.9425925925925925,
|
|
0.9688657407407407,
|
|
0.9814814814814815,
|
|
0.9889467592592592,
|
|
0.9912615740740741,
|
|
0.9951967592592592,
|
|
0.9969328703703704
|
|
],
|
|
"train_auc": [
|
|
0.929881392532579,
|
|
0.9844190904349566,
|
|
0.9947031732253087,
|
|
0.9975638896033379,
|
|
0.9991185753172153,
|
|
0.9989822584447874,
|
|
0.9995797789494741,
|
|
0.9996784890117741
|
|
],
|
|
"train_f1": [
|
|
0.897556655798414,
|
|
0.9611680889376027,
|
|
0.9791327282600264,
|
|
0.9876275904732447,
|
|
0.9926246283353284,
|
|
0.9941719093751206,
|
|
0.9967979630415493,
|
|
0.9979557989740425
|
|
],
|
|
"val_loss": [
|
|
0.11805321856712302,
|
|
0.13345237011671998,
|
|
0.15431126450324276,
|
|
0.16942790824978146,
|
|
0.21927034670079593,
|
|
0.4993017079603002,
|
|
0.2872419604177594,
|
|
0.31290652627665927
|
|
],
|
|
"val_acc": [
|
|
0.8984375,
|
|
0.940625,
|
|
0.9385416666666667,
|
|
0.9322916666666666,
|
|
0.9515625,
|
|
0.91875,
|
|
0.9432291666666667,
|
|
0.9505208333333334
|
|
],
|
|
"val_auc": [
|
|
0.9708716724537036,
|
|
0.9769820601851852,
|
|
0.9798914930555556,
|
|
0.9772757523148148,
|
|
0.9773314525462964,
|
|
0.944581886574074,
|
|
0.9660214120370371,
|
|
0.9617874710648149
|
|
],
|
|
"val_f1": [
|
|
0.9295774647887324,
|
|
0.9607167470709855,
|
|
0.9591695501730104,
|
|
0.9542897327707455,
|
|
0.9681397738951696,
|
|
0.9484467944481163,
|
|
0.9628620102214651,
|
|
0.9677856900644286
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9477083333333334,
|
|
"auc_roc": 0.9851297453703705,
|
|
"f1": 0.9652210059581544,
|
|
"confusion_matrix": [
|
|
[
|
|
1066,
|
|
134
|
|
],
|
|
[
|
|
117,
|
|
3483
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9758333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
29,
|
|
1171
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9758333333333333,
|
|
"pairwise_auc": 0.9876138888888889,
|
|
"pairwise_f1": 0.9349301397205588
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9558333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9558333333333333,
|
|
"pairwise_auc": 0.9802277777777777,
|
|
"pairwise_f1": 0.924627166465135
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9708333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
35,
|
|
1165
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9708333333333333,
|
|
"pairwise_auc": 0.9875475694444444,
|
|
"pairwise_f1": 0.9323729491796718
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8883333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1066,
|
|
134
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.11166666666666669
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9320833333333334,
|
|
"auc_roc": 0.9876138888888889,
|
|
"f1": 0.9349301397205588,
|
|
"confusion_matrix": [
|
|
[
|
|
1066,
|
|
134
|
|
],
|
|
[
|
|
29,
|
|
1171
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9220833333333334,
|
|
"auc_roc": 0.9802277777777777,
|
|
"f1": 0.924627166465135,
|
|
"confusion_matrix": [
|
|
[
|
|
1066,
|
|
134
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9295833333333333,
|
|
"auc_roc": 0.9875475694444444,
|
|
"f1": 0.9323729491796718,
|
|
"confusion_matrix": [
|
|
[
|
|
1066,
|
|
134
|
|
],
|
|
[
|
|
35,
|
|
1165
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17271623247889456,
|
|
0.08797129855525715,
|
|
0.052734318907163534,
|
|
0.03738149641072529,
|
|
0.019922682597563027,
|
|
0.014144108878429472,
|
|
0.010534037952983353,
|
|
0.008255647509284043
|
|
],
|
|
"train_acc": [
|
|
0.8546875,
|
|
0.9425347222222222,
|
|
0.9684606481481481,
|
|
0.9811342592592592,
|
|
0.9903356481481481,
|
|
0.9936921296296296,
|
|
0.9956597222222222,
|
|
0.9969907407407408
|
|
],
|
|
"train_auc": [
|
|
0.9274318772862368,
|
|
0.9826133294753086,
|
|
0.9941157264517604,
|
|
0.9972319030064015,
|
|
0.9990582400977366,
|
|
0.9994191797410836,
|
|
0.9995071284150663,
|
|
0.9997892821930727
|
|
],
|
|
"train_f1": [
|
|
0.8992173389524383,
|
|
0.9611881962087161,
|
|
0.9788587610070212,
|
|
0.98738682968351,
|
|
0.9935473899772034,
|
|
0.9957916682753561,
|
|
0.9971052530008877,
|
|
0.9979935175181355
|
|
],
|
|
"val_loss": [
|
|
0.09013549944696327,
|
|
0.13632765178869402,
|
|
0.1426052969521455,
|
|
0.14648840576519434,
|
|
0.17203345553005722,
|
|
0.263461171353462,
|
|
0.2058163796753585,
|
|
0.29349427083011503
|
|
],
|
|
"val_acc": [
|
|
0.9380208333333333,
|
|
0.9473958333333333,
|
|
0.9557291666666666,
|
|
0.9380208333333333,
|
|
0.946875,
|
|
0.95,
|
|
0.95625,
|
|
0.9520833333333333
|
|
],
|
|
"val_auc": [
|
|
0.9806344039351851,
|
|
0.9816124131944446,
|
|
0.9863541666666669,
|
|
0.9824168113425925,
|
|
0.9814496527777778,
|
|
0.9686270254629631,
|
|
0.9742064525462963,
|
|
0.9637261284722223
|
|
],
|
|
"val_f1": [
|
|
0.9581428068941259,
|
|
0.9655407710678949,
|
|
0.9709996588195156,
|
|
0.9579950582421461,
|
|
0.9645340751043116,
|
|
0.9671232876712329,
|
|
0.970873786407767,
|
|
0.9687287559483345
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9495833333333333,
|
|
"auc_roc": 0.987876736111111,
|
|
"f1": 0.9668038408779149,
|
|
"confusion_matrix": [
|
|
[
|
|
1034,
|
|
166
|
|
],
|
|
[
|
|
76,
|
|
3524
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9858333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9858333333333333,
|
|
"pairwise_auc": 0.9906434027777776,
|
|
"pairwise_f1": 0.9282071400549236
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9558333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9558333333333333,
|
|
"pairwise_auc": 0.9770951388888889,
|
|
"pairwise_f1": 0.9128531635495424
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.995,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
6,
|
|
1194
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.995,
|
|
"pairwise_auc": 0.9958916666666666,
|
|
"pairwise_f1": 0.9328125
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8616666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1034,
|
|
166
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1383333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.92375,
|
|
"auc_roc": 0.9906434027777776,
|
|
"f1": 0.9282071400549236,
|
|
"confusion_matrix": [
|
|
[
|
|
1034,
|
|
166
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90875,
|
|
"auc_roc": 0.9770951388888889,
|
|
"f1": 0.9128531635495424,
|
|
"confusion_matrix": [
|
|
[
|
|
1034,
|
|
166
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9283333333333333,
|
|
"auc_roc": 0.9958916666666666,
|
|
"f1": 0.9328125,
|
|
"confusion_matrix": [
|
|
[
|
|
1034,
|
|
166
|
|
],
|
|
[
|
|
6,
|
|
1194
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17408299346075015,
|
|
0.08351437987078464,
|
|
0.0621230011564438,
|
|
0.03621822881606123,
|
|
0.024109217247024826,
|
|
0.012347555377165783,
|
|
0.010925444532212909,
|
|
0.006826932699009881
|
|
],
|
|
"train_acc": [
|
|
0.8456597222222222,
|
|
0.9428240740740741,
|
|
0.9659143518518518,
|
|
0.9802662037037037,
|
|
0.9885416666666667,
|
|
0.9946759259259259,
|
|
0.9955439814814815,
|
|
0.997337962962963
|
|
],
|
|
"train_auc": [
|
|
0.9258716456618656,
|
|
0.9845440029149519,
|
|
0.9927531918009831,
|
|
0.9975568165580704,
|
|
0.9990276706104252,
|
|
0.9997175068587106,
|
|
0.9996857853223594,
|
|
0.9997132648176726
|
|
],
|
|
"train_f1": [
|
|
0.8922206506364923,
|
|
0.961400218784185,
|
|
0.9771714274640518,
|
|
0.9868059585993423,
|
|
0.9923563928350834,
|
|
0.9964481507219519,
|
|
0.9970285185042257,
|
|
0.9982250347275814
|
|
],
|
|
"val_loss": [
|
|
0.13286577961407603,
|
|
0.10000402753745827,
|
|
0.11289124916474975,
|
|
0.18120637619223695,
|
|
0.19875025467463275,
|
|
0.2659189359733015,
|
|
0.24043363304344895,
|
|
0.26855370177936494
|
|
],
|
|
"val_acc": [
|
|
0.8526041666666667,
|
|
0.928125,
|
|
0.9520833333333333,
|
|
0.9484375,
|
|
0.9458333333333333,
|
|
0.9473958333333333,
|
|
0.9494791666666667,
|
|
0.953125
|
|
],
|
|
"val_auc": [
|
|
0.9709157986111111,
|
|
0.983420138888889,
|
|
0.9864691840277777,
|
|
0.9856539351851853,
|
|
0.9785988136574074,
|
|
0.972962962962963,
|
|
0.9746274594907407,
|
|
0.9746477141203704
|
|
],
|
|
"val_f1": [
|
|
0.8925996204933586,
|
|
0.9507142857142857,
|
|
0.9681219681219682,
|
|
0.9662000682826903,
|
|
0.9640635798203179,
|
|
0.9656111678583589,
|
|
0.9668376068376069,
|
|
0.9693669162695712
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9491666666666667,
|
|
"auc_roc": 0.9871033564814815,
|
|
"f1": 0.9660922734852696,
|
|
"confusion_matrix": [
|
|
[
|
|
1080,
|
|
120
|
|
],
|
|
[
|
|
124,
|
|
3476
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9733333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9733333333333334,
|
|
"pairwise_auc": 0.9887680555555555,
|
|
"pairwise_f1": 0.9389067524115756
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9358333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
77,
|
|
1123
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9358333333333333,
|
|
"pairwise_auc": 0.9784319444444445,
|
|
"pairwise_f1": 0.9193614408514122
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9875,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9875,
|
|
"pairwise_auc": 0.9941100694444444,
|
|
"pairwise_f1": 0.9461077844311377
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1080,
|
|
120
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.09999999999999998
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9366666666666666,
|
|
"auc_roc": 0.9887680555555555,
|
|
"f1": 0.9389067524115756,
|
|
"confusion_matrix": [
|
|
[
|
|
1080,
|
|
120
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9179166666666667,
|
|
"auc_roc": 0.9784319444444445,
|
|
"f1": 0.9193614408514122,
|
|
"confusion_matrix": [
|
|
[
|
|
1080,
|
|
120
|
|
],
|
|
[
|
|
77,
|
|
1123
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.94375,
|
|
"auc_roc": 0.9941100694444444,
|
|
"f1": 0.9461077844311377,
|
|
"confusion_matrix": [
|
|
[
|
|
1080,
|
|
120
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17817864065506944,
|
|
0.08580982140480989,
|
|
0.057252798737371685,
|
|
0.03642915870918336,
|
|
0.024854684943626058,
|
|
0.017833184340997833,
|
|
0.011539986929887568
|
|
],
|
|
"train_acc": [
|
|
0.8463541666666666,
|
|
0.9409143518518519,
|
|
0.9662615740740741,
|
|
0.9804976851851852,
|
|
0.9876736111111111,
|
|
0.9920138888888889,
|
|
0.9961226851851852
|
|
],
|
|
"train_auc": [
|
|
0.9228431855852767,
|
|
0.9834737404263831,
|
|
0.9933516053669411,
|
|
0.9976213580961364,
|
|
0.9989252364826247,
|
|
0.998934515389232,
|
|
0.9994972065043439
|
|
],
|
|
"train_f1": [
|
|
0.8930944231930743,
|
|
0.9600094003368455,
|
|
0.9773513072530204,
|
|
0.9869718173734875,
|
|
0.9917770142454542,
|
|
0.9946714031971581,
|
|
0.9974156219864995
|
|
],
|
|
"val_loss": [
|
|
0.1269730476352076,
|
|
0.11733529190193319,
|
|
0.15899523090483852,
|
|
0.17420920413872712,
|
|
0.17963087618756315,
|
|
0.23321131139291537,
|
|
0.19198723134656273
|
|
],
|
|
"val_acc": [
|
|
0.8635416666666667,
|
|
0.9515625,
|
|
0.9442708333333333,
|
|
0.953125,
|
|
0.9484375,
|
|
0.9484375,
|
|
0.9583333333333334
|
|
],
|
|
"val_auc": [
|
|
0.9744270833333335,
|
|
0.9874949363425927,
|
|
0.985044849537037,
|
|
0.9845326967592594,
|
|
0.9791999421296296,
|
|
0.9724406828703704,
|
|
0.9773676215277778
|
|
],
|
|
"val_f1": [
|
|
0.9009077155824509,
|
|
0.9680741503604532,
|
|
0.9634437991117185,
|
|
0.9691569568197396,
|
|
0.9657320872274143,
|
|
0.9660610215975317,
|
|
0.9724137931034482
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9414583333333333,
|
|
"auc_roc": 0.9857365740740741,
|
|
"f1": 0.9615752769041434,
|
|
"confusion_matrix": [
|
|
[
|
|
1003,
|
|
197
|
|
],
|
|
[
|
|
84,
|
|
3516
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.98,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
24,
|
|
1176
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.98,
|
|
"pairwise_auc": 0.9857819444444444,
|
|
"pairwise_f1": 0.91410804508356
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9625,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
45,
|
|
1155
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9625,
|
|
"pairwise_auc": 0.9810975694444444,
|
|
"pairwise_f1": 0.9051724137931034
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9875,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9875,
|
|
"pairwise_auc": 0.9903302083333333,
|
|
"pairwise_f1": 0.9178931061192874
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8358333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1003,
|
|
197
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.16416666666666668
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9079166666666667,
|
|
"auc_roc": 0.9857819444444444,
|
|
"f1": 0.91410804508356,
|
|
"confusion_matrix": [
|
|
[
|
|
1003,
|
|
197
|
|
],
|
|
[
|
|
24,
|
|
1176
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8991666666666667,
|
|
"auc_roc": 0.9810975694444444,
|
|
"f1": 0.9051724137931034,
|
|
"confusion_matrix": [
|
|
[
|
|
1003,
|
|
197
|
|
],
|
|
[
|
|
45,
|
|
1155
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9116666666666666,
|
|
"auc_roc": 0.9903302083333333,
|
|
"f1": 0.9178931061192874,
|
|
"confusion_matrix": [
|
|
[
|
|
1003,
|
|
197
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17451777354538164,
|
|
0.08983556660823525,
|
|
0.056572918070930796,
|
|
0.036605554571077746,
|
|
0.021891540394473715,
|
|
0.014295070744581742,
|
|
0.009600461349093705,
|
|
0.008624833609823563,
|
|
0.004932643632275391
|
|
],
|
|
"train_acc": [
|
|
0.8482060185185185,
|
|
0.9378472222222223,
|
|
0.9666666666666667,
|
|
0.9800347222222222,
|
|
0.9898148148148148,
|
|
0.9933449074074074,
|
|
0.9960069444444445,
|
|
0.9964120370370371,
|
|
0.9979166666666667
|
|
],
|
|
"train_auc": [
|
|
0.9259183170438958,
|
|
0.9820589974136946,
|
|
0.9931390032007317,
|
|
0.9975000357224508,
|
|
0.9990783250457248,
|
|
0.9994069983853452,
|
|
0.9995170860482396,
|
|
0.9996522687328533,
|
|
0.9996805698445359
|
|
],
|
|
"train_f1": [
|
|
0.894024483859238,
|
|
0.9579581930634933,
|
|
0.9776622973706662,
|
|
0.98665532046571,
|
|
0.9932072558857584,
|
|
0.9955610452773381,
|
|
0.9973366271664029,
|
|
0.9976071015052104,
|
|
0.9986105750675415
|
|
],
|
|
"val_loss": [
|
|
0.13483744577582304,
|
|
0.09238062916168323,
|
|
0.13326148290313236,
|
|
0.10406262401803966,
|
|
0.16612004347983506,
|
|
0.15045142767457947,
|
|
0.14117707508952057,
|
|
0.19352938434567477,
|
|
0.19165282325910388
|
|
],
|
|
"val_acc": [
|
|
0.9192708333333334,
|
|
0.9234375,
|
|
0.946875,
|
|
0.9515625,
|
|
0.9536458333333333,
|
|
0.9453125,
|
|
0.9489583333333333,
|
|
0.9578125,
|
|
0.9578125
|
|
],
|
|
"val_auc": [
|
|
0.9759027777777778,
|
|
0.9818062789351851,
|
|
0.9858745659722222,
|
|
0.9883492476851852,
|
|
0.9827698206018518,
|
|
0.9857269965277778,
|
|
0.9870688657407407,
|
|
0.9748444733796298,
|
|
0.9744748263888889
|
|
],
|
|
"val_f1": [
|
|
0.9467902506007553,
|
|
0.9469888207717274,
|
|
0.9652825051055139,
|
|
0.9672419866150053,
|
|
0.9689787382363193,
|
|
0.9628844114528102,
|
|
0.9652235628105039,
|
|
0.9719237435008665,
|
|
0.9720013826477705
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.94875,
|
|
"auc_roc": 0.9880292824074075,
|
|
"f1": 0.9654202980039359,
|
|
"confusion_matrix": [
|
|
[
|
|
1120,
|
|
80
|
|
],
|
|
[
|
|
166,
|
|
3434
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9616666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9616666666666667,
|
|
"pairwise_auc": 0.9893652777777777,
|
|
"pairwise_f1": 0.9482333607230896
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9158333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
101,
|
|
1099
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9158333333333334,
|
|
"pairwise_auc": 0.9788697916666665,
|
|
"pairwise_f1": 0.9239176124422026
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9841666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9841666666666666,
|
|
"pairwise_auc": 0.9958527777777777,
|
|
"pairwise_f1": 0.9597724502234863
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9333333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1120,
|
|
80
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.06666666666666665
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9475,
|
|
"auc_roc": 0.9893652777777777,
|
|
"f1": 0.9482333607230896,
|
|
"confusion_matrix": [
|
|
[
|
|
1120,
|
|
80
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9245833333333333,
|
|
"auc_roc": 0.9788697916666665,
|
|
"f1": 0.9239176124422026,
|
|
"confusion_matrix": [
|
|
[
|
|
1120,
|
|
80
|
|
],
|
|
[
|
|
101,
|
|
1099
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.95875,
|
|
"auc_roc": 0.9958527777777777,
|
|
"f1": 0.9597724502234863,
|
|
"confusion_matrix": [
|
|
[
|
|
1120,
|
|
80
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9473333333333332,
|
|
"std": 0.0033573354596895836,
|
|
"ci_95": 0.002942834281965474,
|
|
"values": [
|
|
0.9477083333333334,
|
|
0.9495833333333333,
|
|
0.9491666666666667,
|
|
0.9414583333333333,
|
|
0.94875
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9867751388888889,
|
|
"std": 0.0012922902052645558,
|
|
"ci_95": 0.0011327423082864517,
|
|
"values": [
|
|
0.9851297453703705,
|
|
0.987876736111111,
|
|
0.9871033564814815,
|
|
0.9857365740740741,
|
|
0.9880292824074075
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9650225390458838,
|
|
"std": 0.0020247583538202415,
|
|
"ci_95": 0.0017747789483238312,
|
|
"values": [
|
|
0.9652210059581544,
|
|
0.9668038408779149,
|
|
0.9660922734852696,
|
|
0.9615752769041434,
|
|
0.9654202980039359
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9753333333333334,
|
|
"std": 0.008986873143028604,
|
|
"ci_95": 0.007877341627167826,
|
|
"values": [
|
|
0.9758333333333333,
|
|
0.9858333333333333,
|
|
0.9733333333333334,
|
|
0.98,
|
|
0.9616666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9753333333333334,
|
|
"std": 0.008986873143028604,
|
|
"ci_95": 0.007877341627167826,
|
|
"values": [
|
|
0.9758333333333333,
|
|
0.9858333333333333,
|
|
0.9733333333333334,
|
|
0.98,
|
|
0.9616666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9884345138888889,
|
|
"std": 0.0018416237150910859,
|
|
"ci_95": 0.0016142543598403943,
|
|
"values": [
|
|
0.9876138888888889,
|
|
0.9906434027777776,
|
|
0.9887680555555555,
|
|
0.9857819444444444,
|
|
0.9893652777777777
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9328770875987414,
|
|
"std": 0.01275217993166566,
|
|
"ci_95": 0.011177778545897112,
|
|
"values": [
|
|
0.9349301397205588,
|
|
0.9282071400549236,
|
|
0.9389067524115756,
|
|
0.91410804508356,
|
|
0.9482333607230896
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9451666666666666,
|
|
"std": 0.019206480387850562,
|
|
"ci_95": 0.01683522233625415,
|
|
"values": [
|
|
0.9558333333333333,
|
|
0.9558333333333333,
|
|
0.9358333333333333,
|
|
0.9625,
|
|
0.9158333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9451666666666666,
|
|
"std": 0.019206480387850562,
|
|
"ci_95": 0.01683522233625415,
|
|
"values": [
|
|
0.9558333333333333,
|
|
0.9558333333333333,
|
|
0.9358333333333333,
|
|
0.9625,
|
|
0.9158333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9791444444444444,
|
|
"std": 0.0015629341025062142,
|
|
"ci_95": 0.0013699721438421555,
|
|
"values": [
|
|
0.9802277777777777,
|
|
0.9770951388888889,
|
|
0.9784319444444445,
|
|
0.9810975694444444,
|
|
0.9788697916666665
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.917186359420279,
|
|
"std": 0.00819325686071929,
|
|
"ci_95": 0.007181706284692464,
|
|
"values": [
|
|
0.924627166465135,
|
|
0.9128531635495424,
|
|
0.9193614408514122,
|
|
0.9051724137931034,
|
|
0.9239176124422026
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.985,
|
|
"std": 0.00885845484394555,
|
|
"ci_95": 0.00776477802501642,
|
|
"values": [
|
|
0.9708333333333333,
|
|
0.995,
|
|
0.9875,
|
|
0.9875,
|
|
0.9841666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.985,
|
|
"std": 0.00885845484394555,
|
|
"ci_95": 0.00776477802501642,
|
|
"values": [
|
|
0.9708333333333333,
|
|
0.995,
|
|
0.9875,
|
|
0.9875,
|
|
0.9841666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9927464583333332,
|
|
"std": 0.0036833307262198267,
|
|
"ci_95": 0.00322858173187695,
|
|
"values": [
|
|
0.9875475694444444,
|
|
0.9958916666666666,
|
|
0.9941100694444444,
|
|
0.9903302083333333,
|
|
0.9958527777777777
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9377917579907168,
|
|
"std": 0.01583050830010387,
|
|
"ci_95": 0.013876052329543502,
|
|
"values": [
|
|
0.9323729491796718,
|
|
0.9328125,
|
|
0.9461077844311377,
|
|
0.9178931061192874,
|
|
0.9597724502234863
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.8838333333333332,
|
|
"std": 0.03717264525905516,
|
|
"ci_95": 0.03258326018747119,
|
|
"values": [
|
|
0.8883333333333333,
|
|
0.8616666666666667,
|
|
0.9,
|
|
0.8358333333333333,
|
|
0.9333333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.11616666666666667,
|
|
"std": 0.03717264525905516,
|
|
"ci_95": 0.03258326018747119,
|
|
"values": [
|
|
0.11166666666666669,
|
|
0.1383333333333333,
|
|
0.09999999999999998,
|
|
0.16416666666666668,
|
|
0.06666666666666665
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9295833333333332,
|
|
"std": 0.014840003182090097,
|
|
"ci_95": 0.013007836313374925,
|
|
"values": [
|
|
0.9320833333333334,
|
|
0.92375,
|
|
0.9366666666666666,
|
|
0.9079166666666667,
|
|
0.9475
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9884345138888889,
|
|
"std": 0.0018416237150910859,
|
|
"ci_95": 0.0016142543598403943,
|
|
"values": [
|
|
0.9876138888888889,
|
|
0.9906434027777776,
|
|
0.9887680555555555,
|
|
0.9857819444444444,
|
|
0.9893652777777777
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9328770875987414,
|
|
"std": 0.01275217993166566,
|
|
"ci_95": 0.011177778545897112,
|
|
"values": [
|
|
0.9349301397205588,
|
|
0.9282071400549236,
|
|
0.9389067524115756,
|
|
0.91410804508356,
|
|
0.9482333607230896
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9145000000000001,
|
|
"std": 0.010476494854885611,
|
|
"ci_95": 0.0091830526272888,
|
|
"values": [
|
|
0.9220833333333334,
|
|
0.90875,
|
|
0.9179166666666667,
|
|
0.8991666666666667,
|
|
0.9245833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9791444444444444,
|
|
"std": 0.0015629341025062142,
|
|
"ci_95": 0.0013699721438421555,
|
|
"values": [
|
|
0.9802277777777777,
|
|
0.9770951388888889,
|
|
0.9784319444444445,
|
|
0.9810975694444444,
|
|
0.9788697916666665
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.917186359420279,
|
|
"std": 0.00819325686071929,
|
|
"ci_95": 0.007181706284692464,
|
|
"values": [
|
|
0.924627166465135,
|
|
0.9128531635495424,
|
|
0.9193614408514122,
|
|
0.9051724137931034,
|
|
0.9239176124422026
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9344166666666667,
|
|
"std": 0.017728175158330445,
|
|
"ci_95": 0.015539430670251589,
|
|
"values": [
|
|
0.9295833333333333,
|
|
0.9283333333333333,
|
|
0.94375,
|
|
0.9116666666666666,
|
|
0.95875
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9927464583333332,
|
|
"std": 0.0036833307262198267,
|
|
"ci_95": 0.00322858173187695,
|
|
"values": [
|
|
0.9875475694444444,
|
|
0.9958916666666666,
|
|
0.9941100694444444,
|
|
0.9903302083333333,
|
|
0.9958527777777777
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9377917579907168,
|
|
"std": 0.01583050830010387,
|
|
"ci_95": 0.013876052329543502,
|
|
"values": [
|
|
0.9323729491796718,
|
|
0.9328125,
|
|
0.9461077844311377,
|
|
0.9178931061192874,
|
|
0.9597724502234863
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"run_name": "p3_convnext_tiny",
|
|
"backbone": "convnext_tiny"
|
|
}
|
|
} |