Johnny Fernandes
1681 lines
39 KiB
JSON
1681 lines
39 KiB
JSON
{
|
|
"run_name": "p4_convnext_tiny_100pct",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10976590474912276,
|
|
0.06032068864557746,
|
|
0.04213135505780142,
|
|
0.03131574821051159,
|
|
0.022795871932038508,
|
|
0.01803405568006484,
|
|
0.01305549041393331,
|
|
0.009530137429070674,
|
|
0.006397134670442534
|
|
],
|
|
"train_acc": [
|
|
0.9182523148148148,
|
|
0.9625694444444445,
|
|
0.9768402777777778,
|
|
0.9841666666666666,
|
|
0.9890046296296297,
|
|
0.992037037037037,
|
|
0.9942708333333333,
|
|
0.9961226851851852,
|
|
0.9975347222222222
|
|
],
|
|
"train_auc": [
|
|
0.972647155421239,
|
|
0.9926088748856883,
|
|
0.9965618612825788,
|
|
0.9982927962105624,
|
|
0.9989916452331962,
|
|
0.9993695541123686,
|
|
0.9995857988968907,
|
|
0.9998419835248057,
|
|
0.9999009352137631
|
|
],
|
|
"train_f1": [
|
|
0.9443354218386728,
|
|
0.9748694516971279,
|
|
0.984509626326668,
|
|
0.9894296002101717,
|
|
0.9926611457882701,
|
|
0.9946895550958659,
|
|
0.9961790519417363,
|
|
0.9974146247347097,
|
|
0.9983564180716848
|
|
],
|
|
"val_loss": [
|
|
0.1326672952055378,
|
|
0.12021120857016589,
|
|
0.10581790425145907,
|
|
0.06503360926566226,
|
|
0.11248262809961185,
|
|
0.15313717411119795,
|
|
0.11509020431104848,
|
|
0.12211939117706,
|
|
0.12375426999108943
|
|
],
|
|
"val_acc": [
|
|
0.9532291666666667,
|
|
0.9578125,
|
|
0.9578125,
|
|
0.9759375,
|
|
0.9711458333333334,
|
|
0.9647916666666667,
|
|
0.9713541666666666,
|
|
0.9715625,
|
|
0.973125
|
|
],
|
|
"val_auc": [
|
|
0.9914556423611111,
|
|
0.992825954861111,
|
|
0.9922774305555555,
|
|
0.995397974537037,
|
|
0.9921208622685185,
|
|
0.9870002314814815,
|
|
0.9896727141203704,
|
|
0.9885880497685184,
|
|
0.9905859664351852
|
|
],
|
|
"val_f1": [
|
|
0.9695034979284113,
|
|
0.9723869912047454,
|
|
0.9722088794345708,
|
|
0.9839794715306194,
|
|
0.9809477955842906,
|
|
0.9768651608487338,
|
|
0.9810253225695164,
|
|
0.9811581199530679,
|
|
0.9822485207100592
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.972375,
|
|
"auc_roc": 0.9951469351851852,
|
|
"f1": 0.9815674608690818,
|
|
"confusion_matrix": [
|
|
[
|
|
5684,
|
|
316
|
|
],
|
|
[
|
|
347,
|
|
17653
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9876666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
74,
|
|
5926
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9876666666666667,
|
|
"pairwise_auc": 0.9963563194444445,
|
|
"pairwise_f1": 0.9681424603822905
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.96,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
240,
|
|
5760
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.96,
|
|
"pairwise_auc": 0.9909720277777778,
|
|
"pairwise_f1": 0.9539582643259358
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9945,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
33,
|
|
5967
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9945,
|
|
"pairwise_auc": 0.9981124583333334,
|
|
"pairwise_f1": 0.9715867459089799
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9473333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5684,
|
|
316
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.05266666666666664
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9675,
|
|
"auc_roc": 0.9963563194444445,
|
|
"f1": 0.9681424603822905,
|
|
"confusion_matrix": [
|
|
[
|
|
5684,
|
|
316
|
|
],
|
|
[
|
|
74,
|
|
5926
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9536666666666667,
|
|
"auc_roc": 0.9909720277777778,
|
|
"f1": 0.9539582643259358,
|
|
"confusion_matrix": [
|
|
[
|
|
5684,
|
|
316
|
|
],
|
|
[
|
|
240,
|
|
5760
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9709166666666667,
|
|
"auc_roc": 0.9981124583333334,
|
|
"f1": 0.9715867459089799,
|
|
"confusion_matrix": [
|
|
[
|
|
5684,
|
|
316
|
|
],
|
|
[
|
|
33,
|
|
5967
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10888035357179535,
|
|
0.06245906389745263,
|
|
0.04233236001333959,
|
|
0.03127222566347444,
|
|
0.023119855731737415,
|
|
0.01731506652521636,
|
|
0.013754887645344988,
|
|
0.009147508151029764,
|
|
0.006803555942935367,
|
|
0.004607445764163963
|
|
],
|
|
"train_acc": [
|
|
0.9171180555555556,
|
|
0.9623379629629629,
|
|
0.9762152777777777,
|
|
0.9834722222222222,
|
|
0.9889236111111112,
|
|
0.9923148148148148,
|
|
0.9938078703703703,
|
|
0.9963078703703704,
|
|
0.9972106481481482,
|
|
0.9982291666666666
|
|
],
|
|
"train_auc": [
|
|
0.9728382994684499,
|
|
0.9921846397033607,
|
|
0.9966746824274121,
|
|
0.9982485789609054,
|
|
0.9990945305355511,
|
|
0.9995077924954275,
|
|
0.9995755912065616,
|
|
0.9998351869713077,
|
|
0.9998384480738455,
|
|
0.9999152463420211
|
|
],
|
|
"train_f1": [
|
|
0.9434828933349118,
|
|
0.974741907940697,
|
|
0.9840888854477179,
|
|
0.9889601855431002,
|
|
0.9926092966861538,
|
|
0.9948740910003242,
|
|
0.9958692681269641,
|
|
0.9975382193377116,
|
|
0.9981398436233685,
|
|
0.9988192895673044
|
|
],
|
|
"val_loss": [
|
|
0.07399313345444777,
|
|
0.09379419833732149,
|
|
0.07221478742896276,
|
|
0.06820129759006098,
|
|
0.06364023558270067,
|
|
0.08462312995652307,
|
|
0.09529060832261772,
|
|
0.1369951495661068,
|
|
0.09639229792060178,
|
|
0.12909956996560443
|
|
],
|
|
"val_acc": [
|
|
0.9608333333333333,
|
|
0.9092708333333334,
|
|
0.970625,
|
|
0.9685416666666666,
|
|
0.9670833333333333,
|
|
0.9722916666666667,
|
|
0.9740625,
|
|
0.97375,
|
|
0.9709375,
|
|
0.9746875
|
|
],
|
|
"val_auc": [
|
|
0.9919782118055555,
|
|
0.9920560185185184,
|
|
0.9955509837962963,
|
|
0.9950098379629629,
|
|
0.9957432291666666,
|
|
0.9949310185185185,
|
|
0.9937863715277778,
|
|
0.9865707465277778,
|
|
0.9946733217592593,
|
|
0.9871513020833332
|
|
],
|
|
"val_f1": [
|
|
0.9738997639872276,
|
|
0.9358001031915678,
|
|
0.9804898298048983,
|
|
0.9789898427716711,
|
|
0.9778307843412376,
|
|
0.9815303430079155,
|
|
0.9827884150134789,
|
|
0.9826398456875173,
|
|
0.9806290356175796,
|
|
0.9832563908220217
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9671666666666666,
|
|
"auc_roc": 0.9953128101851851,
|
|
"f1": 0.9778664119993259,
|
|
"confusion_matrix": [
|
|
[
|
|
5805,
|
|
195
|
|
],
|
|
[
|
|
593,
|
|
17407
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.983,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.983,
|
|
"pairwise_auc": 0.9977181250000001,
|
|
"pairwise_f1": 0.9754403373852643
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9246666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
452,
|
|
5548
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9246666666666666,
|
|
"pairwise_auc": 0.9891100416666667,
|
|
"pairwise_f1": 0.9449033466746146
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9935,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
39,
|
|
5961
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9935,
|
|
"pairwise_auc": 0.9991102638888888,
|
|
"pairwise_f1": 0.9807502467917077
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9675,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5805,
|
|
195
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03249999999999997
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.97525,
|
|
"auc_roc": 0.9977181250000001,
|
|
"f1": 0.9754403373852643,
|
|
"confusion_matrix": [
|
|
[
|
|
5805,
|
|
195
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9460833333333334,
|
|
"auc_roc": 0.9891100416666667,
|
|
"f1": 0.9449033466746146,
|
|
"confusion_matrix": [
|
|
[
|
|
5805,
|
|
195
|
|
],
|
|
[
|
|
452,
|
|
5548
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9805,
|
|
"auc_roc": 0.9991102638888888,
|
|
"f1": 0.9807502467917077,
|
|
"confusion_matrix": [
|
|
[
|
|
5805,
|
|
195
|
|
],
|
|
[
|
|
39,
|
|
5961
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10498240089121585,
|
|
0.061208187878359306,
|
|
0.04396705227361289,
|
|
0.03068989886629424,
|
|
0.02260338703439629,
|
|
0.01656382369378089,
|
|
0.011915402103649522,
|
|
0.008797623991720483,
|
|
0.007332696492687942
|
|
],
|
|
"train_acc": [
|
|
0.9198495370370371,
|
|
0.9634143518518519,
|
|
0.9759027777777778,
|
|
0.9840046296296296,
|
|
0.9896412037037037,
|
|
0.9924768518518519,
|
|
0.9949537037037037,
|
|
0.9961805555555555,
|
|
0.9970949074074074
|
|
],
|
|
"train_auc": [
|
|
0.9746708122570873,
|
|
0.9925247167209649,
|
|
0.9963766382315958,
|
|
0.998304158450503,
|
|
0.9989714070358939,
|
|
0.9995099311985597,
|
|
0.9997300557984682,
|
|
0.9998183431212848,
|
|
0.9999147472993828
|
|
],
|
|
"train_f1": [
|
|
0.9453825586989613,
|
|
0.9754704535754471,
|
|
0.9838864466596495,
|
|
0.9893179569626515,
|
|
0.9930875753994918,
|
|
0.994981315049878,
|
|
0.9966351274175375,
|
|
0.9974527209571594,
|
|
0.9980633164355763
|
|
],
|
|
"val_loss": [
|
|
0.05447970101173269,
|
|
0.06689616352358523,
|
|
0.06152612486323657,
|
|
0.05799913888744413,
|
|
0.0905450083331713,
|
|
0.08678834943179634,
|
|
0.08499443884752508,
|
|
0.1242760099450849,
|
|
0.11798079409263967
|
|
],
|
|
"val_acc": [
|
|
0.9563541666666666,
|
|
0.9595833333333333,
|
|
0.9657291666666666,
|
|
0.9689583333333334,
|
|
0.9733333333333334,
|
|
0.9688541666666667,
|
|
0.975625,
|
|
0.973125,
|
|
0.971875
|
|
],
|
|
"val_auc": [
|
|
0.9932748553240741,
|
|
0.9923731481481483,
|
|
0.9950711226851852,
|
|
0.9957342303240742,
|
|
0.9947641493055556,
|
|
0.9942186342592593,
|
|
0.9949234375,
|
|
0.9870554687500002,
|
|
0.9882605324074074
|
|
],
|
|
"val_f1": [
|
|
0.970369846545506,
|
|
0.9728139013452914,
|
|
0.9768814559763895,
|
|
0.9791053148226055,
|
|
0.9823131131684399,
|
|
0.9792288989232373,
|
|
0.9837364470391994,
|
|
0.9821181036872747,
|
|
0.9812552068869759
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9711666666666666,
|
|
"auc_roc": 0.9956308472222222,
|
|
"f1": 0.9806032066375154,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
508,
|
|
17492
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9833333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
100,
|
|
5900
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9833333333333333,
|
|
"pairwise_auc": 0.9972175277777778,
|
|
"pairwise_f1": 0.9764978483945713
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9421666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
347,
|
|
5653
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9421666666666667,
|
|
"pairwise_auc": 0.991956486111111,
|
|
"pairwise_f1": 0.955140660640365
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9898333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
61,
|
|
5939
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9898333333333333,
|
|
"pairwise_auc": 0.9977185277777777,
|
|
"pairwise_f1": 0.9797904809040666
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9693333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03066666666666662
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9763333333333334,
|
|
"auc_roc": 0.9972175277777778,
|
|
"f1": 0.9764978483945713,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
100,
|
|
5900
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.95575,
|
|
"auc_roc": 0.991956486111111,
|
|
"f1": 0.955140660640365,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
347,
|
|
5653
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9795833333333334,
|
|
"auc_roc": 0.9977185277777777,
|
|
"f1": 0.9797904809040666,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
61,
|
|
5939
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10971299468043812,
|
|
0.06252712907555245,
|
|
0.04297509410407252,
|
|
0.029988402510282834,
|
|
0.023590039267197695,
|
|
0.017803969608290504,
|
|
0.012462334047810416,
|
|
0.010320856769535735,
|
|
0.0061555235264323625
|
|
],
|
|
"train_acc": [
|
|
0.9162731481481482,
|
|
0.9619791666666667,
|
|
0.9764467592592593,
|
|
0.9848263888888888,
|
|
0.9891203703703704,
|
|
0.9922106481481482,
|
|
0.9946643518518519,
|
|
0.9958796296296296,
|
|
0.9975810185185185
|
|
],
|
|
"train_auc": [
|
|
0.9725527570587563,
|
|
0.9922214638345908,
|
|
0.9965160136602652,
|
|
0.9982630344078646,
|
|
0.9990331479338134,
|
|
0.9993577481995886,
|
|
0.9996407010888203,
|
|
0.9997498338906036,
|
|
0.9999400037865799
|
|
],
|
|
"train_f1": [
|
|
0.9428918782367058,
|
|
0.9744987074687347,
|
|
0.9842430061401947,
|
|
0.9898672932301771,
|
|
0.9927389578087101,
|
|
0.9948047336364548,
|
|
0.9964420501817565,
|
|
0.9972527048509824,
|
|
0.9983872087908696
|
|
],
|
|
"val_loss": [
|
|
0.05385312837994813,
|
|
0.12721827414050493,
|
|
0.07066376141650835,
|
|
0.07291066528736943,
|
|
0.07251917717887106,
|
|
0.07809772704130107,
|
|
0.11628153800508395,
|
|
0.08125251916594001,
|
|
0.16696450959560935
|
|
],
|
|
"val_acc": [
|
|
0.9654166666666667,
|
|
0.9527083333333334,
|
|
0.9677083333333333,
|
|
0.9608333333333333,
|
|
0.9746875,
|
|
0.9619791666666667,
|
|
0.9685416666666666,
|
|
0.9721875,
|
|
0.9691666666666666
|
|
],
|
|
"val_auc": [
|
|
0.9933230613425925,
|
|
0.9916460069444445,
|
|
0.9946691261574073,
|
|
0.9950530960648148,
|
|
0.9949720486111111,
|
|
0.9943324074074075,
|
|
0.9944564814814815,
|
|
0.9944458333333333,
|
|
0.983377980324074
|
|
],
|
|
"val_f1": [
|
|
0.9767832167832168,
|
|
0.9691408374116368,
|
|
0.9786000276128676,
|
|
0.9734275618374558,
|
|
0.9831238280436141,
|
|
0.9742540734993299,
|
|
0.9792468389224849,
|
|
0.9814053903475173,
|
|
0.9796926454445664
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9595,
|
|
"auc_roc": 0.9955074490740741,
|
|
"f1": 0.9724755054652545,
|
|
"confusion_matrix": [
|
|
[
|
|
5857,
|
|
143
|
|
],
|
|
[
|
|
829,
|
|
17171
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9563333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
262,
|
|
5738
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9563333333333334,
|
|
"pairwise_auc": 0.9961415,
|
|
"pairwise_f1": 0.9659119602727043
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9245,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
453,
|
|
5547
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9245,
|
|
"pairwise_auc": 0.992280638888889,
|
|
"pairwise_f1": 0.94901625320787
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.981,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
114,
|
|
5886
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.981,
|
|
"pairwise_auc": 0.9981002083333333,
|
|
"pairwise_f1": 0.9786349655000416
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9761666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5857,
|
|
143
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.023833333333333373
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.96625,
|
|
"auc_roc": 0.9961415,
|
|
"f1": 0.9659119602727043,
|
|
"confusion_matrix": [
|
|
[
|
|
5857,
|
|
143
|
|
],
|
|
[
|
|
262,
|
|
5738
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9503333333333334,
|
|
"auc_roc": 0.992280638888889,
|
|
"f1": 0.94901625320787,
|
|
"confusion_matrix": [
|
|
[
|
|
5857,
|
|
143
|
|
],
|
|
[
|
|
453,
|
|
5547
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9785833333333334,
|
|
"auc_roc": 0.9981002083333333,
|
|
"f1": 0.9786349655000416,
|
|
"confusion_matrix": [
|
|
[
|
|
5857,
|
|
143
|
|
],
|
|
[
|
|
114,
|
|
5886
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10992730081227987,
|
|
0.05964789071156971,
|
|
0.042661502508129126,
|
|
0.029218906156319363,
|
|
0.022335671255308692,
|
|
0.017047790865896147,
|
|
0.013907947207159883,
|
|
0.009291278975566726,
|
|
0.0067760476432299245,
|
|
0.004159879369507322,
|
|
0.0028330688595543862,
|
|
0.0018391810449822042
|
|
],
|
|
"train_acc": [
|
|
0.916875,
|
|
0.9636689814814815,
|
|
0.9768981481481481,
|
|
0.9857407407407407,
|
|
0.9896412037037037,
|
|
0.9922800925925926,
|
|
0.994386574074074,
|
|
0.9961342592592592,
|
|
0.9972453703703704,
|
|
0.9983796296296297,
|
|
0.9990393518518519,
|
|
0.9994328703703703
|
|
],
|
|
"train_auc": [
|
|
0.9727848290323503,
|
|
0.9930250464391861,
|
|
0.9966752975680157,
|
|
0.998414353995199,
|
|
0.9990329243112711,
|
|
0.999415157750343,
|
|
0.9996311531921582,
|
|
0.9997684356424326,
|
|
0.999890431027092,
|
|
0.9999059263545953,
|
|
0.9999682480995655,
|
|
0.9999500718021264
|
|
],
|
|
"train_f1": [
|
|
0.9433381721787427,
|
|
0.9756275573983058,
|
|
0.9845537137639102,
|
|
0.9904800173090594,
|
|
0.9930868278967736,
|
|
0.9948502559430517,
|
|
0.9962569362444337,
|
|
0.9974220836356339,
|
|
0.9981630699885771,
|
|
0.9989197197444366,
|
|
0.9993595629595907,
|
|
0.9996219164975579
|
|
],
|
|
"val_loss": [
|
|
0.12888806685456075,
|
|
0.0682746632627095,
|
|
0.06378444532970509,
|
|
0.09823442336123359,
|
|
0.08585669315273359,
|
|
0.10028958140515896,
|
|
0.07838637531407888,
|
|
0.1449296971670507,
|
|
0.11981039933865152,
|
|
0.16668833541457084,
|
|
0.16990588693974434,
|
|
0.1120091624956664
|
|
],
|
|
"val_acc": [
|
|
0.9467708333333333,
|
|
0.9613541666666666,
|
|
0.9602083333333333,
|
|
0.96125,
|
|
0.9703125,
|
|
0.9703125,
|
|
0.971875,
|
|
0.9707291666666666,
|
|
0.97125,
|
|
0.9727083333333333,
|
|
0.9704166666666667,
|
|
0.9765625
|
|
],
|
|
"val_auc": [
|
|
0.9882049479166666,
|
|
0.9921254629629629,
|
|
0.9936942418981481,
|
|
0.9912602719907407,
|
|
0.9924844907407406,
|
|
0.99249296875,
|
|
0.9941785011574075,
|
|
0.9863015046296295,
|
|
0.988821412037037,
|
|
0.9804513888888888,
|
|
0.9812513599537038,
|
|
0.9892113136574073
|
|
],
|
|
"val_f1": [
|
|
0.9652451880568591,
|
|
0.9741984839001321,
|
|
0.9730530474040632,
|
|
0.974440016490312,
|
|
0.980201458839875,
|
|
0.980195955805712,
|
|
0.9811662946428571,
|
|
0.9806246983382748,
|
|
0.9808466342817488,
|
|
0.9819559228650138,
|
|
0.9804056851110804,
|
|
0.984393424429493
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.969375,
|
|
"auc_roc": 0.9952890185185185,
|
|
"f1": 0.9794778723998325,
|
|
"confusion_matrix": [
|
|
[
|
|
5725,
|
|
275
|
|
],
|
|
[
|
|
460,
|
|
17540
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9851666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
89,
|
|
5911
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9851666666666666,
|
|
"pairwise_auc": 0.9971204166666667,
|
|
"pairwise_f1": 0.9701296569834236
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.942,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
348,
|
|
5652
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.942,
|
|
"pairwise_auc": 0.9896805833333335,
|
|
"pairwise_f1": 0.9477655739079399
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9961666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
23,
|
|
5977
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9961666666666666,
|
|
"pairwise_auc": 0.9990660555555555,
|
|
"pairwise_f1": 0.975677440417891
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9541666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5725,
|
|
275
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.04583333333333328
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9696666666666667,
|
|
"auc_roc": 0.9971204166666667,
|
|
"f1": 0.9701296569834236,
|
|
"confusion_matrix": [
|
|
[
|
|
5725,
|
|
275
|
|
],
|
|
[
|
|
89,
|
|
5911
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9480833333333333,
|
|
"auc_roc": 0.9896805833333335,
|
|
"f1": 0.9477655739079399,
|
|
"confusion_matrix": [
|
|
[
|
|
5725,
|
|
275
|
|
],
|
|
[
|
|
348,
|
|
5652
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9751666666666666,
|
|
"auc_roc": 0.9990660555555555,
|
|
"f1": 0.975677440417891,
|
|
"confusion_matrix": [
|
|
[
|
|
5725,
|
|
275
|
|
],
|
|
[
|
|
23,
|
|
5977
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9679166666666668,
|
|
"std": 0.005098253377383261,
|
|
"ci_95": 0.004468816118391969,
|
|
"values": [
|
|
0.972375,
|
|
0.9671666666666666,
|
|
0.9711666666666666,
|
|
0.9595,
|
|
0.969375
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9953774120370371,
|
|
"std": 0.0001912102355455466,
|
|
"ci_95": 0.00016760316119204677,
|
|
"values": [
|
|
0.9951469351851852,
|
|
0.9953128101851851,
|
|
0.9956308472222222,
|
|
0.9955074490740741,
|
|
0.9952890185185185
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9783980914742021,
|
|
"std": 0.0035858426551305827,
|
|
"ci_95": 0.003143129669928204,
|
|
"values": [
|
|
0.9815674608690818,
|
|
0.9778664119993259,
|
|
0.9806032066375154,
|
|
0.9724755054652545,
|
|
0.9794778723998325
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9791000000000001,
|
|
"std": 0.012861225101486662,
|
|
"ci_95": 0.011273360851533515,
|
|
"values": [
|
|
0.9876666666666667,
|
|
0.983,
|
|
0.9833333333333333,
|
|
0.9563333333333334,
|
|
0.9851666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9791000000000001,
|
|
"std": 0.012861225101486662,
|
|
"ci_95": 0.011273360851533515,
|
|
"values": [
|
|
0.9876666666666667,
|
|
0.983,
|
|
0.9833333333333333,
|
|
0.9563333333333334,
|
|
0.9851666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9969107777777779,
|
|
"std": 0.0006498203417580526,
|
|
"ci_95": 0.000569592643274595,
|
|
"values": [
|
|
0.9963563194444445,
|
|
0.9977181250000001,
|
|
0.9972175277777778,
|
|
0.9961415,
|
|
0.9971204166666667
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9712244526836507,
|
|
"std": 0.004596254342627748,
|
|
"ci_95": 0.004028794563581748,
|
|
"values": [
|
|
0.9681424603822905,
|
|
0.9754403373852643,
|
|
0.9764978483945713,
|
|
0.9659119602727043,
|
|
0.9701296569834236
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9386666666666666,
|
|
"std": 0.014791607981104238,
|
|
"ci_95": 0.012965416049372779,
|
|
"values": [
|
|
0.96,
|
|
0.9246666666666666,
|
|
0.9421666666666667,
|
|
0.9245,
|
|
0.942
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9386666666666666,
|
|
"std": 0.014791607981104238,
|
|
"ci_95": 0.012965416049372779,
|
|
"values": [
|
|
0.96,
|
|
0.9246666666666666,
|
|
0.9421666666666667,
|
|
0.9245,
|
|
0.942
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9907999555555556,
|
|
"std": 0.0013845936256666454,
|
|
"ci_95": 0.0012136498235357784,
|
|
"values": [
|
|
0.9909720277777778,
|
|
0.9891100416666667,
|
|
0.991956486111111,
|
|
0.992280638888889,
|
|
0.9896805833333335
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9501568197513451,
|
|
"std": 0.004298475862220833,
|
|
"ci_95": 0.0037677802172065788,
|
|
"values": [
|
|
0.9539582643259358,
|
|
0.9449033466746146,
|
|
0.955140660640365,
|
|
0.94901625320787,
|
|
0.9477655739079399
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.991,
|
|
"std": 0.00605300659911164,
|
|
"ci_95": 0.005305694215756431,
|
|
"values": [
|
|
0.9945,
|
|
0.9935,
|
|
0.9898333333333333,
|
|
0.981,
|
|
0.9961666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.991,
|
|
"std": 0.00605300659911164,
|
|
"ci_95": 0.005305694215756431,
|
|
"values": [
|
|
0.9945,
|
|
0.9935,
|
|
0.9898333333333333,
|
|
0.981,
|
|
0.9961666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9984215027777777,
|
|
"std": 0.0006290374771859308,
|
|
"ci_95": 0.000551375659278024,
|
|
"values": [
|
|
0.9981124583333334,
|
|
0.9991102638888888,
|
|
0.9977185277777777,
|
|
0.9981002083333333,
|
|
0.9990660555555555
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9772879759045374,
|
|
"std": 0.00371355893837721,
|
|
"ci_95": 0.0032550779280680507,
|
|
"values": [
|
|
0.9715867459089799,
|
|
0.9807502467917077,
|
|
0.9797904809040666,
|
|
0.9786349655000416,
|
|
0.975677440417891
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9629,
|
|
"std": 0.011801953698905559,
|
|
"ci_95": 0.010344868529318702,
|
|
"values": [
|
|
0.9473333333333334,
|
|
0.9675,
|
|
0.9693333333333334,
|
|
0.9761666666666666,
|
|
0.9541666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.03709999999999998,
|
|
"std": 0.011801953698905559,
|
|
"ci_95": 0.010344868529318702,
|
|
"values": [
|
|
0.05266666666666664,
|
|
0.03249999999999997,
|
|
0.03066666666666662,
|
|
0.023833333333333373,
|
|
0.04583333333333328
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9710000000000001,
|
|
"std": 0.004557883890091477,
|
|
"ci_95": 0.003995161379023922,
|
|
"values": [
|
|
0.9675,
|
|
0.97525,
|
|
0.9763333333333334,
|
|
0.96625,
|
|
0.9696666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9969107777777779,
|
|
"std": 0.0006498203417580526,
|
|
"ci_95": 0.000569592643274595,
|
|
"values": [
|
|
0.9963563194444445,
|
|
0.9977181250000001,
|
|
0.9972175277777778,
|
|
0.9961415,
|
|
0.9971204166666667
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9712244526836507,
|
|
"std": 0.004596254342627748,
|
|
"ci_95": 0.004028794563581748,
|
|
"values": [
|
|
0.9681424603822905,
|
|
0.9754403373852643,
|
|
0.9764978483945713,
|
|
0.9659119602727043,
|
|
0.9701296569834236
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9507833333333334,
|
|
"std": 0.0039548669032247,
|
|
"ci_95": 0.0034665936851292086,
|
|
"values": [
|
|
0.9536666666666667,
|
|
0.9460833333333334,
|
|
0.95575,
|
|
0.9503333333333334,
|
|
0.9480833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9907999555555556,
|
|
"std": 0.0013845936256666454,
|
|
"ci_95": 0.0012136498235357784,
|
|
"values": [
|
|
0.9909720277777778,
|
|
0.9891100416666667,
|
|
0.991956486111111,
|
|
0.992280638888889,
|
|
0.9896805833333335
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9501568197513451,
|
|
"std": 0.004298475862220833,
|
|
"ci_95": 0.0037677802172065788,
|
|
"values": [
|
|
0.9539582643259358,
|
|
0.9449033466746146,
|
|
0.955140660640365,
|
|
0.94901625320787,
|
|
0.9477655739079399
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9769500000000001,
|
|
"std": 0.0039302070909530955,
|
|
"ci_95": 0.0034449784066409453,
|
|
"values": [
|
|
0.9709166666666667,
|
|
0.9805,
|
|
0.9795833333333334,
|
|
0.9785833333333334,
|
|
0.9751666666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9984215027777777,
|
|
"std": 0.0006290374771859308,
|
|
"ci_95": 0.000551375659278024,
|
|
"values": [
|
|
0.9981124583333334,
|
|
0.9991102638888888,
|
|
0.9977185277777777,
|
|
0.9981002083333333,
|
|
0.9990660555555555
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9772879759045374,
|
|
"std": 0.00371355893837721,
|
|
"ci_95": 0.0032550779280680507,
|
|
"values": [
|
|
0.9715867459089799,
|
|
0.9807502467917077,
|
|
0.9797904809040666,
|
|
0.9786349655000416,
|
|
0.975677440417891
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"augment": false,
|
|
"run_name": "p4_convnext_tiny_100pct",
|
|
"backbone": "convnext_tiny",
|
|
"subsample": 1.0
|
|
}
|
|
} |