Johnny Fernandes
1617 lines
37 KiB
JSON
1617 lines
37 KiB
JSON
{
|
|
"run_name": "p1_resnet18_baseline",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.23958949717106642,
|
|
0.143196833671795,
|
|
0.09169542824814993,
|
|
0.05906003904960084,
|
|
0.03961496371506386,
|
|
0.030794697152278214,
|
|
0.019622314044008143,
|
|
0.010980474815140176,
|
|
0.009058485340109555
|
|
],
|
|
"train_acc": [
|
|
0.7507523148148149,
|
|
0.8752893518518519,
|
|
0.9286458333333333,
|
|
0.9586226851851852,
|
|
0.9741898148148148,
|
|
0.9828125,
|
|
0.9880208333333333,
|
|
0.9919560185185186,
|
|
0.9939236111111112
|
|
],
|
|
"train_auc": [
|
|
0.8491616565929355,
|
|
0.9504348851165981,
|
|
0.9801190093449932,
|
|
0.9915338148719707,
|
|
0.9959249346279149,
|
|
0.9974669299411294,
|
|
0.998836760902492,
|
|
0.9996191629515319,
|
|
0.9997816733110425
|
|
],
|
|
"train_f1": [
|
|
0.8176159220834216,
|
|
0.9132273001811959,
|
|
0.9514662468018107,
|
|
0.9721193215051667,
|
|
0.9826890234435647,
|
|
0.9885012969917535,
|
|
0.991993192279426,
|
|
0.9946238638561207,
|
|
0.9959442234153502
|
|
],
|
|
"val_loss": [
|
|
0.19691624582434694,
|
|
0.1774458727488915,
|
|
0.27195345516471814,
|
|
0.23256856029232342,
|
|
0.31288731123010316,
|
|
0.42289219592894367,
|
|
0.358063139153334,
|
|
0.4632530469345511,
|
|
0.474098086887534
|
|
],
|
|
"val_acc": [
|
|
0.8307291666666666,
|
|
0.8380208333333333,
|
|
0.8677083333333333,
|
|
0.8635416666666667,
|
|
0.846875,
|
|
0.8770833333333333,
|
|
0.8697916666666666,
|
|
0.8817708333333333,
|
|
0.8776041666666666
|
|
],
|
|
"val_auc": [
|
|
0.9075520833333334,
|
|
0.928791232638889,
|
|
0.9198922164351852,
|
|
0.9320630787037036,
|
|
0.9189192708333334,
|
|
0.9173234953703704,
|
|
0.9229318576388889,
|
|
0.9175361689814816,
|
|
0.9185105613425926
|
|
],
|
|
"val_f1": [
|
|
0.8822890257153205,
|
|
0.8846011131725418,
|
|
0.9121107266435986,
|
|
0.906761565836299,
|
|
0.8943965517241379,
|
|
0.9186767746381805,
|
|
0.9119718309859155,
|
|
0.9215347390252333,
|
|
0.9198225861480723
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8745833333333334,
|
|
"auc_roc": 0.9403236111111113,
|
|
"f1": 0.9146583498724128,
|
|
"confusion_matrix": [
|
|
[
|
|
972,
|
|
228
|
|
],
|
|
[
|
|
374,
|
|
3226
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9141666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
103,
|
|
1097
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9141666666666667,
|
|
"pairwise_auc": 0.9481944444444446,
|
|
"pairwise_f1": 0.8689108910891089
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8275,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
207,
|
|
993
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8275,
|
|
"pairwise_auc": 0.9102131944444445,
|
|
"pairwise_f1": 0.8203221809169765
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9466666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
64,
|
|
1136
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9466666666666667,
|
|
"pairwise_auc": 0.9625631944444444,
|
|
"pairwise_f1": 0.8861154446177847
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.81,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
972,
|
|
228
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.18999999999999995
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8620833333333333,
|
|
"auc_roc": 0.9481944444444446,
|
|
"f1": 0.8689108910891089,
|
|
"confusion_matrix": [
|
|
[
|
|
972,
|
|
228
|
|
],
|
|
[
|
|
103,
|
|
1097
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.81875,
|
|
"auc_roc": 0.9102131944444445,
|
|
"f1": 0.8203221809169765,
|
|
"confusion_matrix": [
|
|
[
|
|
972,
|
|
228
|
|
],
|
|
[
|
|
207,
|
|
993
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8783333333333333,
|
|
"auc_roc": 0.9625631944444444,
|
|
"f1": 0.8861154446177847,
|
|
"confusion_matrix": [
|
|
[
|
|
972,
|
|
228
|
|
],
|
|
[
|
|
64,
|
|
1136
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.23669152959353393,
|
|
0.13688683253540485,
|
|
0.08269713706233435,
|
|
0.05615546396260874,
|
|
0.03753572171316827,
|
|
0.031078304888527513,
|
|
0.018910827419474825,
|
|
0.011221361623675976,
|
|
0.00673896867634472
|
|
],
|
|
"train_acc": [
|
|
0.7619212962962963,
|
|
0.8825810185185186,
|
|
0.9376736111111111,
|
|
0.9609953703703704,
|
|
0.9765625,
|
|
0.9829282407407407,
|
|
0.9889467592592592,
|
|
0.9935185185185185,
|
|
0.9955439814814815
|
|
],
|
|
"train_auc": [
|
|
0.8549236164694787,
|
|
0.9544764785522405,
|
|
0.9836320355366941,
|
|
0.9923031603652263,
|
|
0.996373626471765,
|
|
0.9974486400462963,
|
|
0.9989359174954275,
|
|
0.9995332683184728,
|
|
0.9997378507944673
|
|
],
|
|
"train_f1": [
|
|
0.8274184075845289,
|
|
0.9185892549051077,
|
|
0.9577597364395811,
|
|
0.973719098494892,
|
|
0.9843005000581463,
|
|
0.9885910971883822,
|
|
0.9926126474569715,
|
|
0.9956709956709957,
|
|
0.997024154589372
|
|
],
|
|
"val_loss": [
|
|
0.19221198732654254,
|
|
0.20249952741432936,
|
|
0.36335805027047174,
|
|
0.2351263621201118,
|
|
0.333601384274516,
|
|
0.3226143382489681,
|
|
0.421671195969005,
|
|
0.40155297291930764,
|
|
0.45411645335928674
|
|
],
|
|
"val_acc": [
|
|
0.8057291666666667,
|
|
0.8567708333333334,
|
|
0.8578125,
|
|
0.8729166666666667,
|
|
0.8791666666666667,
|
|
0.8546875,
|
|
0.8755208333333333,
|
|
0.8739583333333333,
|
|
0.8755208333333333
|
|
],
|
|
"val_auc": [
|
|
0.9083029513888888,
|
|
0.9268113425925926,
|
|
0.9027604166666667,
|
|
0.9329014756944443,
|
|
0.9275462962962964,
|
|
0.9208470775462965,
|
|
0.9181640624999998,
|
|
0.9201468460648148,
|
|
0.9158521412037037
|
|
],
|
|
"val_f1": [
|
|
0.8588724933787363,
|
|
0.9019607843137255,
|
|
0.9068577277379734,
|
|
0.9143859649122807,
|
|
0.9193884642112579,
|
|
0.9003215434083601,
|
|
0.9172724125995154,
|
|
0.9156794425087108,
|
|
0.9170426935091982
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8727083333333333,
|
|
"auc_roc": 0.9351839120370371,
|
|
"f1": 0.9141733389521001,
|
|
"confusion_matrix": [
|
|
[
|
|
935,
|
|
265
|
|
],
|
|
[
|
|
346,
|
|
3254
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
95,
|
|
1105
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9208333333333333,
|
|
"pairwise_auc": 0.9446645833333335,
|
|
"pairwise_f1": 0.8599221789883269
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8508333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
179,
|
|
1021
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8508333333333333,
|
|
"pairwise_auc": 0.906173263888889,
|
|
"pairwise_f1": 0.8213998390989542
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.94,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.94,
|
|
"pairwise_auc": 0.9547138888888889,
|
|
"pairwise_f1": 0.8700347088314694
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.7791666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
935,
|
|
265
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.22083333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.85,
|
|
"auc_roc": 0.9446645833333335,
|
|
"f1": 0.8599221789883269,
|
|
"confusion_matrix": [
|
|
[
|
|
935,
|
|
265
|
|
],
|
|
[
|
|
95,
|
|
1105
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.815,
|
|
"auc_roc": 0.906173263888889,
|
|
"f1": 0.8213998390989542,
|
|
"confusion_matrix": [
|
|
[
|
|
935,
|
|
265
|
|
],
|
|
[
|
|
179,
|
|
1021
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8595833333333334,
|
|
"auc_roc": 0.9547138888888889,
|
|
"f1": 0.8700347088314694,
|
|
"confusion_matrix": [
|
|
[
|
|
935,
|
|
265
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.23555645521868157,
|
|
0.14330073559863699,
|
|
0.08902234849864962,
|
|
0.0596706295705021,
|
|
0.04015614009652011,
|
|
0.027025900888973546,
|
|
0.020126515977770192,
|
|
0.014928802549049444,
|
|
0.010074229028281546
|
|
],
|
|
"train_acc": [
|
|
0.7581597222222223,
|
|
0.8776041666666666,
|
|
0.9295717592592593,
|
|
0.9575231481481481,
|
|
0.9728587962962963,
|
|
0.9838541666666667,
|
|
0.9872106481481482,
|
|
0.9912615740740741,
|
|
0.9932291666666667
|
|
],
|
|
"train_auc": [
|
|
0.8572669020776176,
|
|
0.950374871399177,
|
|
0.981142037822931,
|
|
0.9912892232510288,
|
|
0.9959913515946502,
|
|
0.9978648155292638,
|
|
0.9987725230052584,
|
|
0.9994318790723594,
|
|
0.9996619405864198
|
|
],
|
|
"train_f1": [
|
|
0.8229462356480108,
|
|
0.9149817100132652,
|
|
0.9521111242277575,
|
|
0.9713437963613649,
|
|
0.9817984243412117,
|
|
0.9892048752176437,
|
|
0.9914490230218611,
|
|
0.994165604111124,
|
|
0.9954789597743344
|
|
],
|
|
"val_loss": [
|
|
0.16975419651716947,
|
|
0.18313004115285972,
|
|
0.19825653905281798,
|
|
0.2172380828880705,
|
|
0.3135680136232016,
|
|
0.27533191804541274,
|
|
0.37277434571054374,
|
|
0.3967378988515217,
|
|
0.3754097615553292
|
|
],
|
|
"val_acc": [
|
|
0.7963541666666667,
|
|
0.8666666666666667,
|
|
0.8807291666666667,
|
|
0.8796875,
|
|
0.8838541666666667,
|
|
0.8739583333333333,
|
|
0.8838541666666667,
|
|
0.8833333333333333,
|
|
0.8885416666666667
|
|
],
|
|
"val_auc": [
|
|
0.9309266493055555,
|
|
0.9361125578703705,
|
|
0.9378175636574075,
|
|
0.9413896122685185,
|
|
0.9364113136574075,
|
|
0.9379557291666667,
|
|
0.9314489293981483,
|
|
0.9296419270833334,
|
|
0.934572482638889
|
|
],
|
|
"val_f1": [
|
|
0.8475633528265107,
|
|
0.9092198581560283,
|
|
0.9189380530973451,
|
|
0.918804920913884,
|
|
0.923394022672621,
|
|
0.9138790035587189,
|
|
0.9233413544173256,
|
|
0.9231297185998627,
|
|
0.925642807505212
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8754166666666666,
|
|
"auc_roc": 0.9345337962962963,
|
|
"f1": 0.9168982768204558,
|
|
"confusion_matrix": [
|
|
[
|
|
903,
|
|
297
|
|
],
|
|
[
|
|
301,
|
|
3299
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.925,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.925,
|
|
"pairwise_auc": 0.9392815972222223,
|
|
"pairwise_f1": 0.8515535097813579
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8683333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
158,
|
|
1042
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8683333333333333,
|
|
"pairwise_auc": 0.9052475694444445,
|
|
"pairwise_f1": 0.8207955888144939
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9558333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9558333333333333,
|
|
"pairwise_auc": 0.9590722222222223,
|
|
"pairwise_f1": 0.867624810892587
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.7525,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
903,
|
|
297
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.24750000000000005
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.83875,
|
|
"auc_roc": 0.9392815972222223,
|
|
"f1": 0.8515535097813579,
|
|
"confusion_matrix": [
|
|
[
|
|
903,
|
|
297
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8104166666666667,
|
|
"auc_roc": 0.9052475694444445,
|
|
"f1": 0.8207955888144939,
|
|
"confusion_matrix": [
|
|
[
|
|
903,
|
|
297
|
|
],
|
|
[
|
|
158,
|
|
1042
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8541666666666666,
|
|
"auc_roc": 0.9590722222222223,
|
|
"f1": 0.867624810892587,
|
|
"confusion_matrix": [
|
|
[
|
|
903,
|
|
297
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.23815842890353114,
|
|
0.14449559769558687,
|
|
0.08685155709757021,
|
|
0.054759029798744314,
|
|
0.04165420382557419,
|
|
0.02688520766279436,
|
|
0.022700919520435424
|
|
],
|
|
"train_acc": [
|
|
0.7540509259259259,
|
|
0.8746527777777777,
|
|
0.9336805555555555,
|
|
0.9614583333333333,
|
|
0.9749421296296297,
|
|
0.983738425925926,
|
|
0.9868055555555556
|
|
],
|
|
"train_auc": [
|
|
0.8530149748513945,
|
|
0.9498039016060815,
|
|
0.9820211405464107,
|
|
0.9928622345821903,
|
|
0.9958028263603109,
|
|
0.9981013606681528,
|
|
0.9986642839791953
|
|
],
|
|
"train_f1": [
|
|
0.8199457719030673,
|
|
0.9131027842413544,
|
|
0.9549492884660744,
|
|
0.9740674402305116,
|
|
0.9832046856211939,
|
|
0.9891224402895521,
|
|
0.991183976490604
|
|
],
|
|
"val_loss": [
|
|
0.18737150949115555,
|
|
0.16689071032839517,
|
|
0.20768918559576074,
|
|
0.25156240473346164,
|
|
0.37945238465520864,
|
|
0.31712872185744345,
|
|
0.3442687667479428
|
|
],
|
|
"val_acc": [
|
|
0.80625,
|
|
0.8416666666666667,
|
|
0.8677083333333333,
|
|
0.86875,
|
|
0.8729166666666667,
|
|
0.8635416666666667,
|
|
0.8791666666666667
|
|
],
|
|
"val_auc": [
|
|
0.9110677083333334,
|
|
0.9324168113425926,
|
|
0.9296209490740741,
|
|
0.9280975115740739,
|
|
0.92453125,
|
|
0.9245305266203704,
|
|
0.9291030092592594
|
|
],
|
|
"val_f1": [
|
|
0.8589840788476119,
|
|
0.8883994126284875,
|
|
0.910056657223796,
|
|
0.9105748757984387,
|
|
0.9170068027210885,
|
|
0.906761565836299,
|
|
0.9193884642112579
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8420833333333333,
|
|
"auc_roc": 0.9320532407407407,
|
|
"f1": 0.8883981154299175,
|
|
"confusion_matrix": [
|
|
[
|
|
1025,
|
|
175
|
|
],
|
|
[
|
|
583,
|
|
3017
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.8416666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
190,
|
|
1010
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8416666666666667,
|
|
"pairwise_auc": 0.9318243055555555,
|
|
"pairwise_f1": 0.8469601677148847
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.7483333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
302,
|
|
898
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.7483333333333333,
|
|
"pairwise_auc": 0.9008222222222222,
|
|
"pairwise_f1": 0.7901451825780906
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9241666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9241666666666667,
|
|
"pairwise_auc": 0.9635131944444445,
|
|
"pairwise_f1": 0.892914653784219
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8541666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1025,
|
|
175
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.14583333333333337
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8479166666666667,
|
|
"auc_roc": 0.9318243055555555,
|
|
"f1": 0.8469601677148847,
|
|
"confusion_matrix": [
|
|
[
|
|
1025,
|
|
175
|
|
],
|
|
[
|
|
190,
|
|
1010
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.80125,
|
|
"auc_roc": 0.9008222222222222,
|
|
"f1": 0.7901451825780906,
|
|
"confusion_matrix": [
|
|
[
|
|
1025,
|
|
175
|
|
],
|
|
[
|
|
302,
|
|
898
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8891666666666667,
|
|
"auc_roc": 0.9635131944444445,
|
|
"f1": 0.892914653784219,
|
|
"confusion_matrix": [
|
|
[
|
|
1025,
|
|
175
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.23552175492600158,
|
|
0.14149037405020662,
|
|
0.08607236144684807,
|
|
0.05892389668194853,
|
|
0.04063106574294916,
|
|
0.026183983484683,
|
|
0.019621156936153527
|
|
],
|
|
"train_acc": [
|
|
0.7596643518518519,
|
|
0.8807291666666667,
|
|
0.9306712962962963,
|
|
0.9576388888888889,
|
|
0.9722800925925926,
|
|
0.983912037037037,
|
|
0.9885416666666667
|
|
],
|
|
"train_auc": [
|
|
0.8568183888460219,
|
|
0.9515533282607452,
|
|
0.9821654056641518,
|
|
0.9914928322902378,
|
|
0.9959625771604939,
|
|
0.9977760005858483,
|
|
0.998805682370256
|
|
],
|
|
"train_f1": [
|
|
0.8240776041004787,
|
|
0.9173782321106434,
|
|
0.9528197857592943,
|
|
0.971444175704143,
|
|
0.9814218671217468,
|
|
0.9892298155896482,
|
|
0.9923463471202165
|
|
],
|
|
"val_loss": [
|
|
0.19069901450226703,
|
|
0.15564801689858238,
|
|
0.21845358079299332,
|
|
0.254469917776684,
|
|
0.24185769132260854,
|
|
0.30473897442764913,
|
|
0.35159544888107724
|
|
],
|
|
"val_acc": [
|
|
0.8317708333333333,
|
|
0.8557291666666667,
|
|
0.8572916666666667,
|
|
0.878125,
|
|
0.8802083333333334,
|
|
0.8755208333333333,
|
|
0.8828125
|
|
],
|
|
"val_auc": [
|
|
0.9150571469907408,
|
|
0.9409252025462964,
|
|
0.9299233217592591,
|
|
0.934763454861111,
|
|
0.9404072627314813,
|
|
0.9310828993055555,
|
|
0.930873119212963
|
|
],
|
|
"val_f1": [
|
|
0.882417182380779,
|
|
0.8987202925045704,
|
|
0.9016511127063891,
|
|
0.91875,
|
|
0.9192982456140351,
|
|
0.9155178508306823,
|
|
0.9222260629104736
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8604166666666667,
|
|
"auc_roc": 0.9406895833333333,
|
|
"f1": 0.9021612149532711,
|
|
"confusion_matrix": [
|
|
[
|
|
1041,
|
|
159
|
|
],
|
|
[
|
|
511,
|
|
3089
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.8758333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
149,
|
|
1051
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8758333333333334,
|
|
"pairwise_auc": 0.9438777777777778,
|
|
"pairwise_f1": 0.8721991701244813
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.7725,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
273,
|
|
927
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.7725,
|
|
"pairwise_auc": 0.9121888888888889,
|
|
"pairwise_f1": 0.8110236220472441
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9258333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
89,
|
|
1111
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9258333333333333,
|
|
"pairwise_auc": 0.9660020833333334,
|
|
"pairwise_f1": 0.8995951417004049
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8675,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1041,
|
|
159
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.13249999999999995
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8716666666666667,
|
|
"auc_roc": 0.9438777777777778,
|
|
"f1": 0.8721991701244813,
|
|
"confusion_matrix": [
|
|
[
|
|
1041,
|
|
159
|
|
],
|
|
[
|
|
149,
|
|
1051
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.82,
|
|
"auc_roc": 0.9121888888888889,
|
|
"f1": 0.8110236220472441,
|
|
"confusion_matrix": [
|
|
[
|
|
1041,
|
|
159
|
|
],
|
|
[
|
|
273,
|
|
927
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8966666666666666,
|
|
"auc_roc": 0.9660020833333334,
|
|
"f1": 0.8995951417004049,
|
|
"confusion_matrix": [
|
|
[
|
|
1041,
|
|
159
|
|
],
|
|
[
|
|
89,
|
|
1111
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.8650416666666667,
|
|
"std": 0.014194519188757335,
|
|
"ci_95": 0.012442044647082734,
|
|
"values": [
|
|
0.8745833333333334,
|
|
0.8727083333333333,
|
|
0.8754166666666666,
|
|
0.8420833333333333,
|
|
0.8604166666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9365568287037037,
|
|
"std": 0.003792366522656302,
|
|
"ci_95": 0.003324155821379563,
|
|
"values": [
|
|
0.9403236111111113,
|
|
0.9351839120370371,
|
|
0.9345337962962963,
|
|
0.9320532407407407,
|
|
0.9406895833333333
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9072578592056315,
|
|
"std": 0.012012436180286749,
|
|
"ci_95": 0.010529364558803643,
|
|
"values": [
|
|
0.9146583498724128,
|
|
0.9141733389521001,
|
|
0.9168982768204558,
|
|
0.8883981154299175,
|
|
0.9021612149532711
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.8955,
|
|
"std": 0.035865295822625584,
|
|
"ci_95": 0.03143731788107178,
|
|
"values": [
|
|
0.9141666666666667,
|
|
0.9208333333333333,
|
|
0.925,
|
|
0.8416666666666667,
|
|
0.8758333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.8955,
|
|
"std": 0.035865295822625584,
|
|
"ci_95": 0.03143731788107178,
|
|
"values": [
|
|
0.9141666666666667,
|
|
0.9208333333333333,
|
|
0.925,
|
|
0.8416666666666667,
|
|
0.8758333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9415685416666667,
|
|
"std": 0.0063047776590499085,
|
|
"ci_95": 0.005526381280033774,
|
|
"values": [
|
|
0.9481944444444446,
|
|
0.9446645833333335,
|
|
0.9392815972222223,
|
|
0.9318243055555555,
|
|
0.9438777777777778
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.8599091835396319,
|
|
"std": 0.010834770102868352,
|
|
"ci_95": 0.00949709472847364,
|
|
"values": [
|
|
0.8689108910891089,
|
|
0.8599221789883269,
|
|
0.8515535097813579,
|
|
0.8469601677148847,
|
|
0.8721991701244813
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.8135,
|
|
"std": 0.0512936805724318,
|
|
"ci_95": 0.044960893377838186,
|
|
"values": [
|
|
0.8275,
|
|
0.8508333333333333,
|
|
0.8683333333333333,
|
|
0.7483333333333333,
|
|
0.7725
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.8135,
|
|
"std": 0.0512936805724318,
|
|
"ci_95": 0.044960893377838186,
|
|
"values": [
|
|
0.8275,
|
|
0.8508333333333333,
|
|
0.8683333333333333,
|
|
0.7483333333333333,
|
|
0.7725
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9069290277777778,
|
|
"std": 0.004448130081238437,
|
|
"ci_95": 0.003898957923889036,
|
|
"values": [
|
|
0.9102131944444445,
|
|
0.906173263888889,
|
|
0.9052475694444445,
|
|
0.9008222222222222,
|
|
0.9121888888888889
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.812737282691152,
|
|
"std": 0.013330852209377185,
|
|
"ci_95": 0.01168500716136289,
|
|
"values": [
|
|
0.8203221809169765,
|
|
0.8213998390989542,
|
|
0.8207955888144939,
|
|
0.7901451825780906,
|
|
0.8110236220472441
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9385000000000001,
|
|
"std": 0.01355800296667781,
|
|
"ci_95": 0.011884113578873076,
|
|
"values": [
|
|
0.9466666666666667,
|
|
0.94,
|
|
0.9558333333333333,
|
|
0.9241666666666667,
|
|
0.9258333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9385000000000001,
|
|
"std": 0.01355800296667781,
|
|
"ci_95": 0.011884113578873076,
|
|
"values": [
|
|
0.9466666666666667,
|
|
0.94,
|
|
0.9558333333333333,
|
|
0.9241666666666667,
|
|
0.9258333333333333
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9611729166666667,
|
|
"std": 0.0043835653312689635,
|
|
"ci_95": 0.003842364425295284,
|
|
"values": [
|
|
0.9625631944444444,
|
|
0.9547138888888889,
|
|
0.9590722222222223,
|
|
0.9635131944444445,
|
|
0.9660020833333334
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.883256951965293,
|
|
"std": 0.014031840368382521,
|
|
"ci_95": 0.01229945037394657,
|
|
"values": [
|
|
0.8861154446177847,
|
|
0.8700347088314694,
|
|
0.867624810892587,
|
|
0.892914653784219,
|
|
0.8995951417004049
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.8126666666666666,
|
|
"std": 0.04867836617362312,
|
|
"ci_95": 0.04266846923275626,
|
|
"values": [
|
|
0.81,
|
|
0.7791666666666667,
|
|
0.7525,
|
|
0.8541666666666666,
|
|
0.8675
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.18733333333333332,
|
|
"std": 0.04867836617362312,
|
|
"ci_95": 0.04266846923275626,
|
|
"values": [
|
|
0.18999999999999995,
|
|
0.22083333333333333,
|
|
0.24750000000000005,
|
|
0.14583333333333337,
|
|
0.13249999999999995
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.8540833333333333,
|
|
"std": 0.012874258338940466,
|
|
"ci_95": 0.011284784987859647,
|
|
"values": [
|
|
0.8620833333333333,
|
|
0.85,
|
|
0.83875,
|
|
0.8479166666666667,
|
|
0.8716666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9415685416666667,
|
|
"std": 0.0063047776590499085,
|
|
"ci_95": 0.005526381280033774,
|
|
"values": [
|
|
0.9481944444444446,
|
|
0.9446645833333335,
|
|
0.9392815972222223,
|
|
0.9318243055555555,
|
|
0.9438777777777778
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8599091835396319,
|
|
"std": 0.010834770102868352,
|
|
"ci_95": 0.00949709472847364,
|
|
"values": [
|
|
0.8689108910891089,
|
|
0.8599221789883269,
|
|
0.8515535097813579,
|
|
0.8469601677148847,
|
|
0.8721991701244813
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.8130833333333334,
|
|
"std": 0.0075954573412153285,
|
|
"ci_95": 0.006657711902581657,
|
|
"values": [
|
|
0.81875,
|
|
0.815,
|
|
0.8104166666666667,
|
|
0.80125,
|
|
0.82
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9069290277777778,
|
|
"std": 0.004448130081238437,
|
|
"ci_95": 0.003898957923889036,
|
|
"values": [
|
|
0.9102131944444445,
|
|
0.906173263888889,
|
|
0.9052475694444445,
|
|
0.9008222222222222,
|
|
0.9121888888888889
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.812737282691152,
|
|
"std": 0.013330852209377185,
|
|
"ci_95": 0.01168500716136289,
|
|
"values": [
|
|
0.8203221809169765,
|
|
0.8213998390989542,
|
|
0.8207955888144939,
|
|
0.7901451825780906,
|
|
0.8110236220472441
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.8755833333333334,
|
|
"std": 0.018379676275712787,
|
|
"ci_95": 0.01611049657831811,
|
|
"values": [
|
|
0.8783333333333333,
|
|
0.8595833333333334,
|
|
0.8541666666666666,
|
|
0.8891666666666667,
|
|
0.8966666666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9611729166666667,
|
|
"std": 0.0043835653312689635,
|
|
"ci_95": 0.003842364425295284,
|
|
"values": [
|
|
0.9625631944444444,
|
|
0.9547138888888889,
|
|
0.9590722222222223,
|
|
0.9635131944444445,
|
|
0.9660020833333334
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.883256951965293,
|
|
"std": 0.014031840368382521,
|
|
"ci_95": 0.01229945037394657,
|
|
"values": [
|
|
0.8861154446177847,
|
|
0.8700347088314694,
|
|
0.867624810892587,
|
|
0.892914653784219,
|
|
0.8995951417004049
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "data",
|
|
"run_name": "p1_resnet18_baseline",
|
|
"backbone": "resnet18",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 128,
|
|
"subsample": 0.2,
|
|
"augment": false
|
|
}
|
|
} |