Johnny Fernandes
1725 lines
40 KiB
JSON
1725 lines
40 KiB
JSON
{
|
|
"run_name": "p2a_t3_holdout_text2img",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.25936111174983745,
|
|
0.13471967871358365,
|
|
0.07450783130494348,
|
|
0.04490341250727206,
|
|
0.02855289393200907,
|
|
0.020640523931985996,
|
|
0.01199944190863149,
|
|
0.008351340312605945,
|
|
0.005777361075798794
|
|
],
|
|
"train_acc": [
|
|
0.8108796296296297,
|
|
0.9157407407407407,
|
|
0.9574845679012346,
|
|
0.9771604938271605,
|
|
0.9857253086419753,
|
|
0.9901234567901235,
|
|
0.9948302469135802,
|
|
0.9959876543209877,
|
|
0.9974537037037037
|
|
],
|
|
"train_auc": [
|
|
0.9029168006258572,
|
|
0.975353531700103,
|
|
0.9923160070516119,
|
|
0.9969480747385115,
|
|
0.9986825247556583,
|
|
0.9992779197745199,
|
|
0.999671156978738,
|
|
0.9998737836505487,
|
|
0.9999460010502401
|
|
],
|
|
"train_f1": [
|
|
0.8516433629925549,
|
|
0.9358627980735346,
|
|
0.9678773392409491,
|
|
0.9828166724718449,
|
|
0.9892772271489017,
|
|
0.9925814303929523,
|
|
0.9961197660276828,
|
|
0.9969907407407408,
|
|
0.9980901672550495
|
|
],
|
|
"val_loss": [
|
|
0.19289982169866562,
|
|
0.17769995480775833,
|
|
0.25123097085290486,
|
|
0.22796319822470348,
|
|
0.2647772492633926,
|
|
0.30438034501340655,
|
|
0.33032024684879513,
|
|
0.3406846645805571,
|
|
0.5371278474728266
|
|
],
|
|
"val_acc": [
|
|
0.8833333333333333,
|
|
0.9111111111111111,
|
|
0.8548611111111111,
|
|
0.9,
|
|
0.8909722222222223,
|
|
0.8993055555555556,
|
|
0.9041666666666667,
|
|
0.9048611111111111,
|
|
0.8861111111111111
|
|
],
|
|
"val_auc": [
|
|
0.9526030815972222,
|
|
0.9636534288194445,
|
|
0.9576529947916667,
|
|
0.9643446180555555,
|
|
0.9597515190972223,
|
|
0.9584884982638889,
|
|
0.9564192708333333,
|
|
0.9562858072916667,
|
|
0.9418001302083334
|
|
],
|
|
"val_f1": [
|
|
0.9132231404958677,
|
|
0.9333333333333333,
|
|
0.8822535211267606,
|
|
0.9242105263157895,
|
|
0.9149051490514906,
|
|
0.924518479958355,
|
|
0.9287190082644629,
|
|
0.9281594126900892,
|
|
0.9188118811881189
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.7866666666666666,
|
|
"auc_roc": 0.894214699074074,
|
|
"f1": 0.8427035330261137,
|
|
"confusion_matrix": [
|
|
[
|
|
1033,
|
|
167
|
|
],
|
|
[
|
|
857,
|
|
2743
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.935,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
78,
|
|
1122
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.935,
|
|
"pairwise_auc": 0.9670590277777777,
|
|
"pairwise_f1": 0.9015668943350743
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.91,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
108,
|
|
1092
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.91,
|
|
"pairwise_auc": 0.9627385416666667,
|
|
"pairwise_f1": 0.8881659211061407
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.44083333333333335,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
671,
|
|
529
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.44083333333333335,
|
|
"pairwise_auc": 0.7528465277777776,
|
|
"pairwise_f1": 0.5580168776371308
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8608333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1033,
|
|
167
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.13916666666666666
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8979166666666667,
|
|
"auc_roc": 0.9670590277777777,
|
|
"f1": 0.9015668943350743,
|
|
"confusion_matrix": [
|
|
[
|
|
1033,
|
|
167
|
|
],
|
|
[
|
|
78,
|
|
1122
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8854166666666666,
|
|
"auc_roc": 0.9627385416666667,
|
|
"f1": 0.8881659211061407,
|
|
"confusion_matrix": [
|
|
[
|
|
1033,
|
|
167
|
|
],
|
|
[
|
|
108,
|
|
1092
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.6508333333333334,
|
|
"auc_roc": 0.7528465277777776,
|
|
"f1": 0.5580168776371308,
|
|
"confusion_matrix": [
|
|
[
|
|
1033,
|
|
167
|
|
],
|
|
[
|
|
671,
|
|
529
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.2583258535574984,
|
|
0.13611753616068098,
|
|
0.07462463752125148,
|
|
0.0452532290659247,
|
|
0.02999016788657352,
|
|
0.019122767252064264,
|
|
0.010874123830203811,
|
|
0.006297654124936354,
|
|
0.003966155544324544,
|
|
0.0032256692653727776,
|
|
0.0021348577125019893,
|
|
0.0017390118794032281,
|
|
0.0006646750832945223,
|
|
0.0012557536934831727
|
|
],
|
|
"train_acc": [
|
|
0.8185956790123456,
|
|
0.9189814814814815,
|
|
0.957716049382716,
|
|
0.9763117283950618,
|
|
0.9847222222222223,
|
|
0.9912037037037037,
|
|
0.9947530864197531,
|
|
0.9972222222222222,
|
|
0.9982253086419753,
|
|
0.9983024691358025,
|
|
0.9992283950617284,
|
|
0.9993055555555556,
|
|
0.999537037037037,
|
|
0.9994598765432099
|
|
],
|
|
"train_auc": [
|
|
0.9041926681455761,
|
|
0.974750862697188,
|
|
0.9923605886702676,
|
|
0.9969843508873458,
|
|
0.9986152504501029,
|
|
0.9993941963520233,
|
|
0.999769616983882,
|
|
0.9998085187328533,
|
|
0.9999604150591564,
|
|
0.9999880642361111,
|
|
0.9999943201303155,
|
|
0.9999955793467077,
|
|
0.9999997320816187,
|
|
0.9999982719264403
|
|
],
|
|
"train_f1": [
|
|
0.8596334109499074,
|
|
0.9381698268755152,
|
|
0.9680130749474667,
|
|
0.9821646430023819,
|
|
0.9885217391304347,
|
|
0.9933974284721417,
|
|
0.9960634479564664,
|
|
0.9979154603358426,
|
|
0.9986684420772304,
|
|
0.9987268518518518,
|
|
0.9994212962962963,
|
|
0.999479015918958,
|
|
0.9996526571726294,
|
|
0.9995948370666203
|
|
],
|
|
"val_loss": [
|
|
0.1964837079246839,
|
|
0.20594968348741532,
|
|
0.26393358078267837,
|
|
0.309774564868874,
|
|
0.30068880518277485,
|
|
0.30512222581439546,
|
|
0.33635254121488994,
|
|
0.31080355329646003,
|
|
0.3263464295201831,
|
|
0.3539930683043268,
|
|
0.37549908343288635,
|
|
0.32930633922417957,
|
|
0.3372042354610231,
|
|
0.3207292776968744
|
|
],
|
|
"val_acc": [
|
|
0.8729166666666667,
|
|
0.8895833333333333,
|
|
0.8888888888888888,
|
|
0.8930555555555556,
|
|
0.8965277777777778,
|
|
0.9013888888888889,
|
|
0.8743055555555556,
|
|
0.8888888888888888,
|
|
0.9055555555555556,
|
|
0.9069444444444444,
|
|
0.9118055555555555,
|
|
0.9118055555555555,
|
|
0.9125,
|
|
0.9048611111111111
|
|
],
|
|
"val_auc": [
|
|
0.9484678819444444,
|
|
0.9535058593750001,
|
|
0.9505425347222223,
|
|
0.9517469618055554,
|
|
0.9594184027777778,
|
|
0.9555544704861112,
|
|
0.9591558159722221,
|
|
0.9600748697916668,
|
|
0.9611469184027779,
|
|
0.9561469184027778,
|
|
0.9521885850694445,
|
|
0.9576985677083334,
|
|
0.9570789930555555,
|
|
0.960546875
|
|
],
|
|
"val_f1": [
|
|
0.8999453253143794,
|
|
0.9163598106259864,
|
|
0.917610710607621,
|
|
0.9211065573770492,
|
|
0.9237851662404092,
|
|
0.9260416666666667,
|
|
0.9009304871373837,
|
|
0.9137001078748651,
|
|
0.9298245614035088,
|
|
0.9307851239669421,
|
|
0.9351047521716913,
|
|
0.934299017071909,
|
|
0.9351184346035015,
|
|
0.9286829776158251
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.8116666666666666,
|
|
"auc_roc": 0.910004398148148,
|
|
"f1": 0.8645489961042853,
|
|
"confusion_matrix": [
|
|
[
|
|
1011,
|
|
189
|
|
],
|
|
[
|
|
715,
|
|
2885
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9575,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
51,
|
|
1149
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9575,
|
|
"pairwise_auc": 0.9744965277777777,
|
|
"pairwise_f1": 0.9054373522458629
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9241666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9241666666666667,
|
|
"pairwise_auc": 0.9579416666666667,
|
|
"pairwise_f1": 0.8879103282626101
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.5225,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
573,
|
|
627
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.5225,
|
|
"pairwise_auc": 0.7975750000000001,
|
|
"pairwise_f1": 0.6220238095238095
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8425,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1011,
|
|
189
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.15749999999999997
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9,
|
|
"auc_roc": 0.9744965277777777,
|
|
"f1": 0.9054373522458629,
|
|
"confusion_matrix": [
|
|
[
|
|
1011,
|
|
189
|
|
],
|
|
[
|
|
51,
|
|
1149
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8833333333333333,
|
|
"auc_roc": 0.9579416666666667,
|
|
"f1": 0.8879103282626101,
|
|
"confusion_matrix": [
|
|
[
|
|
1011,
|
|
189
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.6825,
|
|
"auc_roc": 0.7975750000000001,
|
|
"f1": 0.6220238095238095,
|
|
"confusion_matrix": [
|
|
[
|
|
1011,
|
|
189
|
|
],
|
|
[
|
|
573,
|
|
627
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.26232991450362736,
|
|
0.13593779744373427,
|
|
0.07640465086578956,
|
|
0.04920590366294354,
|
|
0.027937425581384796,
|
|
0.021169344729181225,
|
|
0.010735498867188092,
|
|
0.010091174576473912,
|
|
0.0058576608773207375,
|
|
0.00232485462857531,
|
|
0.0011369886758849908,
|
|
0.0009826418625941446,
|
|
0.0010492766609505777
|
|
],
|
|
"train_acc": [
|
|
0.8170524691358024,
|
|
0.917283950617284,
|
|
0.9560185185185185,
|
|
0.9757716049382716,
|
|
0.9856481481481482,
|
|
0.9905092592592593,
|
|
0.9955246913580247,
|
|
0.9959876543209877,
|
|
0.9975308641975309,
|
|
0.9991512345679012,
|
|
0.9994598765432099,
|
|
0.999537037037037,
|
|
0.9996913580246913
|
|
],
|
|
"train_auc": [
|
|
0.9024782584233539,
|
|
0.9747301927940671,
|
|
0.99196964216821,
|
|
0.9962500401877572,
|
|
0.998842472029321,
|
|
0.9991242149991427,
|
|
0.9997307152349109,
|
|
0.9998100190757888,
|
|
0.999917682077332,
|
|
0.9999897119341563,
|
|
0.999999410579561,
|
|
0.9999993034122086,
|
|
0.9999988211591221
|
|
],
|
|
"train_f1": [
|
|
0.8566938652160774,
|
|
0.9367476988435214,
|
|
0.9667715984610004,
|
|
0.9817653890824622,
|
|
0.9892111368909513,
|
|
0.9928724575534565,
|
|
0.9966415749855241,
|
|
0.9969886495251332,
|
|
0.998147933788633,
|
|
0.9993633153904035,
|
|
0.9995947901591896,
|
|
0.9996527375853687,
|
|
0.9997684917235791
|
|
],
|
|
"val_loss": [
|
|
0.18554263181156583,
|
|
0.15960322005881203,
|
|
0.19698696964316897,
|
|
0.24289254877302383,
|
|
0.2467615756723616,
|
|
0.27950658169057635,
|
|
0.33689615113867655,
|
|
0.25568568557500837,
|
|
0.3190272374285592,
|
|
0.3059672231475512,
|
|
0.33635250528653465,
|
|
0.3490314172373878,
|
|
0.31362664004166924
|
|
],
|
|
"val_acc": [
|
|
0.86875,
|
|
0.9083333333333333,
|
|
0.9097222222222222,
|
|
0.8798611111111111,
|
|
0.9083333333333333,
|
|
0.9041666666666667,
|
|
0.9076388888888889,
|
|
0.9083333333333333,
|
|
0.9145833333333333,
|
|
0.9138888888888889,
|
|
0.9138888888888889,
|
|
0.9125,
|
|
0.9145833333333333
|
|
],
|
|
"val_auc": [
|
|
0.9542849392361111,
|
|
0.967917751736111,
|
|
0.9659657118055556,
|
|
0.9679568142361112,
|
|
0.9660481770833333,
|
|
0.9649891493055556,
|
|
0.9617361111111111,
|
|
0.9687163628472222,
|
|
0.9629090711805555,
|
|
0.9656477864583333,
|
|
0.9625965711805556,
|
|
0.9636642795138888,
|
|
0.9665223524305555
|
|
],
|
|
"val_f1": [
|
|
0.8963247394404827,
|
|
0.9303062302006336,
|
|
0.9322210636079249,
|
|
0.9034059184812954,
|
|
0.930890052356021,
|
|
0.9275210084033614,
|
|
0.9317598768599281,
|
|
0.9291084854994629,
|
|
0.9363024339720352,
|
|
0.935349322210636,
|
|
0.9357512953367876,
|
|
0.9351184346035015,
|
|
0.9360374414976599
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.7704166666666666,
|
|
"auc_roc": 0.8972921296296297,
|
|
"f1": 0.8268384663733501,
|
|
"confusion_matrix": [
|
|
[
|
|
1067,
|
|
133
|
|
],
|
|
[
|
|
969,
|
|
2631
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9291666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
85,
|
|
1115
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9291666666666667,
|
|
"pairwise_auc": 0.9711552083333335,
|
|
"pairwise_f1": 0.9109477124183006
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8883333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
134,
|
|
1066
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8883333333333333,
|
|
"pairwise_auc": 0.9588736111111111,
|
|
"pairwise_f1": 0.8887036265110463
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.375,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
750,
|
|
450
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.375,
|
|
"pairwise_auc": 0.7618475694444444,
|
|
"pairwise_f1": 0.5047672462142456
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8891666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1067,
|
|
133
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.11083333333333334
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9091666666666667,
|
|
"auc_roc": 0.9711552083333335,
|
|
"f1": 0.9109477124183006,
|
|
"confusion_matrix": [
|
|
[
|
|
1067,
|
|
133
|
|
],
|
|
[
|
|
85,
|
|
1115
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.88875,
|
|
"auc_roc": 0.9588736111111111,
|
|
"f1": 0.8887036265110463,
|
|
"confusion_matrix": [
|
|
[
|
|
1067,
|
|
133
|
|
],
|
|
[
|
|
134,
|
|
1066
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.6320833333333333,
|
|
"auc_roc": 0.7618475694444444,
|
|
"f1": 0.5047672462142456,
|
|
"confusion_matrix": [
|
|
[
|
|
1067,
|
|
133
|
|
],
|
|
[
|
|
750,
|
|
450
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.2620197121191908,
|
|
0.13751073586352078,
|
|
0.07180561355234664,
|
|
0.044450114913037575,
|
|
0.033807506547223226,
|
|
0.017647162451154413,
|
|
0.014270167727954686
|
|
],
|
|
"train_acc": [
|
|
0.8123456790123457,
|
|
0.9162037037037037,
|
|
0.9591049382716049,
|
|
0.9753086419753086,
|
|
0.9849537037037037,
|
|
0.9907407407407407,
|
|
0.9925925925925926
|
|
],
|
|
"train_auc": [
|
|
0.9014318496013374,
|
|
0.9744243505658436,
|
|
0.9928130358367626,
|
|
0.9971750953789438,
|
|
0.9982361727323388,
|
|
0.9994681150334361,
|
|
0.9994378402563444
|
|
],
|
|
"train_f1": [
|
|
0.8537230843257548,
|
|
0.9361552028218695,
|
|
0.969056515646894,
|
|
0.9814277423099246,
|
|
0.9886949968114094,
|
|
0.9930450909933929,
|
|
0.9944392956441149
|
|
],
|
|
"val_loss": [
|
|
0.19311593059036467,
|
|
0.18973327411545649,
|
|
0.22272994501723184,
|
|
0.37533411615424683,
|
|
0.2943312370114856,
|
|
0.3369398739602831,
|
|
0.37520144234100977
|
|
],
|
|
"val_acc": [
|
|
0.8729166666666667,
|
|
0.8930555555555556,
|
|
0.89375,
|
|
0.8833333333333333,
|
|
0.8729166666666667,
|
|
0.9048611111111111,
|
|
0.9041666666666667
|
|
],
|
|
"val_auc": [
|
|
0.9499533420138889,
|
|
0.9603559027777778,
|
|
0.9574663628472222,
|
|
0.9462217881944445,
|
|
0.959501953125,
|
|
0.950947265625,
|
|
0.9460405815972223
|
|
],
|
|
"val_f1": [
|
|
0.9036334913112164,
|
|
0.91995841995842,
|
|
0.9188328912466843,
|
|
0.9154078549848943,
|
|
0.8986149584487535,
|
|
0.9296353364149974,
|
|
0.9293032786885246
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.791875,
|
|
"auc_roc": 0.8873210648148148,
|
|
"f1": 0.8480608365019011,
|
|
"confusion_matrix": [
|
|
[
|
|
1013,
|
|
187
|
|
],
|
|
[
|
|
812,
|
|
2788
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9375,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
75,
|
|
1125
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9375,
|
|
"pairwise_auc": 0.9665180555555555,
|
|
"pairwise_f1": 0.8957006369426752
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9025,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
117,
|
|
1083
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9025,
|
|
"pairwise_auc": 0.9520479166666668,
|
|
"pairwise_f1": 0.8769230769230769
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.48333333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
620,
|
|
580
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.48333333333333334,
|
|
"pairwise_auc": 0.7433972222222223,
|
|
"pairwise_f1": 0.5897305541433655
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8441666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1013,
|
|
187
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.15583333333333338
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8908333333333334,
|
|
"auc_roc": 0.9665180555555555,
|
|
"f1": 0.8957006369426752,
|
|
"confusion_matrix": [
|
|
[
|
|
1013,
|
|
187
|
|
],
|
|
[
|
|
75,
|
|
1125
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8733333333333333,
|
|
"auc_roc": 0.9520479166666668,
|
|
"f1": 0.8769230769230769,
|
|
"confusion_matrix": [
|
|
[
|
|
1013,
|
|
187
|
|
],
|
|
[
|
|
117,
|
|
1083
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.66375,
|
|
"auc_roc": 0.7433972222222223,
|
|
"f1": 0.5897305541433655,
|
|
"confusion_matrix": [
|
|
[
|
|
1013,
|
|
187
|
|
],
|
|
[
|
|
620,
|
|
580
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 12960,
|
|
"val_size": 1440,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.26192283198053457,
|
|
0.13976006851428086,
|
|
0.07379607345368483,
|
|
0.04615930510155949,
|
|
0.03257456865322627,
|
|
0.019680384536385882,
|
|
0.014805444832772654,
|
|
0.007178295066770249,
|
|
0.00486855545719147,
|
|
0.0034230970272790414
|
|
],
|
|
"train_acc": [
|
|
0.8118055555555556,
|
|
0.9132716049382716,
|
|
0.9589506172839506,
|
|
0.9753858024691358,
|
|
0.9844907407407407,
|
|
0.9921296296296296,
|
|
0.9925925925925926,
|
|
0.9970679012345679,
|
|
0.9976851851851852,
|
|
0.9986882716049382
|
|
],
|
|
"train_auc": [
|
|
0.9018450869127228,
|
|
0.9735350088948902,
|
|
0.9923912921167696,
|
|
0.9968518250600138,
|
|
0.9983943517446845,
|
|
0.9992084083504801,
|
|
0.9995807479209533,
|
|
0.9998065629286694,
|
|
0.9999489481524348,
|
|
0.9999726053455076
|
|
],
|
|
"train_f1": [
|
|
0.8530457311562331,
|
|
0.9337498526464694,
|
|
0.9689832089552238,
|
|
0.981461033300401,
|
|
0.9883376849434291,
|
|
0.9940917516218721,
|
|
0.9944360727947142,
|
|
0.997801180418933,
|
|
0.9982622798887859,
|
|
0.9990161467677527
|
|
],
|
|
"val_loss": [
|
|
0.2104154187772009,
|
|
0.16755928165382808,
|
|
0.22552042288912666,
|
|
0.2703928468955888,
|
|
0.232618890537156,
|
|
0.30671322064267265,
|
|
0.34424640751547286,
|
|
0.2897042198313607,
|
|
0.29592826449208787,
|
|
0.31122492684258357
|
|
],
|
|
"val_acc": [
|
|
0.8534722222222222,
|
|
0.8965277777777778,
|
|
0.8798611111111111,
|
|
0.9076388888888889,
|
|
0.9048611111111111,
|
|
0.9118055555555555,
|
|
0.9090277777777778,
|
|
0.9041666666666667,
|
|
0.9159722222222222,
|
|
0.925
|
|
],
|
|
"val_auc": [
|
|
0.9383192274305556,
|
|
0.9631792534722222,
|
|
0.9636555989583334,
|
|
0.9636729600694445,
|
|
0.9671006944444446,
|
|
0.9630620659722223,
|
|
0.9626171875,
|
|
0.9637207031250001,
|
|
0.9643988715277777,
|
|
0.966296657986111
|
|
],
|
|
"val_f1": [
|
|
0.8845101258894362,
|
|
0.9203634420096205,
|
|
0.9046831955922865,
|
|
0.9326582278481013,
|
|
0.9276281035393555,
|
|
0.9354346720894764,
|
|
0.933871781928319,
|
|
0.9255663430420712,
|
|
0.9365495542737283,
|
|
0.9442148760330579
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.7647916666666666,
|
|
"auc_roc": 0.8910119212962961,
|
|
"f1": 0.8215584005057689,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
1001,
|
|
2599
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9108333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
107,
|
|
1093
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9108333333333334,
|
|
"pairwise_auc": 0.9666881944444444,
|
|
"pairwise_f1": 0.9029326724494011
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9058333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
113,
|
|
1087
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9058333333333334,
|
|
"pairwise_auc": 0.9646850694444445,
|
|
"pairwise_f1": 0.9002070393374741
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.3491666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
781,
|
|
419
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.3491666666666667,
|
|
"pairwise_auc": 0.7416625,
|
|
"pairwise_f1": 0.47967945048654836
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8933333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.10666666666666669
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9020833333333333,
|
|
"auc_roc": 0.9666881944444444,
|
|
"f1": 0.9029326724494011,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
107,
|
|
1093
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8995833333333333,
|
|
"auc_roc": 0.9646850694444445,
|
|
"f1": 0.9002070393374741,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
113,
|
|
1087
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.62125,
|
|
"auc_roc": 0.7416625,
|
|
"f1": 0.47967945048654836,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
781,
|
|
419
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.7850833333333334,
|
|
"std": 0.01858877442765451,
|
|
"ci_95": 0.01629377918954893,
|
|
"values": [
|
|
0.7866666666666666,
|
|
0.8116666666666666,
|
|
0.7704166666666666,
|
|
0.791875,
|
|
0.7647916666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.8959688425925926,
|
|
"std": 0.008677269977512222,
|
|
"ci_95": 0.007605962487303476,
|
|
"values": [
|
|
0.894214699074074,
|
|
0.910004398148148,
|
|
0.8972921296296297,
|
|
0.8873210648148148,
|
|
0.8910119212962961
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8407420465022838,
|
|
"std": 0.01721558000704002,
|
|
"ci_95": 0.015090121209788492,
|
|
"values": [
|
|
0.8427035330261137,
|
|
0.8645489961042853,
|
|
0.8268384663733501,
|
|
0.8480608365019011,
|
|
0.8215584005057689
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9339999999999999,
|
|
"std": 0.016774650186781495,
|
|
"ci_95": 0.014703629181637803,
|
|
"values": [
|
|
0.935,
|
|
0.9575,
|
|
0.9291666666666667,
|
|
0.9375,
|
|
0.9108333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9339999999999999,
|
|
"std": 0.016774650186781495,
|
|
"ci_95": 0.014703629181637803,
|
|
"values": [
|
|
0.935,
|
|
0.9575,
|
|
0.9291666666666667,
|
|
0.9375,
|
|
0.9108333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9691834027777777,
|
|
"std": 0.003534134405937126,
|
|
"ci_95": 0.00309780539113213,
|
|
"values": [
|
|
0.9670590277777777,
|
|
0.9744965277777777,
|
|
0.9711552083333335,
|
|
0.9665180555555555,
|
|
0.9666881944444444
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9033170536782628,
|
|
"std": 0.005566489678614175,
|
|
"ci_95": 0.004879243332433443,
|
|
"values": [
|
|
0.9015668943350743,
|
|
0.9054373522458629,
|
|
0.9109477124183006,
|
|
0.8957006369426752,
|
|
0.9029326724494011
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9061666666666668,
|
|
"std": 0.012944861700475792,
|
|
"ci_95": 0.011346671562866178,
|
|
"values": [
|
|
0.91,
|
|
0.9241666666666667,
|
|
0.8883333333333333,
|
|
0.9025,
|
|
0.9058333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9061666666666668,
|
|
"std": 0.012944861700475792,
|
|
"ci_95": 0.011346671562866178,
|
|
"values": [
|
|
0.91,
|
|
0.9241666666666667,
|
|
0.8883333333333333,
|
|
0.9025,
|
|
0.9058333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9592573611111112,
|
|
"std": 0.004884491809371957,
|
|
"ci_95": 0.004281445842748283,
|
|
"values": [
|
|
0.9627385416666667,
|
|
0.9579416666666667,
|
|
0.9588736111111111,
|
|
0.9520479166666668,
|
|
0.9646850694444445
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.8883819984280695,
|
|
"std": 0.008238796655830128,
|
|
"ci_95": 0.007221623675091769,
|
|
"values": [
|
|
0.8881659211061407,
|
|
0.8879103282626101,
|
|
0.8887036265110463,
|
|
0.8769230769230769,
|
|
0.9002070393374741
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.4341666666666667,
|
|
"std": 0.07244010936245501,
|
|
"ci_95": 0.0634965554621316,
|
|
"values": [
|
|
0.44083333333333335,
|
|
0.5225,
|
|
0.375,
|
|
0.48333333333333334,
|
|
0.3491666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.4341666666666667,
|
|
"std": 0.07244010936245501,
|
|
"ci_95": 0.0634965554621316,
|
|
"values": [
|
|
0.44083333333333335,
|
|
0.5225,
|
|
0.375,
|
|
0.48333333333333334,
|
|
0.3491666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.7594657638888889,
|
|
"std": 0.022786813060201933,
|
|
"ci_95": 0.01997352229333108,
|
|
"values": [
|
|
0.7528465277777776,
|
|
0.7975750000000001,
|
|
0.7618475694444444,
|
|
0.7433972222222223,
|
|
0.7416625
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.55084358760102,
|
|
"std": 0.058774228826551034,
|
|
"ci_95": 0.051517883024667956,
|
|
"values": [
|
|
0.5580168776371308,
|
|
0.6220238095238095,
|
|
0.5047672462142456,
|
|
0.5897305541433655,
|
|
0.47967945048654836
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.866,
|
|
"std": 0.02418390190004729,
|
|
"ci_95": 0.021198124654999286,
|
|
"values": [
|
|
0.8608333333333333,
|
|
0.8425,
|
|
0.8891666666666667,
|
|
0.8441666666666666,
|
|
0.8933333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.134,
|
|
"std": 0.02418390190004729,
|
|
"ci_95": 0.021198124654999286,
|
|
"values": [
|
|
0.13916666666666666,
|
|
0.15749999999999997,
|
|
0.11083333333333334,
|
|
0.15583333333333338,
|
|
0.10666666666666669
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9,
|
|
"std": 0.006647106722309916,
|
|
"ci_95": 0.005826445934034069,
|
|
"values": [
|
|
0.8979166666666667,
|
|
0.9,
|
|
0.9091666666666667,
|
|
0.8908333333333334,
|
|
0.9020833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9691834027777777,
|
|
"std": 0.003534134405937126,
|
|
"ci_95": 0.00309780539113213,
|
|
"values": [
|
|
0.9670590277777777,
|
|
0.9744965277777777,
|
|
0.9711552083333335,
|
|
0.9665180555555555,
|
|
0.9666881944444444
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9033170536782628,
|
|
"std": 0.005566489678614175,
|
|
"ci_95": 0.004879243332433443,
|
|
"values": [
|
|
0.9015668943350743,
|
|
0.9054373522458629,
|
|
0.9109477124183006,
|
|
0.8957006369426752,
|
|
0.9029326724494011
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.8860833333333332,
|
|
"std": 0.009485917925477162,
|
|
"ci_95": 0.008314773665657482,
|
|
"values": [
|
|
0.8854166666666666,
|
|
0.8833333333333333,
|
|
0.88875,
|
|
0.8733333333333333,
|
|
0.8995833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9592573611111112,
|
|
"std": 0.004884491809371957,
|
|
"ci_95": 0.004281445842748283,
|
|
"values": [
|
|
0.9627385416666667,
|
|
0.9579416666666667,
|
|
0.9588736111111111,
|
|
0.9520479166666668,
|
|
0.9646850694444445
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8883819984280695,
|
|
"std": 0.008238796655830128,
|
|
"ci_95": 0.007221623675091769,
|
|
"values": [
|
|
0.8881659211061407,
|
|
0.8879103282626101,
|
|
0.8887036265110463,
|
|
0.8769230769230769,
|
|
0.9002070393374741
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.6500833333333333,
|
|
"std": 0.02446191768170817,
|
|
"ci_95": 0.02144181623215098,
|
|
"values": [
|
|
0.6508333333333334,
|
|
0.6825,
|
|
0.6320833333333333,
|
|
0.66375,
|
|
0.62125
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.7594657638888889,
|
|
"std": 0.022786813060201933,
|
|
"ci_95": 0.01997352229333108,
|
|
"values": [
|
|
0.7528465277777776,
|
|
0.7975750000000001,
|
|
0.7618475694444444,
|
|
0.7433972222222223,
|
|
0.7416625
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.55084358760102,
|
|
"std": 0.058774228826551034,
|
|
"ci_95": 0.051517883024667956,
|
|
"values": [
|
|
0.5580168776371308,
|
|
0.6220238095238095,
|
|
0.5047672462142456,
|
|
0.5897305541433655,
|
|
0.47967945048654836
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "data",
|
|
"run_name": "p2a_t3_holdout_text2img",
|
|
"backbone": "resnet18",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"normalization": "imagenet",
|
|
"train_sources": [
|
|
"wiki",
|
|
"inpainting",
|
|
"insight"
|
|
],
|
|
"eval_sources": [
|
|
"wiki",
|
|
"inpainting",
|
|
"insight",
|
|
"text2img"
|
|
]
|
|
}
|
|
} |