Johnny Fernandes
1649 lines
38 KiB
JSON
1649 lines
38 KiB
JSON
{
|
|
"run_name": "p3_efficientnet_b0",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.18377161330922887,
|
|
0.09145856008313044,
|
|
0.0607653069568591,
|
|
0.04045726633781602,
|
|
0.030708745907435917,
|
|
0.02172367741931575,
|
|
0.019949010676807828,
|
|
0.014107607568035342,
|
|
0.01139849259136638,
|
|
0.011594377487664233
|
|
],
|
|
"train_acc": [
|
|
0.8245370370370371,
|
|
0.9293981481481481,
|
|
0.9565972222222222,
|
|
0.9735532407407408,
|
|
0.9802662037037037,
|
|
0.9866898148148148,
|
|
0.9887152777777778,
|
|
0.9912615740740741,
|
|
0.9924768518518519,
|
|
0.9936342592592593
|
|
],
|
|
"train_auc": [
|
|
0.9162158850594422,
|
|
0.979842017461134,
|
|
0.990933784865112,
|
|
0.9957966820987655,
|
|
0.9975816704532465,
|
|
0.9986849226251715,
|
|
0.9988981231424325,
|
|
0.9994862754343851,
|
|
0.9997144525891632,
|
|
0.9996553855166896
|
|
],
|
|
"train_f1": [
|
|
0.8751646903820817,
|
|
0.9520327121176378,
|
|
0.9707237098914826,
|
|
0.9822682652388158,
|
|
0.9867896021384573,
|
|
0.9910997600804892,
|
|
0.992464932957224,
|
|
0.9941665056982809,
|
|
0.9949768160741885,
|
|
0.9957509270704573
|
|
],
|
|
"val_loss": [
|
|
0.10881604209231834,
|
|
0.09755098982326066,
|
|
0.09397131755249574,
|
|
0.11721024793757048,
|
|
0.10389954849767188,
|
|
0.12676911439411925,
|
|
0.13487400793431636,
|
|
0.12308311458279302,
|
|
0.17003116869661122,
|
|
0.15836464502798056
|
|
],
|
|
"val_acc": [
|
|
0.9197916666666667,
|
|
0.9380208333333333,
|
|
0.9354166666666667,
|
|
0.9432291666666667,
|
|
0.9479166666666666,
|
|
0.9510416666666667,
|
|
0.95,
|
|
0.9505208333333334,
|
|
0.9505208333333334,
|
|
0.9489583333333333
|
|
],
|
|
"val_auc": [
|
|
0.9714532696759259,
|
|
0.978736255787037,
|
|
0.980892650462963,
|
|
0.9795565682870371,
|
|
0.983509837962963,
|
|
0.9804875578703703,
|
|
0.9783485243055556,
|
|
0.9834461805555555,
|
|
0.9768388310185185,
|
|
0.9773119212962964
|
|
],
|
|
"val_f1": [
|
|
0.9456598447424136,
|
|
0.9584642233856894,
|
|
0.9559659090909091,
|
|
0.9619811649808162,
|
|
0.9650104968509448,
|
|
0.9674289674289674,
|
|
0.9665038381018841,
|
|
0.9668643181025462,
|
|
0.9673202614379085,
|
|
0.9661134163208852
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.94125,
|
|
"auc_roc": 0.9812567129629629,
|
|
"f1": 0.9602816901408451,
|
|
"confusion_matrix": [
|
|
[
|
|
1109,
|
|
91
|
|
],
|
|
[
|
|
191,
|
|
3409
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.95,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
60,
|
|
1140
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.95,
|
|
"pairwise_auc": 0.9825836805555557,
|
|
"pairwise_f1": 0.9378856437679967
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9166666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
100,
|
|
1100
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9166666666666666,
|
|
"pairwise_auc": 0.9727503472222222,
|
|
"pairwise_f1": 0.9201171058134672
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9741666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
31,
|
|
1169
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9741666666666666,
|
|
"pairwise_auc": 0.9884361111111111,
|
|
"pairwise_f1": 0.9504065040650407
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9241666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1109,
|
|
91
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.07583333333333331
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9370833333333334,
|
|
"auc_roc": 0.9825836805555557,
|
|
"f1": 0.9378856437679967,
|
|
"confusion_matrix": [
|
|
[
|
|
1109,
|
|
91
|
|
],
|
|
[
|
|
60,
|
|
1140
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9204166666666667,
|
|
"auc_roc": 0.9727503472222222,
|
|
"f1": 0.9201171058134672,
|
|
"confusion_matrix": [
|
|
[
|
|
1109,
|
|
91
|
|
],
|
|
[
|
|
100,
|
|
1100
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9491666666666667,
|
|
"auc_roc": 0.9884361111111111,
|
|
"f1": 0.9504065040650407,
|
|
"confusion_matrix": [
|
|
[
|
|
1109,
|
|
91
|
|
],
|
|
[
|
|
31,
|
|
1169
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.18018175760904948,
|
|
0.09254055120292361,
|
|
0.057912689697049144,
|
|
0.04303662302274326,
|
|
0.03112416608615873,
|
|
0.02174639940823437,
|
|
0.015770501833483448,
|
|
0.013207592615044969,
|
|
0.012600847927657284
|
|
],
|
|
"train_acc": [
|
|
0.8336226851851852,
|
|
0.9306712962962963,
|
|
0.9585069444444444,
|
|
0.9693287037037037,
|
|
0.9799189814814815,
|
|
0.9863425925925926,
|
|
0.9897569444444444,
|
|
0.9924189814814814,
|
|
0.9925925925925926
|
|
],
|
|
"train_auc": [
|
|
0.9200014378286465,
|
|
0.9789969046496343,
|
|
0.9916078675125743,
|
|
0.995400529049497,
|
|
0.9975886452617742,
|
|
0.9988423246742113,
|
|
0.9992880069015775,
|
|
0.9994391307298812,
|
|
0.9994656993026978
|
|
],
|
|
"train_f1": [
|
|
0.8824563555337503,
|
|
0.9528235016145546,
|
|
0.9720500526254239,
|
|
0.9793918656194105,
|
|
0.9865665285896791,
|
|
0.9908704061895551,
|
|
0.9931546583130294,
|
|
0.9949414990153299,
|
|
0.9950560061799922
|
|
],
|
|
"val_loss": [
|
|
0.10321292195779581,
|
|
0.085910510395964,
|
|
0.09859157150882918,
|
|
0.09290146874457908,
|
|
0.10666261696896981,
|
|
0.13372382425538187,
|
|
0.12958144192719676,
|
|
0.13287445369484582,
|
|
0.12583209901980202
|
|
],
|
|
"val_acc": [
|
|
0.9192708333333334,
|
|
0.93125,
|
|
0.9390625,
|
|
0.9421875,
|
|
0.9552083333333333,
|
|
0.9416666666666667,
|
|
0.9515625,
|
|
0.9505208333333334,
|
|
0.946875
|
|
],
|
|
"val_auc": [
|
|
0.9742614293981482,
|
|
0.982859519675926,
|
|
0.9833369502314815,
|
|
0.9853566261574074,
|
|
0.9847844328703703,
|
|
0.9815422453703704,
|
|
0.9853016493055556,
|
|
0.9801164641203705,
|
|
0.9828927951388888
|
|
],
|
|
"val_f1": [
|
|
0.945054945054945,
|
|
0.9529579472558802,
|
|
0.9589329589329589,
|
|
0.96095673584242,
|
|
0.9699930216329379,
|
|
0.9607568325157674,
|
|
0.9678978253365551,
|
|
0.9667250437828371,
|
|
0.964260686755431
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9452083333333333,
|
|
"auc_roc": 0.9830223379629629,
|
|
"f1": 0.9630046420030947,
|
|
"confusion_matrix": [
|
|
[
|
|
1114,
|
|
86
|
|
],
|
|
[
|
|
177,
|
|
3423
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9458333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
65,
|
|
1135
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9458333333333333,
|
|
"pairwise_auc": 0.982111111111111,
|
|
"pairwise_f1": 0.9376290788930194
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9283333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
86,
|
|
1114
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9283333333333333,
|
|
"pairwise_auc": 0.9754020833333332,
|
|
"pairwise_f1": 0.9283333333333333
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9783333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9783333333333334,
|
|
"pairwise_auc": 0.9915538194444444,
|
|
"pairwise_f1": 0.9544715447154472
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9283333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1114,
|
|
86
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.07166666666666666
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9370833333333334,
|
|
"auc_roc": 0.982111111111111,
|
|
"f1": 0.9376290788930194,
|
|
"confusion_matrix": [
|
|
[
|
|
1114,
|
|
86
|
|
],
|
|
[
|
|
65,
|
|
1135
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9283333333333333,
|
|
"auc_roc": 0.9754020833333332,
|
|
"f1": 0.9283333333333333,
|
|
"confusion_matrix": [
|
|
[
|
|
1114,
|
|
86
|
|
],
|
|
[
|
|
86,
|
|
1114
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9533333333333334,
|
|
"auc_roc": 0.9915538194444444,
|
|
"f1": 0.9544715447154472,
|
|
"confusion_matrix": [
|
|
[
|
|
1114,
|
|
86
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.18167055828013906,
|
|
0.09439204159641156,
|
|
0.06150248942301712,
|
|
0.043100381485634935,
|
|
0.030508471781299534,
|
|
0.02616083418154934,
|
|
0.018010351766066312,
|
|
0.014479163717334343
|
|
],
|
|
"train_acc": [
|
|
0.8221643518518519,
|
|
0.9282986111111111,
|
|
0.9568865740740741,
|
|
0.9706018518518519,
|
|
0.9804976851851852,
|
|
0.9837962962962963,
|
|
0.9894097222222222,
|
|
0.9905092592592593
|
|
],
|
|
"train_auc": [
|
|
0.9187080350508688,
|
|
0.9786505754886832,
|
|
0.9907588430927069,
|
|
0.9952692222508002,
|
|
0.9974463627400548,
|
|
0.998317061756973,
|
|
0.9992033268318472,
|
|
0.999453714420439
|
|
],
|
|
"train_f1": [
|
|
0.8723996179877922,
|
|
0.951333516634589,
|
|
0.9709427044736534,
|
|
0.980268779616251,
|
|
0.9869475967310896,
|
|
0.9891783257323955,
|
|
0.9929280828534992,
|
|
0.9936606107460378
|
|
],
|
|
"val_loss": [
|
|
0.0951305437212189,
|
|
0.09423504600611826,
|
|
0.07776860665956822,
|
|
0.0992886533747272,
|
|
0.11139404144487344,
|
|
0.10963014076502683,
|
|
0.14097284133640642,
|
|
0.12554817825791056
|
|
],
|
|
"val_acc": [
|
|
0.9119791666666667,
|
|
0.9364583333333333,
|
|
0.9526041666666667,
|
|
0.9463541666666667,
|
|
0.9536458333333333,
|
|
0.9432291666666667,
|
|
0.9458333333333333,
|
|
0.9588541666666667
|
|
],
|
|
"val_auc": [
|
|
0.9793135127314815,
|
|
0.9799580439814816,
|
|
0.9875325520833333,
|
|
0.983644386574074,
|
|
0.9828262442129629,
|
|
0.984712818287037,
|
|
0.9811270254629629,
|
|
0.9851244212962963
|
|
],
|
|
"val_f1": [
|
|
0.9387902933719667,
|
|
0.9570119802677942,
|
|
0.9681037504381353,
|
|
0.9639986018874519,
|
|
0.9689787382363193,
|
|
0.9614432260346657,
|
|
0.9637377963737797,
|
|
0.9724834552420759
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.944375,
|
|
"auc_roc": 0.9849731481481482,
|
|
"f1": 0.9625683443151549,
|
|
"confusion_matrix": [
|
|
[
|
|
1100,
|
|
100
|
|
],
|
|
[
|
|
167,
|
|
3433
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9575,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
51,
|
|
1149
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9575,
|
|
"pairwise_auc": 0.9853777777777778,
|
|
"pairwise_f1": 0.9383421804818293
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.925,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.925,
|
|
"pairwise_auc": 0.9767972222222223,
|
|
"pairwise_f1": 0.921161825726141
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9783333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9783333333333334,
|
|
"pairwise_auc": 0.9927444444444444,
|
|
"pairwise_f1": 0.9490703314470493
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9166666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1100,
|
|
100
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.08333333333333337
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9370833333333334,
|
|
"auc_roc": 0.9853777777777778,
|
|
"f1": 0.9383421804818293,
|
|
"confusion_matrix": [
|
|
[
|
|
1100,
|
|
100
|
|
],
|
|
[
|
|
51,
|
|
1149
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": 0.9767972222222223,
|
|
"f1": 0.921161825726141,
|
|
"confusion_matrix": [
|
|
[
|
|
1100,
|
|
100
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9475,
|
|
"auc_roc": 0.9927444444444444,
|
|
"f1": 0.9490703314470493,
|
|
"confusion_matrix": [
|
|
[
|
|
1100,
|
|
100
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17916183584818132,
|
|
0.08699236897820675,
|
|
0.060961770671592266,
|
|
0.03990008551239346,
|
|
0.03000792982698099,
|
|
0.02349684396871857,
|
|
0.01548156029314635,
|
|
0.016230142107680742
|
|
],
|
|
"train_acc": [
|
|
0.8315393518518519,
|
|
0.9328125,
|
|
0.9565972222222222,
|
|
0.9738425925925925,
|
|
0.9789930555555556,
|
|
0.9847222222222223,
|
|
0.9898148148148148,
|
|
0.9898148148148148
|
|
],
|
|
"train_auc": [
|
|
0.9203963048696846,
|
|
0.9815779946130544,
|
|
0.9906308048982625,
|
|
0.9958646529921125,
|
|
0.9976785765317787,
|
|
0.9985926961877001,
|
|
0.9993483153292181,
|
|
0.999352530578418
|
|
],
|
|
"train_f1": [
|
|
0.880446835598998,
|
|
0.9543326908704716,
|
|
0.9707259953161592,
|
|
0.9824547783557177,
|
|
0.9859100260062881,
|
|
0.9897848630242997,
|
|
0.9931899086828664,
|
|
0.9932009580468206
|
|
],
|
|
"val_loss": [
|
|
0.10325937091062466,
|
|
0.09537656117851535,
|
|
0.09071044682447488,
|
|
0.11239791214854146,
|
|
0.12008310534001794,
|
|
0.12762126658781198,
|
|
0.15743482241086895,
|
|
0.13834097606037782
|
|
],
|
|
"val_acc": [
|
|
0.9119791666666667,
|
|
0.94375,
|
|
0.9421875,
|
|
0.94375,
|
|
0.9427083333333334,
|
|
0.9453125,
|
|
0.9515625,
|
|
0.9479166666666666
|
|
],
|
|
"val_auc": [
|
|
0.9742657696759259,
|
|
0.9797844328703703,
|
|
0.9824291087962962,
|
|
0.9799204282407408,
|
|
0.9785394965277778,
|
|
0.9775122974537037,
|
|
0.9757595486111111,
|
|
0.9799377893518517
|
|
],
|
|
"val_f1": [
|
|
0.9392304926285509,
|
|
0.9621848739495799,
|
|
0.9608465608465608,
|
|
0.9622377622377623,
|
|
0.9612948627726953,
|
|
0.9631190727081138,
|
|
0.9676971170545329,
|
|
0.9649859943977591
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.94625,
|
|
"auc_roc": 0.985225462962963,
|
|
"f1": 0.9636824324324325,
|
|
"confusion_matrix": [
|
|
[
|
|
1119,
|
|
81
|
|
],
|
|
[
|
|
177,
|
|
3423
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9466666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
64,
|
|
1136
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9466666666666667,
|
|
"pairwise_auc": 0.9826763888888889,
|
|
"pairwise_f1": 0.9400082747207281
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9283333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
86,
|
|
1114
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9283333333333333,
|
|
"pairwise_auc": 0.9806555555555555,
|
|
"pairwise_f1": 0.9302713987473904
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9775,
|
|
"pairwise_auc": 0.9923444444444444,
|
|
"pairwise_f1": 0.9559902200488998
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9325,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1119,
|
|
81
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.0675
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9395833333333333,
|
|
"auc_roc": 0.9826763888888889,
|
|
"f1": 0.9400082747207281,
|
|
"confusion_matrix": [
|
|
[
|
|
1119,
|
|
81
|
|
],
|
|
[
|
|
64,
|
|
1136
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9304166666666667,
|
|
"auc_roc": 0.9806555555555555,
|
|
"f1": 0.9302713987473904,
|
|
"confusion_matrix": [
|
|
[
|
|
1119,
|
|
81
|
|
],
|
|
[
|
|
86,
|
|
1114
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.955,
|
|
"auc_roc": 0.9923444444444444,
|
|
"f1": 0.9559902200488998,
|
|
"confusion_matrix": [
|
|
[
|
|
1119,
|
|
81
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.18033534770624504,
|
|
0.09273003415825466,
|
|
0.06080178875838303,
|
|
0.04101538560439453,
|
|
0.028531413568783102,
|
|
0.025501290097519006,
|
|
0.01986067992475,
|
|
0.014265834640361762,
|
|
0.011887943167163973,
|
|
0.008337455909424063
|
|
],
|
|
"train_acc": [
|
|
0.8253472222222222,
|
|
0.93125,
|
|
0.9561342592592592,
|
|
0.9719907407407408,
|
|
0.9802662037037037,
|
|
0.9853587962962963,
|
|
0.9873842592592592,
|
|
0.991550925925926,
|
|
0.993287037037037,
|
|
0.9951388888888889
|
|
],
|
|
"train_auc": [
|
|
0.9196097322245085,
|
|
0.9791677919238684,
|
|
0.9908107656750115,
|
|
0.9957474744227253,
|
|
0.9978721207704617,
|
|
0.9982988700988797,
|
|
0.9989265046296296,
|
|
0.9992963838163008,
|
|
0.9995847265089164,
|
|
0.9996307459562186
|
|
],
|
|
"train_f1": [
|
|
0.8752273854804035,
|
|
0.9533642144932087,
|
|
0.9704091193004373,
|
|
0.9812126387702819,
|
|
0.9867742310824962,
|
|
0.9902176854966555,
|
|
0.9915686881188119,
|
|
0.9943620636391721,
|
|
0.9955212355212355,
|
|
0.996756005252182
|
|
],
|
|
"val_loss": [
|
|
0.10515171031778058,
|
|
0.09159585210339477,
|
|
0.09799439595080912,
|
|
0.10880452486065527,
|
|
0.09861235054850112,
|
|
0.11025987912435084,
|
|
0.13374663864378816,
|
|
0.14156846931406714,
|
|
0.14357659897153402,
|
|
0.13649705853489044
|
|
],
|
|
"val_acc": [
|
|
0.9088541666666666,
|
|
0.9338541666666667,
|
|
0.9375,
|
|
0.9359375,
|
|
0.9411458333333333,
|
|
0.946875,
|
|
0.9479166666666666,
|
|
0.9453125,
|
|
0.9473958333333333,
|
|
0.9453125
|
|
],
|
|
"val_auc": [
|
|
0.9732530381944443,
|
|
0.9801772280092593,
|
|
0.9815407986111111,
|
|
0.9793938078703703,
|
|
0.9851222511574074,
|
|
0.9821021412037036,
|
|
0.9791550925925925,
|
|
0.980906394675926,
|
|
0.9806539351851853,
|
|
0.9817122395833332
|
|
],
|
|
"val_f1": [
|
|
0.9364791288566243,
|
|
0.955076052352317,
|
|
0.9577167019027484,
|
|
0.956398440269408,
|
|
0.9601691928093056,
|
|
0.9643356643356643,
|
|
0.9651567944250871,
|
|
0.9632224168126094,
|
|
0.9646482324116206,
|
|
0.9630151461782318
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9479166666666666,
|
|
"auc_roc": 0.9879561342592593,
|
|
"f1": 0.9647092038396387,
|
|
"confusion_matrix": [
|
|
[
|
|
1133,
|
|
67
|
|
],
|
|
[
|
|
183,
|
|
3417
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.935,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
78,
|
|
1122
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.935,
|
|
"pairwise_auc": 0.9863218749999999,
|
|
"pairwise_f1": 0.9393051485977396
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.93,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
84,
|
|
1116
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.93,
|
|
"pairwise_auc": 0.98235625,
|
|
"pairwise_f1": 0.936634494334872
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9825,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
21,
|
|
1179
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9825,
|
|
"pairwise_auc": 0.9951902777777777,
|
|
"pairwise_f1": 0.964022894521668
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9441666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1133,
|
|
67
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.05583333333333329
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9395833333333333,
|
|
"auc_roc": 0.9863218749999999,
|
|
"f1": 0.9393051485977396,
|
|
"confusion_matrix": [
|
|
[
|
|
1133,
|
|
67
|
|
],
|
|
[
|
|
78,
|
|
1122
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9370833333333334,
|
|
"auc_roc": 0.98235625,
|
|
"f1": 0.936634494334872,
|
|
"confusion_matrix": [
|
|
[
|
|
1133,
|
|
67
|
|
],
|
|
[
|
|
84,
|
|
1116
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9633333333333334,
|
|
"auc_roc": 0.9951902777777777,
|
|
"f1": 0.964022894521668,
|
|
"confusion_matrix": [
|
|
[
|
|
1133,
|
|
67
|
|
],
|
|
[
|
|
21,
|
|
1179
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9450000000000001,
|
|
"std": 0.0024782035944521334,
|
|
"ci_95": 0.002172241226117482,
|
|
"values": [
|
|
0.94125,
|
|
0.9452083333333333,
|
|
0.944375,
|
|
0.94625,
|
|
0.9479166666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9844867592592592,
|
|
"std": 0.0025197494425532535,
|
|
"ci_95": 0.00220865776760798,
|
|
"values": [
|
|
0.9812567129629629,
|
|
0.9830223379629629,
|
|
0.9849731481481482,
|
|
0.985225462962963,
|
|
0.9879561342592593
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9628492625462333,
|
|
"std": 0.0016468940129271584,
|
|
"ci_95": 0.001443566250139877,
|
|
"values": [
|
|
0.9602816901408451,
|
|
0.9630046420030947,
|
|
0.9625683443151549,
|
|
0.9636824324324325,
|
|
0.9647092038396387
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9470000000000001,
|
|
"std": 0.008135143241243837,
|
|
"ci_95": 0.007130767451294722,
|
|
"values": [
|
|
0.95,
|
|
0.9458333333333333,
|
|
0.9575,
|
|
0.9466666666666667,
|
|
0.935
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9470000000000001,
|
|
"std": 0.008135143241243837,
|
|
"ci_95": 0.007130767451294722,
|
|
"values": [
|
|
0.95,
|
|
0.9458333333333333,
|
|
0.9575,
|
|
0.9466666666666667,
|
|
0.935
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9838141666666667,
|
|
"std": 0.0019001653453324834,
|
|
"ci_95": 0.0016655683612158958,
|
|
"values": [
|
|
0.9825836805555557,
|
|
0.982111111111111,
|
|
0.9853777777777778,
|
|
0.9826763888888889,
|
|
0.9863218749999999
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9386340652922627,
|
|
"std": 0.0009992664233556023,
|
|
"ci_95": 0.0008758956389004342,
|
|
"values": [
|
|
0.9378856437679967,
|
|
0.9376290788930194,
|
|
0.9383421804818293,
|
|
0.9400082747207281,
|
|
0.9393051485977396
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9256666666666666,
|
|
"std": 0.005348935511960431,
|
|
"ci_95": 0.004688548697506415,
|
|
"values": [
|
|
0.9166666666666666,
|
|
0.9283333333333333,
|
|
0.925,
|
|
0.9283333333333333,
|
|
0.93
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9256666666666666,
|
|
"std": 0.005348935511960431,
|
|
"ci_95": 0.004688548697506415,
|
|
"values": [
|
|
0.9166666666666666,
|
|
0.9283333333333333,
|
|
0.925,
|
|
0.9283333333333333,
|
|
0.93
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9775922916666666,
|
|
"std": 0.003903602778930183,
|
|
"ci_95": 0.003421658698971229,
|
|
"values": [
|
|
0.9727503472222222,
|
|
0.9754020833333332,
|
|
0.9767972222222223,
|
|
0.9806555555555555,
|
|
0.98235625
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9273036315910408,
|
|
"std": 0.0068246036359346975,
|
|
"ci_95": 0.005982028838581346,
|
|
"values": [
|
|
0.9201171058134672,
|
|
0.9283333333333333,
|
|
0.921161825726141,
|
|
0.9302713987473904,
|
|
0.936634494334872
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9781666666666669,
|
|
"std": 0.002969754946867605,
|
|
"ci_95": 0.002603104983583198,
|
|
"values": [
|
|
0.9741666666666666,
|
|
0.9783333333333334,
|
|
0.9783333333333334,
|
|
0.9775,
|
|
0.9825
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9781666666666669,
|
|
"std": 0.002969754946867605,
|
|
"ci_95": 0.002603104983583198,
|
|
"values": [
|
|
0.9741666666666666,
|
|
0.9783333333333334,
|
|
0.9783333333333334,
|
|
0.9775,
|
|
0.9825
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9920538194444444,
|
|
"std": 0.0024360116093245137,
|
|
"ci_95": 0.002135258320551883,
|
|
"values": [
|
|
0.9884361111111111,
|
|
0.9915538194444444,
|
|
0.9927444444444444,
|
|
0.9923444444444444,
|
|
0.9951902777777777
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9547922989596209,
|
|
"std": 0.005888928207314737,
|
|
"ci_95": 0.005161873164179317,
|
|
"values": [
|
|
0.9504065040650407,
|
|
0.9544715447154472,
|
|
0.9490703314470493,
|
|
0.9559902200488998,
|
|
0.964022894521668
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9291666666666666,
|
|
"std": 0.010223203455321104,
|
|
"ci_95": 0.008961032926572219,
|
|
"values": [
|
|
0.9241666666666667,
|
|
0.9283333333333333,
|
|
0.9166666666666666,
|
|
0.9325,
|
|
0.9441666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.07083333333333333,
|
|
"std": 0.010223203455321104,
|
|
"ci_95": 0.008961032926572219,
|
|
"values": [
|
|
0.07583333333333331,
|
|
0.07166666666666666,
|
|
0.08333333333333337,
|
|
0.0675,
|
|
0.05583333333333329
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9380833333333334,
|
|
"std": 0.001369306393762886,
|
|
"ci_95": 0.0012002499739637315,
|
|
"values": [
|
|
0.9370833333333334,
|
|
0.9370833333333334,
|
|
0.9370833333333334,
|
|
0.9395833333333333,
|
|
0.9395833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9838141666666667,
|
|
"std": 0.0019001653453324834,
|
|
"ci_95": 0.0016655683612158958,
|
|
"values": [
|
|
0.9825836805555557,
|
|
0.982111111111111,
|
|
0.9853777777777778,
|
|
0.9826763888888889,
|
|
0.9863218749999999
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9386340652922627,
|
|
"std": 0.0009992664233556023,
|
|
"ci_95": 0.0008758956389004342,
|
|
"values": [
|
|
0.9378856437679967,
|
|
0.9376290788930194,
|
|
0.9383421804818293,
|
|
0.9400082747207281,
|
|
0.9393051485977396
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9274166666666668,
|
|
"std": 0.006993300365349712,
|
|
"ci_95": 0.006129898041566459,
|
|
"values": [
|
|
0.9204166666666667,
|
|
0.9283333333333333,
|
|
0.9208333333333333,
|
|
0.9304166666666667,
|
|
0.9370833333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9775922916666666,
|
|
"std": 0.003903602778930183,
|
|
"ci_95": 0.003421658698971229,
|
|
"values": [
|
|
0.9727503472222222,
|
|
0.9754020833333332,
|
|
0.9767972222222223,
|
|
0.9806555555555555,
|
|
0.98235625
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9273036315910408,
|
|
"std": 0.0068246036359346975,
|
|
"ci_95": 0.005982028838581346,
|
|
"values": [
|
|
0.9201171058134672,
|
|
0.9283333333333333,
|
|
0.921161825726141,
|
|
0.9302713987473904,
|
|
0.936634494334872
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9536666666666667,
|
|
"std": 0.006196997480859405,
|
|
"ci_95": 0.00543190778844986,
|
|
"values": [
|
|
0.9491666666666667,
|
|
0.9533333333333334,
|
|
0.9475,
|
|
0.955,
|
|
0.9633333333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9920538194444444,
|
|
"std": 0.0024360116093245137,
|
|
"ci_95": 0.002135258320551883,
|
|
"values": [
|
|
0.9884361111111111,
|
|
0.9915538194444444,
|
|
0.9927444444444444,
|
|
0.9923444444444444,
|
|
0.9951902777777777
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9547922989596209,
|
|
"std": 0.005888928207314737,
|
|
"ci_95": 0.005161873164179317,
|
|
"values": [
|
|
0.9504065040650407,
|
|
0.9544715447154472,
|
|
0.9490703314470493,
|
|
0.9559902200488998,
|
|
0.964022894521668
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"run_name": "p3_efficientnet_b0",
|
|
"backbone": "efficientnet_b0"
|
|
}
|
|
} |