Johnny Fernandes
1657 lines
38 KiB
JSON
1657 lines
38 KiB
JSON
{
|
|
"run_name": "p3_mobilenetv3_small",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.21218074295255873,
|
|
0.13277151396636058,
|
|
0.1021481558887495,
|
|
0.07996781544449429,
|
|
0.060434623978842326,
|
|
0.05049107022393771,
|
|
0.039388784910638436,
|
|
0.028939813164039083,
|
|
0.02396829739980738,
|
|
0.01571510569449233
|
|
],
|
|
"train_acc": [
|
|
0.7969328703703704,
|
|
0.8913773148148149,
|
|
0.9219328703703704,
|
|
0.939699074074074,
|
|
0.9557291666666666,
|
|
0.9663194444444444,
|
|
0.9740162037037037,
|
|
0.9804976851851852,
|
|
0.9833333333333333,
|
|
0.9894097222222222
|
|
],
|
|
"train_auc": [
|
|
0.8846377743484225,
|
|
0.9575780803469364,
|
|
0.9752000278635117,
|
|
0.9848739533321901,
|
|
0.99124736546925,
|
|
0.9942682434556471,
|
|
0.9963542381115684,
|
|
0.9979329293124143,
|
|
0.9986672489426155,
|
|
0.9993458236882716
|
|
],
|
|
"train_f1": [
|
|
0.8545853880900087,
|
|
0.9251744070161451,
|
|
0.9468541937517236,
|
|
0.9591820745847697,
|
|
0.9701556587211797,
|
|
0.9774138466314809,
|
|
0.982577315587288,
|
|
0.9869435511991012,
|
|
0.9888484473011694,
|
|
0.9929187787795535
|
|
],
|
|
"val_loss": [
|
|
0.1599124413449317,
|
|
0.13173955893144013,
|
|
0.1388424592364269,
|
|
0.13763202510308475,
|
|
0.1327046855983402,
|
|
0.1615519176547726,
|
|
0.20638603230618174,
|
|
0.20039482988055776,
|
|
0.1953433026203129,
|
|
0.25831733345609487
|
|
],
|
|
"val_acc": [
|
|
0.8895833333333333,
|
|
0.8723958333333334,
|
|
0.9135416666666667,
|
|
0.9067708333333333,
|
|
0.9239583333333333,
|
|
0.9078125,
|
|
0.9192708333333334,
|
|
0.9213541666666667,
|
|
0.9161458333333333,
|
|
0.9223958333333333
|
|
],
|
|
"val_auc": [
|
|
0.9496961805555554,
|
|
0.9599197048611112,
|
|
0.965716869212963,
|
|
0.9661451099537037,
|
|
0.9714438657407407,
|
|
0.9656416377314816,
|
|
0.9629955150462963,
|
|
0.9664308449074074,
|
|
0.9661820023148149,
|
|
0.9618026620370371
|
|
],
|
|
"val_f1": [
|
|
0.9263888888888889,
|
|
0.9095607235142119,
|
|
0.9421199442119944,
|
|
0.9370826010544816,
|
|
0.9491289198606272,
|
|
0.937477922995408,
|
|
0.9461992363762582,
|
|
0.9471473573678684,
|
|
0.9432499118787452,
|
|
0.9486737857388908
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9210416666666666,
|
|
"auc_roc": 0.9712759259259258,
|
|
"f1": 0.9468368635152196,
|
|
"confusion_matrix": [
|
|
[
|
|
1046,
|
|
154
|
|
],
|
|
[
|
|
225,
|
|
3375
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.925,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.925,
|
|
"pairwise_auc": 0.9666506944444445,
|
|
"pairwise_f1": 0.900974025974026
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9141666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
103,
|
|
1097
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9141666666666667,
|
|
"pairwise_auc": 0.9610736111111111,
|
|
"pairwise_f1": 0.8951448388412893
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9733333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9733333333333334,
|
|
"pairwise_auc": 0.9861034722222222,
|
|
"pairwise_f1": 0.9262490087232356
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8716666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1046,
|
|
154
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1283333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8983333333333333,
|
|
"auc_roc": 0.9666506944444445,
|
|
"f1": 0.900974025974026,
|
|
"confusion_matrix": [
|
|
[
|
|
1046,
|
|
154
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8929166666666667,
|
|
"auc_roc": 0.9610736111111111,
|
|
"f1": 0.8951448388412893,
|
|
"confusion_matrix": [
|
|
[
|
|
1046,
|
|
154
|
|
],
|
|
[
|
|
103,
|
|
1097
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9225,
|
|
"auc_roc": 0.9861034722222222,
|
|
"f1": 0.9262490087232356,
|
|
"confusion_matrix": [
|
|
[
|
|
1046,
|
|
154
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.21668534226439617,
|
|
0.1314439894390051,
|
|
0.10657521119272267,
|
|
0.07994892044441292,
|
|
0.061171490301084874,
|
|
0.05102915394257892,
|
|
0.036623445882556824,
|
|
0.03262595429065793
|
|
],
|
|
"train_acc": [
|
|
0.7876157407407407,
|
|
0.8909722222222223,
|
|
0.9174189814814815,
|
|
0.9410879629629629,
|
|
0.9556134259259259,
|
|
0.9633101851851852,
|
|
0.9760416666666667,
|
|
0.9790509259259259
|
|
],
|
|
"train_auc": [
|
|
0.8797282503858024,
|
|
0.958232095907636,
|
|
0.9731424504172382,
|
|
0.9847290273491084,
|
|
0.9907720603995198,
|
|
0.9936988007973251,
|
|
0.9968317311814128,
|
|
0.9973761145404665
|
|
],
|
|
"train_f1": [
|
|
0.8465205754432921,
|
|
0.924730323611666,
|
|
0.943723626611981,
|
|
0.9601315892535442,
|
|
0.9700495919403335,
|
|
0.975326899128269,
|
|
0.9839609483960948,
|
|
0.985980946479746
|
|
],
|
|
"val_loss": [
|
|
0.1493083606318881,
|
|
0.1332949800649658,
|
|
0.12987089032928148,
|
|
0.16270457729697227,
|
|
0.17270868228612624,
|
|
0.18637286826463725,
|
|
0.2565473230487745,
|
|
0.26296898201999286
|
|
],
|
|
"val_acc": [
|
|
0.89375,
|
|
0.903125,
|
|
0.9057291666666667,
|
|
0.9177083333333333,
|
|
0.9057291666666667,
|
|
0.9078125,
|
|
0.9098958333333333,
|
|
0.9171875
|
|
],
|
|
"val_auc": [
|
|
0.9521195023148148,
|
|
0.9618192997685185,
|
|
0.9651041666666667,
|
|
0.9634252025462964,
|
|
0.9611053240740741,
|
|
0.9581018518518518,
|
|
0.950601851851852,
|
|
0.9547410300925927
|
|
],
|
|
"val_f1": [
|
|
0.9278642149929278,
|
|
0.9340425531914893,
|
|
0.935929203539823,
|
|
0.9452908587257618,
|
|
0.9363796133567662,
|
|
0.9372562920950017,
|
|
0.9397002439874521,
|
|
0.9450777202072539
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9085416666666667,
|
|
"auc_roc": 0.9634663194444445,
|
|
"f1": 0.9376863023420866,
|
|
"confusion_matrix": [
|
|
[
|
|
1058,
|
|
142
|
|
],
|
|
[
|
|
297,
|
|
3303
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.89,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
132,
|
|
1068
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.89,
|
|
"pairwise_auc": 0.9511392361111112,
|
|
"pairwise_f1": 0.8863070539419087
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8908333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
131,
|
|
1069
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8908333333333334,
|
|
"pairwise_auc": 0.9538555555555556,
|
|
"pairwise_f1": 0.8867689755288262
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9716666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
34,
|
|
1166
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9716666666666667,
|
|
"pairwise_auc": 0.9854041666666667,
|
|
"pairwise_f1": 0.9298245614035088
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8816666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1058,
|
|
142
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.11833333333333329
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8858333333333334,
|
|
"auc_roc": 0.9511392361111112,
|
|
"f1": 0.8863070539419087,
|
|
"confusion_matrix": [
|
|
[
|
|
1058,
|
|
142
|
|
],
|
|
[
|
|
132,
|
|
1068
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.88625,
|
|
"auc_roc": 0.9538555555555556,
|
|
"f1": 0.8867689755288262,
|
|
"confusion_matrix": [
|
|
[
|
|
1058,
|
|
142
|
|
],
|
|
[
|
|
131,
|
|
1069
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9266666666666666,
|
|
"auc_roc": 0.9854041666666667,
|
|
"f1": 0.9298245614035088,
|
|
"confusion_matrix": [
|
|
[
|
|
1058,
|
|
142
|
|
],
|
|
[
|
|
34,
|
|
1166
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.2204104232981249,
|
|
0.13763563461877681,
|
|
0.10055311655591208,
|
|
0.08203031489780793,
|
|
0.0636024187905369,
|
|
0.05151092803097088,
|
|
0.040638054257983135,
|
|
0.02824621118429436
|
|
],
|
|
"train_acc": [
|
|
0.7753472222222222,
|
|
0.8875,
|
|
0.9191550925925925,
|
|
0.938136574074074,
|
|
0.9546296296296296,
|
|
0.9646412037037037,
|
|
0.9731481481481481,
|
|
0.9814236111111111
|
|
],
|
|
"train_auc": [
|
|
0.8725610585991084,
|
|
0.9546696387745771,
|
|
0.9754571402034751,
|
|
0.9838291341592363,
|
|
0.9900558163294466,
|
|
0.9937081332876085,
|
|
0.9961024216249428,
|
|
0.9976673507516003
|
|
],
|
|
"train_f1": [
|
|
0.8361333896158717,
|
|
0.9222337787022962,
|
|
0.9447454811533441,
|
|
0.9579812114303683,
|
|
0.9693390692217442,
|
|
0.9762266059686393,
|
|
0.9819959646127581,
|
|
0.9875557278542353
|
|
],
|
|
"val_loss": [
|
|
0.17078705870856842,
|
|
0.13740695872499298,
|
|
0.13146800048028429,
|
|
0.13284930942463688,
|
|
0.17224184745864476,
|
|
0.20362560852663592,
|
|
0.21552294058892585,
|
|
0.24551390389557734
|
|
],
|
|
"val_acc": [
|
|
0.8921875,
|
|
0.9161458333333333,
|
|
0.9151041666666667,
|
|
0.9067708333333333,
|
|
0.9151041666666667,
|
|
0.9161458333333333,
|
|
0.9135416666666667,
|
|
0.9177083333333333
|
|
],
|
|
"val_auc": [
|
|
0.9481980613425927,
|
|
0.9650180844907408,
|
|
0.9681850405092592,
|
|
0.9675397858796296,
|
|
0.9641753472222222,
|
|
0.9640125868055555,
|
|
0.9599811921296295,
|
|
0.9606141493055556
|
|
],
|
|
"val_f1": [
|
|
0.9282992726013163,
|
|
0.9437259699405802,
|
|
0.9425044091710758,
|
|
0.9360942520528382,
|
|
0.9429471473573678,
|
|
0.9440389294403893,
|
|
0.942240779401531,
|
|
0.9451388888888889
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9160416666666666,
|
|
"auc_roc": 0.9662616898148148,
|
|
"f1": 0.9434544689210046,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
238,
|
|
3362
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9183333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
98,
|
|
1102
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9183333333333333,
|
|
"pairwise_auc": 0.9588486111111112,
|
|
"pairwise_f1": 0.8933927847588163
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9083333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9083333333333333,
|
|
"pairwise_auc": 0.9560364583333332,
|
|
"pairwise_f1": 0.8879837067209776
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.975,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
30,
|
|
1170
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.975,
|
|
"pairwise_auc": 0.9839000000000001,
|
|
"pairwise_f1": 0.9230769230769231
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8625,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.13749999999999996
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8904166666666666,
|
|
"auc_roc": 0.9588486111111112,
|
|
"f1": 0.8933927847588163,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
98,
|
|
1102
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8854166666666666,
|
|
"auc_roc": 0.9560364583333332,
|
|
"f1": 0.8879837067209776,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.91875,
|
|
"auc_roc": 0.9839000000000001,
|
|
"f1": 0.9230769230769231,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
30,
|
|
1170
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.21376656876807965,
|
|
0.13005489186832198,
|
|
0.1016177341117765,
|
|
0.07947396896089669,
|
|
0.06092459237271989,
|
|
0.047029034814288564,
|
|
0.038066642236339654,
|
|
0.029511018184819948,
|
|
0.01922926662668437
|
|
],
|
|
"train_acc": [
|
|
0.783912037037037,
|
|
0.8907407407407407,
|
|
0.9197916666666667,
|
|
0.9390046296296296,
|
|
0.9552083333333333,
|
|
0.9673611111111111,
|
|
0.9744212962962963,
|
|
0.9803240740740741,
|
|
0.9870949074074075
|
|
],
|
|
"train_auc": [
|
|
0.8817323602537723,
|
|
0.9587023194587334,
|
|
0.9750108239026063,
|
|
0.9850611479052355,
|
|
0.99081112289952,
|
|
0.994710558842021,
|
|
0.9965596868569959,
|
|
0.9978446412751486,
|
|
0.9990664384002057
|
|
],
|
|
"train_f1": [
|
|
0.8441829410782841,
|
|
0.924425586422224,
|
|
0.945273631840796,
|
|
0.9587055320482682,
|
|
0.9697963006321705,
|
|
0.9780766539687475,
|
|
0.9828535960896889,
|
|
0.9868380303499535,
|
|
0.9913783104581481
|
|
],
|
|
"val_loss": [
|
|
0.15234526932860415,
|
|
0.14175559830230972,
|
|
0.12728040980485578,
|
|
0.1402192492717101,
|
|
0.16635804822532615,
|
|
0.1776896538756167,
|
|
0.19778957384405657,
|
|
0.25235894794022895,
|
|
0.2533506410869298
|
|
],
|
|
"val_acc": [
|
|
0.8729166666666667,
|
|
0.9046875,
|
|
0.9,
|
|
0.9125,
|
|
0.903125,
|
|
0.8921875,
|
|
0.915625,
|
|
0.9229166666666667,
|
|
0.915625
|
|
],
|
|
"val_auc": [
|
|
0.9429166666666666,
|
|
0.9582993344907407,
|
|
0.9664554398148149,
|
|
0.9671216724537037,
|
|
0.9616059027777778,
|
|
0.9615646701388889,
|
|
0.9622113715277778,
|
|
0.9621426504629629,
|
|
0.9627756076388889
|
|
],
|
|
"val_f1": [
|
|
0.9115942028985508,
|
|
0.936214708957825,
|
|
0.9319631467044649,
|
|
0.9409282700421941,
|
|
0.9335239456754825,
|
|
0.9255127743792732,
|
|
0.9431977559607293,
|
|
0.9488243430152143,
|
|
0.9431578947368421
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.919375,
|
|
"auc_roc": 0.9686056712962963,
|
|
"f1": 0.9455619637079758,
|
|
"confusion_matrix": [
|
|
[
|
|
1052,
|
|
148
|
|
],
|
|
[
|
|
239,
|
|
3361
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9083333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9083333333333333,
|
|
"pairwise_auc": 0.9585697916666668,
|
|
"pairwise_f1": 0.8941755537325676
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9166666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
100,
|
|
1100
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9166666666666666,
|
|
"pairwise_auc": 0.96234375,
|
|
"pairwise_f1": 0.8986928104575164
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9758333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
29,
|
|
1171
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9758333333333333,
|
|
"pairwise_auc": 0.9849034722222222,
|
|
"pairwise_f1": 0.9297340214370782
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8766666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1052,
|
|
148
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1233333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8925,
|
|
"auc_roc": 0.9585697916666668,
|
|
"f1": 0.8941755537325676,
|
|
"confusion_matrix": [
|
|
[
|
|
1052,
|
|
148
|
|
],
|
|
[
|
|
110,
|
|
1090
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8966666666666666,
|
|
"auc_roc": 0.96234375,
|
|
"f1": 0.8986928104575164,
|
|
"confusion_matrix": [
|
|
[
|
|
1052,
|
|
148
|
|
],
|
|
[
|
|
100,
|
|
1100
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.92625,
|
|
"auc_roc": 0.9849034722222222,
|
|
"f1": 0.9297340214370782,
|
|
"confusion_matrix": [
|
|
[
|
|
1052,
|
|
148
|
|
],
|
|
[
|
|
29,
|
|
1171
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.22211545130444899,
|
|
0.13726882947677815,
|
|
0.10590094614049626,
|
|
0.0792858611671599,
|
|
0.06587177645760002,
|
|
0.050882136166695055,
|
|
0.03713606289648278,
|
|
0.028020461664170767,
|
|
0.026683380827927504,
|
|
0.02203933680822148,
|
|
0.016704037210205216
|
|
],
|
|
"train_acc": [
|
|
0.7792824074074074,
|
|
0.8861111111111111,
|
|
0.9171296296296296,
|
|
0.9403935185185185,
|
|
0.9514467592592593,
|
|
0.9649305555555555,
|
|
0.9751157407407407,
|
|
0.9820023148148148,
|
|
0.9846643518518519,
|
|
0.9869791666666666,
|
|
0.9890625
|
|
],
|
|
"train_auc": [
|
|
0.8722807355967077,
|
|
0.9540933820587563,
|
|
0.9732068222736625,
|
|
0.9846696209133516,
|
|
0.9896924118369914,
|
|
0.9937222793781435,
|
|
0.996372912022748,
|
|
0.9978854273834018,
|
|
0.9983555259059214,
|
|
0.9988915502114768,
|
|
0.9993319544467307
|
|
],
|
|
"train_f1": [
|
|
0.8403649757240917,
|
|
0.921191734742912,
|
|
0.9434171013118382,
|
|
0.9595634422110553,
|
|
0.9672431968141179,
|
|
0.9763926762758084,
|
|
0.9833178150217257,
|
|
0.9879490060836207,
|
|
0.9897544944906244,
|
|
0.9913050199018434,
|
|
0.9926962167175484
|
|
],
|
|
"val_loss": [
|
|
0.16186182308010758,
|
|
0.1564141693059355,
|
|
0.13674026299268008,
|
|
0.12408088110387325,
|
|
0.17875064068163435,
|
|
0.1664262376570453,
|
|
0.15911365750556192,
|
|
0.21940859338792507,
|
|
0.22924538743973244,
|
|
0.2207129465416074,
|
|
0.24139637431362643
|
|
],
|
|
"val_acc": [
|
|
0.8942708333333333,
|
|
0.8927083333333333,
|
|
0.909375,
|
|
0.9026041666666667,
|
|
0.9177083333333333,
|
|
0.925,
|
|
0.9125,
|
|
0.9223958333333333,
|
|
0.9244791666666666,
|
|
0.91875,
|
|
0.9265625
|
|
],
|
|
"val_auc": [
|
|
0.9505396412037036,
|
|
0.9564916087962964,
|
|
0.9657769097222222,
|
|
0.9682183159722223,
|
|
0.9637311921296297,
|
|
0.9698820891203703,
|
|
0.9698314525462963,
|
|
0.9637536168981482,
|
|
0.9638317418981481,
|
|
0.9665335648148148,
|
|
0.9630324074074075
|
|
],
|
|
"val_f1": [
|
|
0.9294894060437652,
|
|
0.9281729428172942,
|
|
0.9389901823281908,
|
|
0.93270960777258,
|
|
0.9452529452529452,
|
|
0.9498956158663883,
|
|
0.9400428265524625,
|
|
0.9479930191972077,
|
|
0.9495301079011487,
|
|
0.9451476793248945,
|
|
0.9507853403141361
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9225,
|
|
"auc_roc": 0.9704186342592592,
|
|
"f1": 0.9481894150417828,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
196,
|
|
3404
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
95,
|
|
1105
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9208333333333333,
|
|
"pairwise_auc": 0.9615777777777779,
|
|
"pairwise_f1": 0.890769850866586
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9383333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
74,
|
|
1126
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9383333333333334,
|
|
"pairwise_auc": 0.9672996527777777,
|
|
"pairwise_f1": 0.9000799360511591
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9775,
|
|
"pairwise_auc": 0.9823784722222222,
|
|
"pairwise_f1": 0.9203609258532758
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8533333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1466666666666666
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8870833333333333,
|
|
"auc_roc": 0.9615777777777779,
|
|
"f1": 0.890769850866586,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
95,
|
|
1105
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8958333333333334,
|
|
"auc_roc": 0.9672996527777777,
|
|
"f1": 0.9000799360511591,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
74,
|
|
1126
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9154166666666667,
|
|
"auc_roc": 0.9823784722222222,
|
|
"f1": 0.9203609258532758,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9175000000000001,
|
|
"std": 0.005555121510822233,
|
|
"ci_95": 0.004869278694016179,
|
|
"values": [
|
|
0.9210416666666666,
|
|
0.9085416666666667,
|
|
0.9160416666666666,
|
|
0.919375,
|
|
0.9225
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.968005648148148,
|
|
"std": 0.003182928770928824,
|
|
"ci_95": 0.0027899600789399893,
|
|
"values": [
|
|
0.9712759259259258,
|
|
0.9634663194444445,
|
|
0.9662616898148148,
|
|
0.9686056712962963,
|
|
0.9704186342592592
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9443458027056139,
|
|
"std": 0.0041109976994330374,
|
|
"ci_95": 0.0036034483620207875,
|
|
"values": [
|
|
0.9468368635152196,
|
|
0.9376863023420866,
|
|
0.9434544689210046,
|
|
0.9455619637079758,
|
|
0.9481894150417828
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9125,
|
|
"std": 0.013994046353122219,
|
|
"ci_95": 0.012266322458938808,
|
|
"values": [
|
|
0.925,
|
|
0.89,
|
|
0.9183333333333333,
|
|
0.9083333333333333,
|
|
0.9208333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9125,
|
|
"std": 0.013994046353122219,
|
|
"ci_95": 0.012266322458938808,
|
|
"values": [
|
|
0.925,
|
|
0.89,
|
|
0.9183333333333333,
|
|
0.9083333333333333,
|
|
0.9208333333333333
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9593572222222223,
|
|
"std": 0.00562449179695862,
|
|
"ci_95": 0.004930084430780652,
|
|
"values": [
|
|
0.9666506944444445,
|
|
0.9511392361111112,
|
|
0.9588486111111112,
|
|
0.9585697916666668,
|
|
0.9615777777777779
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.893123853854781,
|
|
"std": 0.005357557509873788,
|
|
"ci_95": 0.004696106211893378,
|
|
"values": [
|
|
0.900974025974026,
|
|
0.8863070539419087,
|
|
0.8933927847588163,
|
|
0.8941755537325676,
|
|
0.890769850866586
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9136666666666666,
|
|
"std": 0.01708434956326988,
|
|
"ci_95": 0.014975092654137396,
|
|
"values": [
|
|
0.9141666666666667,
|
|
0.8908333333333334,
|
|
0.9083333333333333,
|
|
0.9166666666666666,
|
|
0.9383333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9136666666666666,
|
|
"std": 0.01708434956326988,
|
|
"ci_95": 0.014975092654137396,
|
|
"values": [
|
|
0.9141666666666667,
|
|
0.8908333333333334,
|
|
0.9083333333333333,
|
|
0.9166666666666666,
|
|
0.9383333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9601218055555556,
|
|
"std": 0.005322603044187283,
|
|
"ci_95": 0.004665467271827631,
|
|
"values": [
|
|
0.9610736111111111,
|
|
0.9538555555555556,
|
|
0.9560364583333332,
|
|
0.96234375,
|
|
0.9672996527777777
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.8937340535199537,
|
|
"std": 0.006091564052432748,
|
|
"ci_95": 0.005339491313729217,
|
|
"values": [
|
|
0.8951448388412893,
|
|
0.8867689755288262,
|
|
0.8879837067209776,
|
|
0.8986928104575164,
|
|
0.9000799360511591
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9746666666666666,
|
|
"std": 0.0022515426810177145,
|
|
"ci_95": 0.001973564175687112,
|
|
"values": [
|
|
0.9733333333333334,
|
|
0.9716666666666667,
|
|
0.975,
|
|
0.9758333333333333,
|
|
0.9775
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9746666666666666,
|
|
"std": 0.0022515426810177145,
|
|
"ci_95": 0.001973564175687112,
|
|
"values": [
|
|
0.9733333333333334,
|
|
0.9716666666666667,
|
|
0.975,
|
|
0.9758333333333333,
|
|
0.9775
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9845379166666668,
|
|
"std": 0.001449579430945527,
|
|
"ci_95": 0.001270612393380827,
|
|
"values": [
|
|
0.9861034722222222,
|
|
0.9854041666666667,
|
|
0.9839000000000001,
|
|
0.9849034722222222,
|
|
0.9823784722222222
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9258490880988044,
|
|
"std": 0.004149152181075107,
|
|
"ci_95": 0.003636892239743178,
|
|
"values": [
|
|
0.9262490087232356,
|
|
0.9298245614035088,
|
|
0.9230769230769231,
|
|
0.9297340214370782,
|
|
0.9203609258532758
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.8691666666666669,
|
|
"std": 0.01133455875727953,
|
|
"ci_95": 0.009935178799487087,
|
|
"values": [
|
|
0.8716666666666667,
|
|
0.8816666666666667,
|
|
0.8625,
|
|
0.8766666666666667,
|
|
0.8533333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.1308333333333333,
|
|
"std": 0.01133455875727953,
|
|
"ci_95": 0.009935178799487087,
|
|
"values": [
|
|
0.1283333333333333,
|
|
0.11833333333333329,
|
|
0.13749999999999996,
|
|
0.1233333333333333,
|
|
0.1466666666666666
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.8908333333333334,
|
|
"std": 0.004956407188904283,
|
|
"ci_95": 0.004344482452234979,
|
|
"values": [
|
|
0.8983333333333333,
|
|
0.8858333333333334,
|
|
0.8904166666666666,
|
|
0.8925,
|
|
0.8870833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9593572222222223,
|
|
"std": 0.00562449179695862,
|
|
"ci_95": 0.004930084430780652,
|
|
"values": [
|
|
0.9666506944444445,
|
|
0.9511392361111112,
|
|
0.9588486111111112,
|
|
0.9585697916666668,
|
|
0.9615777777777779
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.893123853854781,
|
|
"std": 0.005357557509873788,
|
|
"ci_95": 0.004696106211893378,
|
|
"values": [
|
|
0.900974025974026,
|
|
0.8863070539419087,
|
|
0.8933927847588163,
|
|
0.8941755537325676,
|
|
0.890769850866586
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.8914166666666666,
|
|
"std": 0.005291830706118851,
|
|
"ci_95": 0.004638494128246565,
|
|
"values": [
|
|
0.8929166666666667,
|
|
0.88625,
|
|
0.8854166666666666,
|
|
0.8966666666666666,
|
|
0.8958333333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9601218055555556,
|
|
"std": 0.005322603044187283,
|
|
"ci_95": 0.004665467271827631,
|
|
"values": [
|
|
0.9610736111111111,
|
|
0.9538555555555556,
|
|
0.9560364583333332,
|
|
0.96234375,
|
|
0.9672996527777777
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8937340535199537,
|
|
"std": 0.006091564052432748,
|
|
"ci_95": 0.005339491313729217,
|
|
"values": [
|
|
0.8951448388412893,
|
|
0.8867689755288262,
|
|
0.8879837067209776,
|
|
0.8986928104575164,
|
|
0.9000799360511591
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9219166666666666,
|
|
"std": 0.004846605341198456,
|
|
"ci_95": 0.004248236889189057,
|
|
"values": [
|
|
0.9225,
|
|
0.9266666666666666,
|
|
0.91875,
|
|
0.92625,
|
|
0.9154166666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9845379166666668,
|
|
"std": 0.001449579430945527,
|
|
"ci_95": 0.001270612393380827,
|
|
"values": [
|
|
0.9861034722222222,
|
|
0.9854041666666667,
|
|
0.9839000000000001,
|
|
0.9849034722222222,
|
|
0.9823784722222222
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9258490880988044,
|
|
"std": 0.004149152181075107,
|
|
"ci_95": 0.003636892239743178,
|
|
"values": [
|
|
0.9262490087232356,
|
|
0.9298245614035088,
|
|
0.9230769230769231,
|
|
0.9297340214370782,
|
|
0.9203609258532758
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"run_name": "p3_mobilenetv3_small",
|
|
"backbone": "mobilenet_v3_small"
|
|
}
|
|
} |