Johnny Fernandes
1641 lines
38 KiB
JSON
1641 lines
38 KiB
JSON
{
|
|
"run_name": "p4_resnet50_50pct",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 43200,
|
|
"val_size": 4800,
|
|
"test_size": 12000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.1310014289824499,
|
|
0.06310213710750556,
|
|
0.038888596181080815,
|
|
0.02649530682997364,
|
|
0.01996485122527466,
|
|
0.0127978444466084,
|
|
0.008492352869334354,
|
|
0.005699059869553838,
|
|
0.002868689212677643,
|
|
0.0024933067745233524
|
|
],
|
|
"train_acc": [
|
|
0.8962731481481482,
|
|
0.9561805555555556,
|
|
0.9754166666666667,
|
|
0.9845601851851852,
|
|
0.9885648148148148,
|
|
0.9927777777777778,
|
|
0.9951388888888889,
|
|
0.9968981481481481,
|
|
0.9984259259259259,
|
|
0.9987731481481481
|
|
],
|
|
"train_auc": [
|
|
0.9595598193872886,
|
|
0.9908889531893004,
|
|
0.9964860496684956,
|
|
0.9983529835390947,
|
|
0.9989396790695015,
|
|
0.9993673125285779,
|
|
0.9997186914151805,
|
|
0.9998417038180156,
|
|
0.9999775605852766,
|
|
0.9999427869227252
|
|
],
|
|
"train_f1": [
|
|
0.929029601355739,
|
|
0.9704896565700656,
|
|
0.9835287548855388,
|
|
0.9896798749825935,
|
|
0.9923626356637086,
|
|
0.9951796804993357,
|
|
0.9967567567567568,
|
|
0.9979308214947499,
|
|
0.9989503581131144,
|
|
0.999182035650899
|
|
],
|
|
"val_loss": [
|
|
0.08767707792265962,
|
|
0.13368320959309737,
|
|
0.06841182627521145,
|
|
0.21279425378099404,
|
|
0.07938659159221667,
|
|
0.11103857491679567,
|
|
0.11696882055493309,
|
|
0.10955535782945541,
|
|
0.1455932674087914,
|
|
0.11922037690638414
|
|
],
|
|
"val_acc": [
|
|
0.9208333333333333,
|
|
0.8714583333333333,
|
|
0.9483333333333334,
|
|
0.949375,
|
|
0.9627083333333334,
|
|
0.96125,
|
|
0.9597916666666667,
|
|
0.9691666666666666,
|
|
0.966875,
|
|
0.9675
|
|
],
|
|
"val_auc": [
|
|
0.9838140046296295,
|
|
0.9848105324074073,
|
|
0.990100925925926,
|
|
0.979960300925926,
|
|
0.991436574074074,
|
|
0.988432175925926,
|
|
0.9853523148148149,
|
|
0.9874761574074074,
|
|
0.9840141203703704,
|
|
0.9863858796296296
|
|
],
|
|
"val_f1": [
|
|
0.9451817657241778,
|
|
0.907008289374529,
|
|
0.9647527003979534,
|
|
0.9670240195413218,
|
|
0.9749965078921637,
|
|
0.974188176519567,
|
|
0.9729881035689293,
|
|
0.9794387329813837,
|
|
0.977962577962578,
|
|
0.9782183747556549
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.963,
|
|
"auc_roc": 0.9923082037037039,
|
|
"f1": 0.975231507307821,
|
|
"confusion_matrix": [
|
|
[
|
|
2815,
|
|
185
|
|
],
|
|
[
|
|
259,
|
|
8741
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.98,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
60,
|
|
2940
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.98,
|
|
"pairwise_auc": 0.9937080555555556,
|
|
"pairwise_f1": 0.96
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9396666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
181,
|
|
2819
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9396666666666667,
|
|
"pairwise_auc": 0.9859274999999998,
|
|
"pairwise_f1": 0.9390406395736176
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.994,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
18,
|
|
2982
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.994,
|
|
"pairwise_auc": 0.9972890555555555,
|
|
"pairwise_f1": 0.9670828603859251
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9383333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
2815,
|
|
185
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"false_alarm_rate": 0.06166666666666665
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": 0.9937080555555556,
|
|
"f1": 0.96,
|
|
"confusion_matrix": [
|
|
[
|
|
2815,
|
|
185
|
|
],
|
|
[
|
|
60,
|
|
2940
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.939,
|
|
"auc_roc": 0.9859274999999998,
|
|
"f1": 0.9390406395736176,
|
|
"confusion_matrix": [
|
|
[
|
|
2815,
|
|
185
|
|
],
|
|
[
|
|
181,
|
|
2819
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9661666666666666,
|
|
"auc_roc": 0.9972890555555555,
|
|
"f1": 0.9670828603859251,
|
|
"confusion_matrix": [
|
|
[
|
|
2815,
|
|
185
|
|
],
|
|
[
|
|
18,
|
|
2982
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 43200,
|
|
"val_size": 4800,
|
|
"test_size": 12000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.12803853407088253,
|
|
0.06124075772430472,
|
|
0.03885383722314145,
|
|
0.02501593027866976,
|
|
0.01730566528073487,
|
|
0.01223388925149535,
|
|
0.006877621188353226,
|
|
0.005629530146293081
|
|
],
|
|
"train_acc": [
|
|
0.9024305555555555,
|
|
0.9589351851851852,
|
|
0.9743518518518518,
|
|
0.9850694444444444,
|
|
0.9902083333333334,
|
|
0.9930787037037037,
|
|
0.9962268518518519,
|
|
0.9971064814814815
|
|
],
|
|
"train_auc": [
|
|
0.9612873914037494,
|
|
0.9912515560699587,
|
|
0.9964169081504344,
|
|
0.9985149748513945,
|
|
0.9992661536922725,
|
|
0.9996025934499314,
|
|
0.9998924168381345,
|
|
0.9998858924897118
|
|
],
|
|
"train_f1": [
|
|
0.9334827276026955,
|
|
0.9723968382398706,
|
|
0.9828233032586116,
|
|
0.9900190334710552,
|
|
0.9934618297602671,
|
|
0.9953813120781007,
|
|
0.9974833639549785,
|
|
0.9980698855828174
|
|
],
|
|
"val_loss": [
|
|
0.0803564018686302,
|
|
0.06945386185039146,
|
|
0.06300256718920233,
|
|
0.069996998704349,
|
|
0.14107923867991,
|
|
0.1766739479874074,
|
|
0.12269854744713787,
|
|
0.11194367747741732
|
|
],
|
|
"val_acc": [
|
|
0.9535416666666666,
|
|
0.9539583333333334,
|
|
0.9485416666666666,
|
|
0.958125,
|
|
0.9520833333333333,
|
|
0.954375,
|
|
0.964375,
|
|
0.9608333333333333
|
|
],
|
|
"val_auc": [
|
|
0.9865824074074074,
|
|
0.9888171296296298,
|
|
0.9907064814814814,
|
|
0.9906428240740741,
|
|
0.9849028935185185,
|
|
0.9847343749999999,
|
|
0.9869310185185185,
|
|
0.9880565972222223
|
|
],
|
|
"val_f1": [
|
|
0.9690320788779336,
|
|
0.9689737470167065,
|
|
0.9650883392226148,
|
|
0.9719077568134172,
|
|
0.9684326104858633,
|
|
0.970200027214587,
|
|
0.9763256264709954,
|
|
0.9736547085201793
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.95625,
|
|
"auc_roc": 0.9913704629629632,
|
|
"f1": 0.9704475091471996,
|
|
"confusion_matrix": [
|
|
[
|
|
2855,
|
|
145
|
|
],
|
|
[
|
|
380,
|
|
8620
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.962,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
114,
|
|
2886
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.962,
|
|
"pairwise_auc": 0.9922814444444444,
|
|
"pairwise_f1": 0.9570552147239264
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9336666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
199,
|
|
2801
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9336666666666666,
|
|
"pairwise_auc": 0.9870954444444443,
|
|
"pairwise_f1": 0.9421459804910864
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9776666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
67,
|
|
2933
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9776666666666667,
|
|
"pairwise_auc": 0.9947345000000001,
|
|
"pairwise_f1": 0.9651201052977953
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9516666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
2855,
|
|
145
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"false_alarm_rate": 0.04833333333333334
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9568333333333333,
|
|
"auc_roc": 0.9922814444444444,
|
|
"f1": 0.9570552147239264,
|
|
"confusion_matrix": [
|
|
[
|
|
2855,
|
|
145
|
|
],
|
|
[
|
|
114,
|
|
2886
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9426666666666667,
|
|
"auc_roc": 0.9870954444444443,
|
|
"f1": 0.9421459804910864,
|
|
"confusion_matrix": [
|
|
[
|
|
2855,
|
|
145
|
|
],
|
|
[
|
|
199,
|
|
2801
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9646666666666667,
|
|
"auc_roc": 0.9947345000000001,
|
|
"f1": 0.9651201052977953,
|
|
"confusion_matrix": [
|
|
[
|
|
2855,
|
|
145
|
|
],
|
|
[
|
|
67,
|
|
2933
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 43200,
|
|
"val_size": 4800,
|
|
"test_size": 12000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.12894397541535674,
|
|
0.0633250668713892,
|
|
0.039845863456484276,
|
|
0.02533684753324013,
|
|
0.01833436753669014,
|
|
0.011535352683215563,
|
|
0.009289013643897642,
|
|
0.00523130823270494,
|
|
0.0038449670582292373,
|
|
0.0027505036443522227
|
|
],
|
|
"train_acc": [
|
|
0.8950694444444445,
|
|
0.9575231481481481,
|
|
0.9740740740740741,
|
|
0.9844444444444445,
|
|
0.9893287037037037,
|
|
0.9936342592592593,
|
|
0.995162037037037,
|
|
0.9967361111111112,
|
|
0.9981944444444445,
|
|
0.9988657407407407
|
|
],
|
|
"train_auc": [
|
|
0.9606062299954276,
|
|
0.9907223208161866,
|
|
0.99623664980567,
|
|
0.9985272305098307,
|
|
0.9992490026291724,
|
|
0.9995602866369456,
|
|
0.9997321987882946,
|
|
0.9999320373228165,
|
|
0.9998753429355282,
|
|
0.9998478709419296
|
|
],
|
|
"train_f1": [
|
|
0.927893104271057,
|
|
0.9714294611300544,
|
|
0.9826248836487744,
|
|
0.9896023518489865,
|
|
0.9928740358307699,
|
|
0.9957519116397621,
|
|
0.996772250621612,
|
|
0.9978224947106697,
|
|
0.9987958503149315,
|
|
0.9992437921508712
|
|
],
|
|
"val_loss": [
|
|
0.0721046452006946,
|
|
0.06456572874992465,
|
|
0.0747598591519636,
|
|
0.10288976659513234,
|
|
0.07962197108463442,
|
|
0.10778757501742803,
|
|
0.13586275433523648,
|
|
0.13340375898282825,
|
|
0.11878780224452688,
|
|
0.1416812105385543
|
|
],
|
|
"val_acc": [
|
|
0.9404166666666667,
|
|
0.9495833333333333,
|
|
0.9654166666666667,
|
|
0.9535416666666666,
|
|
0.9575,
|
|
0.9516666666666667,
|
|
0.9610416666666667,
|
|
0.9633333333333334,
|
|
0.965,
|
|
0.9679166666666666
|
|
],
|
|
"val_auc": [
|
|
0.9873201388888889,
|
|
0.9899142361111111,
|
|
0.9915755787037036,
|
|
0.988799074074074,
|
|
0.9916518518518519,
|
|
0.9892256944444444,
|
|
0.9852898148148148,
|
|
0.9867555555555556,
|
|
0.9866866898148148,
|
|
0.9848246527777779
|
|
],
|
|
"val_f1": [
|
|
0.9595015576323987,
|
|
0.9658288619034171,
|
|
0.9769252154573256,
|
|
0.9687980971036798,
|
|
0.971396522714526,
|
|
0.9672408924032759,
|
|
0.9741534208707671,
|
|
0.9756838905775076,
|
|
0.9765821020351269,
|
|
0.9786703601108033
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9618333333333333,
|
|
"auc_roc": 0.9928141296296296,
|
|
"f1": 0.9743129556926529,
|
|
"confusion_matrix": [
|
|
[
|
|
2856,
|
|
144
|
|
],
|
|
[
|
|
314,
|
|
8686
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.972,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
84,
|
|
2916
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.972,
|
|
"pairwise_auc": 0.9946737222222223,
|
|
"pairwise_f1": 0.9623762376237623
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9403333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
179,
|
|
2821
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9403333333333334,
|
|
"pairwise_auc": 0.9875953888888889,
|
|
"pairwise_f1": 0.9458507963118189
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.983,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
51,
|
|
2949
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.983,
|
|
"pairwise_auc": 0.9961732777777779,
|
|
"pairwise_f1": 0.9679960610536681
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.952,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
2856,
|
|
144
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"false_alarm_rate": 0.04800000000000004
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.962,
|
|
"auc_roc": 0.9946737222222223,
|
|
"f1": 0.9623762376237623,
|
|
"confusion_matrix": [
|
|
[
|
|
2856,
|
|
144
|
|
],
|
|
[
|
|
84,
|
|
2916
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9461666666666667,
|
|
"auc_roc": 0.9875953888888889,
|
|
"f1": 0.9458507963118189,
|
|
"confusion_matrix": [
|
|
[
|
|
2856,
|
|
144
|
|
],
|
|
[
|
|
179,
|
|
2821
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9675,
|
|
"auc_roc": 0.9961732777777779,
|
|
"f1": 0.9679960610536681,
|
|
"confusion_matrix": [
|
|
[
|
|
2856,
|
|
144
|
|
],
|
|
[
|
|
51,
|
|
2949
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 43200,
|
|
"val_size": 4800,
|
|
"test_size": 12000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.1295232505020168,
|
|
0.06170430574824826,
|
|
0.037615918873106474,
|
|
0.024598854103549256,
|
|
0.016743204425551488,
|
|
0.01205475128056504,
|
|
0.008438206393399452
|
|
],
|
|
"train_acc": [
|
|
0.8965277777777778,
|
|
0.9588194444444444,
|
|
0.9754166666666667,
|
|
0.9843518518518518,
|
|
0.9911111111111112,
|
|
0.9932407407407408,
|
|
0.9955092592592593
|
|
],
|
|
"train_auc": [
|
|
0.9603803883744856,
|
|
0.9911017889803384,
|
|
0.9967371799268405,
|
|
0.9985225465820761,
|
|
0.9991573702560586,
|
|
0.9995904020919067,
|
|
0.9997432584590762
|
|
],
|
|
"train_f1": [
|
|
0.9289935188715212,
|
|
0.9722936037004158,
|
|
0.9835292657960855,
|
|
0.9895391662282195,
|
|
0.9940671157530437,
|
|
0.9954895115697118,
|
|
0.9970040460820954
|
|
],
|
|
"val_loss": [
|
|
0.10778514720315191,
|
|
0.06575797121937892,
|
|
0.07033019703017393,
|
|
0.08894823268909628,
|
|
0.09753751006637079,
|
|
0.12266531725380748,
|
|
0.11800297263565275
|
|
],
|
|
"val_acc": [
|
|
0.940625,
|
|
0.959375,
|
|
0.9539583333333334,
|
|
0.9602083333333333,
|
|
0.956875,
|
|
0.955,
|
|
0.96375
|
|
],
|
|
"val_auc": [
|
|
0.9809686342592594,
|
|
0.9922035879629629,
|
|
0.9914534722222222,
|
|
0.9891475694444445,
|
|
0.990147800925926,
|
|
0.9855612268518518,
|
|
0.9898511574074075
|
|
],
|
|
"val_f1": [
|
|
0.9606081548030407,
|
|
0.9729129045700792,
|
|
0.9687809012572397,
|
|
0.9733649421280156,
|
|
0.9709310490099705,
|
|
0.9697563707644917,
|
|
0.9759202878494326
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9590833333333333,
|
|
"auc_roc": 0.9911175185185186,
|
|
"f1": 0.9727962768020388,
|
|
"confusion_matrix": [
|
|
[
|
|
2730,
|
|
270
|
|
],
|
|
[
|
|
221,
|
|
8779
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9833333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
50,
|
|
2950
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9833333333333333,
|
|
"pairwise_auc": 0.9927475555555554,
|
|
"pairwise_f1": 0.9485530546623794
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9553333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
134,
|
|
2866
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9553333333333334,
|
|
"pairwise_auc": 0.9855584444444443,
|
|
"pairwise_f1": 0.9341590612777053
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9876666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
37,
|
|
2963
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9876666666666667,
|
|
"pairwise_auc": 0.9950465555555554,
|
|
"pairwise_f1": 0.9507460291994224
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.91,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
2730,
|
|
270
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"false_alarm_rate": 0.08999999999999997
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9466666666666667,
|
|
"auc_roc": 0.9927475555555554,
|
|
"f1": 0.9485530546623794,
|
|
"confusion_matrix": [
|
|
[
|
|
2730,
|
|
270
|
|
],
|
|
[
|
|
50,
|
|
2950
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9326666666666666,
|
|
"auc_roc": 0.9855584444444443,
|
|
"f1": 0.9341590612777053,
|
|
"confusion_matrix": [
|
|
[
|
|
2730,
|
|
270
|
|
],
|
|
[
|
|
134,
|
|
2866
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9488333333333333,
|
|
"auc_roc": 0.9950465555555554,
|
|
"f1": 0.9507460291994224,
|
|
"confusion_matrix": [
|
|
[
|
|
2730,
|
|
270
|
|
],
|
|
[
|
|
37,
|
|
2963
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 43200,
|
|
"val_size": 4800,
|
|
"test_size": 12000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.12732069519589898,
|
|
0.06170257507538837,
|
|
0.03860741297643269,
|
|
0.025365771362277748,
|
|
0.01670022751770173,
|
|
0.012464827809432824,
|
|
0.008374597468172464,
|
|
0.006437417975058385,
|
|
0.0034875015909215192
|
|
],
|
|
"train_acc": [
|
|
0.9009027777777778,
|
|
0.9566898148148149,
|
|
0.9751620370370371,
|
|
0.9845138888888889,
|
|
0.9905324074074074,
|
|
0.9931944444444445,
|
|
0.9955787037037037,
|
|
0.9963425925925926,
|
|
0.9982638888888888
|
|
],
|
|
"train_auc": [
|
|
0.9617297125057156,
|
|
0.9911038951760403,
|
|
0.9964552040466392,
|
|
0.9983521819272976,
|
|
0.9992034793667124,
|
|
0.9995827817786922,
|
|
0.9996301911865568,
|
|
0.9998173682556012,
|
|
0.9999299397005029
|
|
],
|
|
"train_f1": [
|
|
0.932434778491501,
|
|
0.970832618828628,
|
|
0.9833532432474362,
|
|
0.9896463669426604,
|
|
0.9936790047137006,
|
|
0.9954594594594595,
|
|
0.9970505119137699,
|
|
0.9975598455598456,
|
|
0.998842503279574
|
|
],
|
|
"val_loss": [
|
|
0.14045082565508588,
|
|
0.08130352705096205,
|
|
0.10408762747037448,
|
|
0.0898400347368503,
|
|
0.09015464449475985,
|
|
0.14536626939580552,
|
|
0.1540585432428573,
|
|
0.15825259659189517,
|
|
0.13043432375607153
|
|
],
|
|
"val_acc": [
|
|
0.9389583333333333,
|
|
0.9297916666666667,
|
|
0.9570833333333333,
|
|
0.9547916666666667,
|
|
0.949375,
|
|
0.9441666666666667,
|
|
0.9552083333333333,
|
|
0.956875,
|
|
0.9577083333333334
|
|
],
|
|
"val_auc": [
|
|
0.9839943287037036,
|
|
0.9862025462962964,
|
|
0.9870587962962962,
|
|
0.9889800925925927,
|
|
0.9889494212962964,
|
|
0.9834181712962962,
|
|
0.9811201388888888,
|
|
0.9818784722222222,
|
|
0.9862002314814815
|
|
],
|
|
"val_f1": [
|
|
0.960292722591137,
|
|
0.9515177672277371,
|
|
0.9715076071922545,
|
|
0.9695096248419278,
|
|
0.965614829489175,
|
|
0.9622747747747747,
|
|
0.9698245614035088,
|
|
0.9713415478333103,
|
|
0.9714607057500352
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9634166666666667,
|
|
"auc_roc": 0.9929008518518518,
|
|
"f1": 0.9754570358360821,
|
|
"confusion_matrix": [
|
|
[
|
|
2837,
|
|
163
|
|
],
|
|
[
|
|
276,
|
|
8724
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9736666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
79,
|
|
2921
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9736666666666667,
|
|
"pairwise_auc": 0.9941697777777777,
|
|
"pairwise_f1": 0.960223537146614
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.946,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
162,
|
|
2838
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.946,
|
|
"pairwise_auc": 0.9881915555555555,
|
|
"pairwise_f1": 0.9458423596067322
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9883333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
35,
|
|
2965
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"detection_rate": 0.9883333333333333,
|
|
"pairwise_auc": 0.9963412222222222,
|
|
"pairwise_f1": 0.9676892950391645
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9456666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
2837,
|
|
163
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 3000,
|
|
"false_alarm_rate": 0.054333333333333345
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9596666666666667,
|
|
"auc_roc": 0.9941697777777777,
|
|
"f1": 0.960223537146614,
|
|
"confusion_matrix": [
|
|
[
|
|
2837,
|
|
163
|
|
],
|
|
[
|
|
79,
|
|
2921
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.9458333333333333,
|
|
"auc_roc": 0.9881915555555555,
|
|
"f1": 0.9458423596067322,
|
|
"confusion_matrix": [
|
|
[
|
|
2837,
|
|
163
|
|
],
|
|
[
|
|
162,
|
|
2838
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 6000,
|
|
"accuracy": 0.967,
|
|
"auc_roc": 0.9963412222222222,
|
|
"f1": 0.9676892950391645,
|
|
"confusion_matrix": [
|
|
[
|
|
2837,
|
|
163
|
|
],
|
|
[
|
|
35,
|
|
2965
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9607166666666667,
|
|
"std": 0.0030153542264734086,
|
|
"ci_95": 0.002643074514441248,
|
|
"values": [
|
|
0.963,
|
|
0.95625,
|
|
0.9618333333333333,
|
|
0.9590833333333333,
|
|
0.9634166666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9921022333333335,
|
|
"std": 0.000820390579196715,
|
|
"ci_95": 0.0007191040484482374,
|
|
"values": [
|
|
0.9923082037037039,
|
|
0.9913704629629632,
|
|
0.9928141296296296,
|
|
0.9911175185185186,
|
|
0.9929008518518518
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.973649056957159,
|
|
"std": 0.0020730919324981494,
|
|
"ci_95": 0.0018171451979915288,
|
|
"values": [
|
|
0.975231507307821,
|
|
0.9704475091471996,
|
|
0.9743129556926529,
|
|
0.9727962768020388,
|
|
0.9754570358360821
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9742,
|
|
"std": 0.008231376285625357,
|
|
"ci_95": 0.007215119433830905,
|
|
"values": [
|
|
0.98,
|
|
0.962,
|
|
0.972,
|
|
0.9833333333333333,
|
|
0.9736666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 3000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9742,
|
|
"std": 0.008231376285625357,
|
|
"ci_95": 0.007215119433830905,
|
|
"values": [
|
|
0.98,
|
|
0.962,
|
|
0.972,
|
|
0.9833333333333333,
|
|
0.9736666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.993516111111111,
|
|
"std": 0.0009898567680712113,
|
|
"ci_95": 0.000867647712387025,
|
|
"values": [
|
|
0.9937080555555556,
|
|
0.9922814444444444,
|
|
0.9946737222222223,
|
|
0.9927475555555554,
|
|
0.9941697777777777
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9576416088313364,
|
|
"std": 0.005421963377389286,
|
|
"ci_95": 0.004752560443875861,
|
|
"values": [
|
|
0.96,
|
|
0.9570552147239264,
|
|
0.9623762376237623,
|
|
0.9485530546623794,
|
|
0.960223537146614
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.943,
|
|
"std": 0.00816156303113021,
|
|
"ci_95": 0.0071539254181805,
|
|
"values": [
|
|
0.9396666666666667,
|
|
0.9336666666666666,
|
|
0.9403333333333334,
|
|
0.9553333333333334,
|
|
0.946
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 3000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.943,
|
|
"std": 0.00816156303113021,
|
|
"ci_95": 0.0071539254181805,
|
|
"values": [
|
|
0.9396666666666667,
|
|
0.9336666666666666,
|
|
0.9403333333333334,
|
|
0.9553333333333334,
|
|
0.946
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9868736666666666,
|
|
"std": 0.0011103986139176464,
|
|
"ci_95": 0.0009733072988738293,
|
|
"values": [
|
|
0.9859274999999998,
|
|
0.9870954444444443,
|
|
0.9875953888888889,
|
|
0.9855584444444443,
|
|
0.9881915555555555
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.941407767452192,
|
|
"std": 0.004952224427288878,
|
|
"ci_95": 0.004340816100027134,
|
|
"values": [
|
|
0.9390406395736176,
|
|
0.9421459804910864,
|
|
0.9458507963118189,
|
|
0.9341590612777053,
|
|
0.9458423596067322
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9861333333333334,
|
|
"std": 0.006135506861249879,
|
|
"ci_95": 0.005378008883923071,
|
|
"values": [
|
|
0.994,
|
|
0.9776666666666667,
|
|
0.983,
|
|
0.9876666666666667,
|
|
0.9883333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 3000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9861333333333334,
|
|
"std": 0.006135506861249879,
|
|
"ci_95": 0.005378008883923071,
|
|
"values": [
|
|
0.994,
|
|
0.9776666666666667,
|
|
0.983,
|
|
0.9876666666666667,
|
|
0.9883333333333333
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9959169222222222,
|
|
"std": 0.0010349105023331555,
|
|
"ci_95": 0.000907139051667393,
|
|
"values": [
|
|
0.9972890555555555,
|
|
0.9947345000000001,
|
|
0.9961732777777779,
|
|
0.9950465555555554,
|
|
0.9963412222222222
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9637268701951951,
|
|
"std": 0.007342220515605291,
|
|
"ci_95": 0.006435740038045298,
|
|
"values": [
|
|
0.9670828603859251,
|
|
0.9651201052977953,
|
|
0.9679960610536681,
|
|
0.9507460291994224,
|
|
0.9676892950391645
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9395333333333333,
|
|
"std": 0.017420613587867017,
|
|
"ci_95": 0.015269841067353043,
|
|
"values": [
|
|
0.9383333333333334,
|
|
0.9516666666666667,
|
|
0.952,
|
|
0.91,
|
|
0.9456666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 3000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000,
|
|
3000
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.06046666666666667,
|
|
"std": 0.017420613587867017,
|
|
"ci_95": 0.015269841067353043,
|
|
"values": [
|
|
0.06166666666666665,
|
|
0.04833333333333334,
|
|
0.04800000000000004,
|
|
0.08999999999999997,
|
|
0.054333333333333345
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9568666666666668,
|
|
"std": 0.005990038026219488,
|
|
"ci_95": 0.005250499828058693,
|
|
"values": [
|
|
0.9591666666666666,
|
|
0.9568333333333333,
|
|
0.962,
|
|
0.9466666666666667,
|
|
0.9596666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.993516111111111,
|
|
"std": 0.0009898567680712113,
|
|
"ci_95": 0.000867647712387025,
|
|
"values": [
|
|
0.9937080555555556,
|
|
0.9922814444444444,
|
|
0.9946737222222223,
|
|
0.9927475555555554,
|
|
0.9941697777777777
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9576416088313364,
|
|
"std": 0.005421963377389286,
|
|
"ci_95": 0.004752560443875861,
|
|
"values": [
|
|
0.96,
|
|
0.9570552147239264,
|
|
0.9623762376237623,
|
|
0.9485530546623794,
|
|
0.960223537146614
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9412666666666667,
|
|
"std": 0.005610753563965873,
|
|
"ci_95": 0.004918042338618547,
|
|
"values": [
|
|
0.939,
|
|
0.9426666666666667,
|
|
0.9461666666666667,
|
|
0.9326666666666666,
|
|
0.9458333333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9868736666666666,
|
|
"std": 0.0011103986139176464,
|
|
"ci_95": 0.0009733072988738293,
|
|
"values": [
|
|
0.9859274999999998,
|
|
0.9870954444444443,
|
|
0.9875953888888889,
|
|
0.9855584444444443,
|
|
0.9881915555555555
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.941407767452192,
|
|
"std": 0.004952224427288878,
|
|
"ci_95": 0.004340816100027134,
|
|
"values": [
|
|
0.9390406395736176,
|
|
0.9421459804910864,
|
|
0.9458507963118189,
|
|
0.9341590612777053,
|
|
0.9458423596067322
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9628333333333334,
|
|
"std": 0.007899542884102719,
|
|
"ci_95": 0.00692425463297114,
|
|
"values": [
|
|
0.9661666666666666,
|
|
0.9646666666666667,
|
|
0.9675,
|
|
0.9488333333333333,
|
|
0.967
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9959169222222222,
|
|
"std": 0.0010349105023331555,
|
|
"ci_95": 0.000907139051667393,
|
|
"values": [
|
|
0.9972890555555555,
|
|
0.9947345000000001,
|
|
0.9961732777777779,
|
|
0.9950465555555554,
|
|
0.9963412222222222
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9637268701951951,
|
|
"std": 0.007342220515605291,
|
|
"ci_95": 0.006435740038045298,
|
|
"values": [
|
|
0.9670828603859251,
|
|
0.9651201052977953,
|
|
0.9679960610536681,
|
|
0.9507460291994224,
|
|
0.9676892950391645
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"augment": false,
|
|
"run_name": "p4_resnet50_50pct",
|
|
"backbone": "resnet50",
|
|
"subsample": 0.5
|
|
}
|
|
} |