Johnny Fernandes
1729 lines
40 KiB
JSON
1729 lines
40 KiB
JSON
{
|
|
"run_name": "p3_resnet34",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.18143130752982364,
|
|
0.09914765522504847,
|
|
0.0632085224350535,
|
|
0.04726770077185291,
|
|
0.02548843655111139,
|
|
0.020165729296557536,
|
|
0.01242467272374294,
|
|
0.008408245080691915,
|
|
0.0034914202498890112,
|
|
0.002486942817081736,
|
|
0.0015942691031080325
|
|
],
|
|
"train_acc": [
|
|
0.8414351851851852,
|
|
0.9276041666666667,
|
|
0.9576388888888889,
|
|
0.970949074074074,
|
|
0.9844907407407407,
|
|
0.9883680555555555,
|
|
0.9931134259259259,
|
|
0.9952546296296296,
|
|
0.997800925925926,
|
|
0.9987847222222223,
|
|
0.9992476851851851
|
|
],
|
|
"train_auc": [
|
|
0.9199058445501829,
|
|
0.9771592881944445,
|
|
0.9908215985082305,
|
|
0.9949528820873342,
|
|
0.9983226612511431,
|
|
0.9990990976508916,
|
|
0.9994629218821446,
|
|
0.9996075620856196,
|
|
0.9999645722593735,
|
|
0.9999848804726794,
|
|
0.9999924714934842
|
|
],
|
|
"train_f1": [
|
|
0.8889429312581063,
|
|
0.9508853205606376,
|
|
0.9715374445913368,
|
|
0.9805350911205893,
|
|
0.9896212531949501,
|
|
0.9922294815788456,
|
|
0.9954030980801174,
|
|
0.9968332432223681,
|
|
0.9985330450895614,
|
|
0.9991897835564644,
|
|
0.9994983600231526
|
|
],
|
|
"val_loss": [
|
|
0.15404950051258007,
|
|
0.14179302051779813,
|
|
0.1187218560759599,
|
|
0.14628465867135673,
|
|
0.16971934595834076,
|
|
0.13480250174313066,
|
|
0.16385206343329628,
|
|
0.1861495381036851,
|
|
0.23939787570687562,
|
|
0.2579460703409647,
|
|
0.27772147155884946
|
|
],
|
|
"val_acc": [
|
|
0.825,
|
|
0.9151041666666667,
|
|
0.8911458333333333,
|
|
0.925,
|
|
0.9364583333333333,
|
|
0.9270833333333334,
|
|
0.9380208333333333,
|
|
0.9395833333333333,
|
|
0.9453125,
|
|
0.9411458333333333,
|
|
0.9432291666666667
|
|
],
|
|
"val_auc": [
|
|
0.9604861111111112,
|
|
0.9638483796296295,
|
|
0.974782986111111,
|
|
0.9769769965277778,
|
|
0.9766833043981481,
|
|
0.9795203993055555,
|
|
0.9777213541666667,
|
|
0.9694089988425926,
|
|
0.9671520543981481,
|
|
0.9674833622685186,
|
|
0.9596607349537037
|
|
],
|
|
"val_f1": [
|
|
0.8699690402476781,
|
|
0.9432648799164636,
|
|
0.9233027522935779,
|
|
0.9508532423208191,
|
|
0.9578146611341632,
|
|
0.9504950495049505,
|
|
0.9583770549143057,
|
|
0.9595254710397767,
|
|
0.9640287769784173,
|
|
0.9609402004839267,
|
|
0.9627350427350427
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9314583333333334,
|
|
"auc_roc": 0.9790402777777778,
|
|
"f1": 0.9538375192928301,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
201,
|
|
3399
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9541666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
55,
|
|
1145
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9541666666666667,
|
|
"pairwise_auc": 0.9817479166666666,
|
|
"pairwise_f1": 0.9260008087343308
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9116666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
106,
|
|
1094
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9116666666666666,
|
|
"pairwise_auc": 0.9687749999999999,
|
|
"pairwise_f1": 0.9033856317093312
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9666666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
40,
|
|
1160
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9666666666666667,
|
|
"pairwise_auc": 0.9865979166666666,
|
|
"pairwise_f1": 0.932475884244373
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8933333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.10666666666666669
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.92375,
|
|
"auc_roc": 0.9817479166666666,
|
|
"f1": 0.9260008087343308,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
55,
|
|
1145
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9025,
|
|
"auc_roc": 0.9687749999999999,
|
|
"f1": 0.9033856317093312,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
106,
|
|
1094
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.93,
|
|
"auc_roc": 0.9865979166666666,
|
|
"f1": 0.932475884244373,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
40,
|
|
1160
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.1798529501521477,
|
|
0.1021667472742222,
|
|
0.06341963815847756,
|
|
0.039253244310384613,
|
|
0.029152714926075768,
|
|
0.01912316902404806,
|
|
0.011262987689163614,
|
|
0.008215954819028141,
|
|
0.003789046630271076,
|
|
0.003531608089493872
|
|
],
|
|
"train_acc": [
|
|
0.835474537037037,
|
|
0.9259259259259259,
|
|
0.9570023148148148,
|
|
0.9748842592592593,
|
|
0.9820601851851852,
|
|
0.9891203703703704,
|
|
0.9939236111111112,
|
|
0.9950810185185185,
|
|
0.9976273148148148,
|
|
0.9980324074074074
|
|
],
|
|
"train_auc": [
|
|
0.9216952624885689,
|
|
0.9756210080161181,
|
|
0.9904438157293096,
|
|
0.996355175825903,
|
|
0.9979366444473022,
|
|
0.9988179798239598,
|
|
0.9996395426097393,
|
|
0.9998288537379973,
|
|
0.9999600801611797,
|
|
0.9999696984310699
|
|
],
|
|
"train_f1": [
|
|
0.8837741711295531,
|
|
0.9497605777533559,
|
|
0.971079366315052,
|
|
0.9831730769230769,
|
|
0.9880077369439072,
|
|
0.9927396308025025,
|
|
0.995945163158911,
|
|
0.9967187801582706,
|
|
0.998417538307152,
|
|
0.9986878666255017
|
|
],
|
|
"val_loss": [
|
|
0.30760438268383344,
|
|
0.1784577751532197,
|
|
0.1753600399143276,
|
|
0.18356385634445663,
|
|
0.12388940138819939,
|
|
0.18397595811838982,
|
|
0.1666869391252779,
|
|
0.19786467835095511,
|
|
0.18487099432192433,
|
|
0.1899753529316513
|
|
],
|
|
"val_acc": [
|
|
0.7046875,
|
|
0.853125,
|
|
0.925,
|
|
0.9182291666666667,
|
|
0.9317708333333333,
|
|
0.9270833333333334,
|
|
0.9213541666666667,
|
|
0.9359375,
|
|
0.9322916666666666,
|
|
0.93125
|
|
],
|
|
"val_auc": [
|
|
0.9431061921296295,
|
|
0.9447417534722222,
|
|
0.970240162037037,
|
|
0.9670826099537038,
|
|
0.9778392650462963,
|
|
0.9689626736111112,
|
|
0.971880787037037,
|
|
0.9755403645833334,
|
|
0.9723394097222222,
|
|
0.9725542534722222
|
|
],
|
|
"val_f1": [
|
|
0.7580025608194623,
|
|
0.896399706098457,
|
|
0.9510204081632653,
|
|
0.9459552495697074,
|
|
0.9535954658165072,
|
|
0.9511173184357542,
|
|
0.9462824617573817,
|
|
0.9574835810577256,
|
|
0.9541607898448519,
|
|
0.9536191145467322
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.928125,
|
|
"auc_roc": 0.9817493055555554,
|
|
"f1": 0.9508056466562099,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
266,
|
|
3334
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.925,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.925,
|
|
"pairwise_auc": 0.9821319444444443,
|
|
"pairwise_f1": 0.9292591042277103
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8966666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
124,
|
|
1076
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8966666666666666,
|
|
"pairwise_auc": 0.9734770833333333,
|
|
"pairwise_f1": 0.9138004246284501
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9566666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
52,
|
|
1148
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9566666666666667,
|
|
"pairwise_auc": 0.9896388888888888,
|
|
"pairwise_f1": 0.9460238978162341
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9341666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.0658333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9295833333333333,
|
|
"auc_roc": 0.9821319444444443,
|
|
"f1": 0.9292591042277103,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
90,
|
|
1110
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9154166666666667,
|
|
"auc_roc": 0.9734770833333333,
|
|
"f1": 0.9138004246284501,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
124,
|
|
1076
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9454166666666667,
|
|
"auc_roc": 0.9896388888888888,
|
|
"f1": 0.9460238978162341,
|
|
"confusion_matrix": [
|
|
[
|
|
1121,
|
|
79
|
|
],
|
|
[
|
|
52,
|
|
1148
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17717904072530843,
|
|
0.0940379478054604,
|
|
0.06369061694540842,
|
|
0.04127667683782487,
|
|
0.030577845679228496,
|
|
0.021532554604076454,
|
|
0.012030772082477968,
|
|
0.009333743597454924,
|
|
0.003131703046412642,
|
|
0.0014826828477604674
|
|
],
|
|
"train_acc": [
|
|
0.8396990740740741,
|
|
0.9304976851851852,
|
|
0.9571180555555555,
|
|
0.9722800925925926,
|
|
0.9821180555555555,
|
|
0.9873263888888889,
|
|
0.9931712962962963,
|
|
0.9959490740740741,
|
|
0.9978587962962963,
|
|
0.9992476851851851
|
|
],
|
|
"train_auc": [
|
|
0.9238818694272977,
|
|
0.979183527663466,
|
|
0.9908993394918839,
|
|
0.9961150227194787,
|
|
0.9978819444444446,
|
|
0.9987962605738454,
|
|
0.9995584437157066,
|
|
0.999660877843507,
|
|
0.9999563293038408,
|
|
0.9999916052240512
|
|
],
|
|
"train_f1": [
|
|
0.8873250894890986,
|
|
0.9528668419606766,
|
|
0.9711841337740619,
|
|
0.9814276297933388,
|
|
0.9880478087649402,
|
|
0.9915342688159573,
|
|
0.9954429597590175,
|
|
0.9972995910809351,
|
|
0.9985714837264971,
|
|
0.9994983600231526
|
|
],
|
|
"val_loss": [
|
|
0.13040539792273192,
|
|
0.13235231365154806,
|
|
0.14123243255210885,
|
|
0.14862661970352445,
|
|
0.1436936670134249,
|
|
0.13523276704800083,
|
|
0.17793816352035113,
|
|
0.5479134103229474,
|
|
0.34418828572397614,
|
|
0.19769162325305842
|
|
],
|
|
"val_acc": [
|
|
0.9119791666666667,
|
|
0.8963541666666667,
|
|
0.9390625,
|
|
0.9296875,
|
|
0.9390625,
|
|
0.9328125,
|
|
0.9364583333333333,
|
|
0.9005208333333333,
|
|
0.93125,
|
|
0.9395833333333333
|
|
],
|
|
"val_auc": [
|
|
0.9625940393518518,
|
|
0.9660503472222222,
|
|
0.9745341435185186,
|
|
0.9739945023148148,
|
|
0.9792954282407407,
|
|
0.9786284722222223,
|
|
0.9765842013888889,
|
|
0.9321281828703705,
|
|
0.9603030960648149,
|
|
0.9739373553240742
|
|
],
|
|
"val_f1": [
|
|
0.9410533658876875,
|
|
0.9280289330922242,
|
|
0.9597799931247851,
|
|
0.9530761209593326,
|
|
0.9595854922279793,
|
|
0.954465231203671,
|
|
0.9576094510076442,
|
|
0.9369428854407396,
|
|
0.9551630434782609,
|
|
0.9596662030598053
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9379166666666666,
|
|
"auc_roc": 0.9731581018518518,
|
|
"f1": 0.9585881045025014,
|
|
"confusion_matrix": [
|
|
[
|
|
1053,
|
|
147
|
|
],
|
|
[
|
|
151,
|
|
3449
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9433333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
68,
|
|
1132
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9433333333333334,
|
|
"pairwise_auc": 0.9705652777777777,
|
|
"pairwise_f1": 0.9132714804356595
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9425,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
69,
|
|
1131
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9425,
|
|
"pairwise_auc": 0.962487847222222,
|
|
"pairwise_f1": 0.9128329297820823
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9883333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
14,
|
|
1186
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9883333333333333,
|
|
"pairwise_auc": 0.9864211805555555,
|
|
"pairwise_f1": 0.9364390051322542
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1053,
|
|
147
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.12250000000000005
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9104166666666667,
|
|
"auc_roc": 0.9705652777777777,
|
|
"f1": 0.9132714804356595,
|
|
"confusion_matrix": [
|
|
[
|
|
1053,
|
|
147
|
|
],
|
|
[
|
|
68,
|
|
1132
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.91,
|
|
"auc_roc": 0.962487847222222,
|
|
"f1": 0.9128329297820823,
|
|
"confusion_matrix": [
|
|
[
|
|
1053,
|
|
147
|
|
],
|
|
[
|
|
69,
|
|
1131
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9329166666666666,
|
|
"auc_roc": 0.9864211805555555,
|
|
"f1": 0.9364390051322542,
|
|
"confusion_matrix": [
|
|
[
|
|
1053,
|
|
147
|
|
],
|
|
[
|
|
14,
|
|
1186
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17703111189658996,
|
|
0.09576246125313143,
|
|
0.062477437157132144,
|
|
0.04725347257939305,
|
|
0.030324791977235928,
|
|
0.017554738121685256,
|
|
0.013750801918420754,
|
|
0.006766391139438012,
|
|
0.003396348504592155,
|
|
0.0024946073007303883,
|
|
0.0010481267459134095,
|
|
0.0008546544026515861
|
|
],
|
|
"train_acc": [
|
|
0.8507523148148148,
|
|
0.9276620370370371,
|
|
0.9559027777777778,
|
|
0.9689236111111111,
|
|
0.98125,
|
|
0.9903935185185185,
|
|
0.9921875,
|
|
0.9968171296296297,
|
|
0.997800925925926,
|
|
0.9987268518518518,
|
|
0.9994212962962963,
|
|
0.9997685185185186
|
|
],
|
|
"train_auc": [
|
|
0.9249521944301554,
|
|
0.9784094310842478,
|
|
0.9909113690272062,
|
|
0.9949920428240742,
|
|
0.9977274984282121,
|
|
0.9991086623371056,
|
|
0.9994561167552583,
|
|
0.9997079421724966,
|
|
0.9999673496799268,
|
|
0.9999867469707361,
|
|
0.9999969100080018,
|
|
0.9999905156893004
|
|
],
|
|
"train_f1": [
|
|
0.8957853477189154,
|
|
0.9508222519474389,
|
|
0.9703178560299158,
|
|
0.9791707071098871,
|
|
0.9874525598327007,
|
|
0.9935892484745501,
|
|
0.9947866383471713,
|
|
0.9978762018766653,
|
|
0.998533384793516,
|
|
0.9991510380489311,
|
|
0.9996141379842568,
|
|
0.9998456909189106
|
|
],
|
|
"val_loss": [
|
|
0.13616387210786343,
|
|
0.1198919808860713,
|
|
0.1939462376467418,
|
|
0.09981064383173362,
|
|
0.48543022665852126,
|
|
0.16985621102333728,
|
|
0.12654530949221227,
|
|
0.16735582429784396,
|
|
0.23038662116990358,
|
|
0.19745846508264245,
|
|
0.20346919017623197,
|
|
0.1863294637893735
|
|
],
|
|
"val_acc": [
|
|
0.86875,
|
|
0.9171875,
|
|
0.9135416666666667,
|
|
0.9375,
|
|
0.8885416666666667,
|
|
0.9354166666666667,
|
|
0.9416666666666667,
|
|
0.9489583333333333,
|
|
0.9463541666666667,
|
|
0.9489583333333333,
|
|
0.9515625,
|
|
0.9489583333333333
|
|
],
|
|
"val_auc": [
|
|
0.9600057870370371,
|
|
0.9712818287037037,
|
|
0.9635011574074075,
|
|
0.9801779513888889,
|
|
0.9445326967592593,
|
|
0.974488570601852,
|
|
0.9811552372685186,
|
|
0.9771252893518518,
|
|
0.9719784432870371,
|
|
0.972273582175926,
|
|
0.97220703125,
|
|
0.9803291377314813
|
|
],
|
|
"val_f1": [
|
|
0.9061102831594635,
|
|
0.9435970202199362,
|
|
0.9431506849315069,
|
|
0.9579537491240364,
|
|
0.9301566579634465,
|
|
0.9567642956764296,
|
|
0.9608391608391609,
|
|
0.9661368348306841,
|
|
0.9647863247863248,
|
|
0.9660664819944599,
|
|
0.9679199724042773,
|
|
0.9662534435261708
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9339583333333333,
|
|
"auc_roc": 0.976025925925926,
|
|
"f1": 0.9555960218517999,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
189,
|
|
3411
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9533333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
56,
|
|
1144
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9533333333333334,
|
|
"pairwise_auc": 0.9759979166666667,
|
|
"pairwise_f1": 0.9255663430420712
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9241666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9241666666666667,
|
|
"pairwise_auc": 0.9680131944444444,
|
|
"pairwise_f1": 0.9101354123922856
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.965,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
42,
|
|
1158
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.965,
|
|
"pairwise_auc": 0.9840666666666666,
|
|
"pairwise_f1": 0.9316170555108608
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8933333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.10666666666666669
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9233333333333333,
|
|
"auc_roc": 0.9759979166666667,
|
|
"f1": 0.9255663430420712,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
56,
|
|
1144
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90875,
|
|
"auc_roc": 0.9680131944444444,
|
|
"f1": 0.9101354123922856,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
91,
|
|
1109
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9291666666666667,
|
|
"auc_roc": 0.9840666666666666,
|
|
"f1": 0.9316170555108608,
|
|
"confusion_matrix": [
|
|
[
|
|
1072,
|
|
128
|
|
],
|
|
[
|
|
42,
|
|
1158
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.17705109792037144,
|
|
0.09759698148161448,
|
|
0.06119485531881866,
|
|
0.04207795442608012,
|
|
0.0306523953303339,
|
|
0.016180019439842135,
|
|
0.015286858775878634,
|
|
0.008414860934468986,
|
|
0.004228976060405195,
|
|
0.0028073132647360006,
|
|
0.0012995293141313123,
|
|
0.0004747431101838662
|
|
],
|
|
"train_acc": [
|
|
0.8391203703703703,
|
|
0.9288773148148148,
|
|
0.9593171296296297,
|
|
0.9715277777777778,
|
|
0.9802662037037037,
|
|
0.9890625,
|
|
0.9931712962962963,
|
|
0.9957754629629629,
|
|
0.9973958333333334,
|
|
0.9984375,
|
|
0.9990740740740741,
|
|
0.9997106481481481
|
|
],
|
|
"train_auc": [
|
|
0.9232179944701646,
|
|
0.9775116723108139,
|
|
0.991124301626086,
|
|
0.9957172800211476,
|
|
0.9980221193415638,
|
|
0.9993293824302698,
|
|
0.9994119995284637,
|
|
0.9997395565414952,
|
|
0.9999585351651806,
|
|
0.9998539487597166,
|
|
0.9999960705304068,
|
|
0.9999990176326017
|
|
],
|
|
"train_f1": [
|
|
0.8866416571521775,
|
|
0.9516959478048972,
|
|
0.9726214121587413,
|
|
0.9809006211180125,
|
|
0.9868049375072554,
|
|
0.9926916979235142,
|
|
0.9954454222633935,
|
|
0.9971828811793308,
|
|
0.9982631518005326,
|
|
0.9989579715178881,
|
|
0.9993825254708243,
|
|
0.999807076436316
|
|
],
|
|
"val_loss": [
|
|
0.12170412251725793,
|
|
0.1030890600134929,
|
|
0.1750339979909768,
|
|
0.14935705231813093,
|
|
0.2741888238345382,
|
|
0.134656734423091,
|
|
0.14277773238912536,
|
|
0.22981467842995093,
|
|
0.18359658207567311,
|
|
0.17754823352255092,
|
|
0.1682993321162788,
|
|
0.18065465237386888
|
|
],
|
|
"val_acc": [
|
|
0.8942708333333333,
|
|
0.8875,
|
|
0.9354166666666667,
|
|
0.9083333333333333,
|
|
0.9130208333333333,
|
|
0.9229166666666667,
|
|
0.925,
|
|
0.9380208333333333,
|
|
0.9473958333333333,
|
|
0.9494791666666667,
|
|
0.9520833333333333,
|
|
0.9479166666666666
|
|
],
|
|
"val_auc": [
|
|
0.9661964699074075,
|
|
0.9779123263888889,
|
|
0.9717021122685184,
|
|
0.9725115740740741,
|
|
0.9688353587962962,
|
|
0.9805881076388888,
|
|
0.9837131076388888,
|
|
0.9681250000000001,
|
|
0.9726808449074072,
|
|
0.9789026331018519,
|
|
0.9797699652777778,
|
|
0.9766312210648148
|
|
],
|
|
"val_f1": [
|
|
0.925939438161255,
|
|
0.9199406968124537,
|
|
0.9574468085106383,
|
|
0.9371877230549608,
|
|
0.9436762225969646,
|
|
0.9469914040114613,
|
|
0.9483500717360115,
|
|
0.9589513625388065,
|
|
0.9651363479461512,
|
|
0.9663778162911612,
|
|
0.967966573816156,
|
|
0.9652777777777778
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": 0.9796047453703705,
|
|
"f1": 0.9455275229357798,
|
|
"confusion_matrix": [
|
|
[
|
|
1122,
|
|
78
|
|
],
|
|
[
|
|
302,
|
|
3298
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9116666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
106,
|
|
1094
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9116666666666666,
|
|
"pairwise_auc": 0.9793232638888889,
|
|
"pairwise_f1": 0.9224283305227656
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
147,
|
|
1053
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8775,
|
|
"pairwise_auc": 0.9713951388888888,
|
|
"pairwise_f1": 0.9034749034749034
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9591666666666666,
|
|
"pairwise_auc": 0.9880958333333333,
|
|
"pairwise_f1": 0.9477151090983944
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.935,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1122,
|
|
78
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.06499999999999995
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9233333333333333,
|
|
"auc_roc": 0.9793232638888889,
|
|
"f1": 0.9224283305227656,
|
|
"confusion_matrix": [
|
|
[
|
|
1122,
|
|
78
|
|
],
|
|
[
|
|
106,
|
|
1094
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90625,
|
|
"auc_roc": 0.9713951388888888,
|
|
"f1": 0.9034749034749034,
|
|
"confusion_matrix": [
|
|
[
|
|
1122,
|
|
78
|
|
],
|
|
[
|
|
147,
|
|
1053
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9470833333333334,
|
|
"auc_roc": 0.9880958333333333,
|
|
"f1": 0.9477151090983944,
|
|
"confusion_matrix": [
|
|
[
|
|
1122,
|
|
78
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9304583333333334,
|
|
"std": 0.006460685055695644,
|
|
"ci_95": 0.0056630401385749695,
|
|
"values": [
|
|
0.9314583333333334,
|
|
0.928125,
|
|
0.9379166666666666,
|
|
0.9339583333333333,
|
|
0.9208333333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9779156712962962,
|
|
"std": 0.0033548497426566172,
|
|
"ci_95": 0.0029406554549201254,
|
|
"values": [
|
|
0.9790402777777778,
|
|
0.9817493055555554,
|
|
0.9731581018518518,
|
|
0.976025925925926,
|
|
0.9796047453703705
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9528709630478243,
|
|
"std": 0.004980901709262283,
|
|
"ci_95": 0.0043659528459729004,
|
|
"values": [
|
|
0.9538375192928301,
|
|
0.9508056466562099,
|
|
0.9585881045025014,
|
|
0.9555960218517999,
|
|
0.9455275229357798
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9375,
|
|
"std": 0.01861525658640726,
|
|
"ci_95": 0.016316991825156467,
|
|
"values": [
|
|
0.9541666666666667,
|
|
0.925,
|
|
0.9433333333333334,
|
|
0.9533333333333334,
|
|
0.9116666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9375,
|
|
"std": 0.01861525658640726,
|
|
"ci_95": 0.016316991825156467,
|
|
"values": [
|
|
0.9541666666666667,
|
|
0.925,
|
|
0.9433333333333334,
|
|
0.9533333333333334,
|
|
0.9116666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9779532638888888,
|
|
"std": 0.004799567231185986,
|
|
"ci_95": 0.004207006167872827,
|
|
"values": [
|
|
0.9817479166666666,
|
|
0.9821319444444443,
|
|
0.9705652777777777,
|
|
0.9759979166666667,
|
|
0.9793232638888889
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9233052133925075,
|
|
"std": 0.006108855830535924,
|
|
"ci_95": 0.00535464822551511,
|
|
"values": [
|
|
0.9260008087343308,
|
|
0.9292591042277103,
|
|
0.9132714804356595,
|
|
0.9255663430420712,
|
|
0.9224283305227656
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9105000000000001,
|
|
"std": 0.02496247183273326,
|
|
"ci_95": 0.02188057129053081,
|
|
"values": [
|
|
0.9116666666666666,
|
|
0.8966666666666666,
|
|
0.9425,
|
|
0.9241666666666667,
|
|
0.8775
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9105000000000001,
|
|
"std": 0.02496247183273326,
|
|
"ci_95": 0.02188057129053081,
|
|
"values": [
|
|
0.9116666666666666,
|
|
0.8966666666666666,
|
|
0.9425,
|
|
0.9241666666666667,
|
|
0.8775
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9688296527777778,
|
|
"std": 0.004155370579012198,
|
|
"ci_95": 0.003642342905858583,
|
|
"values": [
|
|
0.9687749999999999,
|
|
0.9734770833333333,
|
|
0.962487847222222,
|
|
0.9680131944444444,
|
|
0.9713951388888888
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9087258603974104,
|
|
"std": 0.005017385996082066,
|
|
"ci_95": 0.004397932733385237,
|
|
"values": [
|
|
0.9033856317093312,
|
|
0.9138004246284501,
|
|
0.9128329297820823,
|
|
0.9101354123922856,
|
|
0.9034749034749034
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9671666666666667,
|
|
"std": 0.012522202504174547,
|
|
"ci_95": 0.010976194442722131,
|
|
"values": [
|
|
0.9666666666666667,
|
|
0.9566666666666667,
|
|
0.9883333333333333,
|
|
0.965,
|
|
0.9591666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9671666666666667,
|
|
"std": 0.012522202504174547,
|
|
"ci_95": 0.010976194442722131,
|
|
"values": [
|
|
0.9666666666666667,
|
|
0.9566666666666667,
|
|
0.9883333333333333,
|
|
0.965,
|
|
0.9591666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9869640972222221,
|
|
"std": 0.0020772149392720898,
|
|
"ci_95": 0.001820759172771472,
|
|
"values": [
|
|
0.9865979166666666,
|
|
0.9896388888888888,
|
|
0.9864211805555555,
|
|
0.9840666666666666,
|
|
0.9880958333333333
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9388541903604233,
|
|
"std": 0.007563276659457836,
|
|
"ci_95": 0.006629504291328617,
|
|
"values": [
|
|
0.932475884244373,
|
|
0.9460238978162341,
|
|
0.9364390051322542,
|
|
0.9316170555108608,
|
|
0.9477151090983944
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9066666666666666,
|
|
"std": 0.026292954273805874,
|
|
"ci_95": 0.02304679056952523,
|
|
"values": [
|
|
0.8933333333333333,
|
|
0.9341666666666667,
|
|
0.8775,
|
|
0.8933333333333333,
|
|
0.935
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.09333333333333334,
|
|
"std": 0.026292954273805878,
|
|
"ci_95": 0.023046790569525234,
|
|
"values": [
|
|
0.10666666666666669,
|
|
0.0658333333333333,
|
|
0.12250000000000005,
|
|
0.10666666666666669,
|
|
0.06499999999999995
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9220833333333334,
|
|
"std": 0.00704031052029825,
|
|
"ci_95": 0.006171104259188769,
|
|
"values": [
|
|
0.92375,
|
|
0.9295833333333333,
|
|
0.9104166666666667,
|
|
0.9233333333333333,
|
|
0.9233333333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9779532638888888,
|
|
"std": 0.004799567231185986,
|
|
"ci_95": 0.004207006167872827,
|
|
"values": [
|
|
0.9817479166666666,
|
|
0.9821319444444443,
|
|
0.9705652777777777,
|
|
0.9759979166666667,
|
|
0.9793232638888889
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9233052133925075,
|
|
"std": 0.006108855830535924,
|
|
"ci_95": 0.00535464822551511,
|
|
"values": [
|
|
0.9260008087343308,
|
|
0.9292591042277103,
|
|
0.9132714804356595,
|
|
0.9255663430420712,
|
|
0.9224283305227656
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9085833333333333,
|
|
"std": 0.004774425503543747,
|
|
"ci_95": 0.004184968471937533,
|
|
"values": [
|
|
0.9025,
|
|
0.9154166666666667,
|
|
0.91,
|
|
0.90875,
|
|
0.90625
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9688296527777778,
|
|
"std": 0.004155370579012198,
|
|
"ci_95": 0.003642342905858583,
|
|
"values": [
|
|
0.9687749999999999,
|
|
0.9734770833333333,
|
|
0.962487847222222,
|
|
0.9680131944444444,
|
|
0.9713951388888888
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9087258603974104,
|
|
"std": 0.005017385996082066,
|
|
"ci_95": 0.004397932733385237,
|
|
"values": [
|
|
0.9033856317093312,
|
|
0.9138004246284501,
|
|
0.9128329297820823,
|
|
0.9101354123922856,
|
|
0.9034749034749034
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9369166666666666,
|
|
"std": 0.00865323478372235,
|
|
"ci_95": 0.007584894711054195,
|
|
"values": [
|
|
0.93,
|
|
0.9454166666666667,
|
|
0.9329166666666666,
|
|
0.9291666666666667,
|
|
0.9470833333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9869640972222221,
|
|
"std": 0.0020772149392720898,
|
|
"ci_95": 0.001820759172771472,
|
|
"values": [
|
|
0.9865979166666666,
|
|
0.9896388888888888,
|
|
0.9864211805555555,
|
|
0.9840666666666666,
|
|
0.9880958333333333
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9388541903604233,
|
|
"std": 0.007563276659457836,
|
|
"ci_95": 0.006629504291328617,
|
|
"values": [
|
|
0.932475884244373,
|
|
0.9460238978162341,
|
|
0.9364390051322542,
|
|
0.9316170555108608,
|
|
0.9477151090983944
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"run_name": "p3_resnet34",
|
|
"backbone": "resnet34"
|
|
}
|
|
} |