Johnny Fernandes
1714 lines
40 KiB
JSON
1714 lines
40 KiB
JSON
{
|
|
"run_name": "p2a_t1_original",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19722581385048452,
|
|
0.10264252484603613,
|
|
0.05920427175566416,
|
|
0.0366648331324945,
|
|
0.027175970428840998,
|
|
0.016415510239635802,
|
|
0.010992527571498713,
|
|
0.006630335180751798,
|
|
0.005149287341508305,
|
|
0.0026562969974941518,
|
|
0.0014051771179260362,
|
|
0.0008840179361898124,
|
|
0.0004661208866591719
|
|
],
|
|
"train_acc": [
|
|
0.811863425925926,
|
|
0.9195023148148148,
|
|
0.9572916666666667,
|
|
0.9757523148148148,
|
|
0.9828703703703704,
|
|
0.9909143518518518,
|
|
0.9928819444444444,
|
|
0.9958912037037037,
|
|
0.9965277777777778,
|
|
0.9983217592592593,
|
|
0.9991898148148148,
|
|
0.9995949074074074,
|
|
0.9997685185185186
|
|
],
|
|
"train_auc": [
|
|
0.9015951949731367,
|
|
0.9747272858796296,
|
|
0.9914408382630315,
|
|
0.9967484800097165,
|
|
0.9982575392232511,
|
|
0.9990713323759717,
|
|
0.9995895847622315,
|
|
0.9998830982796068,
|
|
0.9999293499228395,
|
|
0.9999699663494512,
|
|
0.9999906585791039,
|
|
0.9999967135345222,
|
|
0.9999998213877457
|
|
],
|
|
"train_f1": [
|
|
0.8662635237977704,
|
|
0.9450913827813524,
|
|
0.9712078651685393,
|
|
0.9837527628058474,
|
|
0.9885422311682279,
|
|
0.9939328361092863,
|
|
0.9952441712098364,
|
|
0.9972579461630556,
|
|
0.9976841130152848,
|
|
0.9988807842229169,
|
|
0.9994596263702331,
|
|
0.999729886166313,
|
|
0.9998456551937027
|
|
],
|
|
"val_loss": [
|
|
0.14872334962710737,
|
|
0.15591826991488536,
|
|
0.18881629154008503,
|
|
0.18318608577440804,
|
|
0.2586083890666487,
|
|
0.21436443397154412,
|
|
0.4333447159598412,
|
|
0.21543170170140608,
|
|
0.2493383682408118,
|
|
0.28536541290159223,
|
|
0.27383037212348427,
|
|
0.31338123660993916,
|
|
0.26701862792315295
|
|
],
|
|
"val_acc": [
|
|
0.8822916666666667,
|
|
0.8833333333333333,
|
|
0.9041666666666667,
|
|
0.9015625,
|
|
0.9015625,
|
|
0.8963541666666667,
|
|
0.9036458333333334,
|
|
0.9052083333333333,
|
|
0.91875,
|
|
0.9208333333333333,
|
|
0.9270833333333334,
|
|
0.9229166666666667,
|
|
0.9270833333333334
|
|
],
|
|
"val_auc": [
|
|
0.9479615162037038,
|
|
0.9493648726851852,
|
|
0.9572620081018519,
|
|
0.9601273148148147,
|
|
0.9543258101851851,
|
|
0.9585423900462964,
|
|
0.9473763020833333,
|
|
0.9606532118055556,
|
|
0.9584085648148147,
|
|
0.9560619212962964,
|
|
0.95689453125,
|
|
0.9558637152777778,
|
|
0.9571223958333335
|
|
],
|
|
"val_f1": [
|
|
0.919054441260745,
|
|
0.92,
|
|
0.9360222531293463,
|
|
0.9338929695697796,
|
|
0.9347600966517087,
|
|
0.9286993908993192,
|
|
0.9385177799933533,
|
|
0.9352773826458037,
|
|
0.9456824512534819,
|
|
0.9479095270733379,
|
|
0.9519560741249142,
|
|
0.949419002050581,
|
|
0.9516574585635359
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.914375,
|
|
"auc_roc": 0.9687729166666665,
|
|
"f1": 0.9419409521118802,
|
|
"confusion_matrix": [
|
|
[
|
|
1055,
|
|
145
|
|
],
|
|
[
|
|
266,
|
|
3334
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9333333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
80,
|
|
1120
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9333333333333333,
|
|
"pairwise_auc": 0.9714215277777778,
|
|
"pairwise_f1": 0.9087221095334685
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8858333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
137,
|
|
1063
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8858333333333334,
|
|
"pairwise_auc": 0.9548541666666667,
|
|
"pairwise_f1": 0.882890365448505
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9591666666666666,
|
|
"pairwise_auc": 0.9800430555555556,
|
|
"pairwise_f1": 0.9222756410256411
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8791666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1055,
|
|
145
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.12083333333333335
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90625,
|
|
"auc_roc": 0.9714215277777778,
|
|
"f1": 0.9087221095334685,
|
|
"confusion_matrix": [
|
|
[
|
|
1055,
|
|
145
|
|
],
|
|
[
|
|
80,
|
|
1120
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8825,
|
|
"auc_roc": 0.9548541666666667,
|
|
"f1": 0.882890365448505,
|
|
"confusion_matrix": [
|
|
[
|
|
1055,
|
|
145
|
|
],
|
|
[
|
|
137,
|
|
1063
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9191666666666667,
|
|
"auc_roc": 0.9800430555555556,
|
|
"f1": 0.9222756410256411,
|
|
"confusion_matrix": [
|
|
[
|
|
1055,
|
|
145
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.20000588447545414,
|
|
0.10408794169003764,
|
|
0.058041512501788226,
|
|
0.034372955076267114,
|
|
0.025223094800377214,
|
|
0.015196698343168936,
|
|
0.010545113944270405,
|
|
0.007307375633192553
|
|
],
|
|
"train_acc": [
|
|
0.8090277777777778,
|
|
0.9175925925925926,
|
|
0.9587962962962963,
|
|
0.9762731481481481,
|
|
0.9840277777777777,
|
|
0.9907986111111111,
|
|
0.9944444444444445,
|
|
0.9953703703703703
|
|
],
|
|
"train_auc": [
|
|
0.8993097082904664,
|
|
0.9743819390860768,
|
|
0.9915667866941016,
|
|
0.9970922103623686,
|
|
0.9983073184585048,
|
|
0.9992988843878601,
|
|
0.9995584437157066,
|
|
0.9998731674382716
|
|
],
|
|
"train_f1": [
|
|
0.8646098301468779,
|
|
0.9437198640423682,
|
|
0.9722157184109889,
|
|
0.9840974323171204,
|
|
0.9893130953302873,
|
|
0.993854122376406,
|
|
0.9962934362934363,
|
|
0.9969111969111969
|
|
],
|
|
"val_loss": [
|
|
0.14092905839594702,
|
|
0.13848712935190027,
|
|
0.15262191738778105,
|
|
0.2710734717196222,
|
|
0.29285608716936623,
|
|
0.25471851951188,
|
|
0.20351220176477605,
|
|
0.2155651534909945
|
|
],
|
|
"val_acc": [
|
|
0.884375,
|
|
0.8989583333333333,
|
|
0.9020833333333333,
|
|
0.9114583333333334,
|
|
0.9135416666666667,
|
|
0.909375,
|
|
0.9078125,
|
|
0.9104166666666667
|
|
],
|
|
"val_auc": [
|
|
0.9523307291666666,
|
|
0.9595883969907405,
|
|
0.9632103587962961,
|
|
0.9511574074074074,
|
|
0.9564756944444445,
|
|
0.9586291956018518,
|
|
0.9618590856481481,
|
|
0.9612919560185186
|
|
],
|
|
"val_f1": [
|
|
0.9203158650394831,
|
|
0.9312544294826365,
|
|
0.9338959212376934,
|
|
0.9420980926430518,
|
|
0.943728813559322,
|
|
0.9403292181069959,
|
|
0.9373007438894793,
|
|
0.9392226148409893
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.905625,
|
|
"auc_roc": 0.9629736111111112,
|
|
"f1": 0.9361881955204958,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
277,
|
|
3323
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9516666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
58,
|
|
1142
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9516666666666667,
|
|
"pairwise_auc": 0.9716375000000002,
|
|
"pairwise_f1": 0.9070691024622717
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
147,
|
|
1053
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8775,
|
|
"pairwise_auc": 0.9462055555555556,
|
|
"pairwise_f1": 0.8670234664470976
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.94,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.94,
|
|
"pairwise_auc": 0.9710777777777778,
|
|
"pairwise_f1": 0.9009584664536742
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8533333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1466666666666666
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9025,
|
|
"auc_roc": 0.9716375000000002,
|
|
"f1": 0.9070691024622717,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
58,
|
|
1142
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8654166666666666,
|
|
"auc_roc": 0.9462055555555556,
|
|
"f1": 0.8670234664470976,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
147,
|
|
1053
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8966666666666666,
|
|
"auc_roc": 0.9710777777777778,
|
|
"f1": 0.9009584664536742,
|
|
"confusion_matrix": [
|
|
[
|
|
1024,
|
|
176
|
|
],
|
|
[
|
|
72,
|
|
1128
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.1954930212141739,
|
|
0.10281271817921489,
|
|
0.06016657212274839,
|
|
0.03787446872965019,
|
|
0.026305097892300518,
|
|
0.017765212520370603,
|
|
0.010911685225935932,
|
|
0.009139342036825029,
|
|
0.004632529768130351,
|
|
0.004187346089508571,
|
|
0.0017877653261907484,
|
|
0.0007197948908006529,
|
|
0.0007150228552733198
|
|
],
|
|
"train_acc": [
|
|
0.8079282407407408,
|
|
0.9170717592592592,
|
|
0.9578125,
|
|
0.9766782407407407,
|
|
0.9836226851851851,
|
|
0.9901620370370371,
|
|
0.9935763888888889,
|
|
0.9946759259259259,
|
|
0.9974537037037037,
|
|
0.9972222222222222,
|
|
0.9992476851851851,
|
|
0.9995949074074074,
|
|
0.9996527777777777
|
|
],
|
|
"train_auc": [
|
|
0.9030463123713992,
|
|
0.9746345593278463,
|
|
0.9914829728938043,
|
|
0.996350594421582,
|
|
0.9981514256830133,
|
|
0.9991553515803613,
|
|
0.9995712055612711,
|
|
0.9995691515203475,
|
|
0.9999019329418153,
|
|
0.9999532482424555,
|
|
0.9999925340077732,
|
|
0.9999994820244628,
|
|
0.9999990890775035
|
|
],
|
|
"train_f1": [
|
|
0.8626298580356774,
|
|
0.943236284412755,
|
|
0.9716154654830044,
|
|
0.9843780284529209,
|
|
0.9890501064035597,
|
|
0.9934271574389112,
|
|
0.9957121335033028,
|
|
0.9964456807294081,
|
|
0.9983014206300185,
|
|
0.9981467181467182,
|
|
0.999498437439716,
|
|
0.9997299070108423,
|
|
0.9997685006559148
|
|
],
|
|
"val_loss": [
|
|
0.19500481635332106,
|
|
0.1268259635893628,
|
|
0.13148466631925354,
|
|
0.20869781834092768,
|
|
0.2388368578608303,
|
|
0.22051208737151076,
|
|
0.21102181890407035,
|
|
0.19602018855754674,
|
|
0.2702031286339964,
|
|
0.3234737387955799,
|
|
0.2604731284258378,
|
|
0.25106819690430104,
|
|
0.25952417930657246
|
|
],
|
|
"val_acc": [
|
|
0.7515625,
|
|
0.9072916666666667,
|
|
0.9057291666666667,
|
|
0.9171875,
|
|
0.9114583333333334,
|
|
0.9234375,
|
|
0.9083333333333333,
|
|
0.9270833333333334,
|
|
0.9255208333333333,
|
|
0.9197916666666667,
|
|
0.9302083333333333,
|
|
0.9270833333333334,
|
|
0.9348958333333334
|
|
],
|
|
"val_auc": [
|
|
0.9487427662037037,
|
|
0.9660561342592593,
|
|
0.9677799479166665,
|
|
0.9676699942129628,
|
|
0.9617209201388888,
|
|
0.9637868923611111,
|
|
0.9642853009259259,
|
|
0.9688606770833333,
|
|
0.9638433159722222,
|
|
0.9533347800925925,
|
|
0.9644741030092593,
|
|
0.9622547743055555,
|
|
0.9640183738425926
|
|
],
|
|
"val_f1": [
|
|
0.8039457459926017,
|
|
0.9366096866096866,
|
|
0.9354263289332858,
|
|
0.9457523029682702,
|
|
0.9418604651162791,
|
|
0.9491876944348427,
|
|
0.9376328844790929,
|
|
0.9510831586303284,
|
|
0.9508083935328517,
|
|
0.9471879286694102,
|
|
0.9538567493112947,
|
|
0.9514899514899515,
|
|
0.9568221070811744
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9239583333333333,
|
|
"auc_roc": 0.9678178240740741,
|
|
"f1": 0.9493547939503261,
|
|
"confusion_matrix": [
|
|
[
|
|
1014,
|
|
186
|
|
],
|
|
[
|
|
179,
|
|
3421
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9625,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
45,
|
|
1155
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9625,
|
|
"pairwise_auc": 0.9738322916666667,
|
|
"pairwise_f1": 0.9090909090909091
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9158333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
101,
|
|
1099
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9158333333333334,
|
|
"pairwise_auc": 0.9491239583333334,
|
|
"pairwise_f1": 0.8845070422535212
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9725,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
33,
|
|
1167
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9725,
|
|
"pairwise_auc": 0.9804972222222222,
|
|
"pairwise_f1": 0.9142185663924794
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.845,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1014,
|
|
186
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.15500000000000003
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90375,
|
|
"auc_roc": 0.9738322916666667,
|
|
"f1": 0.9090909090909091,
|
|
"confusion_matrix": [
|
|
[
|
|
1014,
|
|
186
|
|
],
|
|
[
|
|
45,
|
|
1155
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8804166666666666,
|
|
"auc_roc": 0.9491239583333334,
|
|
"f1": 0.8845070422535212,
|
|
"confusion_matrix": [
|
|
[
|
|
1014,
|
|
186
|
|
],
|
|
[
|
|
101,
|
|
1099
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.90875,
|
|
"auc_roc": 0.9804972222222222,
|
|
"f1": 0.9142185663924794,
|
|
"confusion_matrix": [
|
|
[
|
|
1014,
|
|
186
|
|
],
|
|
[
|
|
33,
|
|
1167
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19490147646240613,
|
|
0.10359700576022819,
|
|
0.06170826292924445,
|
|
0.04011154976345737,
|
|
0.02456697733448689,
|
|
0.017018179965196868,
|
|
0.014746911858236234,
|
|
0.009014791383426065
|
|
],
|
|
"train_acc": [
|
|
0.8076967592592592,
|
|
0.9194444444444444,
|
|
0.9548032407407407,
|
|
0.975,
|
|
0.9853009259259259,
|
|
0.9898726851851852,
|
|
0.992824074074074,
|
|
0.9953703703703703
|
|
],
|
|
"train_auc": [
|
|
0.9043987822216506,
|
|
0.9743296414180384,
|
|
0.9908415941500914,
|
|
0.9959990587134202,
|
|
0.9984063857453134,
|
|
0.99924799775663,
|
|
0.9991502343392775,
|
|
0.9995339827674896
|
|
],
|
|
"train_f1": [
|
|
0.8621790883828958,
|
|
0.9450106660346054,
|
|
0.9695052906954044,
|
|
0.9832376222256712,
|
|
0.990171800030955,
|
|
0.9932367149758454,
|
|
0.9952123552123552,
|
|
0.9969107198022861
|
|
],
|
|
"val_loss": [
|
|
0.22769305457671482,
|
|
0.17312400991407534,
|
|
0.14870864135834078,
|
|
0.1872468259287416,
|
|
0.2433047288354525,
|
|
0.3240897962289788,
|
|
0.2584284867192613,
|
|
0.23186856481673507
|
|
],
|
|
"val_acc": [
|
|
0.746875,
|
|
0.8453125,
|
|
0.9104166666666667,
|
|
0.9161458333333333,
|
|
0.9234375,
|
|
0.9229166666666667,
|
|
0.9208333333333333,
|
|
0.9223958333333333
|
|
],
|
|
"val_auc": [
|
|
0.9224739583333335,
|
|
0.9507523148148148,
|
|
0.9644314236111111,
|
|
0.9619386574074074,
|
|
0.9605664062500001,
|
|
0.9496766493055555,
|
|
0.9632212094907407,
|
|
0.9596093749999999
|
|
],
|
|
"val_f1": [
|
|
0.8008196721311476,
|
|
0.8879668049792531,
|
|
0.9398180545836249,
|
|
0.943921978404737,
|
|
0.9496402877697842,
|
|
0.9499323410013532,
|
|
0.9481935923653715,
|
|
0.9479566887879847
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9035416666666667,
|
|
"auc_roc": 0.9612076388888888,
|
|
"f1": 0.935126803979263,
|
|
"confusion_matrix": [
|
|
[
|
|
1000,
|
|
200
|
|
],
|
|
[
|
|
263,
|
|
3337
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9475,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
63,
|
|
1137
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9475,
|
|
"pairwise_auc": 0.9671291666666667,
|
|
"pairwise_f1": 0.8963342530547891
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8941666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
127,
|
|
1073
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8941666666666667,
|
|
"pairwise_auc": 0.9488027777777779,
|
|
"pairwise_f1": 0.8677719369187222
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9391666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
73,
|
|
1127
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9391666666666667,
|
|
"pairwise_auc": 0.9676909722222221,
|
|
"pairwise_f1": 0.8919667590027701
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8333333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1000,
|
|
200
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.16666666666666663
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8904166666666666,
|
|
"auc_roc": 0.9671291666666667,
|
|
"f1": 0.8963342530547891,
|
|
"confusion_matrix": [
|
|
[
|
|
1000,
|
|
200
|
|
],
|
|
[
|
|
63,
|
|
1137
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.86375,
|
|
"auc_roc": 0.9488027777777779,
|
|
"f1": 0.8677719369187222,
|
|
"confusion_matrix": [
|
|
[
|
|
1000,
|
|
200
|
|
],
|
|
[
|
|
127,
|
|
1073
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.88625,
|
|
"auc_roc": 0.9676909722222221,
|
|
"f1": 0.8919667590027701,
|
|
"confusion_matrix": [
|
|
[
|
|
1000,
|
|
200
|
|
],
|
|
[
|
|
73,
|
|
1127
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19957470789689707,
|
|
0.10425364971505822,
|
|
0.06086116745007328,
|
|
0.04081417578606677,
|
|
0.028762150054670334,
|
|
0.019413475335063088,
|
|
0.011313071529999703,
|
|
0.006697674415297823,
|
|
0.004091889076434436,
|
|
0.003935401036381302,
|
|
0.0018815870159793727
|
|
],
|
|
"train_acc": [
|
|
0.8087962962962963,
|
|
0.9174189814814815,
|
|
0.9564236111111111,
|
|
0.9734953703703704,
|
|
0.982175925925926,
|
|
0.9881365740740741,
|
|
0.9929976851851852,
|
|
0.9956597222222222,
|
|
0.9976273148148148,
|
|
0.9975694444444444,
|
|
0.9990740740740741
|
|
],
|
|
"train_auc": [
|
|
0.8990732167352539,
|
|
0.97397518539952,
|
|
0.9910111954160951,
|
|
0.9960041848851166,
|
|
0.9979256508630543,
|
|
0.9990071837848651,
|
|
0.9996136527634888,
|
|
0.9998784186385459,
|
|
0.9999477291237998,
|
|
0.9999236879143805,
|
|
0.9999895779749657
|
|
],
|
|
"train_f1": [
|
|
0.8639433371767419,
|
|
0.943696981653186,
|
|
0.9706238052510435,
|
|
0.9822273961971284,
|
|
0.9880712625871417,
|
|
0.9920742315870869,
|
|
0.9953240329249913,
|
|
0.9971021212472471,
|
|
0.9984171717561673,
|
|
0.9983787539566124,
|
|
0.9993825254708243
|
|
],
|
|
"val_loss": [
|
|
0.16892356189588706,
|
|
0.23779966595272223,
|
|
0.15918385590387818,
|
|
0.18508020597219002,
|
|
0.33955128504943183,
|
|
0.17744567066595968,
|
|
0.18761360666442972,
|
|
0.23809361916598087,
|
|
0.24717846481992942,
|
|
0.3055811640280202,
|
|
0.2333151061244583
|
|
],
|
|
"val_acc": [
|
|
0.8296875,
|
|
0.7703125,
|
|
0.9161458333333333,
|
|
0.903125,
|
|
0.9098958333333333,
|
|
0.9177083333333333,
|
|
0.9177083333333333,
|
|
0.9213541666666667,
|
|
0.9182291666666667,
|
|
0.9239583333333333,
|
|
0.9291666666666667
|
|
],
|
|
"val_auc": [
|
|
0.945900607638889,
|
|
0.924205005787037,
|
|
0.9659411168981483,
|
|
0.9623741319444445,
|
|
0.9572952835648147,
|
|
0.9675354456018519,
|
|
0.9642780671296296,
|
|
0.9610481770833332,
|
|
0.9629398148148148,
|
|
0.9584772858796297,
|
|
0.9648719618055557
|
|
],
|
|
"val_f1": [
|
|
0.8744721689059501,
|
|
0.8242327620565962,
|
|
0.9441166261714682,
|
|
0.9346451159522137,
|
|
0.9422370617696161,
|
|
0.9443661971830986,
|
|
0.9439716312056737,
|
|
0.9474416985729203,
|
|
0.945353289244692,
|
|
0.9501706484641638,
|
|
0.9527449617790132
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9185416666666667,
|
|
"auc_roc": 0.9692168981481482,
|
|
"f1": 0.9447974022306932,
|
|
"confusion_matrix": [
|
|
[
|
|
1063,
|
|
137
|
|
],
|
|
[
|
|
254,
|
|
3346
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9375,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
75,
|
|
1125
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9375,
|
|
"pairwise_auc": 0.9709586805555556,
|
|
"pairwise_f1": 0.9138911454102355
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8783333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
146,
|
|
1054
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8783333333333333,
|
|
"pairwise_auc": 0.9507531250000001,
|
|
"pairwise_f1": 0.8816394813885403
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9725,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
33,
|
|
1167
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9725,
|
|
"pairwise_auc": 0.9859388888888889,
|
|
"pairwise_f1": 0.9321086261980831
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8858333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1063,
|
|
137
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.11416666666666664
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9116666666666666,
|
|
"auc_roc": 0.9709586805555556,
|
|
"f1": 0.9138911454102355,
|
|
"confusion_matrix": [
|
|
[
|
|
1063,
|
|
137
|
|
],
|
|
[
|
|
75,
|
|
1125
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8820833333333333,
|
|
"auc_roc": 0.9507531250000001,
|
|
"f1": 0.8816394813885403,
|
|
"confusion_matrix": [
|
|
[
|
|
1063,
|
|
137
|
|
],
|
|
[
|
|
146,
|
|
1054
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9291666666666667,
|
|
"auc_roc": 0.9859388888888889,
|
|
"f1": 0.9321086261980831,
|
|
"confusion_matrix": [
|
|
[
|
|
1063,
|
|
137
|
|
],
|
|
[
|
|
33,
|
|
1167
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9132083333333334,
|
|
"std": 0.008606965848143645,
|
|
"ci_95": 0.007544338200855582,
|
|
"values": [
|
|
0.914375,
|
|
0.905625,
|
|
0.9239583333333333,
|
|
0.9035416666666667,
|
|
0.9185416666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9659977777777777,
|
|
"std": 0.0036560810622455297,
|
|
"ci_95": 0.003204696348280812,
|
|
"values": [
|
|
0.9687729166666665,
|
|
0.9629736111111112,
|
|
0.9678178240740741,
|
|
0.9612076388888888,
|
|
0.9692168981481482
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9414816295585317,
|
|
"std": 0.0059497120774801284,
|
|
"ci_95": 0.005215152575504448,
|
|
"values": [
|
|
0.9419409521118802,
|
|
0.9361881955204958,
|
|
0.9493547939503261,
|
|
0.935126803979263,
|
|
0.9447974022306932
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9465,
|
|
"std": 0.011598012281804541,
|
|
"ci_95": 0.01016610599546902,
|
|
"values": [
|
|
0.9333333333333333,
|
|
0.9516666666666667,
|
|
0.9625,
|
|
0.9475,
|
|
0.9375
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9465,
|
|
"std": 0.011598012281804541,
|
|
"ci_95": 0.01016610599546902,
|
|
"values": [
|
|
0.9333333333333333,
|
|
0.9516666666666667,
|
|
0.9625,
|
|
0.9475,
|
|
0.9375
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9709958333333335,
|
|
"std": 0.0024285252308698644,
|
|
"ci_95": 0.0021286962205089677,
|
|
"values": [
|
|
0.9714215277777778,
|
|
0.9716375000000002,
|
|
0.9738322916666667,
|
|
0.9671291666666667,
|
|
0.9709586805555556
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9070215039103348,
|
|
"std": 0.006492034890839102,
|
|
"ci_95": 0.00569051948066093,
|
|
"values": [
|
|
0.9087221095334685,
|
|
0.9070691024622717,
|
|
0.9090909090909091,
|
|
0.8963342530547891,
|
|
0.9138911454102355
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.8903333333333332,
|
|
"std": 0.015763001688200867,
|
|
"ci_95": 0.013816880175270344,
|
|
"values": [
|
|
0.8858333333333334,
|
|
0.8775,
|
|
0.9158333333333334,
|
|
0.8941666666666667,
|
|
0.8783333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.8903333333333332,
|
|
"std": 0.015763001688200867,
|
|
"ci_95": 0.013816880175270344,
|
|
"values": [
|
|
0.8858333333333334,
|
|
0.8775,
|
|
0.9158333333333334,
|
|
0.8941666666666667,
|
|
0.8783333333333333
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9499479166666667,
|
|
"std": 0.0031904218615427878,
|
|
"ci_95": 0.0027965280624499492,
|
|
"values": [
|
|
0.9548541666666667,
|
|
0.9462055555555556,
|
|
0.9491239583333334,
|
|
0.9488027777777779,
|
|
0.9507531250000001
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.8767664584912772,
|
|
"std": 0.008616735719668613,
|
|
"ci_95": 0.007552901870825199,
|
|
"values": [
|
|
0.882890365448505,
|
|
0.8670234664470976,
|
|
0.8845070422535212,
|
|
0.8677719369187222,
|
|
0.8816394813885403
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9566666666666667,
|
|
"std": 0.01652018966799919,
|
|
"ci_95": 0.014480584702743658,
|
|
"values": [
|
|
0.9591666666666666,
|
|
0.94,
|
|
0.9725,
|
|
0.9391666666666667,
|
|
0.9725
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9566666666666667,
|
|
"std": 0.01652018966799919,
|
|
"ci_95": 0.014480584702743658,
|
|
"values": [
|
|
0.9591666666666666,
|
|
0.94,
|
|
0.9725,
|
|
0.9391666666666667,
|
|
0.9725
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9770495833333334,
|
|
"std": 0.007468482219229771,
|
|
"ci_95": 0.006546413300930931,
|
|
"values": [
|
|
0.9800430555555556,
|
|
0.9710777777777778,
|
|
0.9804972222222222,
|
|
0.9676909722222221,
|
|
0.9859388888888889
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9123056118145296,
|
|
"std": 0.01610626351688794,
|
|
"ci_95": 0.014117762434216218,
|
|
"values": [
|
|
0.9222756410256411,
|
|
0.9009584664536742,
|
|
0.9142185663924794,
|
|
0.8919667590027701,
|
|
0.9321086261980831
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.8593333333333334,
|
|
"std": 0.022433543832593387,
|
|
"ci_95": 0.01966386816247279,
|
|
"values": [
|
|
0.8791666666666667,
|
|
0.8533333333333334,
|
|
0.845,
|
|
0.8333333333333334,
|
|
0.8858333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.14066666666666666,
|
|
"std": 0.022433543832593383,
|
|
"ci_95": 0.019663868162472786,
|
|
"values": [
|
|
0.12083333333333335,
|
|
0.1466666666666666,
|
|
0.15500000000000003,
|
|
0.16666666666666663,
|
|
0.11416666666666664
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9029166666666665,
|
|
"std": 0.007822909731466082,
|
|
"ci_95": 0.006857082713029891,
|
|
"values": [
|
|
0.90625,
|
|
0.9025,
|
|
0.90375,
|
|
0.8904166666666666,
|
|
0.9116666666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9709958333333335,
|
|
"std": 0.0024285252308698644,
|
|
"ci_95": 0.0021286962205089677,
|
|
"values": [
|
|
0.9714215277777778,
|
|
0.9716375000000002,
|
|
0.9738322916666667,
|
|
0.9671291666666667,
|
|
0.9709586805555556
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9070215039103348,
|
|
"std": 0.006492034890839102,
|
|
"ci_95": 0.00569051948066093,
|
|
"values": [
|
|
0.9087221095334685,
|
|
0.9070691024622717,
|
|
0.9090909090909091,
|
|
0.8963342530547891,
|
|
0.9138911454102355
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.8748333333333334,
|
|
"std": 0.009407812946931092,
|
|
"ci_95": 0.008246311633424692,
|
|
"values": [
|
|
0.8825,
|
|
0.8654166666666666,
|
|
0.8804166666666666,
|
|
0.86375,
|
|
0.8820833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9499479166666667,
|
|
"std": 0.0031904218615427878,
|
|
"ci_95": 0.0027965280624499492,
|
|
"values": [
|
|
0.9548541666666667,
|
|
0.9462055555555556,
|
|
0.9491239583333334,
|
|
0.9488027777777779,
|
|
0.9507531250000001
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8767664584912772,
|
|
"std": 0.008616735719668613,
|
|
"ci_95": 0.007552901870825199,
|
|
"values": [
|
|
0.882890365448505,
|
|
0.8670234664470976,
|
|
0.8845070422535212,
|
|
0.8677719369187222,
|
|
0.8816394813885403
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.908,
|
|
"std": 0.01713761700665141,
|
|
"ci_95": 0.015021783626897774,
|
|
"values": [
|
|
0.9191666666666667,
|
|
0.8966666666666666,
|
|
0.90875,
|
|
0.88625,
|
|
0.9291666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9770495833333334,
|
|
"std": 0.007468482219229771,
|
|
"ci_95": 0.006546413300930931,
|
|
"values": [
|
|
0.9800430555555556,
|
|
0.9710777777777778,
|
|
0.9804972222222222,
|
|
0.9676909722222221,
|
|
0.9859388888888889
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9123056118145296,
|
|
"std": 0.01610626351688794,
|
|
"ci_95": 0.014117762434216218,
|
|
"values": [
|
|
0.9222756410256411,
|
|
0.9009584664536742,
|
|
0.9142185663924794,
|
|
0.8919667590027701,
|
|
0.9321086261980831
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "data",
|
|
"run_name": "p2a_t1_original",
|
|
"backbone": "resnet18",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"normalization": "imagenet"
|
|
}
|
|
} |