Johnny Fernandes
1698 lines
40 KiB
JSON
1698 lines
40 KiB
JSON
{
|
|
"run_name": "p2a_t2_real_norm",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19635879239412368,
|
|
0.10250425095049044,
|
|
0.06135965930815372,
|
|
0.036744935404837,
|
|
0.022988288751393728,
|
|
0.017310182141119623,
|
|
0.012009648638006722,
|
|
0.00723629916516007
|
|
],
|
|
"train_acc": [
|
|
0.8133680555555556,
|
|
0.9196180555555555,
|
|
0.9576388888888889,
|
|
0.9769097222222223,
|
|
0.9848958333333333,
|
|
0.9891203703703704,
|
|
0.9923611111111111,
|
|
0.9959490740740741
|
|
],
|
|
"train_auc": [
|
|
0.9026075335076589,
|
|
0.9748221825703017,
|
|
0.9909446087677184,
|
|
0.9966968253457933,
|
|
0.9986351791123687,
|
|
0.9992185267346823,
|
|
0.9993304362425697,
|
|
0.9997806462905807
|
|
],
|
|
"train_f1": [
|
|
0.8675292667898953,
|
|
0.9451486790664613,
|
|
0.9714953271028037,
|
|
0.9845342842745843,
|
|
0.9899005533413303,
|
|
0.9927300850734725,
|
|
0.9948967756900952,
|
|
0.9972960444993819
|
|
],
|
|
"val_loss": [
|
|
0.15108177385603389,
|
|
0.15344043526177606,
|
|
0.1912794195076761,
|
|
0.227002287551295,
|
|
0.2509125987601389,
|
|
0.22423927540269992,
|
|
0.28618341417237997,
|
|
0.24878037015926868
|
|
],
|
|
"val_acc": [
|
|
0.8697916666666666,
|
|
0.8697916666666666,
|
|
0.9052083333333333,
|
|
0.9078125,
|
|
0.9036458333333334,
|
|
0.8958333333333334,
|
|
0.9213541666666667,
|
|
0.9114583333333334
|
|
],
|
|
"val_auc": [
|
|
0.9453624131944445,
|
|
0.948683449074074,
|
|
0.9581669560185186,
|
|
0.9581069155092593,
|
|
0.9493959780092592,
|
|
0.9546412037037038,
|
|
0.9552770543981483,
|
|
0.957332175925926
|
|
],
|
|
"val_f1": [
|
|
0.9095513748191028,
|
|
0.9088256746900073,
|
|
0.9368932038834952,
|
|
0.9394043135912359,
|
|
0.9357861853523082,
|
|
0.9283667621776505,
|
|
0.9484817468440805,
|
|
0.9406838799720866
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9114583333333334,
|
|
"auc_roc": 0.9665831018518519,
|
|
"f1": 0.941241531867828,
|
|
"confusion_matrix": [
|
|
[
|
|
971,
|
|
229
|
|
],
|
|
[
|
|
196,
|
|
3404
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9558333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9558333333333333,
|
|
"pairwise_auc": 0.9706750000000001,
|
|
"pairwise_f1": 0.890527950310559
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9025,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
117,
|
|
1083
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9025,
|
|
"pairwise_auc": 0.946836111111111,
|
|
"pairwise_f1": 0.8622611464968153
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9783333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9783333333333334,
|
|
"pairwise_auc": 0.9822381944444445,
|
|
"pairwise_f1": 0.9020361121782559
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8091666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
971,
|
|
229
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.1908333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8825,
|
|
"auc_roc": 0.9706750000000001,
|
|
"f1": 0.890527950310559,
|
|
"confusion_matrix": [
|
|
[
|
|
971,
|
|
229
|
|
],
|
|
[
|
|
53,
|
|
1147
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8558333333333333,
|
|
"auc_roc": 0.946836111111111,
|
|
"f1": 0.8622611464968153,
|
|
"confusion_matrix": [
|
|
[
|
|
971,
|
|
229
|
|
],
|
|
[
|
|
117,
|
|
1083
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.89375,
|
|
"auc_roc": 0.9822381944444445,
|
|
"f1": 0.9020361121782559,
|
|
"confusion_matrix": [
|
|
[
|
|
971,
|
|
229
|
|
],
|
|
[
|
|
26,
|
|
1174
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19605547449103108,
|
|
0.1016867975462918,
|
|
0.05938513064843223,
|
|
0.039155238193214044,
|
|
0.02408102286992491,
|
|
0.016763887942512087,
|
|
0.012361552757432004,
|
|
0.006711566793920712,
|
|
0.004867987432482079,
|
|
0.0033510766653097124,
|
|
0.0017171694248005493
|
|
],
|
|
"train_acc": [
|
|
0.8157407407407408,
|
|
0.9196180555555555,
|
|
0.9558449074074075,
|
|
0.9753472222222223,
|
|
0.9855902777777777,
|
|
0.9895833333333334,
|
|
0.9928819444444444,
|
|
0.9956597222222222,
|
|
0.997337962962963,
|
|
0.997800925925926,
|
|
0.9990740740740741
|
|
],
|
|
"train_auc": [
|
|
0.9033538469507316,
|
|
0.9751240462105624,
|
|
0.9914933413351623,
|
|
0.9962482763917466,
|
|
0.9984501189557613,
|
|
0.9991688635973937,
|
|
0.9995644718792867,
|
|
0.999859351780407,
|
|
0.9999119173668268,
|
|
0.9999706540066301,
|
|
0.9999934895833333
|
|
],
|
|
"train_f1": [
|
|
0.8693153833524873,
|
|
0.9451226739362333,
|
|
0.9702174167609977,
|
|
0.9834960483496048,
|
|
0.990361912134701,
|
|
0.9930405196412001,
|
|
0.9952485803685247,
|
|
0.9971021212472471,
|
|
0.99822476072862,
|
|
0.998533384793516,
|
|
0.9993825254708243
|
|
],
|
|
"val_loss": [
|
|
0.14180575897917153,
|
|
0.16796943203856549,
|
|
0.1819291006385659,
|
|
0.19614342823624611,
|
|
0.1975471700677493,
|
|
0.20672754806964805,
|
|
0.2890045994245156,
|
|
0.2595113074560686,
|
|
0.30948789948679406,
|
|
0.25677137968485414,
|
|
0.2594324775731062
|
|
],
|
|
"val_acc": [
|
|
0.890625,
|
|
0.8380208333333333,
|
|
0.8984375,
|
|
0.91875,
|
|
0.9088541666666666,
|
|
0.9182291666666667,
|
|
0.9171875,
|
|
0.9197916666666667,
|
|
0.9177083333333333,
|
|
0.9255208333333333,
|
|
0.9182291666666667
|
|
],
|
|
"val_auc": [
|
|
0.9531922743055556,
|
|
0.9427358217592593,
|
|
0.9559722222222222,
|
|
0.9617787905092593,
|
|
0.958308738425926,
|
|
0.9635467303240741,
|
|
0.9556517650462962,
|
|
0.9584063946759258,
|
|
0.947162181712963,
|
|
0.9610836226851852,
|
|
0.9603349247685184
|
|
],
|
|
"val_f1": [
|
|
0.9253731343283582,
|
|
0.8833020637898686,
|
|
0.9316508937960042,
|
|
0.9462809917355371,
|
|
0.9390456287008011,
|
|
0.945391304347826,
|
|
0.9452667814113598,
|
|
0.9468599033816425,
|
|
0.9452908587257618,
|
|
0.9500523925951799,
|
|
0.945391304347826
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9158333333333334,
|
|
"auc_roc": 0.9642475694444445,
|
|
"f1": 0.943732590529248,
|
|
"confusion_matrix": [
|
|
[
|
|
1008,
|
|
192
|
|
],
|
|
[
|
|
212,
|
|
3388
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9591666666666666,
|
|
"pairwise_auc": 0.970038888888889,
|
|
"pairwise_f1": 0.9052300432559969
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8908333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
131,
|
|
1069
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8908333333333334,
|
|
"pairwise_auc": 0.9423524305555556,
|
|
"pairwise_f1": 0.8687525396180414
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9733333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9733333333333334,
|
|
"pairwise_auc": 0.980351388888889,
|
|
"pairwise_f1": 0.9125
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.84,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1008,
|
|
192
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.16000000000000003
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8995833333333333,
|
|
"auc_roc": 0.970038888888889,
|
|
"f1": 0.9052300432559969,
|
|
"confusion_matrix": [
|
|
[
|
|
1008,
|
|
192
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8654166666666666,
|
|
"auc_roc": 0.9423524305555556,
|
|
"f1": 0.8687525396180414,
|
|
"confusion_matrix": [
|
|
[
|
|
1008,
|
|
192
|
|
],
|
|
[
|
|
131,
|
|
1069
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9066666666666666,
|
|
"auc_roc": 0.980351388888889,
|
|
"f1": 0.9125,
|
|
"confusion_matrix": [
|
|
[
|
|
1008,
|
|
192
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19928204308919334,
|
|
0.10327634860382036,
|
|
0.05515722113368274,
|
|
0.035082230208489044,
|
|
0.025447382428898924,
|
|
0.017044535008877842,
|
|
0.012585550207478702,
|
|
0.007383903622266425,
|
|
0.0065240937882107474,
|
|
0.003589426765918486,
|
|
0.0018262106446854886,
|
|
0.0014432606080622126
|
|
],
|
|
"train_acc": [
|
|
0.8112268518518518,
|
|
0.9170138888888889,
|
|
0.960300925925926,
|
|
0.9774305555555556,
|
|
0.9856481481481482,
|
|
0.9886574074074074,
|
|
0.9927662037037037,
|
|
0.9962962962962963,
|
|
0.9962384259259259,
|
|
0.9982060185185185,
|
|
0.998900462962963,
|
|
0.9992476851851851
|
|
],
|
|
"train_auc": [
|
|
0.8996633605538409,
|
|
0.974294213677412,
|
|
0.9924166684527892,
|
|
0.9968912001314587,
|
|
0.9983060145890489,
|
|
0.999246265217764,
|
|
0.9995093878600823,
|
|
0.9997427001171696,
|
|
0.999896556712963,
|
|
0.9998388649548468,
|
|
0.9999534893689986,
|
|
0.9999838087991542
|
|
],
|
|
"train_f1": [
|
|
0.8659709096885528,
|
|
0.9433067130544793,
|
|
0.9732470166133687,
|
|
0.9848837209302326,
|
|
0.9904128653162209,
|
|
0.9924195544554455,
|
|
0.9951698288187333,
|
|
0.9975281940367681,
|
|
0.9974918001157631,
|
|
0.9988035045737003,
|
|
0.9992667206977731,
|
|
0.9994982825826869
|
|
],
|
|
"val_loss": [
|
|
0.12679128739982842,
|
|
0.2473485317993133,
|
|
0.14678561501204967,
|
|
0.1554646889999276,
|
|
0.15785321037013395,
|
|
0.2617460626413049,
|
|
0.17361461978216539,
|
|
0.20700737896950158,
|
|
0.23985129223819968,
|
|
0.22690686947868624,
|
|
0.26887194706073386,
|
|
0.2289405580937455
|
|
],
|
|
"val_acc": [
|
|
0.8609375,
|
|
0.8739583333333333,
|
|
0.8817708333333333,
|
|
0.9171875,
|
|
0.9109375,
|
|
0.9067708333333333,
|
|
0.9208333333333333,
|
|
0.9223958333333333,
|
|
0.9166666666666666,
|
|
0.9234375,
|
|
0.9161458333333333,
|
|
0.9255208333333333
|
|
],
|
|
"val_auc": [
|
|
0.9617122395833333,
|
|
0.9316594328703705,
|
|
0.9648596643518519,
|
|
0.9680758101851852,
|
|
0.9689894386574074,
|
|
0.9582892071759259,
|
|
0.9722294560185186,
|
|
0.9673271122685185,
|
|
0.9659454571759258,
|
|
0.9686675347222222,
|
|
0.9653291377314814,
|
|
0.9667903645833332
|
|
],
|
|
"val_f1": [
|
|
0.9003359462486002,
|
|
0.9181326116373477,
|
|
0.9162670601254149,
|
|
0.9444250262146103,
|
|
0.9396825396825397,
|
|
0.9383396486393386,
|
|
0.9467414155571128,
|
|
0.9483535528596188,
|
|
0.9446366782006921,
|
|
0.9490467937608319,
|
|
0.9446545204537642,
|
|
0.9502954466458116
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.919375,
|
|
"auc_roc": 0.9684114583333332,
|
|
"f1": 0.9460777483628257,
|
|
"confusion_matrix": [
|
|
[
|
|
1018,
|
|
182
|
|
],
|
|
[
|
|
205,
|
|
3395
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9616666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9616666666666667,
|
|
"pairwise_auc": 0.9754291666666666,
|
|
"pairwise_f1": 0.9100946372239748
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9066666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
112,
|
|
1088
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9066666666666666,
|
|
"pairwise_auc": 0.9537607638888889,
|
|
"pairwise_f1": 0.8809716599190284
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9608333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
47,
|
|
1153
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9608333333333333,
|
|
"pairwise_auc": 0.9760444444444445,
|
|
"pairwise_f1": 0.9096646942800789
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8483333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1018,
|
|
182
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.15166666666666662
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.905,
|
|
"auc_roc": 0.9754291666666666,
|
|
"f1": 0.9100946372239748,
|
|
"confusion_matrix": [
|
|
[
|
|
1018,
|
|
182
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8775,
|
|
"auc_roc": 0.9537607638888889,
|
|
"f1": 0.8809716599190284,
|
|
"confusion_matrix": [
|
|
[
|
|
1018,
|
|
182
|
|
],
|
|
[
|
|
112,
|
|
1088
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9045833333333333,
|
|
"auc_roc": 0.9760444444444445,
|
|
"f1": 0.9096646942800789,
|
|
"confusion_matrix": [
|
|
[
|
|
1018,
|
|
182
|
|
],
|
|
[
|
|
47,
|
|
1153
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.1946798356939797,
|
|
0.09956369337246374,
|
|
0.06275961574197195,
|
|
0.037021170328349666,
|
|
0.025708021630567533,
|
|
0.021358401671346252,
|
|
0.01339459377892059,
|
|
0.007324876425533418
|
|
],
|
|
"train_acc": [
|
|
0.8107060185185185,
|
|
0.9216435185185186,
|
|
0.9565393518518519,
|
|
0.9758680555555556,
|
|
0.9834490740740741,
|
|
0.9872685185185185,
|
|
0.9924768518518519,
|
|
0.9950231481481482
|
|
],
|
|
"train_auc": [
|
|
0.9044035154463876,
|
|
0.9761751793267033,
|
|
0.9906578825160036,
|
|
0.9966033129000914,
|
|
0.9982404281692958,
|
|
0.9988388774577046,
|
|
0.9993337941529492,
|
|
0.9998615219192957
|
|
],
|
|
"train_f1": [
|
|
0.8651967854935092,
|
|
0.9465540380516302,
|
|
0.9707383596337424,
|
|
0.9838340763713898,
|
|
0.9889336016096579,
|
|
0.991492652745553,
|
|
0.9949779803754926,
|
|
0.9966774841600989
|
|
],
|
|
"val_loss": [
|
|
0.1366950663117071,
|
|
0.14299837291861575,
|
|
0.14879056867212057,
|
|
0.19593870592943857,
|
|
0.18934340609703212,
|
|
0.24294537242579584,
|
|
0.22201119096826005,
|
|
0.3828427222149609
|
|
],
|
|
"val_acc": [
|
|
0.8875,
|
|
0.875,
|
|
0.9161458333333333,
|
|
0.9119791666666667,
|
|
0.8869791666666667,
|
|
0.9078125,
|
|
0.9213541666666667,
|
|
0.9151041666666667
|
|
],
|
|
"val_auc": [
|
|
0.9550998263888889,
|
|
0.9605808738425925,
|
|
0.9650802951388888,
|
|
0.9626374421296297,
|
|
0.9632081886574075,
|
|
0.9592136863425926,
|
|
0.9635951967592593,
|
|
0.9525991030092593
|
|
],
|
|
"val_f1": [
|
|
0.9228020014295926,
|
|
0.9121522693997072,
|
|
0.9438437391001047,
|
|
0.9414617249740215,
|
|
0.9210622044379775,
|
|
0.9391124871001032,
|
|
0.9476240027748872,
|
|
0.9451732256979481
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.915,
|
|
"auc_roc": 0.9685586805555556,
|
|
"f1": 0.9427127211457456,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
243,
|
|
3357
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9425,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
69,
|
|
1131
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9425,
|
|
"pairwise_auc": 0.9706340277777777,
|
|
"pairwise_f1": 0.90625
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.8816666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
142,
|
|
1058
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.8816666666666667,
|
|
"pairwise_auc": 0.9508114583333334,
|
|
"pairwise_f1": 0.8732975650020636
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9733333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9733333333333334,
|
|
"pairwise_auc": 0.9842305555555555,
|
|
"pairwise_f1": 0.9222266087643111
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8625,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.13749999999999996
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9025,
|
|
"auc_roc": 0.9706340277777777,
|
|
"f1": 0.90625,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
69,
|
|
1131
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8720833333333333,
|
|
"auc_roc": 0.9508114583333334,
|
|
"f1": 0.8732975650020636,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
142,
|
|
1058
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9179166666666667,
|
|
"auc_roc": 0.9842305555555555,
|
|
"f1": 0.9222266087643111,
|
|
"confusion_matrix": [
|
|
[
|
|
1035,
|
|
165
|
|
],
|
|
[
|
|
32,
|
|
1168
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.19603962583674325,
|
|
0.10220566353829647,
|
|
0.0641474825364572,
|
|
0.038550203464735575,
|
|
0.02755131951842621,
|
|
0.016848129556375272,
|
|
0.014130299409757923,
|
|
0.008122794064207415,
|
|
0.004999924033172984,
|
|
0.0031690383527381563,
|
|
0.001879638221826267,
|
|
0.0010143579254055888
|
|
],
|
|
"train_acc": [
|
|
0.8101851851851852,
|
|
0.9185185185185185,
|
|
0.955324074074074,
|
|
0.9729745370370371,
|
|
0.9843171296296296,
|
|
0.9900462962962963,
|
|
0.9925925925925926,
|
|
0.9954282407407408,
|
|
0.9974537037037037,
|
|
0.9979745370370371,
|
|
0.9987847222222223,
|
|
0.9991898148148148
|
|
],
|
|
"train_auc": [
|
|
0.9027364826245999,
|
|
0.9749666263002972,
|
|
0.9903671732110196,
|
|
0.9963543452789209,
|
|
0.9976980274062643,
|
|
0.9991864658350481,
|
|
0.9992046842849793,
|
|
0.999598667195359,
|
|
0.9999064339706218,
|
|
0.9999665727166209,
|
|
0.9999912301383174,
|
|
0.9999978745141747
|
|
],
|
|
"train_f1": [
|
|
0.8645523620746614,
|
|
0.944391785150079,
|
|
0.9699259836384885,
|
|
0.9818605554476597,
|
|
0.9895095420586072,
|
|
0.9933559950556242,
|
|
0.9950552422158696,
|
|
0.9969499247133315,
|
|
0.9983010271063403,
|
|
0.9986488051577037,
|
|
0.9991894082680357,
|
|
0.9994597097869713
|
|
],
|
|
"val_loss": [
|
|
0.16352217756211757,
|
|
0.13888414253791173,
|
|
0.15331077139514188,
|
|
0.17716894903569483,
|
|
0.17669353319021563,
|
|
0.22967796006262992,
|
|
0.17005534635294073,
|
|
0.22448178055371196,
|
|
0.3090995565579002,
|
|
0.25438256103980167,
|
|
0.21965038712469323,
|
|
0.234941724254107
|
|
],
|
|
"val_acc": [
|
|
0.884375,
|
|
0.9046875,
|
|
0.8979166666666667,
|
|
0.9229166666666667,
|
|
0.8979166666666667,
|
|
0.9265625,
|
|
0.91875,
|
|
0.9239583333333333,
|
|
0.9244791666666666,
|
|
0.9223958333333333,
|
|
0.9317708333333333,
|
|
0.9328125
|
|
],
|
|
"val_auc": [
|
|
0.9458608217592593,
|
|
0.9611993634259258,
|
|
0.9629803240740741,
|
|
0.9689583333333334,
|
|
0.9654275173611111,
|
|
0.9623119212962963,
|
|
0.9730598958333334,
|
|
0.965689380787037,
|
|
0.9596838831018518,
|
|
0.9648972800925926,
|
|
0.9698365162037036,
|
|
0.9703088831018518
|
|
],
|
|
"val_f1": [
|
|
0.9217207334273625,
|
|
0.9353128313891834,
|
|
0.93,
|
|
0.949070887818307,
|
|
0.9291907514450867,
|
|
0.9513960703205792,
|
|
0.9451090781140042,
|
|
0.9492350486787204,
|
|
0.9508641138597086,
|
|
0.9487796493640426,
|
|
0.9546869595295745,
|
|
0.95562435500516
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": 0.9711554398148149,
|
|
"f1": 0.9463579898362507,
|
|
"confusion_matrix": [
|
|
[
|
|
1068,
|
|
132
|
|
],
|
|
[
|
|
248,
|
|
3352
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9441666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
67,
|
|
1133
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9441666666666667,
|
|
"pairwise_auc": 0.9743878472222222,
|
|
"pairwise_f1": 0.9192697768762678
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9041666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
115,
|
|
1085
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9041666666666667,
|
|
"pairwise_auc": 0.9620392361111111,
|
|
"pairwise_f1": 0.8978071990070335
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.945,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
66,
|
|
1134
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.945,
|
|
"pairwise_auc": 0.9770392361111111,
|
|
"pairwise_f1": 0.9197080291970803
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.89,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1068,
|
|
132
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.10999999999999999
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9170833333333334,
|
|
"auc_roc": 0.9743878472222222,
|
|
"f1": 0.9192697768762678,
|
|
"confusion_matrix": [
|
|
[
|
|
1068,
|
|
132
|
|
],
|
|
[
|
|
67,
|
|
1133
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.8970833333333333,
|
|
"auc_roc": 0.9620392361111111,
|
|
"f1": 0.8978071990070335,
|
|
"confusion_matrix": [
|
|
[
|
|
1068,
|
|
132
|
|
],
|
|
[
|
|
115,
|
|
1085
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9175,
|
|
"auc_roc": 0.9770392361111111,
|
|
"f1": 0.9197080291970803,
|
|
"confusion_matrix": [
|
|
[
|
|
1068,
|
|
132
|
|
],
|
|
[
|
|
66,
|
|
1134
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9165000000000001,
|
|
"std": 0.0037133626943782138,
|
|
"ci_95": 0.0032549059126186527,
|
|
"values": [
|
|
0.9114583333333334,
|
|
0.9158333333333334,
|
|
0.919375,
|
|
0.915,
|
|
0.9208333333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9677912500000001,
|
|
"std": 0.0025645991388585803,
|
|
"ci_95": 0.0022479702597338817,
|
|
"values": [
|
|
0.9665831018518519,
|
|
0.9642475694444445,
|
|
0.9684114583333332,
|
|
0.9685586805555556,
|
|
0.9711554398148149
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9440245163483796,
|
|
"std": 0.002191568512519297,
|
|
"ci_95": 0.0019209944991657685,
|
|
"values": [
|
|
0.941241531867828,
|
|
0.943732590529248,
|
|
0.9460777483628257,
|
|
0.9427127211457456,
|
|
0.9463579898362507
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9526666666666668,
|
|
"std": 0.008787617550976023,
|
|
"ci_95": 0.007702686400067023,
|
|
"values": [
|
|
0.9558333333333333,
|
|
0.9591666666666666,
|
|
0.9616666666666667,
|
|
0.9425,
|
|
0.9441666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9526666666666668,
|
|
"std": 0.008787617550976023,
|
|
"ci_95": 0.007702686400067023,
|
|
"values": [
|
|
0.9558333333333333,
|
|
0.9591666666666666,
|
|
0.9616666666666667,
|
|
0.9425,
|
|
0.9441666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9722329861111112,
|
|
"std": 0.002482793167311941,
|
|
"ci_95": 0.0021762641641031514,
|
|
"values": [
|
|
0.9706750000000001,
|
|
0.970038888888889,
|
|
0.9754291666666666,
|
|
0.9706340277777777,
|
|
0.9743878472222222
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9062744815333597,
|
|
"std": 0.010398509349436593,
|
|
"ci_95": 0.009114695317842876,
|
|
"values": [
|
|
0.890527950310559,
|
|
0.9052300432559969,
|
|
0.9100946372239748,
|
|
0.90625,
|
|
0.9192697768762678
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.8971666666666668,
|
|
"std": 0.010583661412237646,
|
|
"ci_95": 0.009276988256493082,
|
|
"values": [
|
|
0.9025,
|
|
0.8908333333333334,
|
|
0.9066666666666666,
|
|
0.8816666666666667,
|
|
0.9041666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.8971666666666668,
|
|
"std": 0.010583661412237646,
|
|
"ci_95": 0.009276988256493082,
|
|
"values": [
|
|
0.9025,
|
|
0.8908333333333334,
|
|
0.9066666666666666,
|
|
0.8816666666666667,
|
|
0.9041666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.95116,
|
|
"std": 0.007441648458215133,
|
|
"ci_95": 0.006522892472352412,
|
|
"values": [
|
|
0.946836111111111,
|
|
0.9423524305555556,
|
|
0.9537607638888889,
|
|
0.9508114583333334,
|
|
0.9620392361111111
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.8766180220085964,
|
|
"std": 0.013665161061577277,
|
|
"ci_95": 0.011978041790410631,
|
|
"values": [
|
|
0.8622611464968153,
|
|
0.8687525396180414,
|
|
0.8809716599190284,
|
|
0.8732975650020636,
|
|
0.8978071990070335
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9661666666666667,
|
|
"std": 0.013483529046787299,
|
|
"ci_95": 0.011818834309882061,
|
|
"values": [
|
|
0.9783333333333334,
|
|
0.9733333333333334,
|
|
0.9608333333333333,
|
|
0.9733333333333334,
|
|
0.945
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9661666666666667,
|
|
"std": 0.013483529046787299,
|
|
"ci_95": 0.011818834309882061,
|
|
"values": [
|
|
0.9783333333333334,
|
|
0.9733333333333334,
|
|
0.9608333333333333,
|
|
0.9733333333333334,
|
|
0.945
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.979980763888889,
|
|
"std": 0.0034438815764337577,
|
|
"ci_95": 0.0030186952980550876,
|
|
"values": [
|
|
0.9822381944444445,
|
|
0.980351388888889,
|
|
0.9760444444444445,
|
|
0.9842305555555555,
|
|
0.9770392361111111
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9132270888839453,
|
|
"std": 0.008084726628118197,
|
|
"ci_95": 0.00708657534143018,
|
|
"values": [
|
|
0.9020361121782559,
|
|
0.9125,
|
|
0.9096646942800789,
|
|
0.9222266087643111,
|
|
0.9197080291970803
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.85,
|
|
"std": 0.029691702918866445,
|
|
"ci_95": 0.026025925108971202,
|
|
"values": [
|
|
0.8091666666666667,
|
|
0.84,
|
|
0.8483333333333334,
|
|
0.8625,
|
|
0.89
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.14999999999999997,
|
|
"std": 0.029691702918866445,
|
|
"ci_95": 0.026025925108971202,
|
|
"values": [
|
|
0.1908333333333333,
|
|
0.16000000000000003,
|
|
0.15166666666666662,
|
|
0.13749999999999996,
|
|
0.10999999999999999
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9013333333333333,
|
|
"std": 0.012456173168353141,
|
|
"ci_95": 0.010918317178027048,
|
|
"values": [
|
|
0.8825,
|
|
0.8995833333333333,
|
|
0.905,
|
|
0.9025,
|
|
0.9170833333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9722329861111112,
|
|
"std": 0.002482793167311941,
|
|
"ci_95": 0.0021762641641031514,
|
|
"values": [
|
|
0.9706750000000001,
|
|
0.970038888888889,
|
|
0.9754291666666666,
|
|
0.9706340277777777,
|
|
0.9743878472222222
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9062744815333597,
|
|
"std": 0.010398509349436593,
|
|
"ci_95": 0.009114695317842876,
|
|
"values": [
|
|
0.890527950310559,
|
|
0.9052300432559969,
|
|
0.9100946372239748,
|
|
0.90625,
|
|
0.9192697768762678
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.8735833333333334,
|
|
"std": 0.015423984749589056,
|
|
"ci_95": 0.01351971872652847,
|
|
"values": [
|
|
0.8558333333333333,
|
|
0.8654166666666666,
|
|
0.8775,
|
|
0.8720833333333333,
|
|
0.8970833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.95116,
|
|
"std": 0.007441648458215133,
|
|
"ci_95": 0.006522892472352412,
|
|
"values": [
|
|
0.946836111111111,
|
|
0.9423524305555556,
|
|
0.9537607638888889,
|
|
0.9508114583333334,
|
|
0.9620392361111111
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.8766180220085964,
|
|
"std": 0.013665161061577277,
|
|
"ci_95": 0.011978041790410631,
|
|
"values": [
|
|
0.8622611464968153,
|
|
0.8687525396180414,
|
|
0.8809716599190284,
|
|
0.8732975650020636,
|
|
0.8978071990070335
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9080833333333335,
|
|
"std": 0.010063168542318618,
|
|
"ci_95": 0.008820756140427464,
|
|
"values": [
|
|
0.89375,
|
|
0.9066666666666666,
|
|
0.9045833333333333,
|
|
0.9179166666666667,
|
|
0.9175
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.979980763888889,
|
|
"std": 0.0034438815764337577,
|
|
"ci_95": 0.0030186952980550876,
|
|
"values": [
|
|
0.9822381944444445,
|
|
0.980351388888889,
|
|
0.9760444444444445,
|
|
0.9842305555555555,
|
|
0.9770392361111111
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9132270888839453,
|
|
"std": 0.008084726628118197,
|
|
"ci_95": 0.00708657534143018,
|
|
"values": [
|
|
0.9020361121782559,
|
|
0.9125,
|
|
0.9096646942800789,
|
|
0.9222266087643111,
|
|
0.9197080291970803
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "data",
|
|
"run_name": "p2a_t2_real_norm",
|
|
"backbone": "resnet18",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"normalization": "real_norm"
|
|
}
|
|
} |