Johnny Fernandes
1737 lines
41 KiB
JSON
1737 lines
41 KiB
JSON
{
|
|
"run_name": "p3_resnet50",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16414252044288097,
|
|
0.07211566557805917,
|
|
0.0453435333126604,
|
|
0.02745912457467057,
|
|
0.020166955894658004,
|
|
0.011375110268084056,
|
|
0.008973008060503936,
|
|
0.005261950669481579,
|
|
0.00301203729498209,
|
|
0.0021092872964598225,
|
|
0.0007151615797405847
|
|
],
|
|
"train_acc": [
|
|
0.8653356481481481,
|
|
0.9497106481481481,
|
|
0.9728587962962963,
|
|
0.983738425925926,
|
|
0.9897569444444444,
|
|
0.9938078703703703,
|
|
0.9958912037037037,
|
|
0.9972222222222222,
|
|
0.9986689814814815,
|
|
0.9990162037037037,
|
|
0.9996527777777777
|
|
],
|
|
"train_auc": [
|
|
0.9346535547410837,
|
|
0.9877919238683127,
|
|
0.9952512270661866,
|
|
0.9982134219964562,
|
|
0.9989599765660722,
|
|
0.999440434599337,
|
|
0.9994629129515318,
|
|
0.9999399594907408,
|
|
0.9999841392318245,
|
|
0.9998730334790809,
|
|
0.9999985889631915
|
|
],
|
|
"train_f1": [
|
|
0.9073830845771145,
|
|
0.9661301009471099,
|
|
0.9818421154516241,
|
|
0.9891350578045857,
|
|
0.9931631194715903,
|
|
0.9958685663539133,
|
|
0.9972600625168834,
|
|
0.9981481481481481,
|
|
0.999112825458052,
|
|
0.9993441104980902,
|
|
0.9997685185185186
|
|
],
|
|
"val_loss": [
|
|
0.10033943792805075,
|
|
0.1824329480936285,
|
|
0.18764888900720203,
|
|
0.15092337583434226,
|
|
0.16416337434347952,
|
|
0.15572371911900215,
|
|
0.19316397059158893,
|
|
0.31078116144394696,
|
|
0.1872688261007473,
|
|
0.25701718308205274,
|
|
0.22669576404931224
|
|
],
|
|
"val_acc": [
|
|
0.9223958333333333,
|
|
0.925,
|
|
0.9296875,
|
|
0.9369791666666667,
|
|
0.9364583333333333,
|
|
0.9447916666666667,
|
|
0.9416666666666667,
|
|
0.9380208333333333,
|
|
0.9489583333333333,
|
|
0.9479166666666666,
|
|
0.9479166666666666
|
|
],
|
|
"val_auc": [
|
|
0.9760112847222223,
|
|
0.9681032986111111,
|
|
0.9767173032407406,
|
|
0.9799927662037038,
|
|
0.9765567129629629,
|
|
0.980189525462963,
|
|
0.9800253182870371,
|
|
0.9619646990740741,
|
|
0.9779448784722221,
|
|
0.9740979456018519,
|
|
0.9707732928240741
|
|
],
|
|
"val_f1": [
|
|
0.9474426807760141,
|
|
0.9509870660313138,
|
|
0.9542837792075856,
|
|
0.9586041737940472,
|
|
0.9576094510076442,
|
|
0.9633471645919779,
|
|
0.961826857532379,
|
|
0.959892146949781,
|
|
0.9661602209944752,
|
|
0.9659632402995235,
|
|
0.9658703071672355
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.95125,
|
|
"auc_roc": 0.9858875,
|
|
"f1": 0.967518045530261,
|
|
"confusion_matrix": [
|
|
[
|
|
1081,
|
|
119
|
|
],
|
|
[
|
|
115,
|
|
3485
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9775,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9775,
|
|
"pairwise_auc": 0.9896520833333333,
|
|
"pairwise_f1": 0.9414125200642055
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9408333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
71,
|
|
1129
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9408333333333333,
|
|
"pairwise_auc": 0.9775805555555556,
|
|
"pairwise_f1": 0.9223856209150327
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9858333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9858333333333333,
|
|
"pairwise_auc": 0.9904298611111111,
|
|
"pairwise_f1": 0.9456434852118305
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9008333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1081,
|
|
119
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.09916666666666663
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9391666666666667,
|
|
"auc_roc": 0.9896520833333333,
|
|
"f1": 0.9414125200642055,
|
|
"confusion_matrix": [
|
|
[
|
|
1081,
|
|
119
|
|
],
|
|
[
|
|
27,
|
|
1173
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9208333333333333,
|
|
"auc_roc": 0.9775805555555556,
|
|
"f1": 0.9223856209150327,
|
|
"confusion_matrix": [
|
|
[
|
|
1081,
|
|
119
|
|
],
|
|
[
|
|
71,
|
|
1129
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9433333333333334,
|
|
"auc_roc": 0.9904298611111111,
|
|
"f1": 0.9456434852118305,
|
|
"confusion_matrix": [
|
|
[
|
|
1081,
|
|
119
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16498077511097545,
|
|
0.07493020069518092,
|
|
0.040497788825369856,
|
|
0.027122006811987377,
|
|
0.017663728153079487,
|
|
0.010251278441278879,
|
|
0.011569880251946179,
|
|
0.007077035647746386,
|
|
0.004594202235690587,
|
|
0.003612333799214782
|
|
],
|
|
"train_acc": [
|
|
0.8653356481481481,
|
|
0.95,
|
|
0.973900462962963,
|
|
0.9832175925925926,
|
|
0.9884259259259259,
|
|
0.9949074074074075,
|
|
0.994849537037037,
|
|
0.9969907407407408,
|
|
0.997800925925926,
|
|
0.9984375
|
|
],
|
|
"train_auc": [
|
|
0.933973819015775,
|
|
0.9869939468307042,
|
|
0.996034906192844,
|
|
0.9981377975680157,
|
|
0.9991262288523091,
|
|
0.9995976758973479,
|
|
0.9995999085505259,
|
|
0.9996476694673069,
|
|
0.9998369895261773,
|
|
0.9998555919924554
|
|
],
|
|
"train_f1": [
|
|
0.9075303000198689,
|
|
0.9663682366679642,
|
|
0.982498350731499,
|
|
0.9887796951172328,
|
|
0.9922654497640963,
|
|
0.9966020542126806,
|
|
0.9965654304789102,
|
|
0.9979933626611098,
|
|
0.998533724340176,
|
|
0.9989580519430402
|
|
],
|
|
"val_loss": [
|
|
0.16876822944032027,
|
|
0.10144863610621542,
|
|
0.12092681264281661,
|
|
0.16224593347378685,
|
|
0.14133250266774364,
|
|
0.17814634380332184,
|
|
0.14857266811098574,
|
|
0.20385507235837394,
|
|
0.1918886211836707,
|
|
0.19614593234909988
|
|
],
|
|
"val_acc": [
|
|
0.91875,
|
|
0.9359375,
|
|
0.9369791666666667,
|
|
0.9463541666666667,
|
|
0.9505208333333334,
|
|
0.9447916666666667,
|
|
0.9557291666666666,
|
|
0.9515625,
|
|
0.9526041666666667,
|
|
0.9567708333333333
|
|
],
|
|
"val_auc": [
|
|
0.967865306712963,
|
|
0.9804188368055555,
|
|
0.978923611111111,
|
|
0.9764431423611111,
|
|
0.9805013020833334,
|
|
0.9744010416666666,
|
|
0.978972800925926,
|
|
0.9760083912037036,
|
|
0.9779087094907407,
|
|
0.9782052951388889
|
|
],
|
|
"val_f1": [
|
|
0.9470108695652174,
|
|
0.9572471324296142,
|
|
0.9577661431064572,
|
|
0.9643721895537876,
|
|
0.9671393981321342,
|
|
0.9631944444444445,
|
|
0.9705168227540756,
|
|
0.9680960548885077,
|
|
0.968566493955095,
|
|
0.9715461090161125
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9525,
|
|
"auc_roc": 0.9844932870370371,
|
|
"f1": 0.9682008368200837,
|
|
"confusion_matrix": [
|
|
[
|
|
1101,
|
|
99
|
|
],
|
|
[
|
|
129,
|
|
3471
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.97,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
36,
|
|
1164
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.97,
|
|
"pairwise_auc": 0.9868194444444445,
|
|
"pairwise_f1": 0.9451887941534713
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9416666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
70,
|
|
1130
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9416666666666667,
|
|
"pairwise_auc": 0.9780156249999999,
|
|
"pairwise_f1": 0.9304240428159737
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9808333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
23,
|
|
1177
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9808333333333333,
|
|
"pairwise_auc": 0.9886447916666667,
|
|
"pairwise_f1": 0.9507269789983845
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9175,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1101,
|
|
99
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.08250000000000002
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.94375,
|
|
"auc_roc": 0.9868194444444445,
|
|
"f1": 0.9451887941534713,
|
|
"confusion_matrix": [
|
|
[
|
|
1101,
|
|
99
|
|
],
|
|
[
|
|
36,
|
|
1164
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9295833333333333,
|
|
"auc_roc": 0.9780156249999999,
|
|
"f1": 0.9304240428159737,
|
|
"confusion_matrix": [
|
|
[
|
|
1101,
|
|
99
|
|
],
|
|
[
|
|
70,
|
|
1130
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9491666666666667,
|
|
"auc_roc": 0.9886447916666667,
|
|
"f1": 0.9507269789983845,
|
|
"confusion_matrix": [
|
|
[
|
|
1101,
|
|
99
|
|
],
|
|
[
|
|
23,
|
|
1177
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16858915814922915,
|
|
0.07113397797069478,
|
|
0.046212628759072956,
|
|
0.02708110625330678,
|
|
0.01983529361512208,
|
|
0.012717378701199777,
|
|
0.007865516435558,
|
|
0.005219494796910517
|
|
],
|
|
"train_acc": [
|
|
0.8554398148148148,
|
|
0.9509837962962963,
|
|
0.9720486111111111,
|
|
0.9836226851851851,
|
|
0.9885416666666667,
|
|
0.9930555555555556,
|
|
0.9957754629629629,
|
|
0.9967013888888889
|
|
],
|
|
"train_auc": [
|
|
0.930862563228738,
|
|
0.9882049379143805,
|
|
0.9949234467878373,
|
|
0.9982081261431184,
|
|
0.9991367580446958,
|
|
0.9995974794238682,
|
|
0.9998441876000229,
|
|
0.9999356906578645
|
|
],
|
|
"train_f1": [
|
|
0.8995092123260118,
|
|
0.9670748299319728,
|
|
0.9812870481577622,
|
|
0.9890568810177487,
|
|
0.9923569829383154,
|
|
0.9953667953667954,
|
|
0.9971833159702126,
|
|
0.9977999922806746
|
|
],
|
|
"val_loss": [
|
|
0.10089091673338164,
|
|
0.1244420326858138,
|
|
0.08564881963053873,
|
|
0.13306436915809172,
|
|
0.19317561716384263,
|
|
0.14889613401107152,
|
|
0.15328831906423754,
|
|
0.1559290382703845
|
|
],
|
|
"val_acc": [
|
|
0.9369791666666667,
|
|
0.9390625,
|
|
0.9473958333333333,
|
|
0.9411458333333333,
|
|
0.9427083333333334,
|
|
0.9473958333333333,
|
|
0.946875,
|
|
0.9520833333333333
|
|
],
|
|
"val_auc": [
|
|
0.9792758969907407,
|
|
0.9771636284722223,
|
|
0.9873444733796295,
|
|
0.9819314236111112,
|
|
0.9770666956018518,
|
|
0.9802944155092592,
|
|
0.9832298900462964,
|
|
0.9784244791666667
|
|
],
|
|
"val_f1": [
|
|
0.957558751315328,
|
|
0.9599726308587068,
|
|
0.9646729625743267,
|
|
0.9607775078097883,
|
|
0.9623803009575923,
|
|
0.9647960962007668,
|
|
0.9643605870020965,
|
|
0.9678546470999301
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9479166666666666,
|
|
"auc_roc": 0.9848238425925926,
|
|
"f1": 0.9651227678571429,
|
|
"confusion_matrix": [
|
|
[
|
|
1091,
|
|
109
|
|
],
|
|
[
|
|
141,
|
|
3459
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9591666666666666,
|
|
"pairwise_auc": 0.9845902777777779,
|
|
"pairwise_f1": 0.9357723577235773
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9391666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
73,
|
|
1127
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9391666666666667,
|
|
"pairwise_auc": 0.9786312500000001,
|
|
"pairwise_f1": 0.9252873563218391
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9841666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9841666666666666,
|
|
"pairwise_auc": 0.9912500000000002,
|
|
"pairwise_f1": 0.9485943775100402
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9091666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1091,
|
|
109
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.09083333333333332
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9341666666666667,
|
|
"auc_roc": 0.9845902777777779,
|
|
"f1": 0.9357723577235773,
|
|
"confusion_matrix": [
|
|
[
|
|
1091,
|
|
109
|
|
],
|
|
[
|
|
49,
|
|
1151
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9241666666666667,
|
|
"auc_roc": 0.9786312500000001,
|
|
"f1": 0.9252873563218391,
|
|
"confusion_matrix": [
|
|
[
|
|
1091,
|
|
109
|
|
],
|
|
[
|
|
73,
|
|
1127
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9466666666666667,
|
|
"auc_roc": 0.9912500000000002,
|
|
"f1": 0.9485943775100402,
|
|
"confusion_matrix": [
|
|
[
|
|
1091,
|
|
109
|
|
],
|
|
[
|
|
19,
|
|
1181
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16950011297508522,
|
|
0.07547963091379238,
|
|
0.04220395572041592,
|
|
0.02993595788602424,
|
|
0.01820202965776574,
|
|
0.01421407222692155,
|
|
0.01162999869657142,
|
|
0.004805721047680183,
|
|
0.004482461260008003,
|
|
0.002507560079492224,
|
|
0.0015911430026992295,
|
|
0.0009422235933087635
|
|
],
|
|
"train_acc": [
|
|
0.8539930555555556,
|
|
0.9454282407407407,
|
|
0.9721643518518519,
|
|
0.98125,
|
|
0.9896412037037037,
|
|
0.9927083333333333,
|
|
0.9941550925925926,
|
|
0.9975694444444444,
|
|
0.9980324074074074,
|
|
0.9983217592592593,
|
|
0.9991898148148148,
|
|
0.9994791666666667
|
|
],
|
|
"train_auc": [
|
|
0.9291453760859625,
|
|
0.9864843839306127,
|
|
0.9955841245856196,
|
|
0.9979011006087106,
|
|
0.9991844475165752,
|
|
0.9995047082190215,
|
|
0.999488133001829,
|
|
0.9998861436185414,
|
|
0.9998295146033379,
|
|
0.9999863004401005,
|
|
0.9999882473136718,
|
|
0.999998642546868
|
|
],
|
|
"train_f1": [
|
|
0.898995155930982,
|
|
0.96315110781134,
|
|
0.9813298140744479,
|
|
0.9874729353541603,
|
|
0.9930837293767629,
|
|
0.995135510771369,
|
|
0.9960990305511568,
|
|
0.9983787539566124,
|
|
0.9986880691464732,
|
|
0.9988808706054876,
|
|
0.9994597514856834,
|
|
0.9996527375853687
|
|
],
|
|
"val_loss": [
|
|
0.1064753856199483,
|
|
0.10567577621744324,
|
|
0.19240089396334953,
|
|
0.128021330399982,
|
|
0.2900175292714872,
|
|
0.21754833572190363,
|
|
0.15359407078982865,
|
|
0.2786675535797561,
|
|
0.15161420754611754,
|
|
0.17650435155922725,
|
|
0.14839965747423775,
|
|
0.15718449806096638
|
|
],
|
|
"val_acc": [
|
|
0.9026041666666667,
|
|
0.9291666666666667,
|
|
0.9338541666666667,
|
|
0.9479166666666666,
|
|
0.9354166666666667,
|
|
0.9375,
|
|
0.9578125,
|
|
0.9520833333333333,
|
|
0.9619791666666667,
|
|
0.953125,
|
|
0.9588541666666667,
|
|
0.9619791666666667
|
|
],
|
|
"val_auc": [
|
|
0.9742961516203704,
|
|
0.9765386284722222,
|
|
0.9757588252314815,
|
|
0.9829159432870371,
|
|
0.9680707465277778,
|
|
0.9768171296296297,
|
|
0.9845500578703703,
|
|
0.9693764467592592,
|
|
0.9835706018518517,
|
|
0.9781163194444444,
|
|
0.9822894965277779,
|
|
0.9805707465277778
|
|
],
|
|
"val_f1": [
|
|
0.9319752637322662,
|
|
0.9519094766619519,
|
|
0.9567585972080354,
|
|
0.9653018736988203,
|
|
0.9582772543741588,
|
|
0.9592114208021754,
|
|
0.9721170395869191,
|
|
0.9687287559483345,
|
|
0.9747492217225874,
|
|
0.9688149688149689,
|
|
0.9725599166377215,
|
|
0.9746263468891206
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9577083333333334,
|
|
"auc_roc": 0.9859638888888889,
|
|
"f1": 0.9719884089968263,
|
|
"confusion_matrix": [
|
|
[
|
|
1075,
|
|
125
|
|
],
|
|
[
|
|
78,
|
|
3522
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9858333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9858333333333333,
|
|
"pairwise_auc": 0.9886131944444444,
|
|
"pairwise_f1": 0.9433811802232854
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9616666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9616666666666667,
|
|
"pairwise_auc": 0.9818725694444445,
|
|
"pairwise_f1": 0.9310205728116175
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9875,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9875,
|
|
"pairwise_auc": 0.9874059027777777,
|
|
"pairwise_f1": 0.9442231075697212
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.8958333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1075,
|
|
125
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.10416666666666663
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9408333333333333,
|
|
"auc_roc": 0.9886131944444444,
|
|
"f1": 0.9433811802232854,
|
|
"confusion_matrix": [
|
|
[
|
|
1075,
|
|
125
|
|
],
|
|
[
|
|
17,
|
|
1183
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.92875,
|
|
"auc_roc": 0.9818725694444445,
|
|
"f1": 0.9310205728116175,
|
|
"confusion_matrix": [
|
|
[
|
|
1075,
|
|
125
|
|
],
|
|
[
|
|
46,
|
|
1154
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9416666666666667,
|
|
"auc_roc": 0.9874059027777777,
|
|
"f1": 0.9442231075697212,
|
|
"confusion_matrix": [
|
|
[
|
|
1075,
|
|
125
|
|
],
|
|
[
|
|
15,
|
|
1185
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 17280,
|
|
"val_size": 1920,
|
|
"test_size": 4800,
|
|
"history": {
|
|
"train_loss": [
|
|
0.16163904536860407,
|
|
0.07696866190483517,
|
|
0.04328630314687164,
|
|
0.027568882121650623,
|
|
0.022392604473754074,
|
|
0.015227735451760425,
|
|
0.009244495551618255,
|
|
0.006486422266403611,
|
|
0.005088362156643429,
|
|
0.001930169196702991,
|
|
0.0005552476216090933,
|
|
0.000913119475174224,
|
|
0.0007377632181883369,
|
|
0.0005450681899282649,
|
|
9.194485542265182e-05
|
|
],
|
|
"train_acc": [
|
|
0.86875,
|
|
0.9456597222222223,
|
|
0.9723379629629629,
|
|
0.9828125,
|
|
0.987037037037037,
|
|
0.9920717592592593,
|
|
0.9951967592592592,
|
|
0.9967592592592592,
|
|
0.9972800925925925,
|
|
0.9986689814814815,
|
|
0.9997685185185186,
|
|
0.9995949074074074,
|
|
0.999537037037037,
|
|
0.999537037037037,
|
|
0.9998842592592593
|
|
],
|
|
"train_auc": [
|
|
0.9368317936957019,
|
|
0.9862764078217877,
|
|
0.9955557163065843,
|
|
0.9981067547582305,
|
|
0.9988143807870371,
|
|
0.9993334458590534,
|
|
0.9997675272205074,
|
|
0.9998790973651122,
|
|
0.9999568026263147,
|
|
0.9999900512974393,
|
|
0.9999987675754458,
|
|
0.9999979816815272,
|
|
0.9999991873142433,
|
|
0.9999984103509374,
|
|
1.0
|
|
],
|
|
"train_f1": [
|
|
0.9099285146942018,
|
|
0.9633818195998908,
|
|
0.9814728682170543,
|
|
0.9885093047549038,
|
|
0.9913473423980222,
|
|
0.9947122621482882,
|
|
0.9967967272586932,
|
|
0.9978390059427337,
|
|
0.9981866584358965,
|
|
0.9991125516070533,
|
|
0.9998456671039432,
|
|
0.9997299278521548,
|
|
0.9996913580246913,
|
|
0.9996913103874054,
|
|
0.9999228335519716
|
|
],
|
|
"val_loss": [
|
|
0.10402235621586442,
|
|
0.16845849981740077,
|
|
0.14206883508644144,
|
|
0.1307694342181397,
|
|
0.1493610362069982,
|
|
0.14732107365877406,
|
|
0.18827487041423108,
|
|
0.16436479951992927,
|
|
0.1362784474820728,
|
|
0.14761912005728844,
|
|
0.16536648601192308,
|
|
0.18152935248511615,
|
|
0.16788835483592568,
|
|
0.16435947424007888,
|
|
0.16416529594246942
|
|
],
|
|
"val_acc": [
|
|
0.9135416666666667,
|
|
0.9276041666666667,
|
|
0.940625,
|
|
0.9302083333333333,
|
|
0.9385416666666667,
|
|
0.9505208333333334,
|
|
0.9526041666666667,
|
|
0.9505208333333334,
|
|
0.959375,
|
|
0.9588541666666667,
|
|
0.9598958333333333,
|
|
0.9578125,
|
|
0.959375,
|
|
0.9604166666666667,
|
|
0.959375
|
|
],
|
|
"val_auc": [
|
|
0.974109519675926,
|
|
0.9718764467592592,
|
|
0.977509403935185,
|
|
0.9819104456018519,
|
|
0.9765017361111112,
|
|
0.9815075231481483,
|
|
0.9773169849537037,
|
|
0.9811494502314815,
|
|
0.9823842592592593,
|
|
0.9832957175925927,
|
|
0.9792534722222223,
|
|
0.9764424189814815,
|
|
0.9781090856481481,
|
|
0.9794675925925925,
|
|
0.9792035590277777
|
|
],
|
|
"val_f1": [
|
|
0.9405869720830351,
|
|
0.9523156089193825,
|
|
0.9607438016528925,
|
|
0.9522792022792023,
|
|
0.9585964912280702,
|
|
0.966933518969718,
|
|
0.9686746987951808,
|
|
0.9671621154510889,
|
|
0.9728789986091794,
|
|
0.9724065665385959,
|
|
0.9731987469544031,
|
|
0.9719237435008665,
|
|
0.9728412256267409,
|
|
0.9735191637630662,
|
|
0.9727272727272728
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9566666666666667,
|
|
"auc_roc": 0.9873546296296296,
|
|
"f1": 0.9710225689607133,
|
|
"confusion_matrix": [
|
|
[
|
|
1107,
|
|
93
|
|
],
|
|
[
|
|
115,
|
|
3485
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9741666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
31,
|
|
1169
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9741666666666666,
|
|
"pairwise_auc": 0.9900319444444444,
|
|
"pairwise_f1": 0.9496344435418359
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9383333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
74,
|
|
1126
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9383333333333334,
|
|
"pairwise_auc": 0.9798756944444444,
|
|
"pairwise_f1": 0.9309632079371641
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9916666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
10,
|
|
1190
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"detection_rate": 0.9916666666666667,
|
|
"pairwise_auc": 0.99215625,
|
|
"pairwise_f1": 0.9585179218687072
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9225,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
1107,
|
|
93
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 1200,
|
|
"false_alarm_rate": 0.07750000000000001
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9483333333333334,
|
|
"auc_roc": 0.9900319444444444,
|
|
"f1": 0.9496344435418359,
|
|
"confusion_matrix": [
|
|
[
|
|
1107,
|
|
93
|
|
],
|
|
[
|
|
31,
|
|
1169
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9304166666666667,
|
|
"auc_roc": 0.9798756944444444,
|
|
"f1": 0.9309632079371641,
|
|
"confusion_matrix": [
|
|
[
|
|
1107,
|
|
93
|
|
],
|
|
[
|
|
74,
|
|
1126
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 2400,
|
|
"accuracy": 0.9570833333333333,
|
|
"auc_roc": 0.99215625,
|
|
"f1": 0.9585179218687072,
|
|
"confusion_matrix": [
|
|
[
|
|
1107,
|
|
93
|
|
],
|
|
[
|
|
10,
|
|
1190
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9532083333333334,
|
|
"std": 0.004017107513843505,
|
|
"ci_95": 0.003521149985760667,
|
|
"values": [
|
|
0.95125,
|
|
0.9525,
|
|
0.9479166666666666,
|
|
0.9577083333333334,
|
|
0.9566666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9857046296296297,
|
|
"std": 0.001125421434076174,
|
|
"ci_95": 0.0009864753813324124,
|
|
"values": [
|
|
0.9858875,
|
|
0.9844932870370371,
|
|
0.9848238425925926,
|
|
0.9859638888888889,
|
|
0.9873546296296296
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9687705256330055,
|
|
"std": 0.002767031850003383,
|
|
"ci_95": 0.0024254103545057097,
|
|
"values": [
|
|
0.967518045530261,
|
|
0.9682008368200837,
|
|
0.9651227678571429,
|
|
0.9719884089968263,
|
|
0.9710225689607133
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9733333333333334,
|
|
"std": 0.009824855102126356,
|
|
"ci_95": 0.008611865199956546,
|
|
"values": [
|
|
0.9775,
|
|
0.97,
|
|
0.9591666666666666,
|
|
0.9858333333333333,
|
|
0.9741666666666666
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9733333333333334,
|
|
"std": 0.009824855102126356,
|
|
"ci_95": 0.008611865199956546,
|
|
"values": [
|
|
0.9775,
|
|
0.97,
|
|
0.9591666666666666,
|
|
0.9858333333333333,
|
|
0.9741666666666666
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9879413888888889,
|
|
"std": 0.002249273534556431,
|
|
"ci_95": 0.0019715751811176852,
|
|
"values": [
|
|
0.9896520833333333,
|
|
0.9868194444444445,
|
|
0.9845902777777779,
|
|
0.9886131944444444,
|
|
0.9900319444444444
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9430778591412752,
|
|
"std": 0.005091182439075068,
|
|
"ci_95": 0.004462618167693012,
|
|
"values": [
|
|
0.9414125200642055,
|
|
0.9451887941534713,
|
|
0.9357723577235773,
|
|
0.9433811802232854,
|
|
0.9496344435418359
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9443333333333334,
|
|
"std": 0.00977880360780397,
|
|
"ci_95": 0.008571499285422588,
|
|
"values": [
|
|
0.9408333333333333,
|
|
0.9416666666666667,
|
|
0.9391666666666667,
|
|
0.9616666666666667,
|
|
0.9383333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9443333333333334,
|
|
"std": 0.00977880360780397,
|
|
"ci_95": 0.008571499285422588,
|
|
"values": [
|
|
0.9408333333333333,
|
|
0.9416666666666667,
|
|
0.9391666666666667,
|
|
0.9616666666666667,
|
|
0.9383333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.979195138888889,
|
|
"std": 0.0017282862376782429,
|
|
"ci_95": 0.0015149096807141563,
|
|
"values": [
|
|
0.9775805555555556,
|
|
0.9780156249999999,
|
|
0.9786312500000001,
|
|
0.9818725694444445,
|
|
0.9798756944444444
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9280161601603254,
|
|
"std": 0.003957866453131809,
|
|
"ci_95": 0.003469222906546934,
|
|
"values": [
|
|
0.9223856209150327,
|
|
0.9304240428159737,
|
|
0.9252873563218391,
|
|
0.9310205728116175,
|
|
0.9309632079371641
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.986,
|
|
"std": 0.0040138648595974494,
|
|
"ci_95": 0.0035183076739945577,
|
|
"values": [
|
|
0.9858333333333333,
|
|
0.9808333333333333,
|
|
0.9841666666666666,
|
|
0.9875,
|
|
0.9916666666666667
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.986,
|
|
"std": 0.0040138648595974494,
|
|
"ci_95": 0.0035183076739945577,
|
|
"values": [
|
|
0.9858333333333333,
|
|
0.9808333333333333,
|
|
0.9841666666666666,
|
|
0.9875,
|
|
0.9916666666666667
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9899773611111111,
|
|
"std": 0.001933911972271006,
|
|
"ci_95": 0.0016951485839394737,
|
|
"values": [
|
|
0.9904298611111111,
|
|
0.9886447916666667,
|
|
0.9912500000000002,
|
|
0.9874059027777777,
|
|
0.99215625
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9495411742317368,
|
|
"std": 0.005620461898994915,
|
|
"ci_95": 0.004926552069471273,
|
|
"values": [
|
|
0.9456434852118305,
|
|
0.9507269789983845,
|
|
0.9485943775100402,
|
|
0.9442231075697212,
|
|
0.9585179218687072
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9091666666666667,
|
|
"std": 0.011118053386771918,
|
|
"ci_95": 0.009745403474915157,
|
|
"values": [
|
|
0.9008333333333334,
|
|
0.9175,
|
|
0.9091666666666667,
|
|
0.8958333333333334,
|
|
0.9225
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 1200.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200,
|
|
1200
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.09083333333333332,
|
|
"std": 0.011118053386771918,
|
|
"ci_95": 0.009745403474915157,
|
|
"values": [
|
|
0.09916666666666663,
|
|
0.08250000000000002,
|
|
0.09083333333333332,
|
|
0.10416666666666663,
|
|
0.07750000000000001
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9412499999999999,
|
|
"std": 0.005270462766947291,
|
|
"ci_95": 0.0046197643037521035,
|
|
"values": [
|
|
0.9391666666666667,
|
|
0.94375,
|
|
0.9341666666666667,
|
|
0.9408333333333333,
|
|
0.9483333333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9879413888888889,
|
|
"std": 0.002249273534556431,
|
|
"ci_95": 0.0019715751811176852,
|
|
"values": [
|
|
0.9896520833333333,
|
|
0.9868194444444445,
|
|
0.9845902777777779,
|
|
0.9886131944444444,
|
|
0.9900319444444444
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9430778591412752,
|
|
"std": 0.005091182439075068,
|
|
"ci_95": 0.004462618167693012,
|
|
"values": [
|
|
0.9414125200642055,
|
|
0.9451887941534713,
|
|
0.9357723577235773,
|
|
0.9433811802232854,
|
|
0.9496344435418359
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.92675,
|
|
"std": 0.004097339922979849,
|
|
"ci_95": 0.003591476793125024,
|
|
"values": [
|
|
0.9208333333333333,
|
|
0.9295833333333333,
|
|
0.9241666666666667,
|
|
0.92875,
|
|
0.9304166666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.979195138888889,
|
|
"std": 0.0017282862376782429,
|
|
"ci_95": 0.0015149096807141563,
|
|
"values": [
|
|
0.9775805555555556,
|
|
0.9780156249999999,
|
|
0.9786312500000001,
|
|
0.9818725694444445,
|
|
0.9798756944444444
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9280161601603254,
|
|
"std": 0.003957866453131809,
|
|
"ci_95": 0.003469222906546934,
|
|
"values": [
|
|
0.9223856209150327,
|
|
0.9304240428159737,
|
|
0.9252873563218391,
|
|
0.9310205728116175,
|
|
0.9309632079371641
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9475833333333334,
|
|
"std": 0.0060553007081949675,
|
|
"ci_95": 0.005307705091028813,
|
|
"values": [
|
|
0.9433333333333334,
|
|
0.9491666666666667,
|
|
0.9466666666666667,
|
|
0.9416666666666667,
|
|
0.9570833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9899773611111111,
|
|
"std": 0.001933911972271006,
|
|
"ci_95": 0.0016951485839394737,
|
|
"values": [
|
|
0.9904298611111111,
|
|
0.9886447916666667,
|
|
0.9912500000000002,
|
|
0.9874059027777777,
|
|
0.99215625
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9495411742317368,
|
|
"std": 0.005620461898994915,
|
|
"ci_95": 0.004926552069471273,
|
|
"values": [
|
|
0.9456434852118305,
|
|
0.9507269789983845,
|
|
0.9485943775100402,
|
|
0.9442231075697212,
|
|
0.9585179218687072
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"subsample": 0.2,
|
|
"augment": false,
|
|
"run_name": "p3_resnet50",
|
|
"backbone": "resnet50"
|
|
}
|
|
} |