Johnny Fernandes
1633 lines
38 KiB
JSON
1633 lines
38 KiB
JSON
{
|
|
"run_name": "p4_efficientnet_b0_100pct",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.11441802558420157,
|
|
0.06033454444750937,
|
|
0.04177368487360784,
|
|
0.0307566182437936,
|
|
0.023731412236364255,
|
|
0.017975853132001146,
|
|
0.012896745995631472,
|
|
0.009899961880528364,
|
|
0.0077350404760384165,
|
|
0.00605471685899129,
|
|
0.004333037007848159
|
|
],
|
|
"train_acc": [
|
|
0.9063657407407407,
|
|
0.9572453703703704,
|
|
0.97125,
|
|
0.9795486111111111,
|
|
0.9855324074074074,
|
|
0.9895138888888889,
|
|
0.992337962962963,
|
|
0.9942708333333333,
|
|
0.9954398148148148,
|
|
0.9966666666666667,
|
|
0.9976851851851852
|
|
],
|
|
"train_auc": [
|
|
0.968628149648491,
|
|
0.991249495598994,
|
|
0.9958075402949245,
|
|
0.9977105245484682,
|
|
0.9985693972908092,
|
|
0.9990966067243943,
|
|
0.999491364811957,
|
|
0.99969332133059,
|
|
0.9997986614797668,
|
|
0.9998129804669639,
|
|
0.999864355424097
|
|
],
|
|
"train_f1": [
|
|
0.9356864615629223,
|
|
0.9711609024904364,
|
|
0.9806869955993718,
|
|
0.9862939319428177,
|
|
0.9903265748336171,
|
|
0.9929942314532717,
|
|
0.9948851080926553,
|
|
0.9961763365441807,
|
|
0.9969581866469026,
|
|
0.9977764395237875,
|
|
0.9984561230161181
|
|
],
|
|
"val_loss": [
|
|
0.059839763751466914,
|
|
0.05016021775747504,
|
|
0.04833069565179358,
|
|
0.051100248010219125,
|
|
0.06635792484282622,
|
|
0.06013715935871308,
|
|
0.07639200245896972,
|
|
0.10937643955203688,
|
|
0.0767469448736271,
|
|
0.10151498611954897,
|
|
0.09388219699557643
|
|
],
|
|
"val_acc": [
|
|
0.9472916666666666,
|
|
0.9682291666666667,
|
|
0.9704166666666667,
|
|
0.9719791666666666,
|
|
0.975,
|
|
0.9763541666666666,
|
|
0.9776041666666667,
|
|
0.9770833333333333,
|
|
0.9776041666666667,
|
|
0.9786458333333333,
|
|
0.9796875
|
|
],
|
|
"val_auc": [
|
|
0.9911581886574075,
|
|
0.9946860532407407,
|
|
0.9949300925925926,
|
|
0.9950583043981482,
|
|
0.9940996817129629,
|
|
0.9956887442129629,
|
|
0.9928369502314816,
|
|
0.9887077835648148,
|
|
0.9924605613425925,
|
|
0.9900258101851851,
|
|
0.9897289641203704
|
|
],
|
|
"val_f1": [
|
|
0.9641083841679671,
|
|
0.9787323059758734,
|
|
0.9801481895708094,
|
|
0.981174329904122,
|
|
0.9833472106577852,
|
|
0.9842130885318868,
|
|
0.9850704812165821,
|
|
0.9848171152518979,
|
|
0.9850684075283006,
|
|
0.9858356940509915,
|
|
0.9864836764400083
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9733333333333334,
|
|
"auc_roc": 0.9953927037037038,
|
|
"f1": 0.9821488340957268,
|
|
"confusion_matrix": [
|
|
[
|
|
5754,
|
|
246
|
|
],
|
|
[
|
|
394,
|
|
17606
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.983,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.983,
|
|
"pairwise_auc": 0.9967189027777777,
|
|
"pairwise_f1": 0.9713438735177866
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9591666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
245,
|
|
5755
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9591666666666666,
|
|
"pairwise_auc": 0.9916036111111111,
|
|
"pairwise_f1": 0.9590867427714357
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9921666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
47,
|
|
5953
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9921666666666666,
|
|
"pairwise_auc": 0.9978555972222223,
|
|
"pairwise_f1": 0.9759816378391671
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.959,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5754,
|
|
246
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.041000000000000036
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.971,
|
|
"auc_roc": 0.9967189027777777,
|
|
"f1": 0.9713438735177866,
|
|
"confusion_matrix": [
|
|
[
|
|
5754,
|
|
246
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9590833333333333,
|
|
"auc_roc": 0.9916036111111111,
|
|
"f1": 0.9590867427714357,
|
|
"confusion_matrix": [
|
|
[
|
|
5754,
|
|
246
|
|
],
|
|
[
|
|
245,
|
|
5755
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9755833333333334,
|
|
"auc_roc": 0.9978555972222223,
|
|
"f1": 0.9759816378391671,
|
|
"confusion_matrix": [
|
|
[
|
|
5754,
|
|
246
|
|
],
|
|
[
|
|
47,
|
|
5953
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.11554015011923319,
|
|
0.06046664769002409,
|
|
0.0417201528933408,
|
|
0.0306289203697061,
|
|
0.023736618425165133,
|
|
0.01802243662213119,
|
|
0.01314274858594196
|
|
],
|
|
"train_acc": [
|
|
0.9062962962962963,
|
|
0.9564699074074074,
|
|
0.9707060185185186,
|
|
0.9790393518518519,
|
|
0.9846296296296296,
|
|
0.989386574074074,
|
|
0.9920023148148148
|
|
],
|
|
"train_auc": [
|
|
0.9679964402577732,
|
|
0.9912761741969593,
|
|
0.9958388985339506,
|
|
0.9977882026606081,
|
|
0.9986493505658436,
|
|
0.9992530164037495,
|
|
0.9995165905778465
|
|
],
|
|
"train_f1": [
|
|
0.9356592227608679,
|
|
0.9706265961684148,
|
|
0.9803168282952398,
|
|
0.9859524197364237,
|
|
0.9897172236503856,
|
|
0.9929088982887014,
|
|
0.9946599278201532
|
|
],
|
|
"val_loss": [
|
|
0.05235111558887487,
|
|
0.04760708744429091,
|
|
0.056180445348445573,
|
|
0.049555566390469416,
|
|
0.06840904980878197,
|
|
0.07192396782734553,
|
|
0.06875771623204552
|
|
],
|
|
"val_acc": [
|
|
0.953125,
|
|
0.970625,
|
|
0.969375,
|
|
0.9720833333333333,
|
|
0.9722916666666667,
|
|
0.9741666666666666,
|
|
0.9747916666666666
|
|
],
|
|
"val_auc": [
|
|
0.9930218460648148,
|
|
0.9951485821759259,
|
|
0.994602199074074,
|
|
0.9948726273148149,
|
|
0.9927291666666667,
|
|
0.9934754629629631,
|
|
0.9935115162037038
|
|
],
|
|
"val_f1": [
|
|
0.9680397727272727,
|
|
0.9803703188082974,
|
|
0.9795064826432455,
|
|
0.9812901424183189,
|
|
0.9814634146341463,
|
|
0.9827634139560745,
|
|
0.9831851028349082
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9672916666666667,
|
|
"auc_roc": 0.993924375,
|
|
"f1": 0.9781379675272232,
|
|
"confusion_matrix": [
|
|
[
|
|
5654,
|
|
346
|
|
],
|
|
[
|
|
439,
|
|
17561
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9788333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
127,
|
|
5873
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9788333333333333,
|
|
"pairwise_auc": 0.9947634444444445,
|
|
"pairwise_f1": 0.9612897945822081
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9553333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
268,
|
|
5732
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9553333333333334,
|
|
"pairwise_auc": 0.9895287361111111,
|
|
"pairwise_f1": 0.9491637688359
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9926666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
44,
|
|
5956
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9926666666666667,
|
|
"pairwise_auc": 0.9974809444444445,
|
|
"pairwise_f1": 0.9682978377499594
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9423333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5654,
|
|
346
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.057666666666666644
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9605833333333333,
|
|
"auc_roc": 0.9947634444444445,
|
|
"f1": 0.9612897945822081,
|
|
"confusion_matrix": [
|
|
[
|
|
5654,
|
|
346
|
|
],
|
|
[
|
|
127,
|
|
5873
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9488333333333333,
|
|
"auc_roc": 0.9895287361111111,
|
|
"f1": 0.9491637688359,
|
|
"confusion_matrix": [
|
|
[
|
|
5654,
|
|
346
|
|
],
|
|
[
|
|
268,
|
|
5732
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9675,
|
|
"auc_roc": 0.9974809444444445,
|
|
"f1": 0.9682978377499594,
|
|
"confusion_matrix": [
|
|
[
|
|
5654,
|
|
346
|
|
],
|
|
[
|
|
44,
|
|
5956
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.11380811969111501,
|
|
0.06108702142188257,
|
|
0.041107767606843953,
|
|
0.031056303198360925,
|
|
0.022846656342156133,
|
|
0.017992572683967107,
|
|
0.014158103403110292,
|
|
0.009887270464245085
|
|
],
|
|
"train_acc": [
|
|
0.9060763888888889,
|
|
0.9559027777777778,
|
|
0.9712731481481481,
|
|
0.9798611111111111,
|
|
0.9853240740740741,
|
|
0.9891898148148148,
|
|
0.9921875,
|
|
0.9942129629629629
|
|
],
|
|
"train_auc": [
|
|
0.9688643836448332,
|
|
0.9911421860711019,
|
|
0.9958939507601738,
|
|
0.9976399630629859,
|
|
0.9987717103195016,
|
|
0.9992029688928897,
|
|
0.9993913962477138,
|
|
0.9995723929755373
|
|
],
|
|
"train_f1": [
|
|
0.9354523110697497,
|
|
0.9702515733091807,
|
|
0.9807040457754144,
|
|
0.9865055606396674,
|
|
0.990185910434823,
|
|
0.99277660051662,
|
|
0.9947843825095233,
|
|
0.9961376838462489
|
|
],
|
|
"val_loss": [
|
|
0.05795087730065764,
|
|
0.056404504951203006,
|
|
0.05484667886617899,
|
|
0.06420475673321865,
|
|
0.06930285179106249,
|
|
0.0719728484604002,
|
|
0.08376166993517169,
|
|
0.08409365626914829
|
|
],
|
|
"val_acc": [
|
|
0.9542708333333333,
|
|
0.9627083333333334,
|
|
0.9654166666666667,
|
|
0.9686458333333333,
|
|
0.9698958333333333,
|
|
0.9723958333333333,
|
|
0.9736458333333333,
|
|
0.9760416666666667
|
|
],
|
|
"val_auc": [
|
|
0.9914726273148148,
|
|
0.9927101273148148,
|
|
0.9939305844907409,
|
|
0.9931634548611111,
|
|
0.9926912037037037,
|
|
0.9927416956018519,
|
|
0.9906892650462963,
|
|
0.9920653067129628
|
|
],
|
|
"val_f1": [
|
|
0.9689774574235036,
|
|
0.9748701389863822,
|
|
0.9766788423714526,
|
|
0.9789672280064287,
|
|
0.9798282962239129,
|
|
0.9815163562809514,
|
|
0.9823533514682291,
|
|
0.9839966601725577
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.966875,
|
|
"auc_roc": 0.9954085138888888,
|
|
"f1": 0.977694228556999,
|
|
"confusion_matrix": [
|
|
[
|
|
5782,
|
|
218
|
|
],
|
|
[
|
|
577,
|
|
17423
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9715,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
171,
|
|
5829
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9715,
|
|
"pairwise_auc": 0.9961655416666666,
|
|
"pairwise_f1": 0.9677098032705238
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9463333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
322,
|
|
5678
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9463333333333334,
|
|
"pairwise_auc": 0.9924056111111111,
|
|
"pairwise_f1": 0.9546065904505716
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.986,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
84,
|
|
5916
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.986,
|
|
"pairwise_auc": 0.9976543888888889,
|
|
"pairwise_f1": 0.9751112576232075
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9636666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5782,
|
|
218
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03633333333333333
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9675833333333334,
|
|
"auc_roc": 0.9961655416666666,
|
|
"f1": 0.9677098032705238,
|
|
"confusion_matrix": [
|
|
[
|
|
5782,
|
|
218
|
|
],
|
|
[
|
|
171,
|
|
5829
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.955,
|
|
"auc_roc": 0.9924056111111111,
|
|
"f1": 0.9546065904505716,
|
|
"confusion_matrix": [
|
|
[
|
|
5782,
|
|
218
|
|
],
|
|
[
|
|
322,
|
|
5678
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9748333333333333,
|
|
"auc_roc": 0.9976543888888889,
|
|
"f1": 0.9751112576232075,
|
|
"confusion_matrix": [
|
|
[
|
|
5782,
|
|
218
|
|
],
|
|
[
|
|
84,
|
|
5916
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.11462977295637959,
|
|
0.06104186699932648,
|
|
0.043467226039544955,
|
|
0.03143523881621217,
|
|
0.02269427054027161,
|
|
0.0180365542558849,
|
|
0.01319043671658299,
|
|
0.010663869875008386,
|
|
0.00850035139169142
|
|
],
|
|
"train_acc": [
|
|
0.9067939814814815,
|
|
0.9572800925925926,
|
|
0.970949074074074,
|
|
0.9799537037037037,
|
|
0.9851851851851852,
|
|
0.989386574074074,
|
|
0.9926041666666666,
|
|
0.9940740740740741,
|
|
0.9956597222222222
|
|
],
|
|
"train_auc": [
|
|
0.9685010709590763,
|
|
0.9910457429555326,
|
|
0.995405782035894,
|
|
0.9975748667552584,
|
|
0.9987710905349794,
|
|
0.999104601766118,
|
|
0.9994363629543896,
|
|
0.9996778803012117,
|
|
0.9997134123513944
|
|
],
|
|
"train_f1": [
|
|
0.9360796920268286,
|
|
0.9711795984976849,
|
|
0.9804918236647392,
|
|
0.9865771812080537,
|
|
0.9900890437475803,
|
|
0.9929105429580894,
|
|
0.9950619382858202,
|
|
0.9960457824253565,
|
|
0.9971044036229701
|
|
],
|
|
"val_loss": [
|
|
0.056692553203126106,
|
|
0.04689954964708401,
|
|
0.049250886738451904,
|
|
0.04974133472356395,
|
|
0.06202284398801263,
|
|
0.06754068279969336,
|
|
0.09429209648697906,
|
|
0.08125589334886778,
|
|
0.09298159147614872
|
|
],
|
|
"val_acc": [
|
|
0.958125,
|
|
0.96,
|
|
0.965,
|
|
0.9711458333333334,
|
|
0.9703125,
|
|
0.9709375,
|
|
0.9744791666666667,
|
|
0.9746875,
|
|
0.9744791666666667
|
|
],
|
|
"val_auc": [
|
|
0.9918540798611112,
|
|
0.9947723958333332,
|
|
0.994551707175926,
|
|
0.9949561631944445,
|
|
0.9928728298611111,
|
|
0.9929115162037037,
|
|
0.9903106770833332,
|
|
0.9917626736111111,
|
|
0.9914118055555555
|
|
],
|
|
"val_f1": [
|
|
0.9716981132075472,
|
|
0.9728583545377438,
|
|
0.9764111204717776,
|
|
0.980600882414735,
|
|
0.980037823072074,
|
|
0.9804936027406838,
|
|
0.9829517778860205,
|
|
0.9830555749250401,
|
|
0.9828970331588133
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9725833333333334,
|
|
"auc_roc": 0.9958473518518518,
|
|
"f1": 0.9815861644372307,
|
|
"confusion_matrix": [
|
|
[
|
|
5804,
|
|
196
|
|
],
|
|
[
|
|
462,
|
|
17538
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.979,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
126,
|
|
5874
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.979,
|
|
"pairwise_auc": 0.9963541805555556,
|
|
"pairwise_f1": 0.9733222866611433
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9515,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
291,
|
|
5709
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9515,
|
|
"pairwise_auc": 0.992698638888889,
|
|
"pairwise_f1": 0.9590928181436371
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9925,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
45,
|
|
5955
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9925,
|
|
"pairwise_auc": 0.998489236111111,
|
|
"pairwise_f1": 0.9801662414616081
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9673333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5804,
|
|
196
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03266666666666662
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9731666666666666,
|
|
"auc_roc": 0.9963541805555556,
|
|
"f1": 0.9733222866611433,
|
|
"confusion_matrix": [
|
|
[
|
|
5804,
|
|
196
|
|
],
|
|
[
|
|
126,
|
|
5874
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9594166666666667,
|
|
"auc_roc": 0.992698638888889,
|
|
"f1": 0.9590928181436371,
|
|
"confusion_matrix": [
|
|
[
|
|
5804,
|
|
196
|
|
],
|
|
[
|
|
291,
|
|
5709
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9799166666666667,
|
|
"auc_roc": 0.998489236111111,
|
|
"f1": 0.9801662414616081,
|
|
"confusion_matrix": [
|
|
[
|
|
5804,
|
|
196
|
|
],
|
|
[
|
|
45,
|
|
5955
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.11263081131520232,
|
|
0.05991070367232034,
|
|
0.04136536123918766,
|
|
0.029774578257430996,
|
|
0.022208618058484043,
|
|
0.017700655086389277,
|
|
0.012931080159417484,
|
|
0.011068318802444655
|
|
],
|
|
"train_acc": [
|
|
0.9072800925925926,
|
|
0.9574768518518518,
|
|
0.9718981481481481,
|
|
0.9805439814814815,
|
|
0.9861921296296297,
|
|
0.9891435185185186,
|
|
0.9925462962962963,
|
|
0.9941550925925926
|
|
],
|
|
"train_auc": [
|
|
0.969474386645519,
|
|
0.991394128657979,
|
|
0.9958527045467535,
|
|
0.99777462205647,
|
|
0.9986964173954047,
|
|
0.9991970575417238,
|
|
0.9994723847593736,
|
|
0.9996280460533837
|
|
],
|
|
"train_f1": [
|
|
0.9363251226045417,
|
|
0.97131480324797,
|
|
0.9811303158418304,
|
|
0.9869680830445535,
|
|
0.9907640378109299,
|
|
0.9927455529775715,
|
|
0.9950236454115537,
|
|
0.9960996331338097
|
|
],
|
|
"val_loss": [
|
|
0.06112454482315419,
|
|
0.051939580755036636,
|
|
0.048177479772827304,
|
|
0.05872072634245039,
|
|
0.06941870198988606,
|
|
0.06735328267960237,
|
|
0.08199378193316079,
|
|
0.08169305338272655
|
|
],
|
|
"val_acc": [
|
|
0.9435416666666666,
|
|
0.9580208333333333,
|
|
0.9616666666666667,
|
|
0.965,
|
|
0.9617708333333334,
|
|
0.9692708333333333,
|
|
0.9734375,
|
|
0.9723958333333333
|
|
],
|
|
"val_auc": [
|
|
0.9909473958333334,
|
|
0.9935502604166667,
|
|
0.9945381365740741,
|
|
0.9938249421296297,
|
|
0.9929071469907408,
|
|
0.9938892650462963,
|
|
0.9912416666666668,
|
|
0.9911994502314816
|
|
],
|
|
"val_f1": [
|
|
0.9613464555698189,
|
|
0.9715495940698906,
|
|
0.9740223069320909,
|
|
0.9763845937587855,
|
|
0.9741749349095771,
|
|
0.9793605261316728,
|
|
0.982278129126416,
|
|
0.9815189343747821
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9616666666666667,
|
|
"auc_roc": 0.9941627175925927,
|
|
"f1": 0.9740464906341684,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
736,
|
|
17264
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9653333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
208,
|
|
5792
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9653333333333334,
|
|
"pairwise_auc": 0.994936375,
|
|
"pairwise_f1": 0.9672678690714763
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9265,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
441,
|
|
5559
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9265,
|
|
"pairwise_auc": 0.9897363333333332,
|
|
"pairwise_f1": 0.9467768032019075
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9855,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
87,
|
|
5913
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9855,
|
|
"pairwise_auc": 0.9978154444444445,
|
|
"pairwise_f1": 0.9775977515086385
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9693333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03066666666666662
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9673333333333334,
|
|
"auc_roc": 0.994936375,
|
|
"f1": 0.9672678690714763,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
208,
|
|
5792
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9479166666666666,
|
|
"auc_roc": 0.9897363333333332,
|
|
"f1": 0.9467768032019075,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
441,
|
|
5559
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9774166666666667,
|
|
"auc_roc": 0.9978154444444445,
|
|
"f1": 0.9775977515086385,
|
|
"confusion_matrix": [
|
|
[
|
|
5816,
|
|
184
|
|
],
|
|
[
|
|
87,
|
|
5913
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.96835,
|
|
"std": 0.004762283095089782,
|
|
"ci_95": 0.004174325181657788,
|
|
"values": [
|
|
0.9733333333333334,
|
|
0.9672916666666667,
|
|
0.966875,
|
|
0.9725833333333334,
|
|
0.9616666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9949471324074075,
|
|
"std": 0.0008489907266573434,
|
|
"ci_95": 0.000744173183012523,
|
|
"values": [
|
|
0.9953927037037038,
|
|
0.993924375,
|
|
0.9954085138888888,
|
|
0.9958473518518518,
|
|
0.9941627175925927
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9787227370502695,
|
|
"std": 0.003286503576188645,
|
|
"ci_95": 0.0028807473986243557,
|
|
"values": [
|
|
0.9821488340957268,
|
|
0.9781379675272232,
|
|
0.977694228556999,
|
|
0.9815861644372307,
|
|
0.9740464906341684
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9755333333333335,
|
|
"std": 0.007052777230755718,
|
|
"ci_95": 0.006182031812707942,
|
|
"values": [
|
|
0.983,
|
|
0.9788333333333333,
|
|
0.9715,
|
|
0.979,
|
|
0.9653333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9755333333333335,
|
|
"std": 0.007052777230755718,
|
|
"ci_95": 0.006182031812707942,
|
|
"values": [
|
|
0.983,
|
|
0.9788333333333333,
|
|
0.9715,
|
|
0.979,
|
|
0.9653333333333334
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9957876888888888,
|
|
"std": 0.0008810018241080616,
|
|
"ci_95": 0.00077223214706672,
|
|
"values": [
|
|
0.9967189027777777,
|
|
0.9947634444444445,
|
|
0.9961655416666666,
|
|
0.9963541805555556,
|
|
0.994936375
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9681867254206276,
|
|
"std": 0.004609256449799118,
|
|
"ci_95": 0.004040191413012229,
|
|
"values": [
|
|
0.9713438735177866,
|
|
0.9612897945822081,
|
|
0.9677098032705238,
|
|
0.9733222866611433,
|
|
0.9672678690714763
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9477666666666666,
|
|
"std": 0.012801150121939997,
|
|
"ci_95": 0.011220702810232321,
|
|
"values": [
|
|
0.9591666666666666,
|
|
0.9553333333333334,
|
|
0.9463333333333334,
|
|
0.9515,
|
|
0.9265
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9477666666666666,
|
|
"std": 0.012801150121939997,
|
|
"ci_95": 0.011220702810232321,
|
|
"values": [
|
|
0.9591666666666666,
|
|
0.9553333333333334,
|
|
0.9463333333333334,
|
|
0.9515,
|
|
0.9265
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9911945861111111,
|
|
"std": 0.0014830387002695018,
|
|
"ci_95": 0.0012999407360496922,
|
|
"values": [
|
|
0.9916036111111111,
|
|
0.9895287361111111,
|
|
0.9924056111111111,
|
|
0.992698638888889,
|
|
0.9897363333333332
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9537453446806904,
|
|
"std": 0.0056440047359999496,
|
|
"ci_95": 0.004947188275970443,
|
|
"values": [
|
|
0.9590867427714357,
|
|
0.9491637688359,
|
|
0.9546065904505716,
|
|
0.9590928181436371,
|
|
0.9467768032019075
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9897666666666666,
|
|
"std": 0.003675368462254274,
|
|
"ci_95": 0.003221602499792095,
|
|
"values": [
|
|
0.9921666666666666,
|
|
0.9926666666666667,
|
|
0.986,
|
|
0.9925,
|
|
0.9855
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9897666666666666,
|
|
"std": 0.003675368462254274,
|
|
"ci_95": 0.003221602499792095,
|
|
"values": [
|
|
0.9921666666666666,
|
|
0.9926666666666667,
|
|
0.986,
|
|
0.9925,
|
|
0.9855
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9978591222222221,
|
|
"std": 0.0003820660467281081,
|
|
"ci_95": 0.0003348956557324351,
|
|
"values": [
|
|
0.9978555972222223,
|
|
0.9974809444444445,
|
|
0.9976543888888889,
|
|
0.998489236111111,
|
|
0.9978154444444445
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9754309452365162,
|
|
"std": 0.004427323251349764,
|
|
"ci_95": 0.0038807199333663164,
|
|
"values": [
|
|
0.9759816378391671,
|
|
0.9682978377499594,
|
|
0.9751112576232075,
|
|
0.9801662414616081,
|
|
0.9775977515086385
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9603333333333334,
|
|
"std": 0.010803805913761243,
|
|
"ci_95": 0.009469953420042672,
|
|
"values": [
|
|
0.959,
|
|
0.9423333333333334,
|
|
0.9636666666666667,
|
|
0.9673333333333334,
|
|
0.9693333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.03966666666666665,
|
|
"std": 0.010803805913761243,
|
|
"ci_95": 0.009469953420042672,
|
|
"values": [
|
|
0.041000000000000036,
|
|
0.057666666666666644,
|
|
0.03633333333333333,
|
|
0.03266666666666662,
|
|
0.03066666666666662
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9679333333333332,
|
|
"std": 0.004777478995825659,
|
|
"ci_95": 0.0041876449759314935,
|
|
"values": [
|
|
0.971,
|
|
0.9605833333333333,
|
|
0.9675833333333334,
|
|
0.9731666666666666,
|
|
0.9673333333333334
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9957876888888888,
|
|
"std": 0.0008810018241080616,
|
|
"ci_95": 0.00077223214706672,
|
|
"values": [
|
|
0.9967189027777777,
|
|
0.9947634444444445,
|
|
0.9961655416666666,
|
|
0.9963541805555556,
|
|
0.994936375
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9681867254206276,
|
|
"std": 0.004609256449799118,
|
|
"ci_95": 0.004040191413012229,
|
|
"values": [
|
|
0.9713438735177866,
|
|
0.9612897945822081,
|
|
0.9677098032705238,
|
|
0.9733222866611433,
|
|
0.9672678690714763
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.95405,
|
|
"std": 0.005474245254928869,
|
|
"ci_95": 0.004798387530086433,
|
|
"values": [
|
|
0.9590833333333333,
|
|
0.9488333333333333,
|
|
0.955,
|
|
0.9594166666666667,
|
|
0.9479166666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9911945861111111,
|
|
"std": 0.0014830387002695018,
|
|
"ci_95": 0.0012999407360496922,
|
|
"values": [
|
|
0.9916036111111111,
|
|
0.9895287361111111,
|
|
0.9924056111111111,
|
|
0.992698638888889,
|
|
0.9897363333333332
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9537453446806904,
|
|
"std": 0.0056440047359999496,
|
|
"ci_95": 0.004947188275970443,
|
|
"values": [
|
|
0.9590867427714357,
|
|
0.9491637688359,
|
|
0.9546065904505716,
|
|
0.9590928181436371,
|
|
0.9467768032019075
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9750500000000001,
|
|
"std": 0.0046534783883790684,
|
|
"ci_95": 0.004078953651230771,
|
|
"values": [
|
|
0.9755833333333334,
|
|
0.9675,
|
|
0.9748333333333333,
|
|
0.9799166666666667,
|
|
0.9774166666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9978591222222221,
|
|
"std": 0.0003820660467281081,
|
|
"ci_95": 0.0003348956557324351,
|
|
"values": [
|
|
0.9978555972222223,
|
|
0.9974809444444445,
|
|
0.9976543888888889,
|
|
0.998489236111111,
|
|
0.9978154444444445
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9754309452365162,
|
|
"std": 0.004427323251349764,
|
|
"ci_95": 0.0038807199333663164,
|
|
"values": [
|
|
0.9759816378391671,
|
|
0.9682978377499594,
|
|
0.9751112576232075,
|
|
0.9801662414616081,
|
|
0.9775977515086385
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"augment": false,
|
|
"run_name": "p4_efficientnet_b0_100pct",
|
|
"backbone": "efficientnet_b0",
|
|
"subsample": 1.0
|
|
}
|
|
} |