Johnny Fernandes
1641 lines
38 KiB
JSON
1641 lines
38 KiB
JSON
{
|
|
"run_name": "p4_resnet50_100pct",
|
|
"n_folds": 5,
|
|
"fold_results": [
|
|
{
|
|
"fold": 0,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10569497120687393,
|
|
0.052549266521279975,
|
|
0.034437011861171214,
|
|
0.023376175384457586,
|
|
0.016215483456127104,
|
|
0.011260907598035306,
|
|
0.008502760997301453,
|
|
0.005292619497228762,
|
|
0.004057022843460307
|
|
],
|
|
"train_acc": [
|
|
0.9206712962962963,
|
|
0.9635416666666666,
|
|
0.9775462962962963,
|
|
0.9856365740740741,
|
|
0.9902546296296296,
|
|
0.9935995370370371,
|
|
0.9951736111111111,
|
|
0.9971180555555555,
|
|
0.9980787037037037
|
|
],
|
|
"train_auc": [
|
|
0.9739272026463193,
|
|
0.9935739497599451,
|
|
0.9972107324531321,
|
|
0.9986884848679698,
|
|
0.9993530046153407,
|
|
0.9996074531321445,
|
|
0.9998320516117969,
|
|
0.999912604666781,
|
|
0.9999362700760174
|
|
],
|
|
"train_f1": [
|
|
0.9461061835566459,
|
|
0.9754921030109702,
|
|
0.9849603076159762,
|
|
0.9904006064403345,
|
|
0.9934912340372901,
|
|
0.9957289051940529,
|
|
0.9967798979142696,
|
|
0.9980778583173154,
|
|
0.9987188787874111
|
|
],
|
|
"val_loss": [
|
|
0.05847585745970719,
|
|
0.10475364394138524,
|
|
0.09643271176709883,
|
|
0.06068119355442832,
|
|
0.07942604735202925,
|
|
0.07901297045857367,
|
|
0.09435704887227075,
|
|
0.09795430390264907,
|
|
0.08238918158974229
|
|
],
|
|
"val_acc": [
|
|
0.9469791666666667,
|
|
0.9585416666666666,
|
|
0.96125,
|
|
0.9691666666666666,
|
|
0.9672916666666667,
|
|
0.9666666666666667,
|
|
0.9748958333333333,
|
|
0.9760416666666667,
|
|
0.9727083333333333
|
|
],
|
|
"val_auc": [
|
|
0.9921601273148147,
|
|
0.9881940393518518,
|
|
0.9929584490740742,
|
|
0.9936334201388888,
|
|
0.9921611689814815,
|
|
0.9924208622685184,
|
|
0.9914331886574074,
|
|
0.990408912037037,
|
|
0.9917259837962963
|
|
],
|
|
"val_f1": [
|
|
0.9637593449626202,
|
|
0.9725403615289085,
|
|
0.9745901639344262,
|
|
0.9793900570951121,
|
|
0.978142837254629,
|
|
0.9776754569554904,
|
|
0.9833229534288285,
|
|
0.9840786376851723,
|
|
0.9817192296957856
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9714166666666667,
|
|
"auc_roc": 0.9947762777777778,
|
|
"f1": 0.9808903002952811,
|
|
"confusion_matrix": [
|
|
[
|
|
5708,
|
|
292
|
|
],
|
|
[
|
|
394,
|
|
17606
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9865,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
81,
|
|
5919
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9865,
|
|
"pairwise_auc": 0.9963499166666667,
|
|
"pairwise_f1": 0.9694537711899107
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.956,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
264,
|
|
5736
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.956,
|
|
"pairwise_auc": 0.98983675,
|
|
"pairwise_f1": 0.9537745261057532
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9918333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
49,
|
|
5951
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9918333333333333,
|
|
"pairwise_auc": 0.9981421666666667,
|
|
"pairwise_f1": 0.97214734950584
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9513333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5708,
|
|
292
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.048666666666666636
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9689166666666666,
|
|
"auc_roc": 0.9963499166666667,
|
|
"f1": 0.9694537711899107,
|
|
"confusion_matrix": [
|
|
[
|
|
5708,
|
|
292
|
|
],
|
|
[
|
|
81,
|
|
5919
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9536666666666667,
|
|
"auc_roc": 0.98983675,
|
|
"f1": 0.9537745261057532,
|
|
"confusion_matrix": [
|
|
[
|
|
5708,
|
|
292
|
|
],
|
|
[
|
|
264,
|
|
5736
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9715833333333334,
|
|
"auc_roc": 0.9981421666666667,
|
|
"f1": 0.97214734950584,
|
|
"confusion_matrix": [
|
|
[
|
|
5708,
|
|
292
|
|
],
|
|
[
|
|
49,
|
|
5951
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10650519421117173,
|
|
0.054470051364731736,
|
|
0.034688337860991146,
|
|
0.02357034895810877,
|
|
0.01648028232624138,
|
|
0.012088147137423625,
|
|
0.007825804898118292,
|
|
0.0059702038110493755,
|
|
0.0036449454067095047,
|
|
0.0025550904984324714
|
|
],
|
|
"train_acc": [
|
|
0.9174884259259259,
|
|
0.9634837962962963,
|
|
0.9772569444444444,
|
|
0.9857754629629629,
|
|
0.9901273148148149,
|
|
0.9929398148148149,
|
|
0.9958333333333333,
|
|
0.996875,
|
|
0.9981018518518519,
|
|
0.9985879629629629
|
|
],
|
|
"train_auc": [
|
|
0.9734640742884089,
|
|
0.9931289251828989,
|
|
0.9971670303212162,
|
|
0.9986235950360083,
|
|
0.9993648505372656,
|
|
0.999580423739712,
|
|
0.9997460637431412,
|
|
0.9998633455504115,
|
|
0.999925910565272,
|
|
0.9999609457161638
|
|
],
|
|
"train_f1": [
|
|
0.943818807972071,
|
|
0.9754640826833194,
|
|
0.984762835275781,
|
|
0.9904921051206473,
|
|
0.9934067632850242,
|
|
0.9952880470886311,
|
|
0.9972200772200772,
|
|
0.9979158304257881,
|
|
0.9987342553716967,
|
|
0.99905846761746
|
|
],
|
|
"val_loss": [
|
|
0.13041815453985084,
|
|
0.049168464083013536,
|
|
0.055981301156959186,
|
|
0.048011566337878925,
|
|
0.04965577525331355,
|
|
0.08917092129290116,
|
|
0.07288087250781548,
|
|
0.07718624085746038,
|
|
0.08893142094205359,
|
|
0.08420268112276669
|
|
],
|
|
"val_acc": [
|
|
0.9495833333333333,
|
|
0.9670833333333333,
|
|
0.9658333333333333,
|
|
0.9669791666666666,
|
|
0.9710416666666667,
|
|
0.9725,
|
|
0.973125,
|
|
0.9755208333333333,
|
|
0.9748958333333333,
|
|
0.9780208333333333
|
|
],
|
|
"val_auc": [
|
|
0.986224537037037,
|
|
0.994519994212963,
|
|
0.994018431712963,
|
|
0.9950894965277778,
|
|
0.995901099537037,
|
|
0.9910672453703704,
|
|
0.9939374421296296,
|
|
0.9931692708333333,
|
|
0.9922651909722223,
|
|
0.9922427083333333
|
|
],
|
|
"val_f1": [
|
|
0.9670748299319728,
|
|
0.9778556412053259,
|
|
0.9770275949012467,
|
|
0.9777246855456398,
|
|
0.9805185704274703,
|
|
0.981661572659072,
|
|
0.9820209059233449,
|
|
0.9836202690457936,
|
|
0.9832557493225874,
|
|
0.9853543416394808
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.969875,
|
|
"auc_roc": 0.9955725925925926,
|
|
"f1": 0.9797337070777855,
|
|
"confusion_matrix": [
|
|
[
|
|
5801,
|
|
199
|
|
],
|
|
[
|
|
524,
|
|
17476
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.983,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.983,
|
|
"pairwise_auc": 0.9972167916666667,
|
|
"pairwise_f1": 0.9751177978011077
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9413333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
352,
|
|
5648
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9413333333333334,
|
|
"pairwise_auc": 0.9912267916666667,
|
|
"pairwise_f1": 0.953490335105934
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9883333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
70,
|
|
5930
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9883333333333333,
|
|
"pairwise_auc": 0.9982741944444444,
|
|
"pairwise_f1": 0.9778217495259296
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9668333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5801,
|
|
199
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.03316666666666668
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9749166666666667,
|
|
"auc_roc": 0.9972167916666667,
|
|
"f1": 0.9751177978011077,
|
|
"confusion_matrix": [
|
|
[
|
|
5801,
|
|
199
|
|
],
|
|
[
|
|
102,
|
|
5898
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9540833333333333,
|
|
"auc_roc": 0.9912267916666667,
|
|
"f1": 0.953490335105934,
|
|
"confusion_matrix": [
|
|
[
|
|
5801,
|
|
199
|
|
],
|
|
[
|
|
352,
|
|
5648
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9775833333333334,
|
|
"auc_roc": 0.9982741944444444,
|
|
"f1": 0.9778217495259296,
|
|
"confusion_matrix": [
|
|
[
|
|
5801,
|
|
199
|
|
],
|
|
[
|
|
70,
|
|
5930
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10433356989347549,
|
|
0.05307071479080521,
|
|
0.035350340842043,
|
|
0.02367758955415748,
|
|
0.01681257231187045,
|
|
0.011728319249519064,
|
|
0.009048324235823517,
|
|
0.005870483719392933
|
|
],
|
|
"train_acc": [
|
|
0.919537037037037,
|
|
0.9636458333333333,
|
|
0.9774189814814814,
|
|
0.9852777777777778,
|
|
0.9902893518518519,
|
|
0.9934143518518519,
|
|
0.995,
|
|
0.9966550925925926
|
|
],
|
|
"train_auc": [
|
|
0.9747061281864428,
|
|
0.9935218492798354,
|
|
0.9971303497942385,
|
|
0.998675611925583,
|
|
0.999280640932213,
|
|
0.9996691600937357,
|
|
0.9997561574788524,
|
|
0.9999053251457476
|
|
],
|
|
"train_f1": [
|
|
0.9451371571072319,
|
|
0.9755623157059387,
|
|
0.9848744447114826,
|
|
0.9901581504750704,
|
|
0.9935173809910139,
|
|
0.9956055328581027,
|
|
0.9966641441831015,
|
|
0.9977686326891451
|
|
],
|
|
"val_loss": [
|
|
0.0549071429570904,
|
|
0.05835173849273512,
|
|
0.06112585439510137,
|
|
0.06775237642389281,
|
|
0.0718517620531505,
|
|
0.07416316316131391,
|
|
0.11034522878509051,
|
|
0.08488445833814694
|
|
],
|
|
"val_acc": [
|
|
0.94625,
|
|
0.9640625,
|
|
0.9691666666666666,
|
|
0.96875,
|
|
0.9676041666666667,
|
|
0.966875,
|
|
0.9705208333333334,
|
|
0.9720833333333333
|
|
],
|
|
"val_auc": [
|
|
0.9929142071759258,
|
|
0.9932123842592593,
|
|
0.9945350405092592,
|
|
0.9936048900462963,
|
|
0.9933492766203703,
|
|
0.9931815972222222,
|
|
0.9889990451388888,
|
|
0.9923971064814815
|
|
],
|
|
"val_f1": [
|
|
0.9631533847472151,
|
|
0.976,
|
|
0.9794871794871794,
|
|
0.9791347892613715,
|
|
0.9782593498776652,
|
|
0.9777217318200925,
|
|
0.9804355340477013,
|
|
0.981370777144446
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.969625,
|
|
"auc_roc": 0.9946256666666666,
|
|
"f1": 0.9798021777075887,
|
|
"confusion_matrix": [
|
|
[
|
|
5589,
|
|
411
|
|
],
|
|
[
|
|
318,
|
|
17682
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9901666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
59,
|
|
5941
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9901666666666666,
|
|
"pairwise_auc": 0.996534,
|
|
"pairwise_f1": 0.961949481865285
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9641666666666666,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
215,
|
|
5785
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9641666666666666,
|
|
"pairwise_auc": 0.9892951805555557,
|
|
"pairwise_f1": 0.9486716956379141
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9926666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
44,
|
|
5956
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9926666666666667,
|
|
"pairwise_auc": 0.9980478194444444,
|
|
"pairwise_f1": 0.9632085388534002
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9315,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5589,
|
|
411
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.0685
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9608333333333333,
|
|
"auc_roc": 0.996534,
|
|
"f1": 0.961949481865285,
|
|
"confusion_matrix": [
|
|
[
|
|
5589,
|
|
411
|
|
],
|
|
[
|
|
59,
|
|
5941
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9478333333333333,
|
|
"auc_roc": 0.9892951805555557,
|
|
"f1": 0.9486716956379141,
|
|
"confusion_matrix": [
|
|
[
|
|
5589,
|
|
411
|
|
],
|
|
[
|
|
215,
|
|
5785
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9620833333333333,
|
|
"auc_roc": 0.9980478194444444,
|
|
"f1": 0.9632085388534002,
|
|
"confusion_matrix": [
|
|
[
|
|
5589,
|
|
411
|
|
],
|
|
[
|
|
44,
|
|
5956
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10583155480834346,
|
|
0.05305233907124838,
|
|
0.03383288917838298,
|
|
0.023973344606176263,
|
|
0.01652335237360384,
|
|
0.011552016472293147,
|
|
0.009053840533335463
|
|
],
|
|
"train_acc": [
|
|
0.9190162037037037,
|
|
0.9642708333333333,
|
|
0.978275462962963,
|
|
0.985474537037037,
|
|
0.9903703703703703,
|
|
0.993125,
|
|
0.9952893518518519
|
|
],
|
|
"train_auc": [
|
|
0.9738950274348422,
|
|
0.9935666030807041,
|
|
0.9972492441129401,
|
|
0.9985583990626429,
|
|
0.9993623299611339,
|
|
0.9995694962419981,
|
|
0.9996786679812528
|
|
],
|
|
"train_f1": [
|
|
0.944952757082504,
|
|
0.9759929075256442,
|
|
0.9854494996085241,
|
|
0.9902885575219185,
|
|
0.9935698276528325,
|
|
0.9954107885099742,
|
|
0.9968579435356242
|
|
],
|
|
"val_loss": [
|
|
0.05906614906290391,
|
|
0.05416134513027828,
|
|
0.06403290858775411,
|
|
0.0884968720643307,
|
|
0.08107796501735418,
|
|
0.08898418609101706,
|
|
0.0962938618196616
|
|
],
|
|
"val_acc": [
|
|
0.9532291666666667,
|
|
0.9659375,
|
|
0.9672916666666667,
|
|
0.9634375,
|
|
0.9695833333333334,
|
|
0.9673958333333333,
|
|
0.9705208333333334
|
|
],
|
|
"val_auc": [
|
|
0.991356394675926,
|
|
0.9940768518518519,
|
|
0.993639525462963,
|
|
0.9907980613425926,
|
|
0.9927527777777777,
|
|
0.9902113715277779,
|
|
0.9907717881944444
|
|
],
|
|
"val_f1": [
|
|
0.9683646868174453,
|
|
0.9771918811466834,
|
|
0.9781458797327395,
|
|
0.975657119078993,
|
|
0.9797755921872836,
|
|
0.978156186754135,
|
|
0.9803894394012889
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.966625,
|
|
"auc_roc": 0.9946070370370369,
|
|
"f1": 0.9776100628930817,
|
|
"confusion_matrix": [
|
|
[
|
|
5712,
|
|
288
|
|
],
|
|
[
|
|
513,
|
|
17487
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9785,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
129,
|
|
5871
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9785,
|
|
"pairwise_auc": 0.9955885277777777,
|
|
"pairwise_f1": 0.9657044164816185
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.9506666666666667,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
296,
|
|
5704
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9506666666666667,
|
|
"pairwise_auc": 0.990835638888889,
|
|
"pairwise_f1": 0.9513008672448299
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.9853333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
88,
|
|
5912
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9853333333333333,
|
|
"pairwise_auc": 0.9973969444444445,
|
|
"pairwise_f1": 0.9691803278688524
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.952,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5712,
|
|
288
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.04800000000000004
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.96525,
|
|
"auc_roc": 0.9955885277777777,
|
|
"f1": 0.9657044164816185,
|
|
"confusion_matrix": [
|
|
[
|
|
5712,
|
|
288
|
|
],
|
|
[
|
|
129,
|
|
5871
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9513333333333334,
|
|
"auc_roc": 0.990835638888889,
|
|
"f1": 0.9513008672448299,
|
|
"confusion_matrix": [
|
|
[
|
|
5712,
|
|
288
|
|
],
|
|
[
|
|
296,
|
|
5704
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9686666666666667,
|
|
"auc_roc": 0.9973969444444445,
|
|
"f1": 0.9691803278688524,
|
|
"confusion_matrix": [
|
|
[
|
|
5712,
|
|
288
|
|
],
|
|
[
|
|
88,
|
|
5912
|
|
]
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_size": 86400,
|
|
"val_size": 9600,
|
|
"test_size": 24000,
|
|
"history": {
|
|
"train_loss": [
|
|
0.10572219112767252,
|
|
0.05240473180946773,
|
|
0.03435035509957945,
|
|
0.02263651304385281,
|
|
0.016451685824715106,
|
|
0.011485795361557022,
|
|
0.008304328180988873,
|
|
0.0065383565808591225,
|
|
0.0034714621708454214,
|
|
0.0020390315323359815
|
|
],
|
|
"train_acc": [
|
|
0.9187152777777777,
|
|
0.9649305555555555,
|
|
0.9779976851851852,
|
|
0.9859259259259259,
|
|
0.9903472222222223,
|
|
0.9935300925925926,
|
|
0.9953009259259259,
|
|
0.9967476851851852,
|
|
0.9982291666666666,
|
|
0.99875
|
|
],
|
|
"train_auc": [
|
|
0.9738291702389117,
|
|
0.9935188057270234,
|
|
0.997209082790352,
|
|
0.998729598908322,
|
|
0.9993232910379516,
|
|
0.9995217874799954,
|
|
0.9998343303469364,
|
|
0.9998332879658207,
|
|
0.999904435299497,
|
|
0.9999919938843164
|
|
],
|
|
"train_f1": [
|
|
0.9446794432497578,
|
|
0.9764305049939326,
|
|
0.9852721286073988,
|
|
0.9905918762088974,
|
|
0.9935549682385125,
|
|
0.9956829643130198,
|
|
0.9968653973842282,
|
|
0.9978312044147725,
|
|
0.9988191073068701,
|
|
0.9991665123170957
|
|
],
|
|
"val_loss": [
|
|
0.06827891671515923,
|
|
0.10762092962071317,
|
|
0.05082726004414629,
|
|
0.06780303101354244,
|
|
0.06042233494823146,
|
|
0.0935744174900295,
|
|
0.1393659184104854,
|
|
0.09219689728405077,
|
|
0.10592633409277369,
|
|
0.10129428458755077
|
|
],
|
|
"val_acc": [
|
|
0.9601041666666666,
|
|
0.9520833333333333,
|
|
0.9648958333333333,
|
|
0.9661458333333334,
|
|
0.966875,
|
|
0.9688541666666667,
|
|
0.9648958333333333,
|
|
0.97,
|
|
0.9730208333333333,
|
|
0.9758333333333333
|
|
],
|
|
"val_auc": [
|
|
0.9909356481481482,
|
|
0.9852364872685184,
|
|
0.9945385706018517,
|
|
0.9925354166666667,
|
|
0.9946561053240739,
|
|
0.9915528935185185,
|
|
0.9859638599537037,
|
|
0.9907507523148149,
|
|
0.9893995081018517,
|
|
0.9902642650462964
|
|
],
|
|
"val_f1": [
|
|
0.9734157007010481,
|
|
0.9683108294295949,
|
|
0.9763326076269401,
|
|
0.9773092229281575,
|
|
0.9777061132922041,
|
|
0.9792979297929792,
|
|
0.9767986230636833,
|
|
0.9799861014593467,
|
|
0.9820051413881749,
|
|
0.9839090026355944
|
|
]
|
|
},
|
|
"test_metrics": {
|
|
"accuracy": 0.9685416666666666,
|
|
"auc_roc": 0.995254060185185,
|
|
"f1": 0.9788426509737985,
|
|
"confusion_matrix": [
|
|
[
|
|
5780,
|
|
220
|
|
],
|
|
[
|
|
535,
|
|
17465
|
|
]
|
|
]
|
|
},
|
|
"source_metrics": {
|
|
"inpainting": {
|
|
"accuracy": 0.9808333333333333,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
115,
|
|
5885
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.9808333333333333,
|
|
"pairwise_auc": 0.9968223472222222,
|
|
"pairwise_f1": 0.9723254853366378
|
|
},
|
|
"insight": {
|
|
"accuracy": 0.938,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
372,
|
|
5628
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.938,
|
|
"pairwise_auc": 0.9903760972222222,
|
|
"pairwise_f1": 0.950033760972316
|
|
},
|
|
"text2img": {
|
|
"accuracy": 0.992,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
48,
|
|
5952
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"detection_rate": 0.992,
|
|
"pairwise_auc": 0.9985637361111112,
|
|
"pairwise_f1": 0.9779822543542557
|
|
},
|
|
"wiki": {
|
|
"accuracy": 0.9633333333333334,
|
|
"auc_roc": null,
|
|
"f1": null,
|
|
"confusion_matrix": [
|
|
[
|
|
5780,
|
|
220
|
|
],
|
|
[
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"n": 6000,
|
|
"false_alarm_rate": 0.036666666666666625
|
|
}
|
|
},
|
|
"pair_metrics": {
|
|
"wiki_vs_inpainting": {
|
|
"sources": [
|
|
"wiki",
|
|
"inpainting"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9720833333333333,
|
|
"auc_roc": 0.9968223472222222,
|
|
"f1": 0.9723254853366378,
|
|
"confusion_matrix": [
|
|
[
|
|
5780,
|
|
220
|
|
],
|
|
[
|
|
115,
|
|
5885
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_insight": {
|
|
"sources": [
|
|
"wiki",
|
|
"insight"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9506666666666667,
|
|
"auc_roc": 0.9903760972222222,
|
|
"f1": 0.950033760972316,
|
|
"confusion_matrix": [
|
|
[
|
|
5780,
|
|
220
|
|
],
|
|
[
|
|
372,
|
|
5628
|
|
]
|
|
]
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"sources": [
|
|
"wiki",
|
|
"text2img"
|
|
],
|
|
"n": 12000,
|
|
"accuracy": 0.9776666666666667,
|
|
"auc_roc": 0.9985637361111112,
|
|
"f1": 0.9779822543542557,
|
|
"confusion_matrix": [
|
|
[
|
|
5780,
|
|
220
|
|
],
|
|
[
|
|
48,
|
|
5952
|
|
]
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"aggregated_metrics": {
|
|
"accuracy": {
|
|
"mean": 0.9692166666666667,
|
|
"std": 0.0017757040857080092,
|
|
"ci_95": 0.0015564732570783506,
|
|
"values": [
|
|
0.9714166666666667,
|
|
0.969875,
|
|
0.969625,
|
|
0.966625,
|
|
0.9685416666666666
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9949671268518518,
|
|
"std": 0.00042766885637824786,
|
|
"ci_95": 0.0003748682808107718,
|
|
"values": [
|
|
0.9947762777777778,
|
|
0.9955725925925926,
|
|
0.9946256666666666,
|
|
0.9946070370370369,
|
|
0.995254060185185
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9793757797895071,
|
|
"std": 0.0012253246491348297,
|
|
"ci_95": 0.0010740444103088506,
|
|
"values": [
|
|
0.9808903002952811,
|
|
0.9797337070777855,
|
|
0.9798021777075887,
|
|
0.9776100628930817,
|
|
0.9788426509737985
|
|
]
|
|
}
|
|
},
|
|
"aggregated_per_source": {
|
|
"inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9837999999999999,
|
|
"std": 0.004619403520898425,
|
|
"ci_95": 0.004049085712986453,
|
|
"values": [
|
|
0.9865,
|
|
0.983,
|
|
0.9901666666666666,
|
|
0.9785,
|
|
0.9808333333333333
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9837999999999999,
|
|
"std": 0.004619403520898425,
|
|
"ci_95": 0.004049085712986453,
|
|
"values": [
|
|
0.9865,
|
|
0.983,
|
|
0.9901666666666666,
|
|
0.9785,
|
|
0.9808333333333333
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9965023166666667,
|
|
"std": 0.000606657607135633,
|
|
"ci_95": 0.0005317588382600736,
|
|
"values": [
|
|
0.9963499166666667,
|
|
0.9972167916666667,
|
|
0.996534,
|
|
0.9955885277777777,
|
|
0.9968223472222222
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9689101905349121,
|
|
"std": 0.005225479857072957,
|
|
"ci_95": 0.004580335044784638,
|
|
"values": [
|
|
0.9694537711899107,
|
|
0.9751177978011077,
|
|
0.961949481865285,
|
|
0.9657044164816185,
|
|
0.9723254853366378
|
|
]
|
|
}
|
|
},
|
|
"insight": {
|
|
"accuracy": {
|
|
"mean": 0.9500333333333334,
|
|
"std": 0.01067993965858941,
|
|
"ci_95": 0.009361379860303111,
|
|
"values": [
|
|
0.956,
|
|
0.9413333333333334,
|
|
0.9641666666666666,
|
|
0.9506666666666667,
|
|
0.938
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9500333333333334,
|
|
"std": 0.01067993965858941,
|
|
"ci_95": 0.009361379860303111,
|
|
"values": [
|
|
0.956,
|
|
0.9413333333333334,
|
|
0.9641666666666666,
|
|
0.9506666666666667,
|
|
0.938
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9903140916666668,
|
|
"std": 0.0007705375401317679,
|
|
"ci_95": 0.0006754059330284413,
|
|
"values": [
|
|
0.98983675,
|
|
0.9912267916666667,
|
|
0.9892951805555557,
|
|
0.990835638888889,
|
|
0.9903760972222222
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9514542370133494,
|
|
"std": 0.002197342024470114,
|
|
"ci_95": 0.0019260552055206147,
|
|
"values": [
|
|
0.9537745261057532,
|
|
0.953490335105934,
|
|
0.9486716956379141,
|
|
0.9513008672448299,
|
|
0.950033760972316
|
|
]
|
|
}
|
|
},
|
|
"text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9900333333333332,
|
|
"std": 0.0031233884733660264,
|
|
"ci_95": 0.0027377707070616046,
|
|
"values": [
|
|
0.9918333333333333,
|
|
0.9883333333333333,
|
|
0.9926666666666667,
|
|
0.9853333333333333,
|
|
0.992
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"detection_rate": {
|
|
"mean": 0.9900333333333332,
|
|
"std": 0.0031233884733660264,
|
|
"ci_95": 0.0027377707070616046,
|
|
"values": [
|
|
0.9918333333333333,
|
|
0.9883333333333333,
|
|
0.9926666666666667,
|
|
0.9853333333333333,
|
|
0.992
|
|
]
|
|
},
|
|
"pairwise_auc": {
|
|
"mean": 0.9980849722222223,
|
|
"std": 0.0004310028036603839,
|
|
"ci_95": 0.00037779061445122447,
|
|
"values": [
|
|
0.9981421666666667,
|
|
0.9982741944444444,
|
|
0.9980478194444444,
|
|
0.9973969444444445,
|
|
0.9985637361111112
|
|
]
|
|
},
|
|
"pairwise_f1": {
|
|
"mean": 0.9720680440216556,
|
|
"std": 0.006223323803512732,
|
|
"ci_95": 0.005454983827693629,
|
|
"values": [
|
|
0.97214734950584,
|
|
0.9778217495259296,
|
|
0.9632085388534002,
|
|
0.9691803278688524,
|
|
0.9779822543542557
|
|
]
|
|
}
|
|
},
|
|
"wiki": {
|
|
"accuracy": {
|
|
"mean": 0.9530000000000001,
|
|
"std": 0.013821782325976155,
|
|
"ci_95": 0.012115326381626433,
|
|
"values": [
|
|
0.9513333333333334,
|
|
0.9668333333333333,
|
|
0.9315,
|
|
0.952,
|
|
0.9633333333333334
|
|
]
|
|
},
|
|
"n": {
|
|
"mean": 6000.0,
|
|
"std": 0.0,
|
|
"ci_95": 0.0,
|
|
"values": [
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000,
|
|
6000
|
|
]
|
|
},
|
|
"false_alarm_rate": {
|
|
"mean": 0.047,
|
|
"std": 0.013821782325976157,
|
|
"ci_95": 0.012115326381626434,
|
|
"values": [
|
|
0.048666666666666636,
|
|
0.03316666666666668,
|
|
0.0685,
|
|
0.04800000000000004,
|
|
0.036666666666666625
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"aggregated_pairwise": {
|
|
"wiki_vs_inpainting": {
|
|
"accuracy": {
|
|
"mean": 0.9683999999999999,
|
|
"std": 0.005556027757710027,
|
|
"ci_95": 0.004870073054437218,
|
|
"values": [
|
|
0.9689166666666666,
|
|
0.9749166666666667,
|
|
0.9608333333333333,
|
|
0.96525,
|
|
0.9720833333333333
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9965023166666667,
|
|
"std": 0.000606657607135633,
|
|
"ci_95": 0.0005317588382600736,
|
|
"values": [
|
|
0.9963499166666667,
|
|
0.9972167916666667,
|
|
0.996534,
|
|
0.9955885277777777,
|
|
0.9968223472222222
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9689101905349121,
|
|
"std": 0.005225479857072957,
|
|
"ci_95": 0.004580335044784638,
|
|
"values": [
|
|
0.9694537711899107,
|
|
0.9751177978011077,
|
|
0.961949481865285,
|
|
0.9657044164816185,
|
|
0.9723254853366378
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_insight": {
|
|
"accuracy": {
|
|
"mean": 0.9515166666666668,
|
|
"std": 0.0025265259415516235,
|
|
"ci_95": 0.0022145976308726276,
|
|
"values": [
|
|
0.9536666666666667,
|
|
0.9540833333333333,
|
|
0.9478333333333333,
|
|
0.9513333333333334,
|
|
0.9506666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9903140916666668,
|
|
"std": 0.0007705375401317679,
|
|
"ci_95": 0.0006754059330284413,
|
|
"values": [
|
|
0.98983675,
|
|
0.9912267916666667,
|
|
0.9892951805555557,
|
|
0.990835638888889,
|
|
0.9903760972222222
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9514542370133494,
|
|
"std": 0.002197342024470114,
|
|
"ci_95": 0.0019260552055206147,
|
|
"values": [
|
|
0.9537745261057532,
|
|
0.953490335105934,
|
|
0.9486716956379141,
|
|
0.9513008672448299,
|
|
0.950033760972316
|
|
]
|
|
}
|
|
},
|
|
"wiki_vs_text2img": {
|
|
"accuracy": {
|
|
"mean": 0.9715166666666667,
|
|
"std": 0.0065525122408635755,
|
|
"ci_95": 0.005743530215236409,
|
|
"values": [
|
|
0.9715833333333334,
|
|
0.9775833333333334,
|
|
0.9620833333333333,
|
|
0.9686666666666667,
|
|
0.9776666666666667
|
|
]
|
|
},
|
|
"auc_roc": {
|
|
"mean": 0.9980849722222223,
|
|
"std": 0.0004310028036603839,
|
|
"ci_95": 0.00037779061445122447,
|
|
"values": [
|
|
0.9981421666666667,
|
|
0.9982741944444444,
|
|
0.9980478194444444,
|
|
0.9973969444444445,
|
|
0.9985637361111112
|
|
]
|
|
},
|
|
"f1": {
|
|
"mean": 0.9720680440216556,
|
|
"std": 0.006223323803512732,
|
|
"ci_95": 0.005454983827693629,
|
|
"values": [
|
|
0.97214734950584,
|
|
0.9778217495259296,
|
|
0.9632085388534002,
|
|
0.9691803278688524,
|
|
0.9779822543542557
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"seed": 42,
|
|
"cv_folds": 5,
|
|
"batch_size": 32,
|
|
"num_workers": 4,
|
|
"early_stopping_patience": 5,
|
|
"lr": 0.0001,
|
|
"weight_decay": 0.0001,
|
|
"T_max": 15,
|
|
"data_dir": "cropped/classifier",
|
|
"pretrained": true,
|
|
"epochs": 15,
|
|
"image_size": 224,
|
|
"augment": false,
|
|
"run_name": "p4_resnet50_100pct",
|
|
"backbone": "resnet50",
|
|
"subsample": 1.0
|
|
}
|
|
} |