{ "model_version": "2ba1863c58d4", "model_sha256": "2ba1863c58d4cbfe4294e23d876615d6ee621737d57a28f7122b7d5eb3ee3f9e", "created_at": "2026-09-14T17:41:09.819592+00:00", "architecture": "InceptionV3", "framework": "TensorFlow 2.10.1", "labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration", "Mass", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening", "No Finding" ], "input_shape": [ 1, 299, 299, 3 ], "input_preprocessing": "Grayscale RGB 0..255; bilinear resize to 299; model internally rescales to [-1,1]", "data_audit": { "source": "https://www.kaggle.com/api/v1/datasets/download/nih-chest-xrays/data", "source_page": "https://www.kaggle.com/datasets/nih-chest-xrays/data", "seed": 20260914, "train_images": 100000, "added_train_images": 50000, "validation_images": 2500, "test_images": 2500, "patient_overlap": 0, "unique_images": 105000, "image_size": [ 299, 299 ], "labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration", "Mass", "Nodule", "Pneumonia", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Fibrosis", "Pleural_Thickening", "Hernia" ], "metadata_sha256": "88f75094e25ccc0c6f1f9cdfd4b2f94f9379a0ae07d5ff4dcf94242707b07462", "manifest_sha256": "ae45669ec7b22ab9699b33cefe3ad377ea5a63ad86e5b8666299e3350bd42ee1", "metadata_label_matches": 105000, "unknown_labels": 0, "duplicate_filenames": 0, "exact_pixel_duplicates_within_splits": 23, "exact_pixel_duplicates_across_splits": 0, "prior_image_hashes_reused": 50000, "new_images_decoded_and_hashed": 55000, "evaluation_split_preserved": false, "fresh_evaluation_patients": false, "prior_evaluation_retired": 5000, "new_training_ids_previously_used": 0, "patients": { "train": 25897, "validation": 1880, "test": 1952 }, "label_counts": { "train": { "Atelectasis": 10567, "Cardiomegaly": 2474, "Effusion": 12185, "Infiltration": 17994, "Mass": 5211, "Nodule": 5663, "Pneumonia": 1296, "Pneumothorax": 4904, "Consolidation": 4297, "Edema": 2131, "Emphysema": 2314, "Fibrosis": 1468, "Pleural_Thickening": 3047, "Hernia": 192, "No Finding": 53131 }, "validation": { "Atelectasis": 132, "Cardiomegaly": 58, "Effusion": 118, "Infiltration": 300, "Mass": 79, "Nodule": 120, "Pneumonia": 13, "Pneumothorax": 31, "Consolidation": 27, "Edema": 17, "Emphysema": 18, "Fibrosis": 47, "Pleural_Thickening": 60, "Hernia": 1, "No Finding": 1740 }, "test": { "Atelectasis": 153, "Cardiomegaly": 75, "Effusion": 118, "Infiltration": 304, "Mass": 100, "Nodule": 111, "Pneumonia": 20, "Pneumothorax": 30, "Consolidation": 37, "Edema": 12, "Emphysema": 27, "Fibrosis": 59, "Pleural_Thickening": 60, "Hernia": 6, "No Finding": 1695 } }, "label_limitation": "CSV labels were text-mined from reports, not verified diagnostic ground truth. Matching the CSV does not establish clinical accuracy.", "evaluation_note": "New validation and test patients were absent from every earlier manifest. Old evaluation is retired because additional training scans can belong to its patients. Baseline is re-evaluated on the fresh cohorts; historical metrics are not directly comparable.", "trained_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration", "Mass", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening", "No Finding" ], "excluded_labels": { "Fibrosis": { "training_positives": 1468, "reason": "User-requested rare-label exclusion; fewer than 2% positive training examples" }, "Hernia": { "training_positives": 192, "reason": "User-requested rare-label exclusion; fewer than 2% positive training examples" }, "Pneumonia": { "training_positives": 1296, "reason": "User-requested rare-label exclusion; fewer than 2% positive training examples" } }, "minimum_positive_training_examples": 1000, "no_finding_training_images": 53131 }, "training_config": { "root": "D:/CodexChestXray/v3", "previous": "D:/CodexChestXray/v2", "epochs": 6, "batch_size": 32 }, "history": [ { "phase": "continued fine tuning", "epoch": 1, "loss": 0.2175845056772232, "auc": 0.8031589984893799, "label_accuracy": 0.9176608324050903, "exact_match_accuracy": 0.41743001341819763, "precision": 0.6930813193321228, "recall": 0.36370018124580383, "val_loss": 0.15933620929718018, "val_auc": 0.7827995419502258, "val_label_accuracy": 0.9453666806221008, "val_exact_match_accuracy": 0.6140000224113464, "val_precision": 0.7606879472732544, "val_recall": 0.5733333230018616, "val_exact_auc": 0.7912873084033595, "val_no_finding_auc": 0.7051527525710829, "elapsed_seconds": 2748, "learning_rate": 1.9999999494757503e-05 }, { "phase": "continued fine tuning", "epoch": 2, "loss": 0.21392811834812164, "auc": 0.8144367337226868, "label_accuracy": 0.9184200167655945, "exact_match_accuracy": 0.4219000041484833, "precision": 0.6965882778167725, "recall": 0.3720444142818451, "val_loss": 0.15528982877731323, "val_auc": 0.7886516451835632, "val_label_accuracy": 0.9466000199317932, "val_exact_match_accuracy": 0.6704000234603882, "val_precision": 0.7370466589927673, "val_recall": 0.6322222352027893, "val_exact_auc": 0.7980295546775648, "val_no_finding_auc": 0.7068088324258923, "elapsed_seconds": 2771, "learning_rate": 1.9999999494757503e-05 }, { "phase": "continued fine tuning", "epoch": 3, "loss": 0.21072489023208618, "auc": 0.8232952952384949, "label_accuracy": 0.9194133281707764, "exact_match_accuracy": 0.4293299913406372, "precision": 0.7005748748779297, "recall": 0.38353589177131653, "val_loss": 0.15611110627651215, "val_auc": 0.7868983745574951, "val_label_accuracy": 0.9472000002861023, "val_exact_match_accuracy": 0.6520000100135803, "val_precision": 0.7540983557701111, "val_recall": 0.6133333444595337, "val_exact_auc": 0.7958851549718852, "val_no_finding_auc": 0.7022485632183908, "elapsed_seconds": 2746, "learning_rate": 1.9999999494757503e-05 }, { "phase": "continued fine tuning", "epoch": 4, "loss": 0.20722565054893494, "auc": 0.8323011994361877, "label_accuracy": 0.9201766848564148, "exact_match_accuracy": 0.4349899888038635, "precision": 0.7024906277656555, "recall": 0.39376845955848694, "val_loss": 0.1552238166332245, "val_auc": 0.7841801643371582, "val_label_accuracy": 0.9469666481018066, "val_exact_match_accuracy": 0.6600000262260437, "val_precision": 0.746334969997406, "val_recall": 0.6222222447395325, "val_exact_auc": 0.7934612179390609, "val_no_finding_auc": 0.7046060193587418, "elapsed_seconds": 2734, "learning_rate": 1.9999999494757503e-05 }, { "phase": "continued fine tuning", "epoch": 5, "loss": 0.20084427297115326, "auc": 0.8478360772132874, "label_accuracy": 0.9219216704368591, "exact_match_accuracy": 0.4446299970149994, "precision": 0.7099997401237488, "recall": 0.41231298446655273, "val_loss": 0.15727542340755463, "val_auc": 0.782825767993927, "val_label_accuracy": 0.9466999769210815, "val_exact_match_accuracy": 0.6420000195503235, "val_precision": 0.7514846920967102, "val_recall": 0.6092592477798462, "val_exact_auc": 0.7909769983842355, "val_no_finding_auc": 0.6985518753781005, "elapsed_seconds": 2759, "learning_rate": 9.999999747378752e-06 } ], "metrics": { "per_label": [ { "label": "Atelectasis", "roc_auc": 0.8090734660573503, "average_precision": 0.26160220376924775, "threshold": 0.05119185149669647, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.7647058823529411, "specificity": 0.7136770345121431, "positives": 153, "negatives": 2347, "tp": 117, "fp": 672, "tn": 1675, "fn": 36, "limited_support": false }, { "label": "Cardiomegaly", "roc_auc": 0.9032907216494845, "average_precision": 0.2650983429181318, "threshold": 0.0105612613260746, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.92, "specificity": 0.7678350515463918, "positives": 75, "negatives": 2425, "tp": 69, "fp": 563, "tn": 1862, "fn": 6, "limited_support": false }, { "label": "Effusion", "roc_auc": 0.8997815537434716, "average_precision": 0.5111248081814207, "threshold": 0.07844139635562897, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.7372881355932204, "specificity": 0.9105793450881612, "positives": 118, "negatives": 2382, "tp": 87, "fp": 213, "tn": 2169, "fn": 31, "limited_support": false }, { "label": "Infiltration", "roc_auc": 0.654217296999329, "average_precision": 0.19735709093577103, "threshold": 0.11796284466981888, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.45394736842105265, "specificity": 0.7513661202185792, "positives": 304, "negatives": 2196, "tp": 138, "fp": 546, "tn": 1650, "fn": 166, "limited_support": false }, { "label": "Mass", "roc_auc": 0.8433125, "average_precision": 0.272776724277233, "threshold": 0.029845068231225014, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.75, "specificity": 0.7966666666666666, "positives": 100, "negatives": 2400, "tp": 75, "fp": 488, "tn": 1912, "fn": 25, "limited_support": false }, { "label": "Nodule", "roc_auc": 0.6913707344850082, "average_precision": 0.12771774101934344, "threshold": 0.023530127480626106, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.7747747747747747, "specificity": 0.556718292172457, "positives": 111, "negatives": 2389, "tp": 86, "fp": 1059, "tn": 1330, "fn": 25, "limited_support": false }, { "label": "Pneumothorax", "roc_auc": 0.8216059379217274, "average_precision": 0.14665970652262905, "threshold": 0.06338544934988022, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.5666666666666667, "specificity": 0.9331983805668016, "positives": 30, "negatives": 2470, "tp": 17, "fp": 165, "tn": 2305, "fn": 13, "limited_support": false }, { "label": "Consolidation", "roc_auc": 0.7574261228341619, "average_precision": 0.07196876186974167, "threshold": 0.016633110120892525, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.5675675675675675, "specificity": 0.8440925700365408, "positives": 37, "negatives": 2463, "tp": 21, "fp": 384, "tn": 2079, "fn": 16, "limited_support": false }, { "label": "Edema", "roc_auc": 0.9540125937834942, "average_precision": 0.29024793853250497, "threshold": 0.0028604387771338224, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.8333333333333334, "specificity": 0.8842443729903537, "positives": 12, "negatives": 2488, "tp": 10, "fp": 288, "tn": 2200, "fn": 2, "limited_support": true }, { "label": "Emphysema", "roc_auc": 0.8014706983570712, "average_precision": 0.03832498902973624, "threshold": 0.004394697491079569, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.7777777777777778, "specificity": 0.6878285483218762, "positives": 27, "negatives": 2473, "tp": 21, "fp": 772, "tn": 1701, "fn": 6, "limited_support": true }, { "label": "Pleural_Thickening", "roc_auc": 0.7376502732240438, "average_precision": 0.10809567655936027, "threshold": 0.019354315474629402, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.6333333333333333, "specificity": 0.7622950819672131, "positives": 60, "negatives": 2440, "tp": 38, "fp": 580, "tn": 1860, "fn": 22, "limited_support": false }, { "label": "No Finding", "roc_auc": 0.7265171586141189, "average_precision": 0.8298262230328795, "threshold": 0.780142605304718, "threshold_method": "maximum Youden J on validation only", "sensitivity": 0.6353982300884956, "specificity": 0.684472049689441, "positives": 1695, "negatives": 805, "tp": 1077, "fp": 254, "tn": 551, "fn": 618, "limited_support": false } ], "macro_roc_auc": 0.7999774214724384, "retained_disease_macro_roc_auc": 0.8066556271868311, "macro_average_precision": 0.2600666838873333, "display_policy": { "version": 2, "kind": "research_screening", "beta": 0.5, "method": "Independent per-label maximum F-beta on fixed0.005..0.995 grid, minimum25 selected images and15 distinct patients. Tie: higher precision, then higher threshold. No test predictions read. NF threshold0.76 and NF>=max disease are fixed; no NF retuning. Forced fallback selects argmax raw retained disease score only if no other output qualifies.", "labels": { "Atelectasis": { "enabled": true, "threshold": 0.295, "validation": { "tp": 30, "fp": 48, "fn": 102, "tn": 2320, "predicted": 78, "distinct_predicted_patients": 63, "precision": 0.38461538461538464, "recall": 0.22727272727272727, "specificity": 0.9797297297297297, "precision_wilson_lower_95": 0.28448876404744494 } }, "Cardiomegaly": { "enabled": true, "threshold": 0.085, "validation": { "tp": 24, "fp": 54, "fn": 34, "tn": 2388, "predicted": 78, "distinct_predicted_patients": 67, "precision": 0.3076923076923077, "recall": 0.41379310344827586, "specificity": 0.9778869778869779, "precision_wilson_lower_95": 0.2163178292080744 } }, "Effusion": { "enabled": true, "threshold": 0.39, "validation": { "tp": 52, "fp": 39, "fn": 66, "tn": 2343, "predicted": 91, "distinct_predicted_patients": 56, "precision": 0.5714285714285714, "recall": 0.4406779661016949, "specificity": 0.9836272040302267, "precision_wilson_lower_95": 0.4688956326088687 } }, "Infiltration": { "enabled": true, "threshold": 0.145, "validation": { "tp": 65, "fp": 217, "fn": 235, "tn": 1983, "predicted": 282, "distinct_predicted_patients": 214, "precision": 0.23049645390070922, "recall": 0.21666666666666667, "specificity": 0.9013636363636364, "precision_wilson_lower_95": 0.185160598380427 } }, "Mass": { "enabled": true, "threshold": 0.235, "validation": { "tp": 13, "fp": 22, "fn": 66, "tn": 2399, "predicted": 35, "distinct_predicted_patients": 33, "precision": 0.37142857142857144, "recall": 0.16455696202531644, "specificity": 0.9909128459314333, "precision_wilson_lower_95": 0.2316560736889106 } }, "Nodule": { "enabled": true, "threshold": 0.11, "validation": { "tp": 20, "fp": 36, "fn": 100, "tn": 2344, "predicted": 56, "distinct_predicted_patients": 52, "precision": 0.35714285714285715, "recall": 0.16666666666666666, "specificity": 0.984873949579832, "precision_wilson_lower_95": 0.24456392702705987 } }, "Pneumothorax": { "enabled": true, "threshold": 0.125, "validation": { "tp": 8, "fp": 38, "fn": 23, "tn": 2431, "predicted": 46, "distinct_predicted_patients": 35, "precision": 0.17391304347826086, "recall": 0.25806451612903225, "specificity": 0.9846091535034427, "precision_wilson_lower_95": 0.09085683568889252 } }, "Consolidation": { "enabled": true, "threshold": 0.055, "validation": { "tp": 11, "fp": 98, "fn": 16, "tn": 2375, "predicted": 109, "distinct_predicted_patients": 64, "precision": 0.10091743119266056, "recall": 0.4074074074074074, "specificity": 0.9603720177921553, "precision_wilson_lower_95": 0.05728913809310716 } }, "Edema": { "enabled": true, "threshold": 0.05, "validation": { "tp": 8, "fp": 33, "fn": 9, "tn": 2450, "predicted": 41, "distinct_predicted_patients": 18, "precision": 0.1951219512195122, "recall": 0.47058823529411764, "specificity": 0.9867096254530809, "precision_wilson_lower_95": 0.1023431574920593 } }, "Emphysema": { "enabled": true, "threshold": 0.035, "validation": { "tp": 1, "fp": 28, "fn": 17, "tn": 2454, "predicted": 29, "distinct_predicted_patients": 25, "precision": 0.034482758620689655, "recall": 0.05555555555555555, "specificity": 0.9887187751813054, "precision_wilson_lower_95": 0.006113056614032264 } }, "Pleural_Thickening": { "enabled": true, "threshold": 0.095, "validation": { "tp": 9, "fp": 43, "fn": 51, "tn": 2397, "predicted": 52, "distinct_predicted_patients": 44, "precision": 0.17307692307692307, "recall": 0.15, "specificity": 0.9823770491803279, "precision_wilson_lower_95": 0.09382368756467781 } }, "No Finding": { "enabled": true, "threshold": 0.76, "validation": { "tp": 1113, "fp": 270, "fn": 627, "tn": 490, "predicted": 1383, "distinct_predicted_patients": 1177, "precision": 0.8047722342733189, "recall": 0.6396551724137931, "specificity": 0.6447368421052632, "precision_wilson_lower_95": 0.7830492183220575 } } }, "limitations": "Scores are uncalibrated. F0.5 favors precision but does not guarantee high precision. Validation figures used to choose thresholds are optimistic; patient-group folds assess threshold stability only. Low positive support and repeat scans limit estimates. Raw-score fallback forces an unsupported finding; show its provenance separately.", "no_finding_rule": "All qualifying diseases first; No Finding alone only when no disease qualifies, score >= 0.76 and above all disease scores; otherwise one explicitly unverified highest-scoring disease.", "frozen_policy_sha256": "4f43afb1ca90475aa4fd487505312fda2c615be9b92df7adc2f035119fb609e0" }, "display_evaluation": { "images": 2500, "per_label": [ { "label": "Atelectasis", "tp": 68, "fp": 203, "fn": 85, "tn": 2144, "predicted": 271, "distinct_predicted_patients": 216, "precision": 0.25092250922509224, "recall": 0.4444444444444444, "specificity": 0.9135066041755433, "precision_wilson_lower_95": 0.20302950040830386 }, { "label": "Cardiomegaly", "tp": 24, "fp": 46, "fn": 51, "tn": 2379, "predicted": 70, "distinct_predicted_patients": 64, "precision": 0.34285714285714286, "recall": 0.32, "specificity": 0.9810309278350515, "precision_wilson_lower_95": 0.24245827082896973 }, { "label": "Effusion", "tp": 46, "fp": 38, "fn": 72, "tn": 2344, "predicted": 84, "distinct_predicted_patients": 62, "precision": 0.5476190476190477, "recall": 0.3898305084745763, "specificity": 0.984047019311503, "precision_wilson_lower_95": 0.4414284668698881 }, { "label": "Infiltration", "tp": 110, "fp": 488, "fn": 194, "tn": 1708, "predicted": 598, "distinct_predicted_patients": 497, "precision": 0.18394648829431437, "recall": 0.3618421052631579, "specificity": 0.7777777777777778, "precision_wilson_lower_95": 0.15494397855205103 }, { "label": "Mass", "tp": 24, "fp": 40, "fn": 76, "tn": 2360, "predicted": 64, "distinct_predicted_patients": 54, "precision": 0.375, "recall": 0.24, "specificity": 0.9833333333333333, "precision_wilson_lower_95": 0.26665803888105327 }, { "label": "Nodule", "tp": 14, "fp": 65, "fn": 97, "tn": 2324, "predicted": 79, "distinct_predicted_patients": 66, "precision": 0.17721518987341772, "recall": 0.12612612612612611, "specificity": 0.972791963164504, "precision_wilson_lower_95": 0.10860327950089017 }, { "label": "Pneumothorax", "tp": 11, "fp": 70, "fn": 19, "tn": 2400, "predicted": 81, "distinct_predicted_patients": 56, "precision": 0.13580246913580246, "recall": 0.36666666666666664, "specificity": 0.97165991902834, "precision_wilson_lower_95": 0.07755383801327821 }, { "label": "Consolidation", "tp": 8, "fp": 67, "fn": 29, "tn": 2396, "predicted": 75, "distinct_predicted_patients": 48, "precision": 0.10666666666666667, "recall": 0.21621621621621623, "specificity": 0.9727974015428339, "precision_wilson_lower_95": 0.05504846963856164 }, { "label": "Edema", "tp": 7, "fp": 14, "fn": 5, "tn": 2474, "predicted": 21, "distinct_predicted_patients": 17, "precision": 0.3333333333333333, "recall": 0.5833333333333334, "specificity": 0.9943729903536977, "precision_wilson_lower_95": 0.17194536432683688 }, { "label": "Emphysema", "tp": 2, "fp": 43, "fn": 25, "tn": 2430, "predicted": 45, "distinct_predicted_patients": 36, "precision": 0.044444444444444446, "recall": 0.07407407407407407, "specificity": 0.9826122118883946, "precision_wilson_lower_95": 0.01227386904044434 }, { "label": "Pleural_Thickening", "tp": 10, "fp": 49, "fn": 50, "tn": 2391, "predicted": 59, "distinct_predicted_patients": 43, "precision": 0.1694915254237288, "recall": 0.16666666666666666, "specificity": 0.9799180327868853, "precision_wilson_lower_95": 0.0947573785633763 }, { "label": "No Finding", "tp": 1135, "fp": 282, "fn": 560, "tn": 523, "predicted": 1417, "distinct_predicted_patients": 1238, "precision": 0.8009880028228652, "recall": 0.6696165191740413, "specificity": 0.6496894409937888, "precision_wilson_lower_95": 0.7793978550538907 } ], "disease_count_distribution": { "0": 1417, "1": 891, "2": 97, "3": 44, "4": 31, "5": 15, "6": 4, "7": 1, "8": 0, "9": 0, "10": 0, "11": 0 }, "no_finding_images": 1417, "any_disease_images": 1083, "threshold_supported_disease_images": 484, "ranked_suggestion_images": 599, "ranked_suggestion_correct": 99, "ranked_suggestion_precision": 0.1652754590984975, "note": "All independently qualifying screening flags, otherwise No Finding under its separate rule, otherwise one explicitly unverified top disease suggestion. Existing internal test predictions audit the frozen presentation; weights are unchanged.", "multilabel": { "all": { "images": 2500, "tp": 324, "fp": 1123, "fn": 703, "precision": 0.22391154111955772, "recall": 0.3154819863680623, "f1": 0.26192400970088925, "displayed_multiple_images": 192, "at_least_two_correct_images": 27, "all_annotations_recovered_images": 195, "exact_disease_set_images": 1272 }, "multiannotated": { "images": 198, "tp": 146, "fp": 166, "fn": 312, "precision": 0.46794871794871795, "recall": 0.31877729257641924, "f1": 0.37922077922077924, "displayed_multiple_images": 70, "at_least_two_correct_images": 27, "all_annotations_recovered_images": 17, "exact_disease_set_images": 3 }, "count_bins": [ "0", "1", "2", "3", "4+" ], "count_matrix": [ [ 1147, 538, 30, 10, 8 ], [ 232, 263, 38, 17, 19 ], [ 34, 68, 23, 12, 14 ], [ 4, 16, 4, 4, 6 ], [ 0, 6, 2, 1, 4 ] ], "annotated_counts": [ 1733, 569, 151, 34, 13 ], "displayed_counts": [ 1417, 891, 97, 44, 51 ], "definitions": { "multiannotated": "Scans with at least two retained disease labels in the CSV; excludes No Finding, Hernia, Pneumonia and Fibrosis.", "at_least_two_correct_images": "At least two displayed labels match CSV disease annotations; additional false positives may be present.", "all_annotations_recovered_images": "Every retained CSV disease is displayed; additional false positives may be present.", "exact_disease_set_images": "Displayed disease set equals retained CSV disease set, with neither extra nor missing disease labels.", "count_matrix": "Rows: CSV disease count; columns: displayed disease count. Equal counts do not imply correct labels." } } }, "no_finding_display": { "threshold": 0.76, "threshold_method": "Independent per-label maximum F-beta on fixed0.005..0.995 grid, minimum25 selected images and15 distinct patients. Tie: higher precision, then higher threshold. No test predictions read. NF threshold0.76 and NF>=max disease are fixed; no NF retuning. Forced fallback selects argmax raw retained disease score only if no other output qualifies.", "test_predicted_no_finding": 1417, "test_correct_no_finding": 1135, "test_false_no_finding": 282, "test_precision": 0.8009880028228652 }, "test_label_accuracy_at_0_5": 0.9440666666666667, "test_exact_match_at_0_5": 0.6504, "validation_all_negative_label_accuracy": 0.91, "training_all_negative_label_accuracy": 0.8967350000000001, "threshold_only_evaluation": { "images": 2500, "per_label": [ { "label": "Atelectasis", "tp": 32, "fp": 43, "fn": 121, "tn": 2304, "predicted": 75, "distinct_predicted_patients": 68, "precision": 0.4266666666666667, "recall": 0.20915032679738563, "specificity": 0.9816787388155092, "precision_wilson_lower_95": 0.32100563365956936 }, { "label": "Cardiomegaly", "tp": 23, "fp": 44, "fn": 52, "tn": 2381, "predicted": 67, "distinct_predicted_patients": 61, "precision": 0.34328358208955223, "recall": 0.30666666666666664, "specificity": 0.9818556701030928, "precision_wilson_lower_95": 0.24088841959339713 }, { "label": "Effusion", "tp": 40, "fp": 22, "fn": 78, "tn": 2360, "predicted": 62, "distinct_predicted_patients": 43, "precision": 0.6451612903225806, "recall": 0.3389830508474576, "specificity": 0.9907640638119227, "precision_wilson_lower_95": 0.5208089986340214 }, { "label": "Infiltration", "tp": 58, "fp": 197, "fn": 246, "tn": 1999, "predicted": 255, "distinct_predicted_patients": 204, "precision": 0.22745098039215686, "recall": 0.19078947368421054, "specificity": 0.9102914389799636, "precision_wilson_lower_95": 0.18026840658528517 }, { "label": "Mass", "tp": 20, "fp": 24, "fn": 80, "tn": 2376, "predicted": 44, "distinct_predicted_patients": 38, "precision": 0.45454545454545453, "recall": 0.2, "specificity": 0.99, "precision_wilson_lower_95": 0.3170500778685361 }, { "label": "Nodule", "tp": 14, "fp": 58, "fn": 97, "tn": 2331, "predicted": 72, "distinct_predicted_patients": 60, "precision": 0.19444444444444445, "recall": 0.12612612612612611, "specificity": 0.9757220594390958, "precision_wilson_lower_95": 0.11951371043417634 }, { "label": "Pneumothorax", "tp": 11, "fp": 62, "fn": 19, "tn": 2408, "predicted": 73, "distinct_predicted_patients": 50, "precision": 0.1506849315068493, "recall": 0.36666666666666664, "specificity": 0.9748987854251012, "precision_wilson_lower_95": 0.08627588209726772 }, { "label": "Consolidation", "tp": 8, "fp": 67, "fn": 29, "tn": 2396, "predicted": 75, "distinct_predicted_patients": 48, "precision": 0.10666666666666667, "recall": 0.21621621621621623, "specificity": 0.9727974015428339, "precision_wilson_lower_95": 0.05504846963856164 }, { "label": "Edema", "tp": 7, "fp": 14, "fn": 5, "tn": 2474, "predicted": 21, "distinct_predicted_patients": 17, "precision": 0.3333333333333333, "recall": 0.5833333333333334, "specificity": 0.9943729903536977, "precision_wilson_lower_95": 0.17194536432683688 }, { "label": "Emphysema", "tp": 2, "fp": 43, "fn": 25, "tn": 2430, "predicted": 45, "distinct_predicted_patients": 36, "precision": 0.044444444444444446, "recall": 0.07407407407407407, "specificity": 0.9826122118883946, "precision_wilson_lower_95": 0.01227386904044434 }, { "label": "Pleural_Thickening", "tp": 10, "fp": 49, "fn": 50, "tn": 2391, "predicted": 59, "distinct_predicted_patients": 43, "precision": 0.1694915254237288, "recall": 0.16666666666666666, "specificity": 0.9799180327868853, "precision_wilson_lower_95": 0.0947573785633763 }, { "label": "No Finding", "tp": 1135, "fp": 282, "fn": 560, "tn": 523, "predicted": 1417, "distinct_predicted_patients": 1238, "precision": 0.8009880028228652, "recall": 0.6696165191740413, "specificity": 0.6496894409937888, "precision_wilson_lower_95": 0.7793978550538907 } ], "disease_count_distribution": { "0": 2016, "1": 292, "2": 97, "3": 44, "4": 31, "5": 15, "6": 4, "7": 1, "8": 0, "9": 0, "10": 0, "11": 0 }, "no_finding_images": 1417, "inconclusive_images": 599, "any_disease_images": 484, "note": "Threshold-only screening outputs before the required top-score fallback. Zero-output images are internal audit counts, not a user-facing result option. Existing internal test benchmark is reused." }, "previous_display_policy": { "version": 1, "precision_target": 0.8, "minimum_predictions": 25, "minimum_distinct_patients": 15, "minimum_wilson_lower_95": 0.6, "grid_step": 0.005, "method": "Fixed 0.005 score grid, validation precision >=80%, at least25 predictions and15 distinct patients, approximate 95% Wilson lower bound >=60%; maximize recall then precision. No test tuning.", "limitations": "Validation agreement with weak CSV labels is not clinical certainty. Image-level Wilson intervals are approximate because repeated scans from a patient are correlated and thresholds were searched. Scores are not calibrated probabilities.", "no_finding_rule": "No accepted disease; No Finding passes its own threshold and ranks above every disease.", "labels": { "Atelectasis": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Cardiomegaly": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Effusion": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Infiltration": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Mass": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Nodule": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Pneumothorax": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Consolidation": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Edema": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Emphysema": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "Pleural_Thickening": { "enabled": false, "threshold": null, "validation": null, "reason": "No validation threshold meets the prespecified precision and support requirements." }, "No Finding": { "enabled": true, "threshold": 0.76, "validation": { "tp": 1177, "fp": 292, "fn": 563, "tn": 468, "predicted": 1469, "distinct_predicted_patients": 1239, "precision": 0.8012253233492171, "recall": 0.6764367816091954, "specificity": 0.6157894736842106, "precision_wilson_lower_95": 0.7800430128331355 } } } }, "previous_display_evaluation": { "images": 2500, "per_label": [ { "label": "Atelectasis", "tp": 71, "fp": 216, "fn": 82, "tn": 2131, "predicted": 287, "distinct_predicted_patients": 224, "precision": 0.24738675958188153, "recall": 0.46405228758169936, "specificity": 0.907967618236046, "precision_wilson_lower_95": 0.20102039647556152 }, { "label": "Cardiomegaly", "tp": 16, "fp": 22, "fn": 59, "tn": 2403, "predicted": 38, "distinct_predicted_patients": 35, "precision": 0.42105263157894735, "recall": 0.21333333333333335, "specificity": 0.9909278350515464, "precision_wilson_lower_95": 0.2785227481181953 }, { "label": "Effusion", "tp": 59, "fp": 49, "fn": 59, "tn": 2333, "predicted": 108, "distinct_predicted_patients": 78, "precision": 0.5462962962962963, "recall": 0.5, "specificity": 0.9794290512174643, "precision_wilson_lower_95": 0.45242362376844814 }, { "label": "Infiltration", "tp": 77, "fp": 380, "fn": 227, "tn": 1816, "predicted": 457, "distinct_predicted_patients": 405, "precision": 0.16849015317286653, "recall": 0.2532894736842105, "specificity": 0.8269581056466302, "precision_wilson_lower_95": 0.13696766230644264 }, { "label": "Mass", "tp": 22, "fp": 33, "fn": 78, "tn": 2367, "predicted": 55, "distinct_predicted_patients": 49, "precision": 0.4, "recall": 0.22, "specificity": 0.98625, "precision_wilson_lower_95": 0.2811830378643358 }, { "label": "Nodule", "tp": 5, "fp": 25, "fn": 106, "tn": 2364, "predicted": 30, "distinct_predicted_patients": 27, "precision": 0.16666666666666666, "recall": 0.04504504504504504, "specificity": 0.9895353704478862, "precision_wilson_lower_95": 0.07336434240351682 }, { "label": "Pneumothorax", "tp": 9, "fp": 37, "fn": 21, "tn": 2433, "predicted": 46, "distinct_predicted_patients": 28, "precision": 0.1956521739130435, "recall": 0.3, "specificity": 0.9850202429149798, "precision_wilson_lower_95": 0.106504904441218 }, { "label": "Consolidation", "tp": 0, "fp": 0, "fn": 37, "tn": 2463, "predicted": 0, "distinct_predicted_patients": 0, "precision": null, "recall": 0.0, "specificity": 1.0, "precision_wilson_lower_95": null }, { "label": "Edema", "tp": 0, "fp": 0, "fn": 12, "tn": 2488, "predicted": 0, "distinct_predicted_patients": 0, "precision": null, "recall": 0.0, "specificity": 1.0, "precision_wilson_lower_95": null }, { "label": "Emphysema", "tp": 0, "fp": 0, "fn": 27, "tn": 2473, "predicted": 0, "distinct_predicted_patients": 0, "precision": null, "recall": 0.0, "specificity": 1.0, "precision_wilson_lower_95": null }, { "label": "Pleural_Thickening", "tp": 0, "fp": 2, "fn": 60, "tn": 2438, "predicted": 2, "distinct_predicted_patients": 2, "precision": 0.0, "recall": 0.0, "specificity": 0.9991803278688525, "precision_wilson_lower_95": 0.0 }, { "label": "No Finding", "tp": 1182, "fp": 295, "fn": 513, "tn": 510, "predicted": 1477, "distinct_predicted_patients": 1283, "precision": 0.8002708192281652, "recall": 0.6973451327433628, "specificity": 0.6335403726708074, "precision_wilson_lower_95": 0.7791139975836713 } ], "disease_count_distribution": { "0": 1477, "1": 1023, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0 }, "no_finding_images": 1477, "any_disease_images": 1023, "threshold_supported_disease_images": 0, "ranked_suggestion_images": 1023, "ranked_suggestion_correct": 259, "ranked_suggestion_precision": 0.25317693059628543, "note": "All independently qualifying screening flags, otherwise No Finding under its separate rule, otherwise one explicitly unverified top disease suggestion. Existing internal test predictions audit the frozen presentation; weights are unchanged.", "multilabel": { "all": { "images": 2500, "tp": 259, "fp": 764, "fn": 768, "precision": 0.25317693059628543, "recall": 0.25219084712755596, "f1": 0.2526829268292683, "displayed_multiple_images": 0, "at_least_two_correct_images": 0, "all_annotations_recovered_images": 150, "exact_disease_set_images": 1345 }, "multiannotated": { "images": 198, "tp": 109, "fp": 50, "fn": 349, "precision": 0.6855345911949685, "recall": 0.23799126637554585, "f1": 0.353322528363047, "displayed_multiple_images": 0, "at_least_two_correct_images": 0, "all_annotations_recovered_images": 0, "exact_disease_set_images": 0 }, "count_bins": [ "0", "1", "2", "3", "4+" ], "count_matrix": [ [ 1195, 538, 0, 0, 0 ], [ 243, 326, 0, 0, 0 ], [ 35, 116, 0, 0, 0 ], [ 4, 30, 0, 0, 0 ], [ 0, 13, 0, 0, 0 ] ], "annotated_counts": [ 1733, 569, 151, 34, 13 ], "displayed_counts": [ 1477, 1023, 0, 0, 0 ], "definitions": { "multiannotated": "Scans with at least two retained disease labels in the CSV; excludes No Finding, Hernia, Pneumonia and Fibrosis.", "at_least_two_correct_images": "At least two displayed labels match CSV disease annotations; additional false positives may be present.", "all_annotations_recovered_images": "Every retained CSV disease is displayed; additional false positives may be present.", "exact_disease_set_images": "Displayed disease set equals retained CSV disease set, with neither extra nor missing disease labels.", "count_matrix": "Rows: CSV disease count; columns: displayed disease count. Equal counts do not imply correct labels." } } }, "constant_no_finding_baseline": { "test": { "label_accuracy": 0.9389333333333333, "exact_match": 0.678 }, "train": { "label_accuracy": 0.9019533333333334, "exact_match": 0.53131 }, "validation": { "label_accuracy": 0.9426666666666667, "exact_match": 0.696 } } }, "comparison": { "criterion": "Exact macro AUROC for the 11 retained disease labels on fresh patient-disjoint validation images; no test-set selection", "compared_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration", "Mass", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening" ], "baseline_validation_macro_auc": 0.7819521210614947, "candidate_validation_macro_auc": 0.7980295546775648, "validation_improved": true, "best_epoch": 2, "baseline_test_macro_auc": 0.7968149340977358, "candidate_test_macro_auc": 0.8066556271868311 }, "notes": "The 100,000-image model is unchanged. Reader version 3 uses frozen validation-selected F0.5 thresholds to show all qualifying disease screening flags. It retains the separate No Finding rule and a user-requested unverified top-score fallback. Threshold selection changes precision and recall, not score accuracy or AUROC. Labels are report-derived and can be wrong. Existing internal test predictions were reused for the policy audit, not an independent clinical evaluation. Ranked suggestions matched 99 of 599 test annotations.", "intended_use": "Research and education only. Not for medical decisions or diagnosis.", "sources": [ "https://www.youtube.com/watch?v=8m3LvPg8EuI", "https://www.kaggle.com/datasets/nih-chest-xrays/data", "https://colab.research.google.com/drive/1klBxr93NYXrLFOVMXhm0RbVm5PIjfXQn", "https://keras.io/guides/transfer_learning/", "https://stanfordmlgroup.github.io/projects/chexnet/", "https://keras.io/guides/transfer_learning/", "https://stanfordmlgroup.github.io/projects/chexnet/" ], "browser_export": { "format": "ONNX", "opset": 15, "tensorflow_onnx_parity_images": 8, "max_absolute_error": 2.485513687133789e-05, "onnx_sha256": "efdce74f167d220367f60d011a2b2c34cd3c6c190307be8abef1b088a1f2c0d5", "outputs": [ "findings", "attention" ], "attention_method": "Positive class activation mapping from final convolutional features and the trained linear classifier; not validated lesion localization" }, "browser_verification": { "passed": true, "engine": "onnxruntime-web 1.20.1 WASM", "images": 8, "attention_verified": true, "max_absolute_error": 4.44948673248291e-05, "verified_at": "2026-09-14T19:08:18.332Z" }, "reader_policy": { "version": 3, "minimum_labels": 1, "frozen_policy_sha256": "4f43afb1ca90475aa4fd487505312fda2c615be9b92df7adc2f035119fb609e0", "frozen_at": "2026-09-15T04:25:05.0502965Z", "fallback": "Highest disease score, explicitly labeled as an unverified suggestion; ties use alphabetical label order.", "no_finding": "Only when no disease passes; score at least 0.76 and at least every disease score. Not proof of a healthy chest.", "additional_findings": "All independent validation-selected F0.5 screening flags; no count cap. These are possible findings, not high-confidence diagnoses.", "reason": "Correct the prior rule that disabled all disease thresholds. Weights and ranking accuracy are unchanged.", "evaluation_note": "Compare the frozen policy with the previous reader on all 2500 existing internal test images and the subset with at least two retained disease annotations. Report precision, recall, F1, recovery, false positives and cardinality. This is repeated internal evaluation, not a fresh independent benchmark; weights are unchanged." }, "latest_experiment": { "completed_at": "2026-09-16T00:48:42.978878+00:00", "promoted": false, "candidate_model_version": "95a885d89397", "published_baseline_model_version": "2ba1863c58d4", "gates": { "validation_improved": true, "chestdr_macro_ap_not_lower": true, "nih_macro_ap_within_0_005": true, "chestdr_macro_auc_within_0_01": true, "nih_macro_auc_within_0_01": false, "chestdr_multiple_recovery_not_lower": true }, "selection": { "selected_epoch": 2, "selected_model": "D:\\CodexChestXray\\v4\\models\\stage-b\\epoch-2.h5", "metrics": { "nih_macro_ap": 0.1987845254061533, "nih_macro_auc": 0.7908012783617082, "nih_no_finding_ap": 0.8110338296258761, "chestdr_macro_ap": 0.3054831361230309, "chestdr_macro_auc": 0.7544236066596302, "composite_ap": 0.2521338307645921 } }, "stage_a_selection": { "selected_epoch": 3, "selected_model": "D:\\CodexChestXray\\v4\\models\\stage-a\\epoch-3.h5", "metrics": { "disease_macro_ap": 0.21125598755471128, "disease_micro_ap": 0.2322027041645885, "disease_macro_auc": 0.7991923125287663, "no_finding_ap": 0.8146655065448212 }, "baseline": { "disease_macro_ap": 0.19363851535899856, "disease_micro_ap": 0.2309219678999035, "disease_macro_auc": 0.7980295546775648, "no_finding_ap": 0.8185128092316982 } }, "learning_phases": [ { "id": "nih", "name": "NIH continuation", "rows": [ { "epoch": 1, "loss": 0.3513324558734894, "bce": 0.24765704572200775, "label_accuracy": 0.9086574912071228, "exact_match_accuracy": 0.4378499984741211, "val_loss": 0.26090511679649353, "val_bce": 0.18111693859100342, "val_label_accuracy": 0.9419999718666077, "val_exact_match_accuracy": 0.6467999815940857, "disease_macro_ap": 0.20456425995741628, "disease_micro_ap": 0.22954963622839586, "disease_macro_auc": 0.7996396186515607, "no_finding_ap": 0.8142045632689419, "eligible": true, "improved": true, "stale_epochs": 0, "elapsed_seconds": 3198, "learning_rate": 9.999999747378752e-06, "next_learning_rate": 9.999999747378752e-06 }, { "epoch": 2, "loss": 0.34492015838623047, "bce": 0.24514257907867432, "label_accuracy": 0.9086283445358276, "exact_match_accuracy": 0.43935999274253845, "val_loss": 0.2658851444721222, "val_bce": 0.1856166273355484, "val_label_accuracy": 0.9380333423614502, "val_exact_match_accuracy": 0.6187999844551086, "disease_macro_ap": 0.20459204378326123, "disease_micro_ap": 0.22629019714570794, "disease_macro_auc": 0.7961730628409431, "no_finding_ap": 0.8163761883126819, "eligible": true, "improved": false, "stale_epochs": 1, "elapsed_seconds": 2719, "learning_rate": 9.999999747378752e-06, "next_learning_rate": 4.999999873689376e-06 }, { "epoch": 3, "loss": 0.33662670850753784, "bce": 0.24039334058761597, "label_accuracy": 0.9093966484069824, "exact_match_accuracy": 0.4433700144290924, "val_loss": 0.2618282437324524, "val_bce": 0.17685481905937195, "val_label_accuracy": 0.941100001335144, "val_exact_match_accuracy": 0.6399999856948853, "disease_macro_ap": 0.21125598755471128, "disease_micro_ap": 0.2322027041645885, "disease_macro_auc": 0.7991923125287663, "no_finding_ap": 0.8146655065448212, "eligible": true, "improved": true, "stale_epochs": 0, "elapsed_seconds": 2711, "learning_rate": 4.999999873689376e-06, "next_learning_rate": 4.999999873689376e-06 } ], "training_images": 100000, "validation_images": 2500, "loss_key": "bce", "accuracy_key": "label_accuracy", "exact_key": "exact_match_accuracy", "note": "100,000 NIH training images; 2,500 NIH validation images. Loss is unweighted binary cross-entropy. Label agreement uses score 0.5; negatives dominate.", "baseline": { "disease_macro_ap": 0.19363851535899856, "disease_micro_ap": 0.2309219678999035, "disease_macro_auc": 0.7980295546775648, "no_finding_ap": 0.8185128092316982 } }, { "id": "expert", "name": "Expert annotation adaptation", "rows": [ { "epoch": 1, "loss": 0.36173081398010254, "masked_bce": 0.2558576166629791, "known_label_accuracy": 0.9047931432723999, "known_exact_match_accuracy": 0.4307299852371216, "val_loss": 0.2818027436733246, "val_masked_bce": 0.19365648925304413, "val_known_label_accuracy": 0.9350094199180603, "val_known_exact_match_accuracy": 0.6102362275123596, "nih_macro_ap": 0.19978923353609693, "nih_macro_auc": 0.7955664375560663, "nih_no_finding_ap": 0.8123340781298496, "chestdr_macro_ap": 0.2988494380796268, "chestdr_macro_auc": 0.7474354501482624, "composite_ap": 0.24931933580786186, "eligible": true, "improved": true, "stale_epochs": 0, "elapsed_seconds": 726, "learning_rate": 3.000000106112566e-06, "next_learning_rate": 3.000000106112566e-06 }, { "epoch": 2, "loss": 0.3446730077266693, "masked_bce": 0.24610574543476105, "known_label_accuracy": 0.9066166877746582, "known_exact_match_accuracy": 0.43720564246177673, "val_loss": 0.28278490900993347, "val_masked_bce": 0.19539499282836914, "val_known_label_accuracy": 0.9339556694030762, "val_known_exact_match_accuracy": 0.604151725769043, "nih_macro_ap": 0.1987845254061533, "nih_macro_auc": 0.7908012783617082, "nih_no_finding_ap": 0.8110338296258761, "chestdr_macro_ap": 0.3054831361230309, "chestdr_macro_auc": 0.7544236066596302, "composite_ap": 0.2521338307645921, "eligible": true, "improved": true, "stale_epochs": 0, "elapsed_seconds": 712, "learning_rate": 3.000000106112566e-06, "next_learning_rate": 3.000000106112566e-06 }, { "epoch": 3, "loss": 0.3333965837955475, "masked_bce": 0.23949754238128662, "known_label_accuracy": 0.9088824391365051, "known_exact_match_accuracy": 0.4434850811958313, "val_loss": 0.2837424874305725, "val_masked_bce": 0.19042086601257324, "val_known_label_accuracy": 0.935566246509552, "val_known_exact_match_accuracy": 0.6163206696510315, "nih_macro_ap": 0.19718437621971222, "nih_macro_auc": 0.7895139195469238, "nih_no_finding_ap": 0.8109477513319743, "chestdr_macro_ap": 0.30652617571497903, "chestdr_macro_auc": 0.753081296597412, "composite_ap": 0.25185527596734564, "eligible": true, "improved": false, "stale_epochs": 1, "elapsed_seconds": 712, "learning_rate": 3.000000106112566e-06, "next_learning_rate": 1.500000053056283e-06 } ], "training_images": 20685, "training_presentations": 25480, "validation_images": 2794, "loss_key": "masked_bce", "accuracy_key": "known_label_accuracy", "exact_key": "known_exact_match_accuracy", "note": "20,000 NIH replay images plus 685 ChestDR training images. Each ChestDR image is presented eight times per epoch with augmentation: 25,480 presentations, 20,685 unique images. Validation combines 2,500 NIH and 294 ChestDR images. Unknown targets are masked; agreement is averaged per image.", "baseline": { "nih_macro_ap": 0.19363851535899856, "nih_macro_auc": 0.7980295546775648, "nih_no_finding_ap": 0.8185128092316982, "chestdr_macro_ap": 0.27248078586690594, "chestdr_macro_auc": 0.7078754337467734, "composite_ap": 0.23305965061295225 } } ], "dataset": { "dataset": "ChestDR / MedFMC", "license": "CC0", "source_url": "https://springernature.figshare.com/articles/dataset/ChestDR_Thoracic_Diseases_Screening_in_Chest_Radiography/22302775", "paper": "https://www.nature.com/articles/s41597-023-02460-0", "annotation": "Initial annotations by a radiology resident, supported by signed reports, confirmed by a senior radiologist.", "images_per_patient": 1, "demographic_limit": "Age metadata unavailable; adult-only inclusion is not established. No pediatric performance claim.", "official_split": "Preserved 3869-image test; official 979-image few-shot pool split by deterministic target-only iterative stratification into 685 training and294 validation.", "seed": 20260915, "mapping": { "Atelectasis": "atelectasis", "Cardiomegaly": "cardiomegaly", "Effusion": "pleural_effusion", "Nodule": "nodule", "Pneumothorax": "pneumothorax", "Consolidation": "consolidation", "Edema": "pulmonary_edema", "Emphysema": "emphysema", "Pleural_Thickening": "thickened_pleura" }, "unknown_labels": [ "Infiltration", "Mass", "No Finding" ], "no_finding_note": "No Finding is unannotated. All19negative is not an NIH No Finding label.", "source_csv_sha256": "bd6af813db773de367649537e841e711adcfd500c4f8f7784c1fd4821303f75a", "manifest_sha256": "98432e7eca10c3803146b5c8104434524d6976059ec7d9df496c48836c3eebef", "images": 4848, "quarantined": 0, "quarantine_reasons": {}, "source_label_counts": { "pleural_effusion": 1266, "nodule": 665, "pneumonia": 898, "cardiomegaly": 1300, "hilar_enlargement": 758, "fracture_old": 429, "fibrosis": 532, "aortic_calcification": 645, "tortuous_aorta": 612, "thickened_pleura": 497, "TB": 305, "pneumothorax": 284, "emphysema": 159, "atelectasis": 199, "calcification": 104, "pulmonary_edema": 86, "increased_lung_markings": 53, "elevated_diaphragm": 23, "consolidation": 34 }, "splits": { "test": { "images": 3869, "patients": 3869, "positives": { "Atelectasis": 156, "Cardiomegaly": 1055, "Effusion": 1011, "Nodule": 536, "Pneumothorax": 235, "Consolidation": 24, "Edema": 67, "Emphysema": 125, "Pleural_Thickening": 385 }, "all19negative": 987, "multiannotated": 1004 }, "train": { "images": 685, "patients": 685, "positives": { "Atelectasis": 30, "Cardiomegaly": 171, "Effusion": 178, "Nodule": 90, "Pneumothorax": 34, "Consolidation": 7, "Edema": 13, "Emphysema": 24, "Pleural_Thickening": 78 }, "all19negative": 177, "multiannotated": 182 }, "validation": { "images": 294, "patients": 294, "positives": { "Atelectasis": 13, "Cardiomegaly": 74, "Effusion": 77, "Nodule": 39, "Pneumothorax": 15, "Consolidation": 3, "Edema": 6, "Emphysema": 10, "Pleural_Thickening": 34 }, "all19negative": 84, "multiannotated": 80 } }, "patient_overlap": 0, "duplicate_note": "Exact hashes of299pxgrayscale arrays checked against all105kNIH scans and acrossChestDRsplits. This does not detect every near-duplicate.", "selection_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Nodule", "Pneumothorax", "Emphysema", "Pleural_Thickening" ] }, "paired_comparisons": { "nih": { "status": "ok", "cohort": "nih", "fixed_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration", "Mass", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening" ], "images": 2500, "patients": 1952, "unit_resampled": "patient_cluster", "estimand": "image-weighted macro metrics; patient-cluster uncertainty", "difference": "candidate minus baseline", "seed": 20260917, "requested_draws": 500, "valid_draws": 500, "excluded_draws": 0, "minimum_valid_draws": 475, "undefined_resample_counts": { "Atelectasis": 0, "Cardiomegaly": 0, "Effusion": 0, "Infiltration": 0, "Mass": 0, "Nodule": 0, "Pneumothorax": 0, "Consolidation": 0, "Edema": 0, "Emphysema": 0, "Pleural_Thickening": 0 }, "confidence": 0.95, "ci_method": "paired patient percentile, unadjusted", "per_label": { "Atelectasis": { "positive_images": 153, "negative_images": 2347, "baseline": { "AP": 0.261619419787621, "AUROC": 0.8090790356761935 }, "candidate": { "AP": 0.2991649572209091, "AUROC": 0.8093575166183502 } }, "Cardiomegaly": { "positive_images": 75, "negative_images": 2425, "baseline": { "AP": 0.2651163116145639, "AUROC": 0.9033072164948454 }, "candidate": { "AP": 0.24196296462751452, "AUROC": 0.8899353951890034 } }, "Effusion": { "positive_images": 118, "negative_images": 2382, "baseline": { "AP": 0.5111580918086458, "AUROC": 0.8997886692567135 }, "candidate": { "AP": 0.5047017887319452, "AUROC": 0.8914065946576727 } }, "Infiltration": { "positive_images": 304, "negative_images": 2196, "baseline": { "AP": 0.19735475705694383, "AUROC": 0.6542157990604928 }, "candidate": { "AP": 0.19630552068818422, "AUROC": 0.6333637414917074 } }, "Mass": { "positive_images": 100, "negative_images": 2400, "baseline": { "AP": 0.27279609291847723, "AUROC": 0.8433041666666666 }, "candidate": { "AP": 0.2587887040331392, "AUROC": 0.835525 } }, "Nodule": { "positive_images": 111, "negative_images": 2389, "baseline": { "AP": 0.12771419903676356, "AUROC": 0.6913820475980376 }, "candidate": { "AP": 0.11847515784512436, "AUROC": 0.6408727689598347 } }, "Pneumothorax": { "positive_images": 30, "negative_images": 2470, "baseline": { "AP": 0.14665970652262905, "AUROC": 0.8216059379217274 }, "candidate": { "AP": 0.1467048651264053, "AUROC": 0.8233468286099865 } }, "Consolidation": { "positive_images": 37, "negative_images": 2463, "baseline": { "AP": 0.07198044029631433, "AUROC": 0.7574700156916966 }, "candidate": { "AP": 0.07321556934691427, "AUROC": 0.7359186226421306 } }, "Edema": { "positive_images": 12, "negative_images": 2488, "baseline": { "AP": 0.2902479385325049, "AUROC": 0.9540125937834941 }, "candidate": { "AP": 0.3842699223052681, "AUROC": 0.948787513397642 } }, "Emphysema": { "positive_images": 27, "negative_images": 2473, "baseline": { "AP": 0.03832165288769015, "AUROC": 0.8014257686720283 }, "candidate": { "AP": 0.04253946095113027, "AUROC": 0.8014257686720283 } }, "Pleural_Thickening": { "positive_images": 60, "negative_images": 2440, "baseline": { "AP": 0.10809728946389464, "AUROC": 0.7376502732240438 }, "candidate": { "AP": 0.08593934808128328, "AUROC": 0.741639344262295 } } }, "macro": { "AP": { "baseline": 0.20827871817509527, "candidate": 0.2138243871779834, "difference": 0.005545669002888137, "baseline_ci": [ 0.18964524187213783, 0.2501342405163364 ], "candidate_ci": [ 0.1886944237132882, 0.25776163197662555 ], "difference_ci": [ -0.013323973698272821, 0.024343840472265142 ] }, "AUROC": { "baseline": 0.8066583203678128, "candidate": 0.7955980995000591, "difference": -0.011060220867753712, "baseline_ci": [ 0.7837315358761866, 0.8236750046841083 ], "candidate_ci": [ 0.7723533896582969, 0.8124736106868118 ], "difference_ci": [ -0.019131536741194617, -0.002855025168713824 ] } }, "interpretation": "ChestDR macro AP is primary; NIH macro AP secondary; AUROC supportive. Intervals do not prove clinical utility." }, "chestdr": { "status": "ok", "cohort": "chestdr", "fixed_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening" ], "images": 3869, "unit_resampled": "patient or exact-image dependence cluster", "estimand": "image-weighted macro metrics; patient-cluster uncertainty", "difference": "candidate minus baseline", "seed": 20260917, "requested_draws": 500, "valid_draws": 500, "excluded_draws": 0, "minimum_valid_draws": 475, "undefined_resample_counts": { "Atelectasis": 0, "Cardiomegaly": 0, "Effusion": 0, "Nodule": 0, "Pneumothorax": 0, "Consolidation": 0, "Edema": 0, "Emphysema": 0, "Pleural_Thickening": 0 }, "confidence": 0.95, "ci_method": "paired patient percentile, unadjusted", "per_label": { "Atelectasis": { "positive_images": 156, "negative_images": 3713, "baseline": { "AP": 0.0544703319179438, "AUROC": 0.6181745357613927 }, "candidate": { "AP": 0.0763108158970721, "AUROC": 0.6819801528931613 } }, "Cardiomegaly": { "positive_images": 1055, "negative_images": 2814, "baseline": { "AP": 0.5633163878355484, "AUROC": 0.7814679143214193 }, "candidate": { "AP": 0.6267954611407653, "AUROC": 0.8279477696150257 } }, "Effusion": { "positive_images": 1011, "negative_images": 2858, "baseline": { "AP": 0.661510363714628, "AUROC": 0.8280385320605599 }, "candidate": { "AP": 0.7001756274988085, "AUROC": 0.8548198646241933 } }, "Nodule": { "positive_images": 536, "negative_images": 3333, "baseline": { "AP": 0.1900898937360009, "AUROC": 0.5910882132989418 }, "candidate": { "AP": 0.22237933576019864, "AUROC": 0.6412878228121319 } }, "Pneumothorax": { "positive_images": 235, "negative_images": 3634, "baseline": { "AP": 0.29925341810852657, "AUROC": 0.8235517980304219 }, "candidate": { "AP": 0.32510795273517773, "AUROC": 0.8441831871567583 } }, "Consolidation": { "positive_images": 24, "negative_images": 3845, "baseline": { "AP": 0.03570982917877497, "AUROC": 0.7807650628521889 }, "candidate": { "AP": 0.03763017158716445, "AUROC": 0.7445925444299957 } }, "Edema": { "positive_images": 67, "negative_images": 3802, "baseline": { "AP": 0.39543683485141956, "AUROC": 0.9452173639953835 }, "candidate": { "AP": 0.35220730342391127, "AUROC": 0.928262422762568 } }, "Emphysema": { "positive_images": 125, "negative_images": 3744, "baseline": { "AP": 0.06165304731824346, "AUROC": 0.6890790598290598 }, "candidate": { "AP": 0.10913408442011044, "AUROC": 0.7625384615384615 } }, "Pleural_Thickening": { "positive_images": 385, "negative_images": 3484, "baseline": { "AP": 0.19349205646506226, "AUROC": 0.6744859617993947 }, "candidate": { "AP": 0.20209890154770863, "AUROC": 0.6909985536851209 } } }, "macro": { "AP": { "baseline": 0.27277024034734976, "candidate": 0.2946488504456575, "difference": 0.021878610098307716, "baseline_ci": [ 0.26024450614946165, 0.2926102632940224 ], "candidate_ci": [ 0.28083018115310926, 0.31580342727413246 ], "difference_ci": [ 0.0114170067771829, 0.03282440216700731 ] }, "AUROC": { "baseline": 0.7479853824387513, "candidate": 0.7751789755019352, "difference": 0.027193593063183807, "baseline_ci": [ 0.7320974102715636, 0.7619782453612373 ], "candidate_ci": [ 0.7590644535192911, 0.7892977794592677 ], "difference_ci": [ 0.019955208811465144, 0.03368840559562395 ] } }, "interpretation": "ChestDR macro AP is primary; NIH macro AP secondary; AUROC supportive. Intervals do not prove clinical utility.", "dependence_clusters": 3868, "nominal_patient_ids": 3869 } }, "note": "The experiment completed, but the candidate did not pass every frozen release check. The previous model remains active. These experiment curves do not describe a newly deployed model.", "duplicate_note": "One byte-identical test pair has conflicting Cardiomegaly annotations. All 3869 official rows remain; uncertainty uses 3868 dependence clusters. No Finding remains unannotated." }, "external_evaluation": { "dataset": "ChestDR", "model_evaluated": "published", "images": 3869, "known_labels": [ "Atelectasis", "Cardiomegaly", "Effusion", "Nodule", "Pneumothorax", "Consolidation", "Edema", "Emphysema", "Pleural_Thickening" ], "unknown_labels": [ "Infiltration", "Mass", "No Finding" ], "macro_ap": 0.27277024034734976, "macro_auc": 0.7479853824387513, "evidence_scope": "Official ChestDR test, held out from adaptation and threshold selection. Same dataset as adaptation, not a never-seen hospital. Nine radiologist-confirmed labels; No Finding, Mass and Infiltration unannotated." } }