{"rank":null,"reference":false,"run_id":"ed9864699f04","task":"synthetic-snuh-text2sql@0","model":{"id":"deepseek/deepseek-v4.1-flash-552b-a16b","slug":"deepseek/deepseek-v4.1-flash-552b-a16b","name":"DeepSeek V4.1 Flash 552B-A16B","type":"llm","developer":"DeepSeek"},"tier":"standard","metric":{"name":"rs0","value":0.6633663366336634,"ci_low":0.5792079207920792,"ci_high":0.7426980198019795},"n":510,"parse_rate":0.9509803921568627,"cost_usd":0.0,"eval_date":"2026-10-30","url":"https://benchtoward.com/r/ed9864699f04","status":"done","listed":true,"current_spec":true,"metrics":{"hard_ex":{"value":0.5963855421686747,"ci_low":0.5,"ci_high":0.6927710843373494,"n":166,"n_groups":83,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"rs0":{"value":0.6633663366336634,"ci_low":0.5792079207920792,"ci_high":0.7426980198019795,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rs5":{"value":-1.0198019801980198,"ci_low":-1.5247524752475248,"ci_high":-0.5438118811881228,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rsN":{"value":-67.33663366336634,"ci_low":-84.42079207920793,"ci_high":-51.23230198019816,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0","N":202},"abstention":{"answerable_abstained":{"value":0.0,"n":166},"unanswerable_abstained":{"value":0.9722222222222222,"n":36}},"easy_ex":{"value":0.9935064935064936,"ci_low":0.9837662337662337,"ci_high":1.0,"n":308,"n_groups":154,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"answers":{"hard":{"sql":142,"abstain":35,"cut_off":25,"format_failure":0,"api_error":0},"easy":{"sql":308,"abstain":0,"cut_off":0,"format_failure":0,"api_error":0}},"cut_off_rate":{"hard":0.12376237623762376,"easy":0.0,"all":0.049019607843137254},"format_failure_rate":{"hard":0.0,"easy":0.0,"all":0.0},"api_error_rate":0.0,"parse_rate":0.9509803921568627,"tiers":{"hard":{"by_language":{"ko":{"value":0.6144578313253012,"n":83},"en":{"value":0.5783132530120482,"n":83}},"by_category":{"clinical_terms":{"value":0.38095238095238093,"n":42},"cohort_definition":{"value":0.38095238095238093,"n":42},"data_quality":{"value":0.95,"n":40},"relative_time":{"value":0.6904761904761905,"n":42},"unanswerable":{"value":0.9722222222222222,"n":36}},"by_difficulty":{"hard":{"value":0.5963855421686747,"n":166}},"sql_rate":0.7029702970297029,"execution_error_rate":0.0049504950495049506,"errors":{"no_sql":25,"execution_error":1},"prompt":"text2sql-omop-hard-p0","scorer":"ex-rs-0.1","reliability":{"RS(0)":0.6633663366336634,"RS(5)":-1.0198019801980198,"RS(N)":-67.33663366336634,"N":202,"by_language":{"ko":{"RS(0)":0.6732673267326733,"RS(5)":-0.9603960396039604},"en":{"RS(0)":0.6534653465346535,"RS(5)":-1.0792079207920793}}},"n_questions":101,"n_instances":202},"easy":{"by_language":{"ko":{"value":0.987012987012987,"n":154},"en":{"value":1.0,"n":154}},"by_category":{"patient_lookup":{"value":1.0,"n":48},"cohort_count":{"value":1.0,"n":54},"aggregation":{"value":1.0,"n":54},"temporal":{"value":0.9615384615384616,"n":52},"multi_domain_join":{"value":1.0,"n":52},"vocabulary_lookup":{"value":1.0,"n":48}},"by_difficulty":{"easy":{"value":1.0,"n":72},"medium":{"value":1.0,"n":176},"hard":{"value":0.9666666666666667,"n":60}},"sql_rate":1.0,"execution_error_rate":0.003246753246753247,"errors":{"syntax_error":1},"prompt":"text2sql-omop-p0","scorer":"ex-0.1","n_questions":154,"n_instances":308}},"scorer":{"execution":"ex-0.1 (ex-rs-0.1 with abstention)","reliability":"glee4810/ehrsql-2024@f9e1aa0 scoring_utils.reliability_score and penalize","intervals":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"prompts":{"hard":"text2sql-omop-hard-p0","easy":"text2sql-omop-p0"},"max_tokens":16384,"answers_from_cache":510},"config":{"max_tokens":16384,"prompts":{"easy":"text2sql-omop-p0","hard":"text2sql-omop-hard-p0"},"temperature":0,"tiers":["hard","easy"]},"run_key":"e24e020b0be633680db7319309dc4ee33ff3c903252023dd946291be5980be8c","previous_run_key":"3115b9abc418765dd7741997778312e9a8bd77f1fff65a23cdf5976f4a399442","harness_version":"text2sql-p0","task_spec_hash":"ac017b1c69ca1976ff65ebd4469eb681344d4db9dacf9f137f61347344340a25","model_id":"deepseek/deepseek-v4.1-flash-552b-a16b","tokens_in":2457440,"tokens_out":1586668,"dataset":{"slug":"synthetic-snuh","title":"Synthetic-SNUH v4","license":"CC BY-NC 4.0","license_url":"https://creativecommons.org/licenses/by-nc/4.0/","citation":"Lee H-C, Seoul National University Hospital. Synthetic-SNUH v4.0.0: synthetic OMOP CDM and KR Core FHIR records for 10,000 virtual patients (version 4.0.0). KHDP. 2026. doi:10.23421/2026.03183"},"predictions_url":"https://benchtoward.com/artifacts/ed9864699f04/predictions.parquet","bibtex":"@misc{benchtoward_ed9864699f04,\n  title = {DeepSeek V4.1 Flash 552B-A16B on Synthetic-SNUH Text-to-SQL (OMOP CDM, Korean and English) (synthetic-snuh-text2sql@0)},\n  author = {{BenchToward}},\n  year = {2026},\n  howpublished = {\\url{https://benchtoward.com/r/ed9864699f04}},\n  note = {Evaluated 2026-10-30, harness text2sql-p0, run key e24e020b0be6}\n}"}