{"rank":null,"reference":false,"run_id":"071979f7a992","task":"synthetic-snuh-text2sql@0","model":{"id":"openai/gpt-oss-20b","slug":"openai/gpt-oss-20b","name":"gpt-oss 20B","type":"llm","developer":"OpenAI"},"tier":"standard","metric":{"name":"rs0","value":0.22277227722772278,"ci_low":0.15841584158415842,"ci_high":0.29207920792079206},"n":510,"parse_rate":0.9509803921568627,"cost_usd":0.27080224420000004,"eval_date":"2026-10-30","url":"https://benchtoward.com/r/071979f7a992","status":"done","listed":true,"current_spec":true,"metrics":{"hard_ex":{"value":0.1927710843373494,"ci_low":0.12650602409638553,"ci_high":0.2710843373493976,"n":166,"n_groups":83,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"rs0":{"value":0.22277227722772278,"ci_low":0.15841584158415842,"ci_high":0.29207920792079206,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rs5":{"value":-3.613861386138614,"ci_low":-4.0049504950495045,"ci_high":-3.1829207920792095,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rsN":{"value":-154.77722772277227,"ci_low":-167.84653465346534,"ci_high":-139.71757425742575,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0","N":202},"abstention":{"answerable_abstained":{"value":0.012048192771084338,"n":166},"unanswerable_abstained":{"value":0.3611111111111111,"n":36}},"easy_ex":{"value":0.8993506493506493,"ci_low":0.8571428571428571,"ci_high":0.9383116883116883,"n":308,"n_groups":154,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"answers":{"hard":{"sql":162,"abstain":15,"cut_off":25,"format_failure":0,"api_error":0},"easy":{"sql":308,"abstain":0,"cut_off":0,"format_failure":0,"api_error":0}},"cut_off_rate":{"hard":0.12376237623762376,"easy":0.0,"all":0.049019607843137254},"format_failure_rate":{"hard":0.0,"easy":0.0,"all":0.0},"api_error_rate":0.0,"parse_rate":0.9509803921568627,"tiers":{"hard":{"by_language":{"ko":{"value":0.1927710843373494,"n":83},"en":{"value":0.1927710843373494,"n":83}},"by_category":{"clinical_terms":{"value":0.09523809523809523,"n":42},"cohort_definition":{"value":0.0,"n":42},"data_quality":{"value":0.45,"n":40},"relative_time":{"value":0.23809523809523808,"n":42},"unanswerable":{"value":0.3611111111111111,"n":36}},"by_difficulty":{"hard":{"value":0.1927710843373494,"n":166}},"sql_rate":0.801980198019802,"execution_error_rate":0.09900990099009901,"errors":{"execution_error":15,"syntax_error":5,"no_sql":25,"abstained":2},"prompt":"text2sql-omop-hard-p0","scorer":"ex-rs-0.1","reliability":{"RS(0)":0.22277227722772278,"RS(5)":-3.613861386138614,"RS(N)":-154.77722772277227,"N":202,"by_language":{"ko":{"RS(0)":0.2376237623762376,"RS(5)":-3.5247524752475248},"en":{"RS(0)":0.2079207920792079,"RS(5)":-3.702970297029703}}},"n_questions":101,"n_instances":202},"easy":{"by_language":{"ko":{"value":0.9025974025974026,"n":154},"en":{"value":0.8961038961038961,"n":154}},"by_category":{"patient_lookup":{"value":0.9583333333333334,"n":48},"cohort_count":{"value":0.9259259259259259,"n":54},"aggregation":{"value":0.8703703703703703,"n":54},"temporal":{"value":0.75,"n":52},"multi_domain_join":{"value":0.9423076923076923,"n":52},"vocabulary_lookup":{"value":0.9583333333333334,"n":48}},"by_difficulty":{"easy":{"value":0.9444444444444444,"n":72},"medium":{"value":0.9090909090909091,"n":176},"hard":{"value":0.8166666666666667,"n":60}},"sql_rate":1.0,"execution_error_rate":0.022727272727272728,"errors":{"execution_error":6,"syntax_error":1},"prompt":"text2sql-omop-p0","scorer":"ex-0.1","n_questions":154,"n_instances":308}},"scorer":{"execution":"ex-0.1 (ex-rs-0.1 with abstention)","reliability":"glee4810/ehrsql-2024@f9e1aa0 scoring_utils.reliability_score and penalize","intervals":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"prompts":{"hard":"text2sql-omop-hard-p0","easy":"text2sql-omop-p0"},"max_tokens":16384,"answers_from_cache":510},"config":{"max_tokens":16384,"prompts":{"easy":"text2sql-omop-p0","hard":"text2sql-omop-hard-p0"},"temperature":0,"tiers":["hard","easy"]},"run_key":"c492b3b663fb31442176f740278e4360f8120839406d404e5fcd056204a17994","previous_run_key":"0687d608194fbb1430fd8d6374e3cbf688f353f1cd3e0de420465aea6fe56854","harness_version":"text2sql-p0","task_spec_hash":"ac017b1c69ca1976ff65ebd4469eb681344d4db9dacf9f137f61347344340a25","model_id":"openai/gpt-oss-20b","tokens_in":2399594,"tokens_out":1432101,"dataset":{"slug":"synthetic-snuh","title":"Synthetic-SNUH v4","license":"CC BY-NC 4.0","license_url":"https://creativecommons.org/licenses/by-nc/4.0/","citation":"Lee H-C, Seoul National University Hospital. Synthetic-SNUH v4.0.0: synthetic OMOP CDM and KR Core FHIR records for 10,000 virtual patients (version 4.0.0). KHDP. 2026. doi:10.23421/2026.03183"},"predictions_url":"https://benchtoward.com/artifacts/071979f7a992/predictions.parquet","bibtex":"@misc{benchtoward_071979f7a992,\n  title = {gpt-oss 20B on Synthetic-SNUH Text-to-SQL (OMOP CDM, Korean and English) (synthetic-snuh-text2sql@0)},\n  author = {{BenchToward}},\n  year = {2026},\n  howpublished = {\\url{https://benchtoward.com/r/071979f7a992}},\n  note = {Evaluated 2026-10-30, harness text2sql-p0, run key c492b3b663fb}\n}"}