{"rank":null,"reference":true,"run_id":"780a92bc3ebf","task":"synthetic-snuh-text2sql@0","model":{"id":"benchtoward/prior","slug":"benchtoward/prior","name":"Prior (ignores the input)","type":"baseline","developer":"BenchToward"},"tier":"baseline","metric":{"name":"rs0","value":0.1782178217821782,"ci_low":0.10891089108910891,"ci_high":0.25742574257425743},"n":510,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-30","url":"https://benchtoward.com/r/780a92bc3ebf","status":"done","listed":true,"current_spec":true,"metrics":{"hard_ex":{"value":0.0,"ci_low":0.0,"ci_high":0.0,"n":166,"n_groups":83,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"rs0":{"value":0.1782178217821782,"ci_low":0.10891089108910891,"ci_high":0.25742574257425743,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rs5":{"value":0.1782178217821782,"ci_low":0.10891089108910891,"ci_high":0.25742574257425743,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rsN":{"value":0.1782178217821782,"ci_low":0.10891089108910891,"ci_high":0.25742574257425743,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0","N":202},"abstention":{"answerable_abstained":{"value":1.0,"n":166},"unanswerable_abstained":{"value":1.0,"n":36}},"easy_ex":{"value":0.0,"ci_low":0.0,"ci_high":0.0,"n":308,"n_groups":154,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"answers":{"hard":{"sql":0,"abstain":202,"cut_off":0,"format_failure":0,"api_error":0},"easy":{"sql":0,"abstain":308,"cut_off":0,"format_failure":0,"api_error":0}},"cut_off_rate":{"hard":0.0,"easy":0.0,"all":0.0},"format_failure_rate":{"hard":0.0,"easy":0.0,"all":0.0},"api_error_rate":0.0,"parse_rate":1.0,"tiers":{"hard":{"by_language":{"ko":{"value":0.0,"n":83},"en":{"value":0.0,"n":83}},"by_category":{"clinical_terms":{"value":0.0,"n":42},"cohort_definition":{"value":0.0,"n":42},"data_quality":{"value":0.0,"n":40},"relative_time":{"value":0.0,"n":42},"unanswerable":{"value":1.0,"n":36}},"by_difficulty":{"hard":{"value":0.0,"n":166}},"sql_rate":0.0,"execution_error_rate":0.0,"errors":{"abstained":166},"prompt":"text2sql-omop-hard-p0","scorer":"ex-rs-0.1","reliability":{"RS(0)":0.1782178217821782,"RS(5)":0.1782178217821782,"RS(N)":0.1782178217821782,"N":202,"by_language":{"ko":{"RS(0)":0.1782178217821782,"RS(5)":0.1782178217821782},"en":{"RS(0)":0.1782178217821782,"RS(5)":0.1782178217821782}}},"n_questions":101,"n_instances":202},"easy":{"by_language":{"ko":{"value":0.0,"n":154},"en":{"value":0.0,"n":154}},"by_category":{"patient_lookup":{"value":0.0,"n":48},"cohort_count":{"value":0.0,"n":54},"aggregation":{"value":0.0,"n":54},"temporal":{"value":0.0,"n":52},"multi_domain_join":{"value":0.0,"n":52},"vocabulary_lookup":{"value":0.0,"n":48}},"by_difficulty":{"easy":{"value":0.0,"n":72},"medium":{"value":0.0,"n":176},"hard":{"value":0.0,"n":60}},"sql_rate":0.0,"execution_error_rate":0.0,"errors":{"no_sql":308},"prompt":"text2sql-omop-p0","scorer":"ex-0.1","n_questions":154,"n_instances":308}},"scorer":{"execution":"ex-0.1 (ex-rs-0.1 with abstention)","reliability":"glee4810/ehrsql-2024@f9e1aa0 scoring_utils.reliability_score and penalize","intervals":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"prompts":{"hard":"text2sql-omop-hard-p0","easy":"text2sql-omop-p0"}},"config":{"prompts":{"easy":"text2sql-omop-p0","hard":"text2sql-omop-hard-p0"},"rule":"abstain on every item of both tiers (the easy tier has no abstention: no answer there)","tiers":["hard","easy"]},"run_key":"2a388480532d68dbc508359115f853a02020a451ef08a1402f5290cdc99b633d","previous_run_key":"7fb85f57af7ebae7cd26853d553538a2179f9284f6c109b5345da6c39508fdfc","harness_version":"p0.1","task_spec_hash":"ac017b1c69ca1976ff65ebd4469eb681344d4db9dacf9f137f61347344340a25","model_id":"benchtoward/prior","tokens_in":0,"tokens_out":0,"dataset":{"slug":"synthetic-snuh","title":"Synthetic-SNUH v4","license":"CC BY-NC 4.0","license_url":"https://creativecommons.org/licenses/by-nc/4.0/","citation":"Lee H-C, Seoul National University Hospital. Synthetic-SNUH v4.0.0: synthetic OMOP CDM and KR Core FHIR records for 10,000 virtual patients (version 4.0.0). KHDP. 2026. doi:10.23421/2026.03183"},"predictions_url":"https://benchtoward.com/artifacts/780a92bc3ebf/predictions.parquet","bibtex":"@misc{benchtoward_780a92bc3ebf,\n  title = {Prior (ignores the input) on Synthetic-SNUH Text-to-SQL (OMOP CDM, Korean and English) (synthetic-snuh-text2sql@0)},\n  author = {{BenchToward}},\n  year = {2026},\n  howpublished = {\\url{https://benchtoward.com/r/780a92bc3ebf}},\n  note = {Evaluated 2026-10-30, harness p0.1, run key 2a388480532d}\n}"}