{"summary":{"run_at":"2026-10-23T11:26:25+00:00","questions":113,"run":113,"passed":113,"score_pct":100.0,"by_category":{"values":{"run":42,"passed":42},"calculations":{"run":28,"passed":28},"local":{"run":28,"passed":28},"guardrails":{"run":15,"passed":15}}},"results":[{"id":"V01","category":"values","question":"What is the latest US consumer price index (CPI-U, all items)?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.all_items","timeframe":"2Y"},"ours":"336.94 (2026-09-01)","expected":"336.94 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V02","category":"values","question":"What is the latest core CPI (all items less food and energy)?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.core","timeframe":"2Y"},"ours":"339.287 (2026-09-01)","expected":"339.287 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V03","category":"values","question":"What is the latest CPI for groceries (food at home)?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.food_at_home","timeframe":"2Y"},"ours":"321.758 (2026-09-01)","expected":"321.758 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V04","category":"values","question":"What is the latest CPI for restaurant meals (food away from home)?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.food_away","timeframe":"2Y"},"ours":"398.783 (2026-09-01)","expected":"398.783 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V05","category":"values","question":"What is the latest CPI for limited-service (fast food and counter-service) meals?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.limited_service_meals","timeframe":"2Y"},"ours":"260.209 (2026-09-01)","expected":"260.209 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V06","category":"values","question":"What is the latest CPI for full-service restaurant meals?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.full_service_meals","timeframe":"2Y"},"ours":"246.898 (2026-09-01)","expected":"246.898 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V07","category":"values","question":"What is the latest CPI for alcoholic drinks bought away from home?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.alcohol_away","timeframe":"2Y"},"ours":"458.827 (2026-09-01)","expected":"458.827 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V08","category":"values","question":"What is the latest CPI for eggs?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.eggs","timeframe":"2Y"},"ours":"279.288 (2026-09-01)","expected":"279.288 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V09","category":"values","question":"What is the latest CPI for coffee?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.coffee","timeframe":"2Y"},"ours":"299.101 (2026-09-01)","expected":"299.101 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V10","category":"values","question":"What is the latest CPI for beef and veal?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.beef","timeframe":"2Y"},"ours":"512.606 (2026-09-01)","expected":"512.606 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V11","category":"values","question":"What is the latest CPI for shelter?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.shelter","timeframe":"2Y"},"ours":"432.092 (2026-09-01)","expected":"432.092 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V12","category":"values","question":"What is the latest producer price index for final demand?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.final_demand","timeframe":"2Y"},"ours":"159.613 (2026-09-01)","expected":"159.613 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V13","category":"values","question":"What is the latest producer price index for processed foods and feeds?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.processed_foods","timeframe":"2Y"},"ours":"276.68 (2026-09-01)","expected":"276.68 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V14","category":"values","question":"What is the latest producer price index for truckload freight?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.truckload","timeframe":"2Y"},"ours":"219.384 (2026-09-01)","expected":"219.384 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V15","category":"values","question":"How many people are on US nonfarm payrolls?","tool":"get_metric_series","arguments":{"metric_id":"bls.ces.nonfarm_payrolls","timeframe":"2Y"},"ours":"159044 (2026-09-01)","expected":"159044 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V16","category":"values","question":"What is the US unemployment rate?","tool":"get_metric_series","arguments":{"metric_id":"bls.cps.unemployment_rate","timeframe":"2Y"},"ours":"4.2 (2026-09-01)","expected":"4.2 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V17","category":"values","question":"What is the average hourly wage in restaurants and bars?","tool":"get_metric_series","arguments":{"metric_id":"bls.ces.food_services_ahe","timeframe":"2Y"},"ours":"22.17 (2026-08-01)","expected":"22.17 (2026-08-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V18","category":"values","question":"What are average hourly earnings for private-sector workers?","tool":"get_metric_series","arguments":{"metric_id":"bls.ces.avg_hourly_earnings","timeframe":"2Y"},"ours":"37.81 (2026-09-01)","expected":"37.81 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V19","category":"values","question":"What were US retail sales excluding food services last month?","tool":"get_metric_series","arguments":{"metric_id":"census.retail_sales_ex_food","timeframe":"2Y"},"ours":"652115 (2026-09-01)","expected":"652115 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V20","category":"values","question":"What were US restaurant and bar sales last month?","tool":"get_metric_series","arguments":{"metric_id":"census.food_services_sales","timeframe":"2Y"},"ours":"92906 (2026-09-01)","expected":"92906 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V21","category":"values","question":"What were US grocery store sales last month?","tool":"get_metric_series","arguments":{"metric_id":"census.grocery_store_sales","timeframe":"2Y"},"ours":"77152 (2026-09-01)","expected":"77152 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V22","category":"values","question":"How many housing starts were there (annual rate)?","tool":"get_metric_series","arguments":{"metric_id":"census.housing_starts","timeframe":"2Y"},"ours":"1221 (2026-09-01)","expected":"1221 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V23","category":"values","question":"How many building permits were issued (annual rate)?","tool":"get_metric_series","arguments":{"metric_id":"census.building_permits","timeframe":"2Y"},"ours":"1400 (2026-09-01)","expected":"1400 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V24","category":"values","question":"How many new homes were sold (annual rate)?","tool":"get_metric_series","arguments":{"metric_id":"census.new_home_sales","timeframe":"2Y"},"ours":"684 (2026-08-01)","expected":"684 (2026-08-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V25","category":"values","question":"What is the effective federal funds rate?","tool":"get_metric_series","arguments":{"metric_id":"nyfed.effr","timeframe":"2Y"},"ours":"3.88 (2026-10-21)","expected":"3.88 (2026-10-21)","pass":true,"note":"","source":"Federal Reserve Bank of New York"},{"id":"V26","category":"values","question":"What is SOFR today?","tool":"get_metric_series","arguments":{"metric_id":"nyfed.sofr","timeframe":"2Y"},"ours":"3.88 (2026-10-21)","expected":"3.88 (2026-10-21)","pass":true,"note":"","source":"Federal Reserve Bank of New York"},{"id":"V27","category":"values","question":"What is the average US retail diesel price?","tool":"get_metric_series","arguments":{"metric_id":"eia.diesel_price","timeframe":"2Y"},"ours":"6.276 (2026-10-19)","expected":"6.276 (2026-10-19)","pass":true,"note":"","source":"US Energy Information Administration"},{"id":"V28","category":"values","question":"What is the WTI crude oil spot price?","tool":"get_metric_series","arguments":{"metric_id":"eia.wti_crude","timeframe":"2Y"},"ours":"107.89 (2026-10-20)","expected":"107.89 (2026-10-20)","pass":true,"note":"","source":"US Energy Information Administration"},{"id":"V29","category":"values","question":"What is the average US retail gasoline price?","tool":"get_metric_series","arguments":{"metric_id":"eia.gasoline_price","timeframe":"2Y"},"ours":"4.424 (2026-10-19)","expected":"4.424 (2026-10-19)","pass":true,"note":"","source":"US Energy Information Administration"},{"id":"V30","category":"values","question":"What is the latest industrial production index?","tool":"get_metric_series","arguments":{"metric_id":"frb.industrial_production","timeframe":"2Y"},"ours":"103.951 (2026-09-01)","expected":"103.951 (2026-09-01)","pass":true,"note":"","source":"Board of Governors of the Federal Reserve System"},{"id":"V31","category":"values","question":"What is the latest FHFA house price index?","tool":"get_metric_series","arguments":{"metric_id":"fhfa.hpi","timeframe":"2Y"},"ours":"443.52 (2026-07-01)","expected":"443.52 (2026-07-01)","pass":true,"note":"","source":"Federal Housing Finance Agency"},{"id":"V32","category":"values","question":"How much total public debt does the US have outstanding?","tool":"get_metric_series","arguments":{"metric_id":"fiscal.total_public_debt","timeframe":"2Y"},"ours":"40.4136 (2026-10-21)","expected":"40.4136 (2026-10-21)","pass":true,"note":"","source":"US Department of the Treasury (Fiscal Data)"},{"id":"V33","category":"values","question":"How many initial unemployment claims were filed last week?","tool":"get_metric_series","arguments":{"metric_id":"dol.ui.initial_claims","timeframe":"2Y"},"ours":"191033 (2026-10-10)","expected":"191033 (2026-10-10)","pass":true,"note":"","source":"US Dept. of Labor, Employment & Training Administration (ETA 539)"},{"id":"V34","category":"values","question":"What is the US personal saving rate?","tool":"get_metric_series","arguments":{"metric_id":"bea.saving_rate","timeframe":"2Y"},"ours":"4.1 (2026-08-01)","expected":"4.1 (2026-08-01)","pass":true,"note":"","source":"US Bureau of Economic Analysis"},{"id":"V35","category":"values","question":"What is the 10-year Treasury yield?","tool":"get_metric_series","arguments":{"metric_id":"treasury.yield","timeframe":"2Y","entity":"10Y"},"ours":"4.96 (2026-10-22)","expected":"4.96 (2026-10-22)","pass":true,"note":"","source":"US Treasury"},{"id":"V36","category":"values","question":"What share of US retail sales is e-commerce?","tool":"get_metric_series","arguments":{"metric_id":"census.ecommerce_share","timeframe":"2Y"},"ours":"17.1 (2026-04-01)","expected":"17.1 (2026-04-01)","pass":true,"note":"","source":"US Census Bureau (Quarterly E-Commerce Report)"},{"id":"V37","category":"values","question":"How much did Americans spend online last quarter?","tool":"get_metric_series","arguments":{"metric_id":"census.ecommerce_sales","timeframe":"2Y"},"ours":"340244 (2026-04-01)","expected":"340244 (2026-04-01)","pass":true,"note":"","source":"US Census Bureau (Quarterly E-Commerce Report)"},{"id":"V38","category":"values","question":"What were US clothing and accessories store sales last month?","tool":"get_metric_series","arguments":{"metric_id":"census.clothing_store_sales","timeframe":"2Y"},"ours":"26478 (2026-09-01)","expected":"26478 (2026-09-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V39","category":"values","question":"What were sales of online-first (electronic shopping) retailers?","tool":"get_metric_series","arguments":{"metric_id":"census.electronic_shopping_sales","timeframe":"2Y"},"ours":"116650 (2026-08-01)","expected":"116650 (2026-08-01)","pass":true,"note":"","source":"US Census Bureau (Economic Indicators)"},{"id":"V40","category":"values","question":"What is the latest producer price index for couriers and express delivery?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.couriers_express","timeframe":"2Y"},"ours":"400.821 (2026-09-01)","expected":"400.821 (2026-09-01)","pass":true,"note":"","source":"US Bureau of Labor Statistics (US Dept. of Labor)"},{"id":"V41","category":"values","question":"What does regular gasoline cost in California this week?","tool":"get_metric_series","arguments":{"metric_id":"eia.gasoline_regular_by_area","timeframe":"2Y","entity":"state:CA"},"ours":"6.031 (2026-10-19)","expected":"6.031 (2026-10-19)","pass":true,"note":"","source":"EIA weekly retail gasoline by area"},{"id":"V42","category":"values","question":"What did regular gasoline cost in the Atlanta metro area last month?","tool":"get_metric_series","arguments":{"metric_id":"bls.ap.gasoline_regular_by_area","timeframe":"2Y","entity":"metro:12060"},"ours":"4.023 (2026-09-01)","expected":"4.023 (2026-09-01)","pass":true,"note":"","source":"BLS average price data"},{"id":"C01","category":"calculations","question":"What is headline inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.all_items","timeframe":"2Y"},"ours":3.738,"expected":3.738,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C02","category":"calculations","question":"What is core inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.core","timeframe":"2Y"},"ours":2.564,"expected":2.564,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C03","category":"calculations","question":"What is grocery inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.food_at_home","timeframe":"2Y"},"ours":1.809,"expected":1.809,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C04","category":"calculations","question":"What is restaurant menu inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.food_away","timeframe":"2Y"},"ours":3.459,"expected":3.459,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C05","category":"calculations","question":"What is fast-food price inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.limited_service_meals","timeframe":"2Y"},"ours":3.188,"expected":3.188,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C06","category":"calculations","question":"What is sit-down restaurant price inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.full_service_meals","timeframe":"2Y"},"ours":3.766,"expected":3.766,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C07","category":"calculations","question":"What is inflation in drink prices at bars and restaurants year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.alcohol_away","timeframe":"2Y"},"ours":2.623,"expected":2.623,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C08","category":"calculations","question":"What is egg price inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.eggs","timeframe":"2Y"},"ours":-20.588,"expected":-20.588,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C09","category":"calculations","question":"What is food manufacturers' price inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.processed_foods","timeframe":"2Y"},"ours":-0.863,"expected":-0.863,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C10","category":"calculations","question":"What is producer price inflation year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.final_demand","timeframe":"2Y"},"ours":6.338,"expected":6.338,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the publisher's index values"},{"id":"C11","category":"calculations","question":"What is restaurant wage growth year over year?","tool":"get_metric_series","arguments":{"metric_id":"bls.ces.food_services_ahe","timeframe":"2Y"},"ours":4.329,"expected":4.329,"pass":true,"note":"2025-08-01 to 2026-08-01","source":"computed from the publisher's index values"},{"id":"C12","category":"calculations","question":"How much did restaurant and bar sales change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"census.food_services_sales","timeframe":"2Y"},"ours":6.989,"expected":6.989,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from publisher values"},{"id":"C13","category":"calculations","question":"How much did grocery store sales change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"census.grocery_store_sales","timeframe":"2Y"},"ours":0.913,"expected":0.913,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from publisher values"},{"id":"C14","category":"calculations","question":"How much did retail sales excluding food services change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"census.retail_sales_ex_food","timeframe":"2Y"},"ours":6.188,"expected":6.188,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from publisher values"},{"id":"C15","category":"calculations","question":"How much did the diesel price change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"eia.diesel_price","timeframe":"2Y"},"ours":73.37,"expected":73.37,"pass":true,"note":"2025-10-20 to 2026-10-19","source":"computed from publisher values"},{"id":"C16","category":"calculations","question":"How much did consumer spending on restaurant meals change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"bea.pce_purchased_meals","timeframe":"2Y"},"ours":5.229,"expected":5.229,"pass":true,"note":"2025-08-01 to 2026-08-01","source":"computed from publisher values"},{"id":"C17","category":"calculations","question":"How fast is e-commerce growing year over year?","tool":"get_metric_series","arguments":{"metric_id":"census.ecommerce_sales","timeframe":"2Y"},"ours":12.187,"expected":12.187,"pass":true,"note":"2025-04-01 to 2026-04-01","source":"computed from Census quarterly e-commerce values"},{"id":"C18","category":"calculations","question":"How much have parcel and express shipping prices risen in a year?","tool":"get_metric_series","arguments":{"metric_id":"bls.ppi.couriers_express","timeframe":"2Y"},"ours":11.369,"expected":11.369,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from the BLS producer price index"},{"id":"C19","category":"calculations","question":"How much did furniture store sales change from a year earlier?","tool":"get_metric_series","arguments":{"metric_id":"census.furniture_store_sales","timeframe":"2Y"},"ours":4.852,"expected":4.852,"pass":true,"note":"2025-09-01 to 2026-09-01","source":"computed from Census monthly values"},{"id":"C20","category":"calculations","question":"How many jobs did the US economy add last month?","tool":"get_metric_series","arguments":{"metric_id":"bls.ces.nonfarm_payrolls","timeframe":"2Y"},"ours":29.0,"expected":29.0,"pass":true,"note":"2026-08-01 to 2026-09-01","source":"BLS CES"},{"id":"C21","category":"calculations","question":"How much did the unemployment rate change last month?","tool":"get_metric_series","arguments":{"metric_id":"bls.cps.unemployment_rate","timeframe":"2Y"},"ours":0.1,"expected":0.1,"pass":true,"note":"2026-08-01 to 2026-09-01","source":"BLS CPS"},{"id":"C22","category":"calculations","question":"What is the 10-year minus 2-year Treasury spread?","tool":"get_metric_series","arguments":{"metric_id":"treasury.spread_10y_2y","timeframe":"2Y"},"ours":0.3099999999999996,"expected":0.3099999999999996,"pass":true,"note":"2026-10-22","source":"US Treasury yield curve"},{"id":"C23","category":"calculations","question":"What is the 10-year minus 3-month Treasury spread?","tool":"get_metric_series","arguments":{"metric_id":"treasury.spread_10y_3m","timeframe":"2Y"},"ours":0.7400000000000002,"expected":0.7400000000000002,"pass":true,"note":"2026-10-22","source":"US Treasury yield curve"},{"id":"C24","category":"calculations","question":"What were restaurant and bar sales two years ago in today's dollars?","tool":"get_metric_series","arguments":{"metric_id":"census.food_services_sales","timeframe":"2Y","adjust":"real"},"ours":86582.93,"expected":86582.93,"pass":true,"note":"2024-08-01 in 2026-08 dollars","source":"Census sales restated with BEA's PCE price index"},{"id":"C25","category":"calculations","question":"What were grocery store sales two years ago in today's dollars?","tool":"get_metric_series","arguments":{"metric_id":"census.grocery_store_sales","timeframe":"2Y","adjust":"real"},"ours":78332.9,"expected":78332.9,"pass":true,"note":"2024-08-01 in 2026-08 dollars","source":"Census sales restated with BEA's PCE price index"},{"id":"C26","category":"calculations","question":"Are restaurant costs rising faster than menu prices (cost-price gap)?","tool":"get_metric_series","arguments":{"metric_id":"horizon.restaurant_cost_pressure","timeframe":"2Y"},"ours":0.39,"expected":0.39,"pass":true,"note":"2026-08-01","source":"BEA input-output weights x BLS and EIA prices"},{"id":"C27","category":"calculations","question":"How much did the diesel price change in the latest week?","tool":"get_metric_series","arguments":{"metric_id":"eia.diesel_price","timeframe":"2Y"},"ours":0.097,"expected":0.097,"pass":true,"note":"2026-10-12 to 2026-10-19","source":"EIA weekly retail diesel"},{"id":"C28","category":"calculations","question":"How much did initial unemployment claims change from the week before?","tool":"get_metric_series","arguments":{"metric_id":"dol.ui.initial_claims","timeframe":"2Y"},"ours":20301.0,"expected":20301.0,"pass":true,"note":"2026-10-03 to 2026-10-10","source":"DOL weekly claims"},{"id":"L01","category":"local","question":"What is the median household income in ZIP 60614 (Chicago)?","tool":"get_local_profile","arguments":{"area":"60614"},"ours":"136448.0 ± 7632.0","expected":"136448.0 ± 7632.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L02","category":"local","question":"What is the median household income in ZIP 10021 (Manhattan)?","tool":"get_local_profile","arguments":{"area":"10021"},"ours":"158677.0 ± 22110.0","expected":"158677.0 ± 22110.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L03","category":"local","question":"What is the median household income in ZIP 94110 (San Francisco)?","tool":"get_local_profile","arguments":{"area":"94110"},"ours":"158351.0 ± 9782.0","expected":"158351.0 ± 9782.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L04","category":"local","question":"What is the median household income in Cook County, IL?","tool":"get_local_profile","arguments":{"area":"county:17031"},"ours":"83498.0 ± 630.0","expected":"83498.0 ± 630.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L05","category":"local","question":"What is the median household income in Travis County, TX?","tool":"get_local_profile","arguments":{"area":"county:48453"},"ours":"99611.0 ± 1578.0","expected":"99611.0 ± 1578.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L06","category":"local","question":"What is the median household income in the Austin metro?","tool":"get_local_profile","arguments":{"area":"metro:12420"},"ours":"100431.0 ± 996.0","expected":"100431.0 ± 996.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L07","category":"local","question":"What is the median household income in the city of Austin?","tool":"get_local_profile","arguments":{"area":"city:4805000"},"ours":"93658.0 ± 1670.0","expected":"93658.0 ± 1670.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L08","category":"local","question":"What is the median household income in Mississippi?","tool":"get_local_profile","arguments":{"area":"state:MS"},"ours":"56447.0 ± 539.0","expected":"56447.0 ± 539.0","pass":true,"note":"Census ACS 5-year 2020-2024, B19013_001","source":"Census ACS 5-year"},{"id":"L09","category":"local","question":"How many households are there in ZIP 60614?","tool":"get_local_profile","arguments":{"area":"60614"},"ours":"35386.0 ± 960.0","expected":"35386.0 ± 960.0","pass":true,"note":"Census ACS 5-year 2020-2024, B11001_001","source":"Census ACS 5-year"},{"id":"L10","category":"local","question":"How many households are there in the Chicago metro?","tool":"get_local_profile","arguments":{"area":"metro:16980"},"ours":"3643783.0 ± 5293.0","expected":"3643783.0 ± 5293.0","pass":true,"note":"Census ACS 5-year 2020-2024, B11001_001","source":"Census ACS 5-year"},{"id":"L11","category":"local","question":"How many households are there in Los Angeles County?","tool":"get_local_profile","arguments":{"area":"county:06037"},"ours":"3416449.0 ± 5011.0","expected":"3416449.0 ± 5011.0","pass":true,"note":"Census ACS 5-year 2020-2024, B11001_001","source":"Census ACS 5-year"},{"id":"L12","category":"local","question":"How many households are there in Texas?","tool":"get_local_profile","arguments":{"area":"state:TX"},"ours":"10992816.0 ± 15813.0","expected":"10992816.0 ± 15813.0","pass":true,"note":"Census ACS 5-year 2020-2024, B11001_001","source":"Census ACS 5-year"},{"id":"L13","category":"local","question":"How many people live in the city of Austin?","tool":"get_local_profile","arguments":{"area":"city:4805000"},"ours":"979539.0 ± 484.0","expected":"979539.0 ± 484.0","pass":true,"note":"Census ACS 5-year 2020-2024, B01003_001","source":"Census ACS 5-year"},{"id":"L14","category":"local","question":"What share of households in ZIP 10021 earn $200,000 or more?","tool":"get_local_profile","arguments":{"area":"10021"},"ours":"42.1658 ± 3.2086","expected":"42.1658 ± 3.2086","pass":true,"note":"B19001 brackets 17","source":"Census ACS 5-year B19001"},{"id":"L15","category":"local","question":"What share of households in the San Francisco metro earn $200,000 or more?","tool":"get_local_profile","arguments":{"area":"metro:41860"},"ours":"33.9781 ± 0.3114","expected":"33.9781 ± 0.3114","pass":true,"note":"B19001 brackets 17","source":"Census ACS 5-year B19001"},{"id":"L16","category":"local","question":"What share of households in Mississippi earn $200,000 or more?","tool":"get_local_profile","arguments":{"area":"state:MS"},"ours":"5.7501 ± 0.2023","expected":"5.7501 ± 0.2023","pass":true,"note":"B19001 brackets 17","source":"Census ACS 5-year B19001"},{"id":"L17","category":"local","question":"Is median income in ZIP 10021 higher than in the New York metro area?","tool":"get_local_profile","arguments":{"area":"10021"},"ours":"higher","expected":"higher","pass":true,"note":"z = 4.43 (158,677 ± 22,110 vs 99,155 ± 487)","source":"Census 90% significance test"},{"id":"L18","category":"local","question":"Is Travis County's median income different from the Austin metro's?","tool":"get_local_profile","arguments":{"area":"county:48453"},"ours":"about the same","expected":"about the same","pass":true,"note":"z = 0.72 (99,611 ± 1,578 vs 100,431 ± 996)","source":"Census 90% significance test"},{"id":"L19","category":"local","question":"Is Mississippi's median household income below the US median?","tool":"get_local_profile","arguments":{"area":"state:MS"},"ours":"lower","expected":"lower","pass":true,"note":"z = 70.84 (56,447 ± 539 vs 80,734 ± 166)","source":"Census 90% significance test"},{"id":"L20","category":"local","question":"How much has median household income in the Austin metro changed after inflation over five years?","tool":"get_local_profile","arguments":{"area":"metro:12420"},"ours":6.52,"expected":6.52,"pass":true,"note":"CPI-U factor 1.2270","source":"Census ACS 2015-2019 vs 2020-2024, BLS CPI-U"},{"id":"L21","category":"local","question":"How has Mississippi's median household income changed after inflation over five years?","tool":"get_local_profile","arguments":{"area":"state:MS"},"ours":2.05,"expected":2.05,"pass":true,"note":"CPI-U factor 1.2270","source":"Census ACS 2015-2019 vs 2020-2024, BLS CPI-U"},{"id":"L22","category":"local","question":"What is the average income per tax return in ZIP 60614?","tool":"get_metric_series","arguments":{"metric_id":"irs.zip.agi_per_return","entity":"zip:60614","timeframe":"MAX"},"ours":300277.2,"expected":300277.2,"pass":true,"note":"IRS SOI tax year 2022","source":"IRS Statistics of Income"},{"id":"L23","category":"local","question":"What is the average income per tax return in ZIP 10021?","tool":"get_metric_series","arguments":{"metric_id":"irs.zip.agi_per_return","entity":"zip:10021","timeframe":"MAX"},"ours":571437.59,"expected":571437.59,"pass":true,"note":"IRS SOI tax year 2022","source":"IRS Statistics of Income"},{"id":"L24","category":"local","question":"What is the average income per tax return in ZIP 94301?","tool":"get_metric_series","arguments":{"metric_id":"irs.zip.agi_per_return","entity":"zip:94301","timeframe":"MAX"},"ours":1641171.71,"expected":1641171.71,"pass":true,"note":"IRS SOI tax year 2022","source":"IRS Statistics of Income"},{"id":"L25","category":"local","question":"What does gas cost near ZIP 60614 (Chicago)?","tool":"get_local_profile","arguments":{"area":"60614"},"ours":"4.705 metro:16980 (2026-10-19)","expected":"4.705 metro:16980 (2026-10-19)","pass":true,"note":"its metro area","source":"EIA weekly, Chicago city area"},{"id":"L26","category":"local","question":"What does gas cost in Phoenix?","tool":"get_local_profile","arguments":{"area":"Phoenix city, Arizona"},"ours":"4.691 metro:38060 (2026-09-01)","expected":"4.691 metro:38060 (2026-09-01)","pass":true,"note":"its metro area","source":"BLS monthly average price, Phoenix metro"},{"id":"L27","category":"local","question":"What does gas cost in Missoula, Montana (no city price published)?","tool":"get_local_profile","arguments":{"area":"59801"},"ours":"4.468 padd:4 (2026-10-19)","expected":"4.468 padd:4 (2026-10-19)","pass":true,"note":"its fuel region","source":"EIA weekly, Rocky Mountain fuel region"},{"id":"L28","category":"local","question":"Which US metro area has the highest median household income?","tool":"rank_areas","arguments":{"metric":"census.area.median_household_income","level":"metro","limit":5},"ours":"metro:41940","expected":"metro:41940","pass":true,"note":"San Jose-Sunnyvale-Santa Clara, CA Metro Area: $162,111","source":"Census ACS 5-year, all metro areas"},{"id":"G01","category":"guardrails","question":"Give me the series 'bls.cpi.food_awy' (a typo): it must not silently return a different series.","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.food_awy","timeframe":"2Y"},"ours":"NotFound: Unknown metric 'bls.cpi.food_awy'. Did you mean: bls.cpi.core, bls.cpi.fish_seafood, bls.cpi.food_at_home? Use explore_ontology_nodes or /api/v1/metad","expected":"an error mentioning 'Did you mean'","pass":true,"note":"","source":""},{"id":"G02","category":"guardrails","question":"What is the 30-year mortgage rate from Freddie Mac? (Freddie Mac's terms forbid redistribution.)","tool":"get_metric_series","arguments":{"metric_id":"fmac.mortgage_30y","timeframe":"2Y"},"ours":"Not stored or served by HorizonIQ: Freddie Mac (Primary Mortgage Market Survey)'s terms do not allow storing or commerci","expected":"no values; where to get it yourself","pass":true,"note":"","source":""},{"id":"G03","category":"guardrails","question":"What is the University of Michigan consumer sentiment index? (non-commercial licence)","tool":"get_metric_series","arguments":{"metric_id":"umich.consumer_sentiment","timeframe":"2Y"},"ours":true,"expected":"no values (reference only)","pass":true,"note":"","source":""},{"id":"G04","category":"guardrails","question":"Year-over-year change in ZIP 60614 median income: overlapping 5-year estimates must not be differenced year to year.","tool":"get_metric_series","arguments":{"metric_id":"census.area.median_household_income","timeframe":"MAX","entity":"60614"},"ours":"5-year estimates overlap; compare editions five years apart (e.g. 2015-2019 vs 2","expected":"no YoY; a comparison note","pass":true,"note":"","source":""},{"id":"G05","category":"guardrails","question":"Every number answered carries its source credit (Census notice on local data).","tool":"get_local_profile","arguments":{"area":"60614"},"ours":true,"expected":"Census API notice present","pass":true,"note":"","source":""},{"id":"G06","category":"guardrails","question":"Every number answered carries its source credit (BLS).","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.all_items","timeframe":"2Y"},"ours":true,"expected":"credit line present","pass":true,"note":"","source":""},{"id":"G07","category":"guardrails","question":"Profile of a place that does not exist ('Narnia') must say so, not invent data.","tool":"get_local_profile","arguments":{"area":"Narnia"},"ours":"NotFound: No area matches 'Narnia'. Try a ZIP code (60614), a county ('Cook County, Illinois'), a city ('Austin city, Texas'), a metro area ('Chicago metro') or","expected":"an error mentioning 'No area matches'","pass":true,"note":"","source":""},{"id":"G08","category":"guardrails","question":"Inflation-adjusting an interest rate makes no sense and must be refused.","tool":"get_metric_series","arguments":{"metric_id":"nyfed.effr","adjust":"real"},"ours":"NotFound: nyfed.effr is not a nominal dollar series, so it is not adjusted for inflation","expected":"an error mentioning 'not'","pass":true,"note":"","source":""},{"id":"G09","category":"guardrails","question":"Ranking must leave out estimates too imprecise to rank (CV above 30%).","tool":"rank_areas","arguments":{"metric":"census.area.median_income_pacific_islander","level":"zip","limit":50},"ours":50,"expected":"no unreliable rows","pass":true,"note":"","source":""},{"id":"G10","category":"guardrails","question":"Sentences in a local profile never quote an unreliable figure.","tool":"get_local_profile","arguments":{"area":"94110"},"ours":8,"expected":"no unreliable figure in the sentences","pass":true,"note":"","source":""},{"id":"G11","category":"guardrails","question":"Customer account data can never be read through SQL.","tool":"run_sql","arguments":{"sql":"SELECT * FROM acct_customers"},"ours":"ValueError: Disallowed SQL construct: acct_customers","expected":"an error","pass":true,"note":"","source":""},{"id":"G12","category":"guardrails","question":"SQL cannot write or change data.","tool":"run_sql","arguments":{"sql":"DELETE FROM observations"},"ours":"ValueError: Only SELECT / WITH / DESCRIBE / SUMMARIZE / SHOW / EXPLAIN statements are allowed","expected":"an error","pass":true,"note":"","source":""},{"id":"G13","category":"guardrails","question":"Retired data points explain why they are gone rather than returning old numbers.","tool":"get_metric_series","arguments":{"metric_id":"wb.lpi_overall","timeframe":"2Y"},"ours":"NotFound: 'wb.lpi_overall' is no longer offered. World Bank Logistics Performance Index: last published for 2022 (irregular survey); retired as too old.","expected":"an error mentioning 'no longer offered'","pass":true,"note":"","source":""},{"id":"G14","category":"guardrails","question":"Monthly figures that are not seasonally adjusted are labelled as such.","tool":"get_metric_series","arguments":{"metric_id":"bls.cpi.all_items","timeframe":"2Y"},"ours":false,"expected":"seasonally_adjusted = false (CUUR series)","pass":true,"note":"","source":""},{"id":"G15","category":"guardrails","question":"Yearly local data states which edition it is (no stale numbers passed off as current).","tool":"get_metric_series","arguments":{"metric_id":"census.area.median_household_income","timeframe":"MAX","entity":"US"},"ours":"2020-2024","expected":"edition label like 2020-2024","pass":true,"note":"","source":""}]}