{"degradation_reason":"rolling 30d precision 0.386 < 0.6","degraded":true,"generated_at":"2026-06-08T02:19:00.920017+00:00","methodology_url":"https://voidly.ai/sentinel/methodology","notes":"prod_rolling reflects real alerts evaluated against observed incidents in the 7-day target window. training_holdout is the stratified holdout from the last training run — it sanity-checks the model but is not evidence of live performance. When prod_rolling.n_evaluated is small, defer to training_holdout. When it's large, prod_rolling is authoritative.","prod_rolling":{"accuracy":0.53,"brier_score":0.3344,"calibration_bins":[{"bin_high":0.1,"bin_low":0,"n":550,"observed_rate":0.3764,"predicted_mean":0.0503},{"bin_high":0.2,"bin_low":0.1,"n":26,"observed_rate":0.1154,"predicted_mean":0.1392},{"bin_high":0.3,"bin_low":0.2,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.4,"bin_low":0.3,"n":6,"observed_rate":0.8333,"predicted_mean":0.3283},{"bin_high":0.5,"bin_low":0.4,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.6,"bin_low":0.5,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.7,"bin_low":0.6,"n":6,"observed_rate":0.1667,"predicted_mean":0.6905},{"bin_high":0.8,"bin_low":0.7,"n":1,"observed_rate":0,"predicted_mean":0.703},{"bin_high":0.9,"bin_low":0.8,"n":11,"observed_rate":0.2727,"predicted_mean":0.823},{"bin_high":1,"bin_low":0.9,"n":0,"observed_rate":null,"predicted_mean":null}],"calibration_mae":0.3215,"confusion":{"false_negative":112,"false_positive":170,"total":600,"true_negative":211,"true_positive":107},"n_evaluated":600,"per_country":{"BD":{"false_negative":6,"false_positive":3,"n":20,"precision":0.4,"recall":0.25,"true_negative":9,"true_positive":2},"BR":{"false_negative":4,"false_positive":4,"n":20,"precision":0,"recall":0,"true_negative":12,"true_positive":0},"BY":{"false_negative":4,"false_positive":2,"n":20,"precision":0.6,"recall":0.42857142857142855,"true_negative":11,"true_positive":3},"CN":{"false_negative":1,"false_positive":12,"n":20,"precision":0.3333333333333333,"recall":0.8571428571428571,"true_negative":1,"true_positive":6},"CU":{"false_negative":6,"false_positive":3,"n":20,"precision":0.25,"recall":0.14285714285714285,"true_negative":10,"true_positive":1},"EG":{"false_negative":7,"false_positive":2,"n":20,"precision":0.8461538461538461,"recall":0.6111111111111112,"true_negative":0,"true_positive":11},"ER":{"false_negative":0,"false_positive":4,"n":20,"precision":0,"recall":null,"true_negative":16,"true_positive":0},"ET":{"false_negative":0,"false_positive":8,"n":20,"precision":0.6,"recall":1,"true_negative":0,"true_positive":12},"ID":{"false_negative":5,"false_positive":2,"n":20,"precision":0.7142857142857143,"recall":0.5,"true_negative":8,"true_positive":5},"IN":{"false_negative":10,"false_positive":0,"n":20,"precision":1,"recall":0.5,"true_negative":0,"true_positive":10},"IR":{"false_negative":1,"false_positive":6,"n":20,"precision":0.6842105263157895,"recall":0.9285714285714286,"true_negative":0,"true_positive":13},"KP":{"false_negative":0,"false_positive":5,"n":20,"precision":0,"recall":null,"true_negative":15,"true_positive":0},"KZ":{"false_negative":1,"false_positive":7,"n":20,"precision":0,"recall":0,"true_negative":12,"true_positive":0},"LB":{"false_negative":0,"false_positive":17,"n":20,"precision":0.05555555555555555,"recall":1,"true_negative":2,"true_positive":1},"MM":{"false_negative":1,"false_positive":10,"n":20,"precision":0,"recall":0,"true_negative":9,"true_positive":0},"MY":{"false_negative":1,"false_positive":4,"n":20,"precision":0,"recall":0,"true_negative":15,"true_positive":0},"NG":{"false_negative":4,"false_positive":6,"n":20,"precision":0.3333333333333333,"recall":0.42857142857142855,"true_negative":7,"true_positive":3},"NI":{"false_negative":3,"false_positive":7,"n":20,"precision":0.36363636363636365,"recall":0.5714285714285714,"true_negative":6,"true_positive":4},"PH":{"false_negative":7,"false_positive":3,"n":20,"precision":0,"recall":0,"true_negative":10,"true_positive":0},"PK":{"false_negative":13,"false_positive":0,"n":20,"precision":1,"recall":0.35,"true_negative":0,"true_positive":7},"RU":{"false_negative":8,"false_positive":5,"n":20,"precision":0.375,"recall":0.2727272727272727,"true_negative":4,"true_positive":3},"SA":{"false_negative":1,"false_positive":7,"n":20,"precision":0,"recall":0,"true_negative":12,"true_positive":0},"SD":{"false_negative":1,"false_positive":6,"n":20,"precision":0,"recall":0,"true_negative":13,"true_positive":0},"SY":{"false_negative":6,"false_positive":6,"n":20,"precision":0.14285714285714285,"recall":0.14285714285714285,"true_negative":7,"true_positive":1},"TH":{"false_negative":1,"false_positive":7,"n":20,"precision":0,"recall":0,"true_negative":12,"true_positive":0},"TM":{"false_negative":1,"false_positive":9,"n":20,"precision":0,"recall":0,"true_negative":10,"true_positive":0},"TR":{"false_negative":0,"false_positive":19,"n":20,"precision":0.05,"recall":1,"true_negative":0,"true_positive":1},"UZ":{"false_negative":9,"false_positive":2,"n":20,"precision":0.7142857142857143,"recall":0.35714285714285715,"true_negative":4,"true_positive":5},"VE":{"false_negative":2,"false_positive":0,"n":20,"precision":1,"recall":0.9,"true_negative":0,"true_positive":18},"VN":{"false_negative":9,"false_positive":4,"n":20,"precision":0.2,"recall":0.1,"true_negative":6,"true_positive":1}},"precision":0.3862815884476534,"recall":0.4885844748858447,"window_days":30},"published_warning":"Stratified AUC overstates real-world performance by 47.9pp vs. time-based split. Do not cite the stratified number as a deployment figure; use the loco_median or the prod_rolling block once it populates.","schema":"voidly-sentinel-accuracy/v1","training_holdout":{"as_of":"2026-06-07T02:14:25.651493","f1":0.6813186813186813,"positive_rate":0.05946436677258284,"precision":0.6549295774647887,"recall":0.7099236641221374,"roc_auc":0.9672938341831472,"samples":2203,"source":"training_holdout","split_loco":{"f1_median":0.5528455284552846,"n_countries":20,"n_countries_evaluated":19,"notes":"Leave-country-out cross-validation. For each country, train on all other countries + eval on that country. The median is a more honest generalization figure than stratified. Individual per-country numbers reveal which countries the model struggles on (novel crises).","per_country":{"BD":{"auc":0.8629360129360131,"f1":0.6476190476190476,"n_pos_test":65,"n_test":731,"precision":0.85,"recall":0.5230769230769231,"threshold":0.42000000000000004},"BY":{"auc":0.9164408866995074,"f1":0.3492063492063492,"n_pos_test":35,"n_test":731,"precision":0.39285714285714285,"recall":0.3142857142857143,"threshold":0.42000000000000004},"CN":{"auc":0.9480124391563007,"f1":0.5344827586206896,"n_pos_test":43,"n_test":731,"precision":0.4246575342465753,"recall":0.7209302325581395,"threshold":0.22000000000000003},"CU":{"auc":0.69163438900281,"f1":0.18947368421052632,"n_pos_test":38,"n_test":731,"precision":0.15789473684210525,"recall":0.23684210526315788,"threshold":0.07},"EG":{"auc":0.9435666435666437,"f1":0.7,"n_pos_test":65,"n_test":731,"precision":0.7636363636363637,"recall":0.6461538461538462,"threshold":0.23000000000000004},"ID":{"auc":0.9961107091172214,"f1":0.8395061728395061,"n_pos_test":40,"n_test":731,"precision":0.8292682926829268,"recall":0.85,"threshold":0.43000000000000005},"IR":{"auc":0.8510521164668461,"f1":0.5093167701863354,"n_pos_test":61,"n_test":731,"precision":0.41,"recall":0.6721311475409836,"threshold":0.05},"KP":{"auc":null,"f1":null,"n_pos_test":0,"n_test":731,"note":"no positives in train or test","precision":null,"recall":null},"KZ":{"auc":0.905096011816839,"f1":0.656,"n_pos_test":54,"n_test":731,"precision":0.5774647887323944,"recall":0.7592592592592593,"threshold":0.08000000000000002},"MM":{"auc":0.9297528297528299,"f1":0.784,"n_pos_test":65,"n_test":731,"precision":0.8166666666666667,"recall":0.7538461538461538,"threshold":0.25000000000000006},"MY":{"auc":0.8833882381649585,"f1":0.5882352941176471,"n_pos_test":48,"n_test":731,"precision":1,"recall":0.4166666666666667,"threshold":0.43000000000000005},"PK":{"auc":0.7344099669327633,"f1":0.5180722891566265,"n_pos_test":94,"n_test":731,"precision":0.5972222222222222,"recall":0.4574468085106383,"threshold":0.19},"RU":{"auc":0.7997119815668202,"f1":0.5279187817258884,"n_pos_test":80,"n_test":731,"precision":0.4444444444444444,"recall":0.65,"threshold":0.05},"SA":{"auc":0.802767986730251,"f1":0.3968253968253968,"n_pos_test":42,"n_test":731,"precision":0.2976190476190476,"recall":0.5952380952380952,"threshold":0.08000000000000002},"SY":{"auc":0.9399555696604253,"f1":0.4567901234567901,"n_pos_test":46,"n_test":731,"precision":0.31896551724137934,"recall":0.8043478260869565,"threshold":0.05},"TH":{"auc":0.9196018062397373,"f1":0.574468085106383,"n_pos_test":35,"n_test":731,"precision":0.4576271186440678,"recall":0.7714285714285715,"threshold":0.24000000000000005},"TM":{"auc":0.6979420438517758,"f1":0.11965811965811966,"n_pos_test":22,"n_test":731,"precision":0.07368421052631578,"recall":0.3181818181818182,"threshold":0.060000000000000005},"TR":{"auc":0.7848001848001849,"f1":0.6909090909090909,"n_pos_test":65,"n_test":731,"precision":0.8444444444444444,"recall":0.5846153846153846,"threshold":0.4600000000000001},"UZ":{"auc":0.9631451382948388,"f1":0.7034482758620689,"n_pos_test":63,"n_test":731,"precision":0.6219512195121951,"recall":0.8095238095238095,"threshold":0.05},"VN":{"auc":0.9221879588839941,"f1":0.5528455284552846,"n_pos_test":50,"n_test":731,"precision":0.4657534246575342,"recall":0.68,"threshold":0.08000000000000002}},"precision_median":0.4657534246575342,"recall_median":0.65,"roc_auc_max":0.9961107091172214,"roc_auc_median":0.905096011816839,"roc_auc_min":0.69163438900281,"split":"loco"},"split_stratified":{"f1":0.7945205479452054,"n_test":2100,"notes":"Stratified random 15% holdout. Inflates AUC via within-country temporal leakage. Publish alongside time-based as the training-level sanity check, NOT as the deployment figure.","positive_rate":0.07238095238095238,"precision":0.8285714285714286,"recall":0.7631578947368421,"roc_auc":0.9802986193666918,"split":"stratified","threshold":0.34},"split_summary":{"delta_stratified_vs_time_pp":47.94,"generated_at":"2026-04-18T00:28:52.397285+00:00","loco_median_auc":0.905096011816839,"loco_median_f1":0.5528455284552846,"schema":"voidly-sentinel-evaluation/v1","stratified_auc":0.9802986193666918,"stratified_f1":0.7945205479452054,"time_based_auc":0.500863501950972,"time_based_f1":0},"split_time":{"f1":0,"n_test":2193,"notes":"Time-based split — train on data before T, test on data after T. This is the honest deployment-aligned figure. Treat as the Sentinel baseline when n_evaluated in prod is < 30.","positive_rate":0.38896488828089376,"precision":0,"recall":0,"roc_auc":0.500863501950972,"split":"time","test_date_range":["2025-12-29","2026-04-17"],"threshold":0.5,"train_date_range":["2024-04-17","2025-09-08"],"val_date_range":["2025-09-08","2025-12-29"]},"threshold":0.24000000000000005},"window_days":30,"_voidly_pay":{"claim_did":"https://voidly.ai/pay/claim","for_builders":"https://voidly.ai/pay/for-builders","marketplace":"https://api.voidly.ai/v1/pay/marketplace","live_demo":"https://huggingface.co/spaces/emperor-mew/voidly-pay","universal_proxy":{"url":"https://api.voidly.ai/v1/pay/proxy","example":"https://api.voidly.ai/v1/pay/proxy?u=<https-url>&to=did:voidly:<your-did>&price=0.01","page":"https://voidly.ai/pay/proxy","note":"Paywall any public HTTPS URL with one query param. No SDK install required."},"install":{"typescript":"npm install @voidly/pay","python":"pip install voidly-pay","langchain":"pip install voidly-pay-langchain","vercel_ai":"npm install @voidly/pay-vercel-ai","mcp":"npx @voidly/pay-mcp","cli":"npm install -g @voidly/pay-cli"},"note":"Voidly Pay is the open agent-to-agent payment rail. Claim a DID + 10 starter credits in 60s, or paywall any URL via the universal proxy with one query param."}}