{"degradation_reason":null,"degraded":false,"generated_at":"2026-05-07T02:48:07.672808+00:00","methodology_url":"https://voidly.ai/sentinel/methodology","notes":"prod_rolling reflects real alerts evaluated against observed incidents in the 7-day target window. training_holdout is the stratified holdout from the last training run — it sanity-checks the model but is not evidence of live performance. When prod_rolling.n_evaluated is small, defer to training_holdout. When it's large, prod_rolling is authoritative.","prod_rolling":{"accuracy":0.43846153846153846,"brier_score":0.7285,"calibration_bins":[{"bin_high":0.1,"bin_low":0,"n":390,"observed_rate":0.8026,"predicted_mean":0.0465},{"bin_high":0.2,"bin_low":0.1,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.3,"bin_low":0.2,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.4,"bin_low":0.3,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.5,"bin_low":0.4,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.6,"bin_low":0.5,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.7,"bin_low":0.6,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.8,"bin_low":0.7,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":0.9,"bin_low":0.8,"n":0,"observed_rate":null,"predicted_mean":null},{"bin_high":1,"bin_low":0.9,"n":0,"observed_rate":null,"predicted_mean":null}],"calibration_mae":0.7561,"confusion":{"false_negative":205,"false_positive":14,"total":390,"true_negative":63,"true_positive":108},"n_evaluated":390,"per_country":{"BD":{"false_negative":6,"false_positive":0,"n":13,"precision":1,"recall":0.5384615384615384,"true_negative":0,"true_positive":7},"BR":{"false_negative":9,"false_positive":0,"n":13,"precision":1,"recall":0.3076923076923077,"true_negative":0,"true_positive":4},"BY":{"false_negative":11,"false_positive":0,"n":13,"precision":1,"recall":0.15384615384615385,"true_negative":0,"true_positive":2},"CN":{"false_negative":9,"false_positive":0,"n":13,"precision":1,"recall":0.3076923076923077,"true_negative":0,"true_positive":4},"CU":{"false_negative":2,"false_positive":2,"n":13,"precision":0.3333333333333333,"recall":0.3333333333333333,"true_negative":8,"true_positive":1},"EG":{"false_negative":13,"false_positive":0,"n":13,"precision":null,"recall":0,"true_negative":0,"true_positive":0},"ER":{"false_negative":0,"false_positive":2,"n":13,"precision":0,"recall":null,"true_negative":11,"true_positive":0},"ET":{"false_negative":6,"false_positive":0,"n":13,"precision":1,"recall":0.5,"true_negative":1,"true_positive":6},"ID":{"false_negative":12,"false_positive":0,"n":13,"precision":1,"recall":0.07692307692307693,"true_negative":0,"true_positive":1},"IN":{"false_negative":7,"false_positive":0,"n":13,"precision":1,"recall":0.46153846153846156,"true_negative":0,"true_positive":6},"IR":{"false_negative":2,"false_positive":0,"n":13,"precision":1,"recall":0.8461538461538461,"true_negative":0,"true_positive":11},"KP":{"false_negative":0,"false_positive":1,"n":13,"precision":0,"recall":null,"true_negative":12,"true_positive":0},"KZ":{"false_negative":11,"false_positive":0,"n":13,"precision":1,"recall":0.15384615384615385,"true_negative":0,"true_positive":2},"LB":{"false_negative":0,"false_positive":1,"n":13,"precision":0.9230769230769231,"recall":1,"true_negative":0,"true_positive":12},"MM":{"false_negative":9,"false_positive":0,"n":13,"precision":1,"recall":0.3076923076923077,"true_negative":0,"true_positive":4},"MY":{"false_negative":5,"false_positive":1,"n":13,"precision":0,"recall":0,"true_negative":7,"true_positive":0},"NG":{"false_negative":6,"false_positive":0,"n":13,"precision":1,"recall":0.5384615384615384,"true_negative":0,"true_positive":7},"NI":{"false_negative":10,"false_positive":0,"n":13,"precision":1,"recall":0.16666666666666666,"true_negative":1,"true_positive":2},"PH":{"false_negative":8,"false_positive":1,"n":13,"precision":0.6666666666666666,"recall":0.2,"true_negative":2,"true_positive":2},"PK":{"false_negative":11,"false_positive":0,"n":13,"precision":1,"recall":0.15384615384615385,"true_negative":0,"true_positive":2},"RU":{"false_negative":6,"false_positive":0,"n":13,"precision":1,"recall":0.5384615384615384,"true_negative":0,"true_positive":7},"SA":{"false_negative":11,"false_positive":0,"n":13,"precision":1,"recall":0.08333333333333333,"true_negative":1,"true_positive":1},"SD":{"false_negative":4,"false_positive":1,"n":13,"precision":0,"recall":0,"true_negative":8,"true_positive":0},"SY":{"false_negative":8,"false_positive":0,"n":13,"precision":1,"recall":0.38461538461538464,"true_negative":0,"true_positive":5},"TH":{"false_negative":7,"false_positive":0,"n":13,"precision":1,"recall":0.2222222222222222,"true_negative":4,"true_positive":2},"TM":{"false_negative":2,"false_positive":3,"n":13,"precision":0.25,"recall":0.3333333333333333,"true_negative":7,"true_positive":1},"TR":{"false_negative":7,"false_positive":0,"n":13,"precision":1,"recall":0.46153846153846156,"true_negative":0,"true_positive":6},"UZ":{"false_negative":10,"false_positive":0,"n":13,"precision":1,"recall":0.23076923076923078,"true_negative":0,"true_positive":3},"VE":{"false_negative":8,"false_positive":0,"n":13,"precision":1,"recall":0.38461538461538464,"true_negative":0,"true_positive":5},"VN":{"false_negative":5,"false_positive":2,"n":13,"precision":0.7142857142857143,"recall":0.5,"true_negative":1,"true_positive":5}},"precision":0.8852459016393442,"recall":0.3450479233226837,"window_days":30},"published_warning":"Stratified AUC overstates real-world performance by 47.9pp vs. time-based split. Do not cite the stratified number as a deployment figure; use the loco_median or the prod_rolling block once it populates.","schema":"voidly-sentinel-accuracy/v1","training_holdout":{"as_of":"2026-05-03T02:14:46.605257","f1":0.8214285714285714,"positive_rate":0.09217877094972067,"precision":0.8298969072164949,"recall":0.8131313131313131,"roc_auc":0.9789005439005439,"samples":2148,"source":"training_holdout","split_loco":{"f1_median":0.5528455284552846,"n_countries":20,"n_countries_evaluated":19,"notes":"Leave-country-out cross-validation. For each country, train on all other countries + eval on that country. The median is a more honest generalization figure than stratified. Individual per-country numbers reveal which countries the model struggles on (novel crises).","per_country":{"BD":{"auc":0.8629360129360131,"f1":0.6476190476190476,"n_pos_test":65,"n_test":731,"precision":0.85,"recall":0.5230769230769231,"threshold":0.42000000000000004},"BY":{"auc":0.9164408866995074,"f1":0.3492063492063492,"n_pos_test":35,"n_test":731,"precision":0.39285714285714285,"recall":0.3142857142857143,"threshold":0.42000000000000004},"CN":{"auc":0.9480124391563007,"f1":0.5344827586206896,"n_pos_test":43,"n_test":731,"precision":0.4246575342465753,"recall":0.7209302325581395,"threshold":0.22000000000000003},"CU":{"auc":0.69163438900281,"f1":0.18947368421052632,"n_pos_test":38,"n_test":731,"precision":0.15789473684210525,"recall":0.23684210526315788,"threshold":0.07},"EG":{"auc":0.9435666435666437,"f1":0.7,"n_pos_test":65,"n_test":731,"precision":0.7636363636363637,"recall":0.6461538461538462,"threshold":0.23000000000000004},"ID":{"auc":0.9961107091172214,"f1":0.8395061728395061,"n_pos_test":40,"n_test":731,"precision":0.8292682926829268,"recall":0.85,"threshold":0.43000000000000005},"IR":{"auc":0.8510521164668461,"f1":0.5093167701863354,"n_pos_test":61,"n_test":731,"precision":0.41,"recall":0.6721311475409836,"threshold":0.05},"KP":{"auc":null,"f1":null,"n_pos_test":0,"n_test":731,"note":"no positives in train or test","precision":null,"recall":null},"KZ":{"auc":0.905096011816839,"f1":0.656,"n_pos_test":54,"n_test":731,"precision":0.5774647887323944,"recall":0.7592592592592593,"threshold":0.08000000000000002},"MM":{"auc":0.9297528297528299,"f1":0.784,"n_pos_test":65,"n_test":731,"precision":0.8166666666666667,"recall":0.7538461538461538,"threshold":0.25000000000000006},"MY":{"auc":0.8833882381649585,"f1":0.5882352941176471,"n_pos_test":48,"n_test":731,"precision":1,"recall":0.4166666666666667,"threshold":0.43000000000000005},"PK":{"auc":0.7344099669327633,"f1":0.5180722891566265,"n_pos_test":94,"n_test":731,"precision":0.5972222222222222,"recall":0.4574468085106383,"threshold":0.19},"RU":{"auc":0.7997119815668202,"f1":0.5279187817258884,"n_pos_test":80,"n_test":731,"precision":0.4444444444444444,"recall":0.65,"threshold":0.05},"SA":{"auc":0.802767986730251,"f1":0.3968253968253968,"n_pos_test":42,"n_test":731,"precision":0.2976190476190476,"recall":0.5952380952380952,"threshold":0.08000000000000002},"SY":{"auc":0.9399555696604253,"f1":0.4567901234567901,"n_pos_test":46,"n_test":731,"precision":0.31896551724137934,"recall":0.8043478260869565,"threshold":0.05},"TH":{"auc":0.9196018062397373,"f1":0.574468085106383,"n_pos_test":35,"n_test":731,"precision":0.4576271186440678,"recall":0.7714285714285715,"threshold":0.24000000000000005},"TM":{"auc":0.6979420438517758,"f1":0.11965811965811966,"n_pos_test":22,"n_test":731,"precision":0.07368421052631578,"recall":0.3181818181818182,"threshold":0.060000000000000005},"TR":{"auc":0.7848001848001849,"f1":0.6909090909090909,"n_pos_test":65,"n_test":731,"precision":0.8444444444444444,"recall":0.5846153846153846,"threshold":0.4600000000000001},"UZ":{"auc":0.9631451382948388,"f1":0.7034482758620689,"n_pos_test":63,"n_test":731,"precision":0.6219512195121951,"recall":0.8095238095238095,"threshold":0.05},"VN":{"auc":0.9221879588839941,"f1":0.5528455284552846,"n_pos_test":50,"n_test":731,"precision":0.4657534246575342,"recall":0.68,"threshold":0.08000000000000002}},"precision_median":0.4657534246575342,"recall_median":0.65,"roc_auc_max":0.9961107091172214,"roc_auc_median":0.905096011816839,"roc_auc_min":0.69163438900281,"split":"loco"},"split_stratified":{"f1":0.7945205479452054,"n_test":2100,"notes":"Stratified random 15% holdout. Inflates AUC via within-country temporal leakage. Publish alongside time-based as the training-level sanity check, NOT as the deployment figure.","positive_rate":0.07238095238095238,"precision":0.8285714285714286,"recall":0.7631578947368421,"roc_auc":0.9802986193666918,"split":"stratified","threshold":0.34},"split_summary":{"delta_stratified_vs_time_pp":47.94,"generated_at":"2026-04-18T00:28:52.397285+00:00","loco_median_auc":0.905096011816839,"loco_median_f1":0.5528455284552846,"schema":"voidly-sentinel-evaluation/v1","stratified_auc":0.9802986193666918,"stratified_f1":0.7945205479452054,"time_based_auc":0.500863501950972,"time_based_f1":0},"split_time":{"f1":0,"n_test":2193,"notes":"Time-based split — train on data before T, test on data after T. This is the honest deployment-aligned figure. Treat as the Sentinel baseline when n_evaluated in prod is < 30.","positive_rate":0.38896488828089376,"precision":0,"recall":0,"roc_auc":0.500863501950972,"split":"time","test_date_range":["2025-12-29","2026-04-17"],"threshold":0.5,"train_date_range":["2024-04-17","2025-09-08"],"val_date_range":["2025-09-08","2025-12-29"]},"threshold":0.32000000000000006},"window_days":30,"_voidly_pay":{"claim_did":"https://voidly.ai/pay/claim","for_builders":"https://voidly.ai/pay/for-builders","marketplace":"https://api.voidly.ai/v1/pay/marketplace","live_demo":"https://huggingface.co/spaces/emperor-mew/voidly-pay","universal_proxy":{"url":"https://api.voidly.ai/v1/pay/proxy","example":"https://api.voidly.ai/v1/pay/proxy?u=<https-url>&to=did:voidly:<your-did>&price=0.01","page":"https://voidly.ai/pay/proxy","note":"Paywall any public HTTPS URL with one query param. No SDK install required."},"install":{"typescript":"npm install @voidly/pay","python":"pip install voidly-pay","langchain":"pip install voidly-pay-langchain","vercel_ai":"npm install @voidly/pay-vercel-ai","mcp":"npx @voidly/pay-mcp","cli":"npm install -g @voidly/pay-cli"},"note":"Voidly Pay is the open agent-to-agent payment rail. Claim a DID + 10 starter credits in 60s, or paywall any URL via the universal proxy with one query param."}}