{"degradation_reason":"rolling 30d precision 0.245 < 0.6","degraded":true,"generated_at":"2026-08-05T06:03:47.592334+00:00","label_definition":"A country-day is a POSITIVE outcome iff a citable-censorship incident (incident_type != 'disruption' AND status NOT IN ('suspected','draft')) has first_seen within the 7-day target window. Tightened 2026-06-21: the outcome join previously matched ANY incident, which counted IODA disruption outages (~65% of the incidents table) and single-source 'suspected' rows as ground-truth censorship — inflating the observed/positive rate (e.g. the lowest reliability bin's observed rate fell ~0.34 -> ~0.18 after the fix). These honest numbers are lower than before BY DESIGN; the model no longer gets credit for predicting outages.","methodology_url":"https://voidly.ai/sentinel/methodology","notes":"prod_rolling reflects real alerts evaluated against observed incidents in the 7-day target window. training_holdout is the stratified holdout from the last training run — it sanity-checks the model but is not evidence of live performance. When prod_rolling.n_evaluated is small, defer to training_holdout. When it's large, prod_rolling is authoritative.","prod_rolling":{"accuracy":0.44333333333333336,"brier_score":0.2708,"calibration_bins":[{"bin_high":0.1,"bin_low":0,"n":325,"observed_rate":0.2369,"predicted_mean":0.033},{"bin_high":0.2,"bin_low":0.1,"n":46,"observed_rate":0.3043,"predicted_mean":0.1498},{"bin_high":0.3,"bin_low":0.2,"n":40,"observed_rate":0.425,"predicted_mean":0.2641},{"bin_high":0.4,"bin_low":0.3,"n":119,"observed_rate":0.1597,"predicted_mean":0.3393},{"bin_high":0.5,"bin_low":0.4,"n":54,"observed_rate":0.2407,"predicted_mean":0.4506},{"bin_high":0.6,"bin_low":0.5,"n":74,"observed_rate":0.3784,"predicted_mean":0.5544},{"bin_high":0.7,"bin_low":0.6,"n":124,"observed_rate":0.3145,"predicted_mean":0.6505},{"bin_high":0.8,"bin_low":0.7,"n":95,"observed_rate":0.2211,"predicted_mean":0.7421},{"bin_high":0.9,"bin_low":0.8,"n":23,"observed_rate":0.1739,"predicted_mean":0.8301},{"bin_high":1,"bin_low":0.9,"n":0,"observed_rate":null,"predicted_mean":null}],"calibration_mae":0.2575,"confusion":{"false_negative":103,"false_positive":398,"total":900,"true_negative":270,"true_positive":129},"n_evaluated":900,"per_country":{"BD":{"false_negative":26,"false_positive":0,"n":30,"precision":null,"recall":0,"true_negative":4,"true_positive":0},"BR":{"false_negative":0,"false_positive":22,"n":30,"precision":0,"recall":null,"true_negative":8,"true_positive":0},"BY":{"false_negative":0,"false_positive":28,"n":30,"precision":0.034482758620689655,"recall":1,"true_negative":1,"true_positive":1},"CN":{"false_negative":0,"false_positive":11,"n":30,"precision":0,"recall":null,"true_negative":19,"true_positive":0},"CU":{"false_negative":0,"false_positive":29,"n":30,"precision":0.03333333333333333,"recall":1,"true_negative":0,"true_positive":1},"EG":{"false_negative":0,"false_positive":15,"n":30,"precision":0.4,"recall":1,"true_negative":5,"true_positive":10},"ER":{"false_negative":0,"false_positive":9,"n":30,"precision":0,"recall":null,"true_negative":21,"true_positive":0},"ET":{"false_negative":0,"false_positive":7,"n":30,"precision":0.4166666666666667,"recall":1,"true_negative":18,"true_positive":5},"ID":{"false_negative":14,"false_positive":2,"n":30,"precision":0.3333333333333333,"recall":0.06666666666666667,"true_negative":13,"true_positive":1},"IN":{"false_negative":18,"false_positive":4,"n":30,"precision":0.42857142857142855,"recall":0.14285714285714285,"true_negative":5,"true_positive":3},"IR":{"false_negative":14,"false_positive":0,"n":30,"precision":1,"recall":0.5,"true_negative":2,"true_positive":14},"KP":{"false_negative":0,"false_positive":8,"n":30,"precision":0,"recall":null,"true_negative":22,"true_positive":0},"KZ":{"false_negative":0,"false_positive":29,"n":30,"precision":0,"recall":null,"true_negative":1,"true_positive":0},"LB":{"false_negative":1,"false_positive":23,"n":30,"precision":0,"recall":0,"true_negative":6,"true_positive":0},"MM":{"false_negative":1,"false_positive":8,"n":30,"precision":0.7142857142857143,"recall":0.9523809523809523,"true_negative":1,"true_positive":20},"MY":{"false_negative":0,"false_positive":17,"n":30,"precision":0.2608695652173913,"recall":1,"true_negative":7,"true_positive":6},"NG":{"false_negative":0,"false_positive":30,"n":30,"precision":0,"recall":null,"true_negative":0,"true_positive":0},"NI":{"false_negative":0,"false_positive":24,"n":30,"precision":0,"recall":null,"true_negative":6,"true_positive":0},"PH":{"false_negative":0,"false_positive":24,"n":30,"precision":0,"recall":null,"true_negative":6,"true_positive":0},"PK":{"false_negative":0,"false_positive":25,"n":30,"precision":0.16666666666666666,"recall":1,"true_negative":0,"true_positive":5},"RU":{"false_negative":0,"false_positive":0,"n":30,"precision":1,"recall":1,"true_negative":0,"true_positive":30},"SA":{"false_negative":0,"false_positive":26,"n":30,"precision":0.13333333333333333,"recall":1,"true_negative":0,"true_positive":4},"SD":{"false_negative":0,"false_positive":2,"n":30,"precision":0,"recall":null,"true_negative":28,"true_positive":0},"SY":{"false_negative":0,"false_positive":28,"n":30,"precision":0.06666666666666667,"recall":1,"true_negative":0,"true_positive":2},"TH":{"false_negative":5,"false_positive":0,"n":30,"precision":null,"recall":0,"true_negative":25,"true_positive":0},"TM":{"false_negative":0,"false_positive":0,"n":30,"precision":null,"recall":null,"true_negative":30,"true_positive":0},"TR":{"false_negative":14,"false_positive":0,"n":30,"precision":null,"recall":0,"true_negative":16,"true_positive":0},"UZ":{"false_negative":4,"false_positive":18,"n":30,"precision":0,"recall":0,"true_negative":8,"true_positive":0},"VE":{"false_negative":1,"false_positive":3,"n":30,"precision":0.896551724137931,"recall":0.9629629629629629,"true_negative":0,"true_positive":26},"VN":{"false_negative":5,"false_positive":6,"n":30,"precision":0.14285714285714285,"recall":0.16666666666666666,"true_negative":18,"true_positive":1}},"precision":0.24478178368121442,"recall":0.5560344827586207,"window_days":30},"published_warning":"Stratified AUC overstates real-world performance by 49.6pp vs. time-based split. Do not cite the stratified number as a deployment figure; use the loco_median or the prod_rolling block once it populates.","schema":"voidly-sentinel-accuracy/v1","split_audit_complete":true,"training_holdout":{"as_of":"2026-08-02T02:10:30.609563","f1":0.816414686825054,"positive_rate":0.11782326510234647,"precision":0.8325991189427313,"recall":0.8008474576271186,"roc_auc":0.9792020373514432,"samples":2003,"source":"training_holdout","split_loco":{"f1_median":0.49052529973582604,"n_countries":21,"n_countries_evaluated":20,"notes":"Leave-country-out cross-validation. For each country, train on all other countries + eval on that country. The median is a more honest generalization figure than stratified. Individual per-country numbers reveal which countries the model struggles on (novel crises).","per_country":{"BD":{"auc":0.9350406858471374,"f1":0.6719367588932806,"n_pos_test":111,"n_test":731,"precision":0.5985915492957746,"recall":0.7657657657657657,"threshold":0.05},"BY":{"auc":0.8692889561270802,"f1":0.5,"n_pos_test":70,"n_test":731,"precision":0.41509433962264153,"recall":0.6285714285714286,"threshold":0.35000000000000003},"CN":{"auc":0.8499538319482917,"f1":0.08648648648648649,"n_pos_test":9,"n_test":731,"precision":0.045454545454545456,"recall":0.8888888888888888,"threshold":0.22000000000000003},"CU":{"auc":0.7139199643791394,"f1":0.17777777777777778,"n_pos_test":53,"n_test":731,"precision":0.14634146341463414,"recall":0.22641509433962265,"threshold":0.05},"EG":{"auc":0.8612234999053568,"f1":0.6686746987951807,"n_pos_test":144,"n_test":731,"precision":0.5904255319148937,"recall":0.7708333333333334,"threshold":0.05},"ID":{"auc":0.8979260369815093,"f1":0.5546875,"n_pos_test":87,"n_test":731,"precision":0.42011834319526625,"recall":0.8160919540229885,"threshold":0.05},"IR":{"auc":0.827003300330033,"f1":0.4868421052631579,"n_pos_test":125,"n_test":731,"precision":0.4134078212290503,"recall":0.592,"threshold":0.05},"KP":{"auc":null,"f1":null,"n_pos_test":0,"n_test":731,"note":"no positives in train or test","precision":null,"recall":null},"KZ":{"auc":0.8490803228794496,"f1":0.4235294117647059,"n_pos_test":44,"n_test":731,"precision":0.2857142857142857,"recall":0.8181818181818182,"threshold":0.1},"MM":{"auc":0.9634928603302098,"f1":0.6923076923076923,"n_pos_test":83,"n_test":731,"precision":0.576,"recall":0.8674698795180723,"threshold":0.22000000000000003},"MY":{"auc":0.898464937124731,"f1":0.4075829383886256,"n_pos_test":52,"n_test":731,"precision":0.27044025157232704,"recall":0.8269230769230769,"threshold":0.07},"PK":{"auc":0.8989345710098087,"f1":0.592964824120603,"n_pos_test":82,"n_test":731,"precision":0.5042735042735043,"recall":0.7195121951219512,"threshold":0.27},"RU":{"auc":0.6315987750340268,"f1":0.3491271820448878,"n_pos_test":139,"n_test":731,"precision":0.26717557251908397,"recall":0.5035971223021583,"threshold":0.060000000000000005},"SA":{"auc":0.8612369201936593,"f1":0.4351851851851852,"n_pos_test":57,"n_test":731,"precision":0.29559748427672955,"recall":0.8245614035087719,"threshold":0.08000000000000002},"SY":{"auc":0.9403725205611998,"f1":0.4788732394366197,"n_pos_test":42,"n_test":731,"precision":0.34,"recall":0.8095238095238095,"threshold":0.09000000000000001},"TH":{"auc":0.9280132148553202,"f1":0.5862068965517241,"n_pos_test":66,"n_test":731,"precision":0.4722222222222222,"recall":0.7727272727272727,"threshold":0.29000000000000004},"TM":{"auc":0.5615968801313628,"f1":0.046511627906976744,"n_pos_test":35,"n_test":731,"precision":0.125,"recall":0.02857142857142857,"threshold":0.05},"TR":{"auc":0.8965300917355712,"f1":0.6107784431137725,"n_pos_test":74,"n_test":731,"precision":0.5483870967741935,"recall":0.6891891891891891,"threshold":0.4100000000000001},"UZ":{"auc":0.8140489085836206,"f1":0.3870967741935484,"n_pos_test":54,"n_test":731,"precision":0.46153846153846156,"recall":0.3333333333333333,"threshold":0.23000000000000004},"VE":{"auc":0.9666087012874794,"f1":0.8226950354609929,"n_pos_test":163,"n_test":731,"precision":0.9747899159663865,"recall":0.7116564417177914,"threshold":0.12000000000000001},"VN":{"auc":0.8764957264957264,"f1":0.4942084942084942,"n_pos_test":81,"n_test":731,"precision":0.3595505617977528,"recall":0.7901234567901234,"threshold":0.05}},"precision_median":0.4142510804258459,"recall_median":0.7682995495495495,"roc_auc_max":0.9666087012874794,"roc_auc_median":0.8728923413114034,"roc_auc_min":0.5615968801313628,"split":"loco"},"split_stratified":{"f1":0.816414686825054,"n_test":2003,"notes":"Stratified random 15% holdout. Inflates AUC via within-country temporal leakage. Publish alongside time-based as the training-level sanity check, NOT as the deployment figure.","positive_rate":0.11782326510234647,"precision":0.8325991189427313,"recall":0.8008474576271186,"roc_auc":0.9792020373514432,"split":"stratified","threshold":0.38000000000000006},"split_summary":{"delta_stratified_vs_time_pp":49.6,"generated_at":"2026-08-02T03:00:24.624530+00:00","loco_median_auc":0.8728923413114034,"loco_median_f1":0.49052529973582604,"schema":"voidly-sentinel-evaluation/v1","stratified_auc":0.9792020373514432,"stratified_f1":0.816414686825054,"time_based_auc":0.4832385051784418,"time_based_f1":0.4413595021541407},"split_time":{"f1":0.4413595021541407,"n_test":2303,"notes":"Time-based split — train on data before T, test on data after T. This is the honest deployment-aligned figure. Treat as the Sentinel baseline when n_evaluated in prod is < 30.","positive_rate":0.3538862353452019,"precision":0.3618524332810047,"recall":0.5656441717791411,"roc_auc":0.4832385051784418,"split":"time","test_date_range":["2026-04-15","2026-08-02"],"threshold":0.05,"train_date_range":["2024-08-02","2025-12-24"],"val_date_range":["2025-12-24","2026-04-15"]},"threshold":0.38000000000000006},"window_days":30,"_voidly_pay":{"claim_did":"https://voidly.ai/pay/claim","for_builders":"https://voidly.ai/pay/for-builders","marketplace":"https://api.voidly.ai/v1/pay/marketplace","live_demo":"https://huggingface.co/spaces/emperor-mew/voidly-pay","universal_proxy":{"url":"https://api.voidly.ai/v1/pay/proxy","example":"https://api.voidly.ai/v1/pay/proxy?u=<https-url>&to=did:voidly:<your-did>&price=0.01","page":"https://voidly.ai/pay/proxy","note":"Paywall any public HTTPS URL with one query param. No SDK install required."},"install":{"typescript":"npm install @voidly/pay","python":"pip install voidly-pay","langchain":"pip install voidly-pay-langchain","vercel_ai":"npm install @voidly/pay-vercel-ai","mcp":"npx @voidly/pay-mcp","cli":"npm install -g @voidly/pay-cli"},"note":"Voidly Pay is the open agent-to-agent payment rail. Claim a DID + 10 starter credits in 60s, or paywall any URL via the universal proxy with one query param."}}