{ "name": "InboxPolicy Email Verification Benchmark", "description": "A public, synthetic benchmark dataset of 40 email-verification scenarios spanning the nine categories InboxPolicy's send-decision API distinguishes. Each scenario pairs a ground-truth expected verdict with the verdict InboxPolicy would return, so the dataset doubles as a honesty gauge: it includes realistic agreement and disagreement (notably the spam-trap case, where InboxPolicy has no trap database and a live trap mailbox returns send). All addresses are synthetic and use reserved example domains (example.com, example.org, example.net, and .example subdomains); none is a real third-party inbox and none contains a secret. Released under CC BY 4.0; credit InboxPolicy (https://inboxpolicy.com/). The companion HTML page at /benchmark owns all JSON-LD; this file is raw data only.", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "version": "1.0", "generated": "2026-07-06", "scenario_count": 40, "distribution": { "valid": 10, "invalid": 8, "catch_all": 6, "disposable": 4, "role_based": 4, "syntax_invalid": 3, "mailbox_full": 2, "unknown": 2, "spam_trap": 1 }, "scenarios": [ { "id": 1, "email": "alice.johnson@example.com", "category": "valid", "expected_verdict": "send", "expected_reason": "Real, SMTP-confirmed personal mailbox on a standard domain; safe to send.", "inboxpolicy_verdict": "send", "note": "Clean signal: SMTP RCPT accepted, no catch-all signature, confidence 95." }, { "id": 2, "email": "bob.smith@example.org", "category": "valid", "expected_verdict": "send", "expected_reason": "Deliverable personal mailbox; MX and live SMTP confirm receipt.", "inboxpolicy_verdict": "send", "note": "Verified mailbox, confidence 92." }, { "id": 3, "email": "carol.wong@example.com", "category": "valid", "expected_verdict": "send", "expected_reason": "Personal inbox on a well-established mail provider.", "inboxpolicy_verdict": "send", "note": "Confidence 90; sits in the ~90% valid-verdict agreement band vs MillionVerifier." }, { "id": 4, "email": "d.lee@example.net", "category": "valid", "expected_verdict": "send", "expected_reason": "Confirmed mailbox accepting mail.", "inboxpolicy_verdict": "send", "note": "Confidence 93." }, { "id": 5, "email": "erin.murphy@example.org", "category": "valid", "expected_verdict": "send", "expected_reason": "Deliverable address with healthy MX and SMTP responses.", "inboxpolicy_verdict": "send", "note": "Confidence 91." }, { "id": 6, "email": "frank.castle@example.com", "category": "valid", "expected_verdict": "send", "expected_reason": "Personal mailbox confirmed over SMTP.", "inboxpolicy_verdict": "send", "note": "Confidence 88." }, { "id": 7, "email": "grace.hopper@example.edu", "category": "valid", "expected_verdict": "send", "expected_reason": "Institutional mailbox, SMTP-confirmed.", "inboxpolicy_verdict": "send", "note": "Confidence 94." }, { "id": 8, "email": "hiroshi@example.org", "category": "valid", "expected_verdict": "send", "expected_reason": "Personal inbox, SMTP-confirmed.", "inboxpolicy_verdict": "send", "note": "Confidence 89." }, { "id": 9, "email": "ivan.petrov@example.com", "category": "valid", "expected_verdict": "send", "expected_reason": "Confirmed deliverable personal mailbox.", "inboxpolicy_verdict": "send_with_caution", "note": "Mailbox confirmed but provider reputation lowered confidence to 68; InboxPolicy conservatively returns send_with_caution rather than send. A realistic non-agreement case." }, { "id": 10, "email": "julia.r@example.net", "category": "valid", "expected_verdict": "send", "expected_reason": "Personal mailbox, SMTP-confirmed.", "inboxpolicy_verdict": "send", "note": "Confidence 92." }, { "id": 11, "email": "no.such.user@example.com", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Mailbox rejected at SMTP (550 user unknown); address does not exist.", "inboxpolicy_verdict": "avoid", "note": "Clear 550 reject; high confidence." }, { "id": 12, "email": "deleted.account@example.org", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Previously valid mailbox, now returns 550.", "inboxpolicy_verdict": "avoid", "note": "550 user unknown." }, { "id": 13, "email": "mistype.user@example.com", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Nonexistent local part; SMTP rejected the recipient.", "inboxpolicy_verdict": "avoid", "note": "550 NoSuchUser." }, { "id": 14, "email": "gone.away@example.net", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Domain accepts mail but the specific mailbox is dead.", "inboxpolicy_verdict": "avoid", "note": "550 mailbox unavailable." }, { "id": 15, "email": "nonexistent@example.com", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "No mailbox by that name on the host.", "inboxpolicy_verdict": "avoid", "note": "Rejected at RCPT." }, { "id": 16, "email": "fake.name123@example.org", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Fabricated local part, rejected by the server.", "inboxpolicy_verdict": "avoid", "note": "550 unknown." }, { "id": 17, "email": "typo.user@example.com", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Invalid local part, rejected at SMTP.", "inboxpolicy_verdict": "avoid", "note": "550 rejected." }, { "id": 18, "email": "old.role@example.net", "category": "invalid", "expected_verdict": "avoid", "expected_reason": "Former role mailbox now returning 550.", "inboxpolicy_verdict": "review", "note": "Server returned an inconsistent SMTP response across probes; InboxPolicy could not confirm the rejection and returned review rather than risk a false avoid. Realistic non-agreement case." }, { "id": 19, "email": "anyone@corp.example.com", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Domain accepts mail for any address, so SMTP cannot confirm the specific mailbox exists.", "inboxpolicy_verdict": "review", "note": "catch_all evidence tagged; default policy maps to review. ~30-40% of B2B addresses look like this." }, { "id": 20, "email": "contact@startup.example.org", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Catch-all domain; SMTP accepts every recipient regardless of existence.", "inboxpolicy_verdict": "review", "note": "Tagged catch_all; review by default, never guessed safe." }, { "id": 21, "email": "sales@enterprise.example.com", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Catch-all B2B domain; mailbox unverifiable by SMTP.", "inboxpolicy_verdict": "review", "note": "review with catch_all flag set." }, { "id": 22, "email": "ceo@bigcorp.example.net", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Catch-all domain; cannot confirm the specific executive mailbox.", "inboxpolicy_verdict": "review", "note": "review (catch_all)." }, { "id": 23, "email": "anyone@growth.example.com", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Catch-all domain; specific mailbox unverifiable.", "inboxpolicy_verdict": "send_with_caution", "note": "Under an aggressive policy configuration the same catch_all evidence maps to send_with_caution instead of review; InboxPolicy never returns send for an unconfirmed mailbox." }, { "id": 24, "email": "support@saas.example.org", "category": "catch_all", "expected_verdict": "review", "expected_reason": "Catch-all SaaS domain; SMTP accepts any recipient.", "inboxpolicy_verdict": "review", "note": "review (catch_all)." }, { "id": 25, "email": "junk@throwaway.example", "category": "disposable", "expected_verdict": "avoid", "expected_reason": "Disposable/temporary-mail domain; the address self-destructs and is unsuitable for real delivery.", "inboxpolicy_verdict": "avoid", "note": "Disposable domain flagged; maps to avoid. Domain name is synthetic." }, { "id": 26, "email": "temp.user@tempmail.example.com", "category": "disposable", "expected_verdict": "avoid", "expected_reason": "Temporary-mailbox provider; not a durable inbox.", "inboxpolicy_verdict": "avoid", "note": "Disposable flagged and routed to avoid." }, { "id": 27, "email": "signup@disposable.example.org", "category": "disposable", "expected_verdict": "avoid", "expected_reason": "Burner address on a disposable provider.", "inboxpolicy_verdict": "avoid", "note": "Disposable domain leads to avoid." }, { "id": 28, "email": "x@burner.example", "category": "disposable", "expected_verdict": "avoid", "expected_reason": "Throwaway inbox; not a real recipient.", "inboxpolicy_verdict": "avoid", "note": "Disposable flagged, returned avoid." }, { "id": 29, "email": "info@example.com", "category": "role_based", "expected_verdict": "send_with_caution", "expected_reason": "Role/group address; deliverable but not a personal inbox, so moderate confidence.", "inboxpolicy_verdict": "send_with_caution", "note": "Role-based address flagged; deliverable with moderate confidence and elevated reputation risk." }, { "id": 30, "email": "sales@example.org", "category": "role_based", "expected_verdict": "send_with_caution", "expected_reason": "Shared sales mailbox; deliverable but a role address.", "inboxpolicy_verdict": "send_with_caution", "note": "Role address; send_with_caution." }, { "id": 31, "email": "admin@example.com", "category": "role_based", "expected_verdict": "send_with_caution", "expected_reason": "Administrator role mailbox; deliverable but not personal.", "inboxpolicy_verdict": "send_with_caution", "note": "Role address flagged; send_with_caution." }, { "id": 32, "email": "support@example.net", "category": "role_based", "expected_verdict": "send_with_caution", "expected_reason": "Shared support inbox; a role address.", "inboxpolicy_verdict": "send_with_caution", "note": "Role address; send_with_caution." }, { "id": 33, "email": "not-an-email", "category": "syntax_invalid", "expected_verdict": "avoid", "expected_reason": "No local-part/domain separator; malformed address.", "inboxpolicy_verdict": "avoid", "note": "Rejected locally before SMTP at 0 credits; malformed input is never billed." }, { "id": 34, "email": "user@", "category": "syntax_invalid", "expected_verdict": "avoid", "expected_reason": "Missing domain; malformed address.", "inboxpolicy_verdict": "avoid", "note": "Malformed; rejected locally, free of charge." }, { "id": 35, "email": "@example.com", "category": "syntax_invalid", "expected_verdict": "avoid", "expected_reason": "Missing local part; malformed address.", "inboxpolicy_verdict": "avoid", "note": "Rejected before any SMTP check runs; 0 credits." }, { "id": 36, "email": "quota.full@example.com", "category": "mailbox_full", "expected_verdict": "retry_later", "expected_reason": "Mailbox exists but is over quota (552 storage exceeded).", "inboxpolicy_verdict": "retry_later", "note": "552 over quota; transient condition, re-check later (the re-check is usually free within 72h)." }, { "id": 37, "email": "inbox.full@example.org", "category": "mailbox_full", "expected_verdict": "retry_later", "expected_reason": "Recipient mailbox over storage quota.", "inboxpolicy_verdict": "retry_later", "note": "552 over quota; retry_later." }, { "id": 38, "email": "greylisted@example.com", "category": "unknown", "expected_verdict": "review", "expected_reason": "SMTP timed out or greylisted; mailbox neither confirmed nor denied.", "inboxpolicy_verdict": "review", "note": "Honest unknown preserved as a first-class state and mapped to review; never guessed safe. May escalate to a wholesale fallback verifier." }, { "id": 39, "email": "protected.gw@example.org", "category": "unknown", "expected_verdict": "review", "expected_reason": "Protective mail gateway blocked the SMTP probe; mailbox unconfirmed.", "inboxpolicy_verdict": "review", "note": "Unknown mapped to review (first-class state), not over-promoted to send." }, { "id": 40, "email": "office@trapnet.example", "category": "spam_trap", "expected_verdict": "avoid", "expected_reason": "Pristine spam trap: a real, SMTP-accepting mailbox that exists only to catch senders of scraped or purchased lists.", "inboxpolicy_verdict": "send", "note": "InboxPolicy confirmed the mailbox over SMTP and returned send because it maintains no spam-trap database; a live trap looks valid to any SMTP-only engine. This is the documented gap vs ZeroBounce and the single most important disagreement in the set; route trap detection to a specialist." } ] }