# turingpost.com/robots.txt
# Last updated: September 2026
# Purpose: Maximize crawl budget for content pages, block scrapers,
#          permit AI search and retrieval bots for citation visibility.
#
# NOTE ON STRUCTURE: Crawlers obey exactly ONE group (the most specific
# match) and ignore all others. Per-bot "Allow: /" groups therefore
# OVERRIDE the shared rules below and were removed — AI bots are already
# permitted by default via User-agent: *. Do not re-add them without
# duplicating the full Disallow list inside each group.
#
# NOTE ON RULE ORDER: Disallow rules come BEFORE the closing "Allow: /".
# Under longest-match parsers (Google, Bing) order is irrelevant, but
# first-match parsers would let a leading "Allow: /" shadow everything
# below it, including the state-changing endpoints.
#
# NOTE ON PARAMETERS: Only self-generated tracking parameters are
# blocked. Third-party attribution params (ref=, trk=) are left
# crawlable so Google can read rel=canonical and consolidate link
# equity from external backlinks onto the clean URL.

# -- SEO SCRAPERS: Block from entire site -------------------------
User-agent: DotBot
User-agent: BLEXBot
User-agent: MauiBot
Disallow: /

# -- AGGRESSIVE CRAWLERS: Block from entire site ------------------
# Bytespider (ByteDance/TikTok) — aggressive scraper, no SEO value
# NOTE: Amazonbot removed from this group — it serves Amazon search
# and Rufus retrieval, which fits the citation-visibility goal above.
User-agent: Nutch
User-agent: Bytespider
Disallow: /

# -- ALL OTHER CRAWLERS -------------------------------------------
# Applies to Googlebot, Bingbot, GPTBot, ClaudeBot, PerplexityBot,
# OAI-SearchBot, Google-Extended, CCBot, Amazonbot, and every other
# compliant crawler.
User-agent: *

# Beehiiv technical API endpoints — no content value for indexing
Disallow: /paginated_comments
Disallow: /merge_tags
Disallow: /post_insights
Disallow: /dynamic_content_conditions
Disallow: /recommended_posts
Disallow: /visit_token
Disallow: /post_meta_information
Disallow: /page_view
Disallow: /current_subscription
Disallow: /forms/
Disallow: /login
Disallow: /subscribe/*/referrals
Disallow: /*jwt_token=
Disallow: /*offer_id=
Disallow: /*fbclid=
Disallow: /*via=
Disallow: /locales/
Disallow: /rss_feed

# NOTE: /upgrade was previously disallowed here. Removed August 2026 —
# Google had indexed it anyway via internal links and the sitemap, but
# could never fetch the page, so it ranked #1 on brand queries with no
# snippet: 923 impressions, 0 clicks over three months. The page needs
# to be crawlable for its meta description to be used at all.

# State-changing endpoints — bots hitting these register real
# poll votes and likes against subscriber IDs. Must stay blocked
# for ALL user agents, not just Googlebot.
Disallow: /polls
Disallow: /*/likes

# Platform-generated fallback for non-existent URLs. Beehiiv serves
# /archive?404=<path>&page=N with HTTP 200 instead of a real 404.
# Confirmed present in Common Crawl WARC records (CC-MAIN-2026-30) and
# reported to Beehiiv as a defect. Archive pagination without this
# parameter stays crawlable as a discovery path for older posts.
Disallow: /*404=

# Self-generated tracking parameters — matched in any position
# (? or &). Do NOT anchor these to "?" — Beehiiv appends after "&".
Disallow: /*utm_
Disallow: /*bhlid=
Disallow: /*last_resource_guid=
Disallow: /*post_slug=
Disallow: /*postSlug=
Disallow: /*subscription_id=
Disallow: /*pcid=
Disallow: /*ppid=

# UI state parameters
Disallow: /*comments=true
Disallow: /*modal=login
Disallow: /*?q=
Disallow: /*&q=

# Per-subscriber account endpoints — added September 2026 after GSC
# Crawl Stats (discovery sample, Aug 2026) showed Googlebot fetching
# /subscribe/<uuid>/manage with HTTP 200 and /unsubscribe/<JWT>.
# The URL space is unbounded: one URL per subscriber, and one per
# subscriber x post for unsubscribe tokens. Not linked from the site —
# a grep of post HTML (Sept 2026) found no hrefs to these paths, so
# discovery comes from email clients and browser telemetry. Robots.txt
# cannot stop discovery, but it stops the fetch, which is the point:
# /subscribe/*/manage is a subscriber's own preference page.
# The sibling rule /subscribe/*/referrals already existed above;
# "manage" was simply missing from it.
# _bhba is Beehiiv-generated, so it falls under self-generated
# tracking parameters per the NOTE ON PARAMETERS above.
Disallow: /unsubscribe/
Disallow: /subscribe/*/manage
Disallow: /*_bhba=

Allow: /

# -- SITEMAP -----------------------------------------------------
Sitemap: https://www.turingpost.com/sitemap.xml