Sitemap: https://www.mahdlo.net/sitemap.xml

# AI crawlers and training/grounding controls.
#
# Allowed deliberately, not by omission: Mahdlo wants to be found and cited when
# someone asks an assistant about fractional CMO and CRO leadership.
#
# Each crawler gets its OWN complete group, repeated verbatim, and that
# verbosity is load-bearing. HubSpot does not preserve a group that lists
# several User-agent lines together - it splits them into one group per agent
# and injects only its own system exclusions into each, which silently drops
# "Disallow: /meetings/*" from every agent but the last. Written out this way,
# HubSpot's per-group injection lands on top of a rule set that is already
# correct.
#
# Google-Extended and Applebot-Extended are not crawlers - they are the tokens
# controlling whether Google and Apple may use already-crawled content for AI
# grounding. Listing them at all is how you opt in.
#
# Content-Signal (contentsignals.org) states usage preferences per group:
# search, AI answers (ai-input) and AI training (ai-train) are all permitted.


User-agent: GPTBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: OAI-SearchBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ChatGPT-User
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ClaudeBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-User
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-SearchBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: PerplexityBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Perplexity-User
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Google-Extended
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Applebot-Extended
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: meta-externalagent
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: CCBot
Allow: /
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Everything else. HubSpot appends its own system exclusions to every group
# above and below; this must stay the last User-agent block in the file.

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: *
Disallow: /meetings/*
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*