# Sitemap location
Sitemap: https://jeanscotthomes-napa.flywheelsites.com/sitemap_index.xml

# Flywheel default robots file version:2
User-agent: *
Disallow: /calendar/action:agenda/
Disallow: /calendar/action:oneday/
Disallow: /calendar/action:month/
Disallow: /calendar/action:week/
Disallow: /calendar/action:map/
Disallow: /cdn-cgi*
Disallow: /events/action*
Allow: /*.css
Allow: /*.js

# Disallow: /*?
# I need /properties/features/pdf/?pid=.* to get 
# a 410 error so it drops from GSC.TJS 02/20/25

# Crawl delay setting
# Crawl-delay: 1

# Allow Semrush bots
User-agent: SiteAuditBot
Allow: /

User-agent: SemrushBot
Allow: /

# Allow Googlebot
User-agent: Googlebot
Allow: /

# Block specific bots
User-agent: GoogleOther
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# OpenAI's search bot (if you want to allow it)
User-agent: OAI-SearchBot
Allow: /