# LucyESL — English teaching jobs in Korea
# Search + AI answer engines are welcome; job pages are meant to be found.
#
# RFC 9309 §2.2.1: a crawler obeys exactly ONE group — the most specific
# user-agent match — and ignores every other group. Groups do NOT merge. So the
# two exclusions below have to be repeated inside every named group, or the 17
# bots named further down inherit nothing and crawl the two paths this file is
# trying to keep them out of.

User-agent: *
# Stance, in the syntax closest to the IETF AIPREF vocabulary that may
# eventually standardise it. It has to sit INSIDE the group: a blank line
# ends a group, so a signal after the last one belongs to nobody — which is
# where this line lived until 2026-09-09, declaring intent to no parser.
# Unenforceable either way; it states a position, it does not police one.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /signup.php
# /go.php is the counted redirect out — to an aggregated listing's original
# posting, or to one of the board's own doors out such as the footer's Google
# Preferred Sources link. Every one of them leaves the site. It sends
# noindex/nofollow anyway; this keeps crawl budget on the real pages.
Disallow: /go.php

# AI answer/search engines and their user-initiated fetchers, named explicitly
# so listings and salary data can be surfaced and cited.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-WebIndexer
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
User-agent: Amazonbot
User-agent: Applebot
User-agent: Yeti
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /signup.php
Disallow: /go.php

# Model-training crawlers — allowed deliberately: for a niche board, being in
# the models' world knowledge is brand recall with no licensing downside.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /signup.php
Disallow: /go.php

Sitemap: https://lucyesl.com/sitemap.xml