# robots.txt는 크롤러당 "가장 구체적인 user-agent 그룹 하나"만 적용하며,
# 특정 UA 그룹과 * 그룹을 병합하지 않는다 (RFC 9309 §2.2.1 / Google robots spec).
# → 아래 공통 차단(admin/cart/payment/profile/oauth/assets·sounds 등)은 모든 UA 그룹에
#   그대로 복제한다. 특정 UA 그룹에서 한 줄이라도 빠지면 그 봇에게 해당 경로가 열린다.
# sounds 탭은 오디오 수집에 크롤 예산만 낭비되어 전 UA 차단 (base·query·sub-path).

User-agent: *
Disallow: /admin
Allow: /partners$
Disallow: /partners/
Allow: /artist-center$
Disallow: /artist-center/
Disallow: /cart
Allow: /payment-terms
Disallow: /payment
Disallow: /terms
Disallow: /oauth
Disallow: /public/user/token-refresh
Disallow: /*/sounds$
Disallow: /*/sounds?
Disallow: /*/sounds/

User-agent: Googlebot
User-agent: Bingbot
Disallow: /admin
Allow: /partners$
Disallow: /partners/
Allow: /artist-center$
Disallow: /artist-center/
Allow: /profile/*/post
Disallow: /profile
Disallow: /cart
Allow: /payment-terms
Disallow: /payment
Disallow: /terms
Disallow: /oauth
Disallow: /public/user/token-refresh
Disallow: /*/sounds?
Disallow: /*/sounds/
# 비색인 로케일 크롤 차단 (AI 크롤러 그룹과 동일 범위).
Disallow: /ar/
Disallow: /de/
Disallow: /es/
Disallow: /fr/
Disallow: /hi/
Disallow: /id/
Disallow: /it/
Disallow: /ms/
Disallow: /nl/
Disallow: /pl/
Disallow: /pt/
Disallow: /ru/
Disallow: /th/
Disallow: /tl/
Disallow: /vi/

# ClaudeBot / Applebot / Amazonbot: 공통 차단 + search 쿼리 변형 + minor lang 차단.
# 태그 상세(song/musician/hashtag/theme)는 무프리픽스+en/ko/ja만 허용 —
# 로케일 곱셈 크롤(카탈로그×8~9 변형, 일 450만건) 억제로 de/es/fr/zh-hans/zh-hant 차단.
# genre 는 전 로케일 차단 — 최다 재크롤 표적(경로당 일 1천회+)인데 집계 페이지라
# AI 답변 가치 낮음(meta-externalagent 의 /sheet 차단과 동일 논리)
User-agent: ClaudeBot
User-agent: Applebot
User-agent: Amazonbot
User-agent: GPTBot
User-agent: PerplexityBot
User-agent: meta-webindexer
Disallow: /admin
Allow: /partners$
Disallow: /partners/
Allow: /artist-center$
Disallow: /artist-center/
Allow: /profile/*/post
Disallow: /profile
Disallow: /cart
Allow: /payment-terms
Disallow: /payment
Disallow: /terms
Disallow: /oauth
Disallow: /public/user/token-refresh
Disallow: /*/sounds?
Disallow: /*/sounds/
Disallow: /search?
Disallow: /*/search?
Disallow: /ar/
Disallow: /hi/
Disallow: /id/
Disallow: /it/
Disallow: /ms/
Disallow: /nl/
Disallow: /pl/
Disallow: /pt/
Disallow: /ru/
Disallow: /tl/
Disallow: /th/
Disallow: /vi/
Disallow: /genre/
Disallow: /*/genre/
Disallow: /de/song/
Disallow: /de/musician/
Disallow: /de/hashtag/
Disallow: /de/theme/
Disallow: /es/song/
Disallow: /es/musician/
Disallow: /es/hashtag/
Disallow: /es/theme/
Disallow: /fr/song/
Disallow: /fr/musician/
Disallow: /fr/hashtag/
Disallow: /fr/theme/
Disallow: /zh-hans/song/
Disallow: /zh-hans/musician/
Disallow: /zh-hans/hashtag/
Disallow: /zh-hans/theme/
Disallow: /zh-hant/song/
Disallow: /zh-hant/musician/
Disallow: /zh-hant/hashtag/
Disallow: /zh-hant/theme/

# meta-externalagent (Meta AI training crawler):
# - 개별 sheet PDP (/:artist/:id, /sheetmusic/...) 는 학습 허용
# - listing/aggregate pages (/sheet, /song, /musician) 는 차단
#   filter 조합 폭발 → cache miss 폭증 (Apr 25/29 burst의 60%+ 이 봇),
#   집계 페이지 자체는 AI 학습 가치도 낮음
# - $ matcher 로 /sheet 정확 매칭 — /sheetmusic/ 등 PDP variant 는 매칭 안 됨.
#   $ 미지원 보수적 봇 대비 sub-path / 와 query ? 도 명시 (PDP 와 충돌 없음)
User-agent: meta-externalagent
Disallow: /admin
Allow: /partners$
Disallow: /partners/
Allow: /artist-center$
Disallow: /artist-center/
Allow: /profile/1*/post
Allow: /profile/2*/post
Allow: /profile/3*/post
Allow: /profile/4*/post
Allow: /profile/5*/post
Allow: /profile/6*/post
Allow: /profile/7*/post
Allow: /profile/8*/post
Allow: /profile/9*/post
Disallow: /profile
Disallow: /cart
Allow: /payment-terms
Disallow: /payment
Disallow: /terms
Disallow: /oauth
Disallow: /public/user/token-refresh
Disallow: /*/sounds?
Disallow: /*/sounds/
Disallow: /ar/
Disallow: /hi/
Disallow: /id/
Disallow: /it/
Disallow: /ms/
Disallow: /nl/
Disallow: /pl/
Disallow: /pt/
Disallow: /ru/
Disallow: /tl/
Disallow: /th/
Disallow: /vi/
Disallow: /sheet$
Disallow: /sheet/
Disallow: /sheet?
Disallow: /*/sheet$
Disallow: /*/sheet/
Disallow: /*/sheet?
Disallow: /song/
Disallow: /*/song/
Disallow: /musician/
Disallow: /*/musician/

Sitemap: https://www.mymusic5.com/sitemap_index.xml