# =================================================================
# robots.txt — DataWiper.com
# Diperbarui: 2025-06-24
# Panduan lengkap untuk Googlebot & crawler lainnya
# =================================================================

# ── SEMUA CRAWLER (default) ───────────────────────────────────────
User-agent: *
Allow: /

# Halaman admin & sistem — JANGAN di-crawl
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /masuk/
Disallow: /register/
Disallow: /daftar/
Disallow: /akun/
Disallow: /profil/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cache/
Disallow: /assets/private/

# URL dengan parameter yang menghasilkan duplicate content
Disallow: /search?
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?session=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?lang=
Disallow: /*?print=
Disallow: /*?preview=
Disallow: /*?replytocom=

# Feed / API endpoints
Disallow: /feed/
Disallow: /rss/
Disallow: /api/

# File non-HTML yang tidak perlu di-index
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar$

# Izinkan eksplisit halaman SEO penting
Allow: /sitemap.xml
Allow: /sitemap-pages.xml
Allow: /sitemap-blog.xml

# ── GOOGLEBOT — Aturan khusus Google ─────────────────────────────
User-agent: Googlebot
Allow: /
# Izinkan akses ke CSS dan JS untuk rendering penuh
Allow: /*.css
Allow: /*.js
Allow: /assets/
Allow: /images/
# Blokir halaman admin & sistem
Disallow: /admin/
Disallow: /api/
Disallow: /tmp/
Disallow: /cache/
# Crawl-delay ringan agar tidak membebani server
Crawl-delay: 1

# ── GOOGLEBOT IMAGE — Hanya izinkan folder gambar ────────────────
User-agent: Googlebot-Image
Allow: /assets/images/
Allow: /images/
Allow: /uploads/
Disallow: /

# ── BINGBOT ───────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# ── YANDEX ────────────────────────────────────────────────────────
User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 3

# ── BLOKIR SCRAPER & BOT BERBAHAYA ───────────────────────────────
# Bot SEO 3rd party — konsumsi crawl budget tanpa manfaat indexing
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Bot scraper konten
User-agent: ia_archiver
Disallow: /

User-agent: CCBot
Disallow: /

# ── SITEMAP — WAJIB untuk Google Indexing ────────────────────────
Sitemap: https://www.data-wiper.com/sitemap.xml
Sitemap: https://www.data-wiper.com/sitemap-pages.xml
Sitemap: https://www.data-wiper.com/sitemap-blog.xml