# ========================================= # robots.txt for djstoian.com # WordPress SEO + AI optimized # ========================================= User-agent: * Allow: / # Block WordPress system folders Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Block unnecessary system files Disallow: /readme.html Disallow: /license.txt # Block search/result pages Disallow: /?s= Disallow: /search/ # Block temporary or tracking URLs Disallow: /*?replytocom= Disallow: /*?utm_source= Disallow: /*?utm_medium= Disallow: /*?utm_campaign= # Allow important assets Allow: /wp-content/uploads/ Allow: /wp-content/themes/ Allow: /wp-content/plugins/ # Sitemap Sitemap: https://djstoian.com/sitemap.xml # ========================================= # AI Crawlers # ========================================= User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: Bingbot Allow: / User-agent: Googlebot Allow: / # ========================================= # Crawl-delay for aggressive bots # ========================================= User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10