User-agent: * Allow: / # Prevent indexing of internal search queries, filter/sort permutations & ajax endpoints to protect crawl budget ([SEO5-06]) Disallow: /*?search= Disallow: /*?q= Disallow: /*?ajax= Disallow: /*?sort= Disallow: /*?duration= Disallow: /*?quality= Disallow: /*?gender= Disallow: /*?utm_ # Prevent crawling of backend utilities, admin panels & cron scripts Disallow: /admin-studio.php Disallow: /cron-importer.php Disallow: /cron-sitemaps.php Disallow: /cron-runner.php Disallow: /auto-tagger.php Disallow: /database-editor.php Disallow: /database-query.php Disallow: /search-index.php Disallow: /tagger-test.php Disallow: /build-performer-index.php # Block aggressive non-search scrapers from draining server resources ([SEO5-06]) User-agent: MJ12bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DotBot Disallow: / User-agent: Bytespider Disallow: / User-agent: PetalBot Disallow: / # Sitemaps Sitemap: https://davisporn.com/sitemap.xml