User-agent: Scrapy Allow: / #Default robots.txt file (which asks all bots to crawl slowly but still index everything). User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-includes/ Allow: /wp-includes/js/ Allow: /wp-includes/images/ Disallow: /trackback/ Disallow: /wp-login.php Disallow: /wp-register.php Crawl-delay: 10