# For all robots
User-agent: *
Disallow: /beta/

User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-agent: Mediapartners-Google
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-agent: AdsBot-Google
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-agent: MSNBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-agent: MSNBot-Media
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js

User-Agent: Alexabot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js


# Allow search crawlers to discover the sitemap

User-agent: grapeshot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: SemrushBot-SA
Disallow:

User-agent: Twitterbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php