# Global settings for all bots
User-Agent: *  
Disallow: /wp-admin/  
Allow: /wp-admin/admin-ajax.php  

# Prevent indexing of sensitive or unnecessary files
Disallow: /wp-login.php  
Disallow: /wp-register.php  
Disallow: /xmlrpc.php  
Disallow: /cgi-bin/  

# Block unnecessary WordPress directories
Disallow: /wp-includes/  
Disallow: /wp-content/plugins/  
Disallow: /wp-content/cache/  
Disallow: /wp-content/themes/  
Disallow: /trackback/  
Disallow: /?s=  
Disallow: /search/  

# Allow indexing of media files and assets
Allow: /wp-content/uploads/  

# Sitemaps for better crawl efficiency
Sitemap: https://news.gauteng.net/sitemap_index.xml