User-agent: * Allow: / Disallow: /*.json$ # Crawl-delay for polite crawling Crawl-delay: 1 # Sitemap location Sitemap: https://urbanfit.com.co/sitemap.xml # Block specific bots that waste bandwidth User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10