User-agent: *
Allow: /

# Disallow specific patterns that may cause duplicate content
Disallow: /jupiter/dca$  # Redirected URL
Disallow: /*?*sort=       # Sort parameters
Disallow: /*?*page=       # Pagination parameters
Disallow: /*?*size=       # Page size parameters

# Allow important directories
Allow: /jupiter/
Allow: /kamino/
Allow: /drift/
Allow: /tokens/
Allow: /api/docs/
Allow: /about
Allow: /privacy-policy.html

# Sitemap
Sitemap: https://cosmic.markets/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific user agent rules
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block unwanted bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /