# robots.txt for MarkdownForge - AI-Powered Markdown to Word Converter
# Website: https://mdforgeweb.space/
# Created: 2024
# Purpose: SEO optimization and crawler guidance for a productivity tool

# =====================
# SEARCH ENGINE CRAWLERS
# =====================

# Major search engines with optimized crawl settings
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Slurp
User-agent: Baiduspider
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# =====================
# SOCIAL MEDIA CRAWLERS
# =====================

User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
Allow: /

# =====================
# AI & RESEARCH CRAWLERS
# =====================

User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ChatGPT
User-agent: PerplexityBot
User-agent: YouBot
User-agent: AI2Bot
Allow: /

# =====================
# DEFAULT RULES
# =====================

# Allow all other bots by default
User-agent: *
Allow: /

# =====================
# RESTRICTED AREAS
# =====================

# Block sensitive directories (uncommented for security)
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/

# Block URL parameters to reduce duplicate crawling
Disallow: /*?*

# =====================
# ALLOWED RESOURCES
# =====================

# Explicitly allow important resources for proper rendering
Allow: /css/
Allow: /js/
Allow: /assets/
Allow: /images/
Allow: /fonts/

# =====================
# SITE CONFIGURATION
# =====================

# Sitemap location for improved crawl efficiency
Sitemap: https://mdforgeweb.space/sitemap.xml

# Host directive for consistent domain handling
Host: mdforgeweb.space

# =====================
# ADDITIONAL NOTES
# =====================
# This robots.txt file is optimized for:
# - Document conversion tools
# - Productivity applications  
# - Developer resources
# - Educational content
# - Business tools
#
# The site provides value through:
# - Free document conversion services
# - Educational markdown resources
# - Professional templates
# - Open source tools
# - Technical documentation
#
# Target audiences:
# - Developers and technical writers
# - Students and educators  
# - Business professionals
# - Content creators
# - Documentation teams
#
# Geographic relevance: Global
# Content language: English (primary)
# Update frequency: Regular feature updates
# Content quality: High-value productivity tool