Enhancement: Major code improvements and optimizations

requirements.txt

@@ -20,4 +20,12 @@ asyncio>=3.4.3
 
 # Utilities
 tqdm>=4.66.1
-PyYAML>=6.0.1
+PyYAML>=6.0.1
+requests>=2.26.0
+backoff>=2.1.2
+logging>=0.5.1.2
+typing>=3.7.4.3
+aiohttp>=3.8.0
+cachetools>=5.0.0
+ratelimit>=2.2.1
+backoff>=2.1.2

src/config.py

@@ -18,7 +18,7 @@
 
 # Create directories if they don't exist
 for directory in [INPUT_DIR, RAW_DIR, LOGS_DIR, SECTIONS_DIR, REPORTS_DIR, 
-                 REPORTS_INDIVIDUAL_DIR, REPORTS_CROSS_CASE_DIR, REPORTS_EXECUTIVE_DIR]:
+                  REPORTS_INDIVIDUAL_DIR, REPORTS_CROSS_CASE_DIR, REPORTS_EXECUTIVE_DIR]:
     directory.mkdir(parents=True, exist_ok=True)
 
 # Claude settings
@@ -35,10 +35,49 @@
 REQUEST_TIMEOUT = 30
 RETRY_DELAY = 1
 
-# Logging format
-LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
-
-# Add to existing config.py
+# FireCrawl API settings
 FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY")
 if not FIRECRAWL_API_KEY:
-    raise ValueError("FIRECRAWL_API_KEY environment variable is not set")
+    raise ValueError("FIRECRAWL_API_KEY environment variable is not set")
+
+# Logging configuration
+LOGGING_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
+        },
+    },
+    'handlers': {
+        'default': {
+            'level': 'INFO',
+            'formatter': 'standard',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stdout',
+        },
+        'file': {
+            'level': 'INFO',
+            'formatter': 'standard',
+            'class': 'logging.FileHandler',
+            'filename': LOGS_DIR / 'crawler.log',
+            'mode': 'a',
+        },
+    },
+    'loggers': {
+        '': {  # root logger
+            'handlers': ['default', 'file'],
+            'level': 'INFO',
+            'propagate': True
+        }
+    }
+}
+
+# Crawler configurations
+CRAWLER_CONFIG = {
+    'max_retries': 3,
+    'timeout': 30,
+    'rate_limit_pause': 1.0,
+    'max_pages': 100,
+    'user_agent': 'AI Case Study Analyzer Bot 1.0',
+}

src/firecrawl_config.py

@@ -0,0 +1,34 @@
+"""Firecrawl API configuration settings"""
+
+FIRECRAWL_CONFIG = {
+    # API Settings
+    "base_url": "https://api.firecrawl.com/v1",
+
+    # Cache Settings
+    "cache_ttl": 3600,  # Cache lifetime in seconds
+    "cache_maxsize": 1000,  # Maximum cache entries
+
+    # Rate Limiting
+    "rate_limit_calls": 60,  # Calls allowed per period
+    "rate_limit_period": 60,  # Period in seconds
+
+    # Concurrent Requests
+    "max_concurrent": 5,  # Maximum concurrent requests
+
+    # Timeouts
+    "request_timeout": 30,  # Request timeout in seconds
+    "scrape_timeout": 30000,  # Scrape timeout in milliseconds
+
+    # Retry Settings
+    "max_retries": 3,
+    "retry_delay": 1,  # Initial retry delay in seconds
+
+    # Output Formats
+    "default_formats": ["markdown"],
+
+    # Content Settings
+    "only_main_content": True,
+    "include_subdomains": True,
+    "ignore_sitemap": False,
+    "url_limit": 5000
+}