Initial commit: Market Trends Scraper

main.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Market Trends Scraper - Main Entry Point
+
+This script serves as the main entry point for the Market Trends Scraper application.
+It initializes the scraper, processes configuration, and orchestrates the scraping
+and analysis workflow.
+"""
+
+import sys
+import argparse
+from pathlib import Path
+
+# Add src directory to Python path
+sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
+
+from scraper import MarketTrendsScraper
+from config_manager import ConfigManager
+from logger import setup_logger
+
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Web Scraper for Market Trends - Collect and analyze pricing and product trends"
+    )
+    parser.add_argument(
+        "--config", "-c",
+        type=str,
+        default="config/config.yaml",
+        help="Path to configuration file"
+    )
+    parser.add_argument(
+        "--output", "-o",
+        type=str,
+        default="data/output.csv",
+        help="Path to output file"
+    )
+    parser.add_argument(
+        "--headless",
+        action="store_true",
+        help="Run browser in headless mode"
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Enable verbose logging"
+    )
+    return parser.parse_args()
+
+
+def main():
+    """Main application entry point."""
+    args = parse_arguments()
+    
+    # Setup logging
+    logger = setup_logger(verbose=args.verbose)
+    logger.info("Starting Market Trends Scraper")
+    
+    try:
+        # Load configuration
+        config_manager = ConfigManager(args.config)
+        config = config_manager.load_config()
+        logger.info("Configuration loaded successfully")
+        
+        # Initialize and run scraper
+        scraper = MarketTrendsScraper(config, headless=args.headless)
+        logger.info("Scraper initialized")
+        
+        # Run the scraping process
+        data = scraper.scrape_market_trends()
+        logger.info(f"Scraped {len(data)} product records")
+        
+        # Save results
+        scraper.save_data(data, args.output)
+        logger.info(f"Data saved to {args.output}")
+        
+        # Analyze trends
+        analysis = scraper.analyze_trends(data)
+        logger.info("Trend analysis completed")
+        
+        # Save analysis results
+        analysis_output = args.output.replace('.csv', '_analysis.json')
+        scraper.save_analysis(analysis, analysis_output)
+        logger.info(f"Analysis saved to {analysis_output}")
+        
+        logger.info("Market Trends Scraper completed successfully")
+        return 0
+        
+    except Exception as e:
+        logger.error(f"Error occurred: {str(e)}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())