97 lines
3.0 KiB
Python
97 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Basic Usage Example for Market Trends Scraper
|
|
|
|
This script demonstrates how to use the Market Trends Scraper
|
|
to collect and analyze pricing data from e-commerce websites.
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add src directory to Python path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
|
|
|
from config_manager import ConfigManager
|
|
from scraper import MarketTrendsScraper
|
|
from logger import setup_logger
|
|
|
|
|
|
def main():
|
|
"""Main function demonstrating basic scraper usage."""
|
|
|
|
# Setup logging
|
|
setup_logger(verbose=True)
|
|
|
|
# Initialize configuration manager
|
|
config_manager = ConfigManager("../config/sample_config.yaml")
|
|
|
|
# Load configuration
|
|
try:
|
|
config = config_manager.load_config()
|
|
print("✓ Configuration loaded successfully")
|
|
except Exception as e:
|
|
print(f"✗ Failed to load configuration: {str(e)}")
|
|
return 1
|
|
|
|
# Initialize scraper
|
|
try:
|
|
scraper = MarketTrendsScraper(config, headless=True)
|
|
print("✓ Scraper initialized successfully")
|
|
except Exception as e:
|
|
print(f"✗ Failed to initialize scraper: {str(e)}")
|
|
return 1
|
|
|
|
try:
|
|
# Scrape market trends data
|
|
print("\n🔍 Scraping market trends data...")
|
|
data = scraper.scrape_market_trends()
|
|
print(f"✓ Scraped {len(data)} product records")
|
|
|
|
if not data:
|
|
print("⚠ No data was scraped. Check your configuration and selectors.")
|
|
return 0
|
|
|
|
# Save scraped data
|
|
output_file = "../data/example_output.csv"
|
|
scraper.save_data(data, output_file)
|
|
print(f"✓ Data saved to {output_file}")
|
|
|
|
# Analyze trends
|
|
print("\n📊 Analyzing market trends...")
|
|
analysis = scraper.analyze_trends(data)
|
|
print("✓ Trend analysis completed")
|
|
|
|
# Save analysis results
|
|
analysis_file = "../data/example_analysis.json"
|
|
scraper.save_analysis(analysis, analysis_file)
|
|
print(f"✓ Analysis saved to {analysis_file}")
|
|
|
|
# Print summary
|
|
print("\n📋 Summary:")
|
|
print(f" - Total products: {analysis.get('total_products', 0)}")
|
|
|
|
if 'price_analysis' in analysis:
|
|
price_analysis = analysis['price_analysis']
|
|
print(f" - Average price: ${price_analysis.get('average_price', 0):.2f}")
|
|
print(f" - Price range: ${price_analysis.get('min_price', 0):.2f} - ${price_analysis.get('max_price', 0):.2f}")
|
|
|
|
if 'sources' in analysis:
|
|
print(" - Products by source:")
|
|
for source, count in analysis['sources'].items():
|
|
print(f" * {source}: {count} products")
|
|
|
|
print("\n✅ Market trends analysis completed successfully!")
|
|
return 0
|
|
|
|
except Exception as e:
|
|
print(f"✗ Error during scraping: {str(e)}")
|
|
return 1
|
|
|
|
finally:
|
|
# Close scraper
|
|
scraper.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |