pi-bot-01
94d193a5b0
✅ Configured Brave browser integration (/usr/bin/brave) ✅ Updated Selenium WebDriver to use Brave binary ✅ Added proper Service-based WebDriver initialization ✅ Enhanced error handling and fallback mechanisms ✅ Created comprehensive Brave compatibility test script 🔧 Technical improvements: - Fixed WebDriver initialization for newer Selenium versions - Added detailed browser version detection - Improved error messages for ChromeDriver compatibility issues - Enhanced dynamic content handling with longer wait times 📋 Known compatibility note: - Brave 146 vs ChromeDriver 114 version mismatch (solvable) - Core PDF generation functionality works independently - Graceful fallback to requests-only mode when browser unavailable This allows users with Brave browser to utilize dynamic content scraping while maintaining full functionality for PDF catalog generation.
377 lines
14 KiB
Python
Executable File
377 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Pokemon Discovery - TCG Product Scraper for Dollar General
|
|
Scrapes product information and saves to JSON for PDF generation
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
import requests
|
|
from datetime import datetime
|
|
from urllib.parse import urljoin, urlparse
|
|
import pandas as pd
|
|
from bs4 import BeautifulSoup
|
|
|
|
# Try selenium imports (fallback for dynamic content)
|
|
try:
|
|
from selenium import webdriver
|
|
from selenium.webdriver.chrome.options import Options
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.common.exceptions import TimeoutException
|
|
from webdriver_manager.chrome import ChromeDriverManager
|
|
SELENIUM_AVAILABLE = True
|
|
except ImportError:
|
|
SELENIUM_AVAILABLE = False
|
|
print("Selenium not available, using requests only (install selenium for Brave browser support)")
|
|
|
|
class PokemonTCGScraper:
|
|
def __init__(self):
|
|
self.base_url = "https://www.dollargeneral.com"
|
|
self.search_url = "https://www.dollargeneral.com/c/toys/pokemon?q=&soldAtStore=true"
|
|
self.session = requests.Session()
|
|
|
|
# Headers to appear more like a real browser
|
|
self.headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
}
|
|
self.session.headers.update(self.headers)
|
|
|
|
self.products = []
|
|
|
|
def get_page_with_requests(self, url):
|
|
"""Try to get page content using requests"""
|
|
try:
|
|
response = self.session.get(url, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except requests.RequestException as e:
|
|
print(f"Requests failed for {url}: {e}")
|
|
return None
|
|
|
|
def get_page_with_selenium(self, url):
|
|
"""Fallback to selenium for dynamic content using Brave browser"""
|
|
if not SELENIUM_AVAILABLE:
|
|
return None
|
|
|
|
options = Options()
|
|
options.add_argument('--headless')
|
|
options.add_argument('--no-sandbox')
|
|
options.add_argument('--disable-dev-shm-usage')
|
|
options.add_argument('--disable-gpu')
|
|
options.add_argument('--disable-web-security')
|
|
options.add_argument('--disable-features=VizDisplayCompositor')
|
|
options.add_argument(f'--user-agent={self.headers["User-Agent"]}')
|
|
|
|
# Use Brave browser
|
|
options.binary_location = '/usr/bin/brave'
|
|
|
|
try:
|
|
print("Starting Brave browser with Selenium...")
|
|
from selenium.webdriver.chrome.service import Service
|
|
|
|
# Try to get compatible ChromeDriver
|
|
try:
|
|
# Try with webdriver manager (auto-detects version)
|
|
service = Service(ChromeDriverManager().install())
|
|
except Exception as e:
|
|
print(f"ChromeDriver auto-install failed: {e}")
|
|
print("This usually means ChromeDriver version doesn't match Brave version.")
|
|
print("For best results, ensure ChromeDriver and Brave versions are compatible.")
|
|
print("You can manually install a compatible ChromeDriver or use a different browser.")
|
|
return None
|
|
|
|
driver = webdriver.Chrome(service=service, options=options)
|
|
|
|
print(f"Navigating to: {url}")
|
|
driver.get(url)
|
|
|
|
# Wait for content to load
|
|
print("Waiting for page content to load...")
|
|
WebDriverWait(driver, 15).until(
|
|
EC.presence_of_element_located((By.TAG_NAME, "body"))
|
|
)
|
|
|
|
# Additional wait for dynamic content and JavaScript execution
|
|
print("Waiting for dynamic content...")
|
|
time.sleep(5)
|
|
|
|
# Try to find product-related elements
|
|
print("Looking for product elements...")
|
|
try:
|
|
# Check if we have product elements loaded
|
|
product_elements = driver.find_elements(By.CSS_SELECTOR, 'a[href*="/p/"], .product-item, .product-card')
|
|
print(f"Found {len(product_elements)} potential product elements")
|
|
except:
|
|
print("No specific product elements found, proceeding with full page content")
|
|
|
|
html = driver.page_source
|
|
print(f"Retrieved {len(html)} characters of HTML content")
|
|
driver.quit()
|
|
return html
|
|
|
|
except Exception as e:
|
|
print(f"Brave/Selenium failed for {url}: {e}")
|
|
if 'driver' in locals():
|
|
driver.quit()
|
|
return None
|
|
|
|
def get_page_content(self, url):
|
|
"""Get page content, trying requests first, then selenium"""
|
|
print(f"Fetching: {url}")
|
|
|
|
# Try requests first
|
|
content = self.get_page_with_requests(url)
|
|
if content and len(content) > 1000: # Basic content check
|
|
return content
|
|
|
|
# Fallback to selenium
|
|
print("Falling back to Selenium...")
|
|
return self.get_page_with_selenium(url)
|
|
|
|
def extract_product_links(self, html):
|
|
"""Extract product page links from search results"""
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
links = []
|
|
|
|
# Common selectors for product links
|
|
selectors = [
|
|
'a[href*="/p/"]',
|
|
'.product-item a',
|
|
'.product-card a',
|
|
'.product-link',
|
|
'[data-testid*="product"] a'
|
|
]
|
|
|
|
for selector in selectors:
|
|
elements = soup.select(selector)
|
|
for element in elements:
|
|
href = element.get('href')
|
|
if href and '/p/' in href:
|
|
full_url = urljoin(self.base_url, href)
|
|
if full_url not in links:
|
|
links.append(full_url)
|
|
|
|
return links
|
|
|
|
def extract_product_info(self, url, html):
|
|
"""Extract product information from product page"""
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
product = {'url': url}
|
|
|
|
# Extract title
|
|
title_selectors = [
|
|
'h1',
|
|
'.product-title',
|
|
'.product-name',
|
|
'[data-testid="product-title"]',
|
|
'.pdp-product-name'
|
|
]
|
|
|
|
for selector in title_selectors:
|
|
title_elem = soup.select_one(selector)
|
|
if title_elem:
|
|
product['title'] = title_elem.get_text().strip()
|
|
break
|
|
|
|
# Extract price
|
|
price_selectors = [
|
|
'.price',
|
|
'.product-price',
|
|
'[data-testid="price"]',
|
|
'.price-current',
|
|
'.current-price'
|
|
]
|
|
|
|
for selector in price_selectors:
|
|
price_elem = soup.select_one(selector)
|
|
if price_elem:
|
|
price_text = price_elem.get_text().strip()
|
|
product['price'] = price_text
|
|
break
|
|
|
|
# Extract SKU
|
|
sku_selectors = [
|
|
'[data-sku]',
|
|
'.sku',
|
|
'.product-sku',
|
|
'*[text()*="SKU"]',
|
|
'script[type="application/ld+json"]'
|
|
]
|
|
|
|
# Try data attributes first
|
|
for selector in sku_selectors[:-1]:
|
|
elem = soup.select_one(selector)
|
|
if elem:
|
|
sku = elem.get('data-sku') or elem.get_text().strip()
|
|
if sku and sku.lower() != 'sku':
|
|
product['sku'] = sku
|
|
break
|
|
|
|
# Try JSON-LD structured data
|
|
if 'sku' not in product:
|
|
scripts = soup.find_all('script', type='application/ld+json')
|
|
for script in scripts:
|
|
try:
|
|
data = json.loads(script.string)
|
|
if isinstance(data, dict) and 'sku' in data:
|
|
product['sku'] = data['sku']
|
|
break
|
|
elif isinstance(data, list):
|
|
for item in data:
|
|
if isinstance(item, dict) and 'sku' in item:
|
|
product['sku'] = item['sku']
|
|
break
|
|
except:
|
|
continue
|
|
|
|
# Extract stock information
|
|
stock_selectors = [
|
|
'.stock',
|
|
'.inventory',
|
|
'.availability',
|
|
'[data-testid="stock"]',
|
|
'.in-stock',
|
|
'.out-of-stock'
|
|
]
|
|
|
|
for selector in stock_selectors:
|
|
stock_elem = soup.select_one(selector)
|
|
if stock_elem:
|
|
stock_text = stock_elem.get_text().strip().lower()
|
|
if 'in stock' in stock_text:
|
|
product['stock'] = 'In Stock'
|
|
elif 'out of stock' in stock_text:
|
|
product['stock'] = 'Out of Stock'
|
|
else:
|
|
product['stock'] = stock_text
|
|
break
|
|
|
|
# Extract image URL
|
|
img_selectors = [
|
|
'.product-image img',
|
|
'.product-photo img',
|
|
'.pdp-image img',
|
|
'[data-testid="product-image"] img',
|
|
'img[alt*="Pokemon"]',
|
|
'img[alt*="TCG"]'
|
|
]
|
|
|
|
for selector in img_selectors:
|
|
img_elem = soup.select_one(selector)
|
|
if img_elem:
|
|
src = img_elem.get('src') or img_elem.get('data-src')
|
|
if src:
|
|
product['image_url'] = urljoin(self.base_url, src)
|
|
break
|
|
|
|
return product
|
|
|
|
def is_pokemon_tcg_product(self, product):
|
|
"""Check if product is a Pokemon TCG card pack or tin"""
|
|
if not product.get('title'):
|
|
return False
|
|
|
|
title = product['title'].lower()
|
|
pokemon_keywords = ['pokemon', 'tcg', 'trading card', 'cards']
|
|
tcg_keywords = ['pack', 'tin', 'box', 'booster', 'collection']
|
|
|
|
has_pokemon = any(keyword in title for keyword in pokemon_keywords)
|
|
has_tcg = any(keyword in title for keyword in tcg_keywords)
|
|
|
|
return has_pokemon and has_tcg
|
|
|
|
def scrape_products(self):
|
|
"""Main scraping method"""
|
|
print(f"Starting scrape of: {self.search_url}")
|
|
|
|
# Get search results page
|
|
html = self.get_page_content(self.search_url)
|
|
if not html:
|
|
print("Failed to get search results page")
|
|
return []
|
|
|
|
# Extract product links
|
|
product_links = self.extract_product_links(html)
|
|
print(f"Found {len(product_links)} potential product links")
|
|
|
|
if not product_links:
|
|
print("No product links found with requests. Trying Brave browser for dynamic content...")
|
|
# Try Selenium with Brave as fallback
|
|
selenium_html = self.get_page_with_selenium(self.search_url)
|
|
if selenium_html and len(selenium_html) > len(html):
|
|
print("Got enhanced content from Brave, re-extracting product links...")
|
|
html = selenium_html
|
|
product_links = self.extract_product_links(html)
|
|
print(f"Found {len(product_links)} product links with Brave browser")
|
|
|
|
if not product_links:
|
|
print("No product links found even with Brave browser.")
|
|
print("This could be due to:")
|
|
print("1. No Pokemon TCG products currently in stock")
|
|
print("2. Website structure changes")
|
|
print("3. Enhanced anti-bot protection")
|
|
print("4. Geographic restrictions")
|
|
print("\nFirst 1000 chars of final page content:")
|
|
print(html[:1000])
|
|
return []
|
|
|
|
# Scrape each product page
|
|
for i, link in enumerate(product_links):
|
|
print(f"Scraping product {i+1}/{len(product_links)}: {link}")
|
|
|
|
product_html = self.get_page_content(link)
|
|
if not product_html:
|
|
continue
|
|
|
|
product = self.extract_product_info(link, product_html)
|
|
|
|
# Filter for Pokemon TCG products
|
|
if self.is_pokemon_tcg_product(product):
|
|
print(f"Found Pokemon TCG product: {product.get('title', 'Unknown')}")
|
|
self.products.append(product)
|
|
else:
|
|
print(f"Skipping non-TCG product: {product.get('title', 'Unknown')}")
|
|
|
|
# Be respectful to the server
|
|
time.sleep(1)
|
|
|
|
return self.products
|
|
|
|
def save_to_json(self, filename=None):
|
|
"""Save scraped products to JSON file"""
|
|
if not filename:
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
filename = f"pokemon_tcg_products_{timestamp}.json"
|
|
|
|
with open(filename, 'w') as f:
|
|
json.dump(self.products, f, indent=2)
|
|
|
|
print(f"Saved {len(self.products)} products to {filename}")
|
|
return filename
|
|
|
|
def main():
|
|
scraper = PokemonTCGScraper()
|
|
products = scraper.scrape_products()
|
|
|
|
if products:
|
|
filename = scraper.save_to_json()
|
|
print(f"\nScraping completed successfully!")
|
|
print(f"Found {len(products)} Pokemon TCG products")
|
|
print(f"Data saved to: {filename}")
|
|
else:
|
|
print("\nNo products found. This could be due to:")
|
|
print("1. No Pokemon TCG products in stock")
|
|
print("2. Website structure changes")
|
|
print("3. Anti-bot protection")
|
|
|
|
if __name__ == "__main__":
|
|
main() |