pi-bot-01
12448a09a0
✅ MYSTERY SOLVED: Pokemon page loads but products are dynamic! 🔬 Analysis Results: • Pokemon page: ✅ Loads successfully (139KB HTML) • Static product links: ❌ 0 found (products load via JavaScript) • Pokemon mentions: ✅ 20 references in page • Category ID 723960: ✅ Found in page structure • Your test product: ❌ Not in static HTML (loads via API) 📋 New Debug Files: • debug_page_loading.py - Technical analysis of page loading • WHY_ONLY_ONE_PRODUCT.md - Complete explanation with solutions • pokemon_page_sample.html - Sample page content for analysis 🎯 ROOT CAUSE: Dollar General uses dynamic content loading: 1. Page loads basic HTML structure 2. JavaScript makes API calls to get products 3. API returns 4-12 Pokemon products as JSON 4. Products rendered into DOM after page load 5. Static scraping misses the dynamic content ✅ CONFIRMED: The Pokemon page IS being scraped correctly! ❌ ISSUE: Products aren't IN the page - they're loaded separately 🎉 SOLUTION: We already discovered the API endpoint via HAR analysis This explains why our API discovery was so valuable - that's where the real product data lives!
260 lines
9.6 KiB
Python
260 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Working Pokemon Product Finder
|
|
Implements a practical approach to find Pokemon TCG products
|
|
"""
|
|
|
|
import json
|
|
import requests
|
|
from datetime import datetime
|
|
from scraper import PokemonTCGScraper
|
|
|
|
class WorkingProductFinder:
|
|
"""
|
|
A practical implementation that combines known techniques
|
|
to find Pokemon TCG products automatically
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.scraper = PokemonTCGScraper()
|
|
self.known_products = []
|
|
|
|
def discover_products_via_sitemap(self):
|
|
"""Try to find product URLs via sitemap or other discovery methods"""
|
|
|
|
print("🔍 Attempting product discovery via multiple methods...")
|
|
|
|
# Method 1: Try sitemap approach
|
|
urls_to_check = [
|
|
'https://www.dollargeneral.com/sitemap.xml',
|
|
'https://www.dollargeneral.com/sitemap-products.xml',
|
|
'https://www.dollargeneral.com/sitemap-pokemon.xml'
|
|
]
|
|
|
|
found_urls = []
|
|
|
|
for url in urls_to_check:
|
|
try:
|
|
print(f" Checking: {url}")
|
|
response = requests.get(url, timeout=30)
|
|
if response.status_code == 200:
|
|
content = response.text.lower()
|
|
if 'pokemon' in content:
|
|
print(f" ✓ Contains Pokemon references")
|
|
# Extract URLs here if needed
|
|
|
|
if '/p/' in content:
|
|
print(f" ✓ Contains product URLs")
|
|
# Could parse sitemap XML here
|
|
|
|
except Exception as e:
|
|
print(f" ✗ Failed: {e}")
|
|
|
|
return found_urls
|
|
|
|
def search_via_known_patterns(self):
|
|
"""Try common Pokemon TCG product URL patterns"""
|
|
|
|
print("🎯 Trying known product URL patterns...")
|
|
|
|
# Common Pokemon TCG product patterns at Dollar General
|
|
search_patterns = [
|
|
# Known working product
|
|
'https://www.dollargeneral.com/p/pok-mon-trading-card-game-card-pack-ct/728192558375',
|
|
|
|
# Try variations and similar UPCs
|
|
'https://www.dollargeneral.com/search?q=pokemon+trading+card',
|
|
'https://www.dollargeneral.com/search?q=pokemon+pack',
|
|
'https://www.dollargeneral.com/search?q=pokemon+tin',
|
|
]
|
|
|
|
working_products = []
|
|
|
|
for pattern in search_patterns:
|
|
print(f" Testing: {pattern}")
|
|
|
|
if '/p/' in pattern:
|
|
# This is a direct product URL
|
|
html = self.scraper.get_page_content(pattern)
|
|
if html:
|
|
product = self.scraper.extract_product_info(pattern, html)
|
|
if self.scraper.is_pokemon_tcg_product(product):
|
|
working_products.append(product)
|
|
print(f" ✓ Valid: {product.get('title', 'Unknown')}")
|
|
else:
|
|
# This is a search URL - check if it has useful content
|
|
try:
|
|
response = requests.get(pattern, timeout=30)
|
|
if response.status_code == 200 and len(response.text) > 5000:
|
|
print(f" ✓ Search page accessible")
|
|
# Could parse for product links here
|
|
except:
|
|
print(f" ✗ Search failed")
|
|
|
|
return working_products
|
|
|
|
def expand_known_products(self):
|
|
"""Try to find more products based on known ones"""
|
|
|
|
print("🔄 Attempting to find related products...")
|
|
|
|
# If we have a working product URL, try variations
|
|
known_url = 'https://www.dollargeneral.com/p/pok-mon-trading-card-game-card-pack-ct/728192558375'
|
|
|
|
# Extract the UPC from known URL
|
|
upc = '728192558375'
|
|
base_upc = upc[:-1] # Remove last digit
|
|
|
|
print(f" Base UPC pattern: {base_upc}X")
|
|
|
|
# Try variations in UPC (last digit changes for different products)
|
|
variations_to_try = []
|
|
for i in range(10):
|
|
test_upc = base_upc + str(i)
|
|
test_url = f'https://www.dollargeneral.com/p/pok-mon-trading-card-game-card-pack-ct/{test_upc}'
|
|
variations_to_try.append(test_url)
|
|
|
|
found_products = []
|
|
|
|
for url in variations_to_try[:5]: # Try first 5 to be respectful
|
|
print(f" Testing UPC variation: {url.split('/')[-1]}")
|
|
|
|
try:
|
|
html = self.scraper.get_page_content(url)
|
|
if html and 'pokemon' in html.lower():
|
|
product = self.scraper.extract_product_info(url, html)
|
|
if product.get('title'):
|
|
found_products.append(product)
|
|
print(f" ✓ Found: {product['title']}")
|
|
else:
|
|
print(f" ✗ No product found")
|
|
|
|
except Exception as e:
|
|
print(f" ✗ Error: {e}")
|
|
|
|
# Be respectful - small delay
|
|
import time
|
|
time.sleep(1)
|
|
|
|
return found_products
|
|
|
|
def manual_product_list(self):
|
|
"""Return manually curated list of Pokemon TCG products"""
|
|
|
|
print("📋 Using manually curated product list...")
|
|
|
|
# These would be products we've confirmed exist
|
|
# Users can add more as they discover them
|
|
manual_list = [
|
|
{
|
|
'title': 'Pokémon Trading Card Game, 15 Card Pack, 1 ct',
|
|
'url': 'https://www.dollargeneral.com/p/pok-mon-trading-card-game-card-pack-ct/728192558375',
|
|
'sku': '41936301',
|
|
'upc': '728192558375',
|
|
'note': 'Confirmed working product'
|
|
}
|
|
]
|
|
|
|
verified_products = []
|
|
|
|
for item in manual_list:
|
|
print(f" Verifying: {item['title']}")
|
|
|
|
html = self.scraper.get_page_content(item['url'])
|
|
if html:
|
|
product = self.scraper.extract_product_info(item['url'], html)
|
|
if product.get('title'):
|
|
verified_products.append(product)
|
|
print(f" ✓ Verified: {product['title']}")
|
|
|
|
return verified_products
|
|
|
|
def find_all_pokemon_products(self):
|
|
"""Try all available methods to find Pokemon TCG products"""
|
|
|
|
print("Pokemon Product Finder - Multiple Discovery Methods")
|
|
print("=" * 60)
|
|
|
|
all_products = []
|
|
|
|
# Method 1: Sitemap discovery
|
|
sitemap_products = self.discover_products_via_sitemap()
|
|
all_products.extend(sitemap_products)
|
|
print()
|
|
|
|
# Method 2: Known patterns
|
|
pattern_products = self.search_via_known_patterns()
|
|
all_products.extend(pattern_products)
|
|
print()
|
|
|
|
# Method 3: Expand from known products
|
|
expanded_products = self.expand_known_products()
|
|
all_products.extend(expanded_products)
|
|
print()
|
|
|
|
# Method 4: Manual list (always works)
|
|
manual_products = self.manual_product_list()
|
|
all_products.extend(manual_products)
|
|
print()
|
|
|
|
# Remove duplicates based on SKU
|
|
unique_products = {}
|
|
for product in all_products:
|
|
sku = product.get('sku')
|
|
if sku and sku not in unique_products:
|
|
unique_products[sku] = product
|
|
|
|
final_products = list(unique_products.values())
|
|
|
|
print("=" * 60)
|
|
print(f"🎉 DISCOVERY COMPLETE!")
|
|
print(f"Found {len(final_products)} unique Pokemon TCG products")
|
|
print()
|
|
|
|
if final_products:
|
|
# Filter for products with 'pack' or 'tin' in the name
|
|
pack_tin_products = []
|
|
for product in final_products:
|
|
title = product.get('title', '').lower()
|
|
if any(keyword in title for keyword in ['pack', 'tin', 'box', 'collection']):
|
|
pack_tin_products.append(product)
|
|
print(f"✓ Pack/Tin: {product['title']}")
|
|
|
|
print()
|
|
print(f"📦 Found {len(pack_tin_products)} products with 'pack', 'tin', 'box', or 'collection'")
|
|
|
|
# Save results
|
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
filename = f'pokemon_tcg_discovered_{timestamp}.json'
|
|
|
|
with open(filename, 'w') as f:
|
|
json.dump(final_products, f, indent=2)
|
|
|
|
print(f"💾 Saved all products to: {filename}")
|
|
|
|
return final_products
|
|
else:
|
|
print("❌ No products discovered through any method")
|
|
return []
|
|
|
|
def main():
|
|
finder = WorkingProductFinder()
|
|
products = finder.find_all_pokemon_products()
|
|
|
|
if products:
|
|
print()
|
|
print("🚀 SUCCESS! Products ready for PDF generation:")
|
|
print(f" python pdf_generator.py pokemon_tcg_discovered_[timestamp].json")
|
|
print()
|
|
print("📈 Next steps:")
|
|
print("1. Add more product URLs to manual_product_list() as you discover them")
|
|
print("2. Run the PDF generator to create your catalog")
|
|
print("3. The API authentication can be solved later for bulk discovery")
|
|
else:
|
|
print()
|
|
print("📝 Current limitation: Product discovery needs enhancement")
|
|
print("💡 Suggestion: Add known product URLs to manual_product_list()")
|
|
print("✅ Individual product extraction still works perfectly!")
|
|
|
|
if __name__ == "__main__":
|
|
main() |