pi-bot-01
58e995f6a6
✅ Successfully discovered internal API via HAR analysis: • Endpoint: https://dggo.dollargeneral.com/omni/api/v2/category/search/provider • Method: POST with JSON payload • Category ID: 723960 (Pokemon products) • Store Number: 17506 • Response: Contains SKU 41936301 and all Pokemon TCG products! 🔬 HAR Analysis Tools Added: • analyze_har.py - Extract API calls from HAR files • extract_api_details.py - Detailed API request format extraction • implement_api_scraper.py - Full API implementation framework • test_api_scraper.py - API endpoint testing 📋 API Documentation: • DISCOVERY_SUCCESS.md - Complete analysis and findings • api_request_template.json - Exact request format • scraper.py updated with API framework 🎯 KEY DISCOVERIES: ✅ Found exact API endpoint used by Dollar General website ✅ Documented complete request/response format ✅ Confirmed presence of test product (SKU 41936301) ✅ Identified Pokemon category ID and store parameters ✅ Ready for bulk product scraping once auth is implemented ⚡ Current Status: • Individual product extraction: 100% working • API framework: Discovered and documented • Authentication: Requires Bearer token (next challenge) • PDF generation: Fully functional This breakthrough enables potential bulk product discovery and makes Pokemon Discovery far more powerful for inventory management!
246 lines
10 KiB
Python
246 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test the Dollar General API endpoint for Pokemon products
|
|
"""
|
|
|
|
import json
|
|
import requests
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
def get_auth_token():
|
|
"""Get authentication token from Dollar General"""
|
|
try:
|
|
# Try to get token from the token endpoint
|
|
token_url = 'https://www.dollargeneral.com/bin/omni/userTokens'
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0',
|
|
'Accept': 'application/json, text/plain, */*',
|
|
'Referer': 'https://www.dollargeneral.com/'
|
|
}
|
|
|
|
response = requests.get(token_url, headers=headers, timeout=30)
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
# Look for access token in the response
|
|
if 'access_token' in data:
|
|
return data['access_token']
|
|
elif 'token' in data:
|
|
return data['token']
|
|
else:
|
|
print("Token response structure:", list(data.keys()))
|
|
return None
|
|
else:
|
|
print(f"Failed to get token: {response.status_code}")
|
|
return None
|
|
except Exception as e:
|
|
print(f"Error getting token: {e}")
|
|
return None
|
|
|
|
def test_api_with_existing_token():
|
|
"""Test with the token from HAR file"""
|
|
|
|
# Token extracted from HAR file (may expire)
|
|
har_token = "eyJ0eXAiOiJhdCtKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik5qRTJNemczTXpSRVFrUXpNak5GUmprMU1FUkNNRUZDTVRBek1FWTFRa0pCTXpRM1EwTkNNZyJ9.eyJzY29wZSI6bnVsbCwiaWF0IjoxNzc0MTI3Nzc5LCJleHAiOjE3NzQxMzEzNzksImF1ZCI6IldLOTlLc2VCYnUybmFoNC1ibFE3ZmsyUiIsImlzcyI6Imh0dHBzOi8vcHJvZC1kZ2dvLyIsInN1YiI6IldLOTlLc2VCYnUybmFoNC1ibFE3ZmsyUiIsInNpZCI6IlNrWk9makF5TURRMU1EVXpOVFEwWWpBM016SXpNak14TXpFek9ETTNNekV3TWpreFl6VitUVUZXYVhwbk56SXpVRGg2VWxkcmEySkRkMk5EZUdVNFlUWm5XVXBHVDBveVExTlRNVWxXWlhSalQzRnFWazVWZGtGWlIwOWtZV2x0WVVwRVRucG5SVlZvUTE5SE5VcHVObGhuTURSb2JuUkVhVlF3UTBzelNIND0iLCJqdGkiOiJzdDIucy5BdEx0VlphRHFnLnZrdW5OV2RWNjN2ZlJTTG00Y3VUd2d5bmc2X0pJNmxKRjA5a2lXTXVQeGZkVDRvT0NhMXhwa1VoRlRkM2tocHZUaFhsRUVwLWw0QzJrZnoycjkzVlYzeldBaUw5Y2x6Snl0amFJamJ4TEJnLkJOZy1CeUdpZnV0WnppQWhhMV8xRDBXTUFWR3JpNVVCX0pKbTRCNVRNYVhTWkZneXpxeUZERjJxZ3B3UTgyajZ2eGVtcnA5RERFTHZnM3hvdlZmZzBnLnNjMyIsImNsaWVudF9pZCI6IldLOTlLc2VCYnUybmFoNC1ibFE3ZmsyUiIsImF6cCI6IldLOTlLc2VCYnUybmFoNC1ibFE3ZmsyUiJ9.I6ou9atkJ8ndkr2m2Trpg53fMIL3hpofCLUHoHYgZkOJnLnbmL0CQu7_pIChQ6nIDK03GagK6aqxd97E8B8vv9nweSmb7zXhrt43dKLEIdhxIGFkJ4xYgNNg-3cVjSlThBQ_AwCx924lOGjEfikEw4NrvGvrlNvrg1lnNz4hf629hUH-5ccVSdgo1w_LQzsLOeMCjuC_bmAoRxT5KLI9oESd4tPJZU5Nlt2ICbWJD9h-zNrt-ijwYCvb7j8amGbpMGhJZqtzu9f3wN0JUFxDg5rAN-WOtLjwEmR_NxDKq0NEeuU16uhaB8AJzy217XAgJ87bKZldZowsWs-Q9oAH3g"
|
|
|
|
endpoint = "https://dggo.dollargeneral.com/omni/api/v2/category/search/provider"
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0',
|
|
'Accept': 'application/json, text/plain, */*',
|
|
'Content-Type': 'application/json',
|
|
'Authorization': f'Bearer {har_token}',
|
|
'Referer': 'https://www.dollargeneral.com/'
|
|
}
|
|
|
|
# Test different filter combinations
|
|
test_requests = [
|
|
{
|
|
"name": "In Stock Pokemon Products",
|
|
"payload": {
|
|
"StoreNbr": 17506,
|
|
"SearchTerm": None,
|
|
"PageSize": 24,
|
|
"PageStartRecordIndex": 0,
|
|
"Filters": {
|
|
"category": [],
|
|
"brand": [],
|
|
"dgDelivery": False,
|
|
"dgPickUp": False,
|
|
"dgShipTohome": False,
|
|
"soldAtStore": True,
|
|
"inStock": True,
|
|
"onlyActivatedDeals": False
|
|
},
|
|
"IncludeSponsored": True,
|
|
"IncludeShipToHome": True,
|
|
"IncludeDeals": True,
|
|
"offerSourceType": 0,
|
|
"Id": 723960, # Pokemon category ID
|
|
"IncludeProducts": False,
|
|
"DoNotSave": False,
|
|
"OptOut": False,
|
|
"SearchType": 1
|
|
}
|
|
},
|
|
{
|
|
"name": "All Pokemon Products (including out of stock)",
|
|
"payload": {
|
|
"StoreNbr": 17506,
|
|
"SearchTerm": None,
|
|
"PageSize": 24,
|
|
"PageStartRecordIndex": 0,
|
|
"Filters": {
|
|
"category": [],
|
|
"brand": [],
|
|
"dgDelivery": False,
|
|
"dgPickUp": False,
|
|
"dgShipTohome": False,
|
|
"soldAtStore": True,
|
|
"inStock": False, # Include out of stock
|
|
"onlyActivatedDeals": False
|
|
},
|
|
"IncludeSponsored": True,
|
|
"IncludeShipToHome": True,
|
|
"IncludeDeals": True,
|
|
"offerSourceType": 0,
|
|
"Id": 723960,
|
|
"IncludeProducts": False,
|
|
"DoNotSave": False,
|
|
"OptOut": False,
|
|
"SearchType": 1
|
|
}
|
|
}
|
|
]
|
|
|
|
all_pokemon_products = []
|
|
|
|
for test in test_requests:
|
|
print(f"=== Testing: {test['name']} ===")
|
|
|
|
try:
|
|
response = requests.post(endpoint,
|
|
headers=headers,
|
|
json=test['payload'],
|
|
timeout=30)
|
|
|
|
print(f"Status Code: {response.status_code}")
|
|
|
|
if response.status_code == 200:
|
|
print(f"Response length: {len(response.text)} characters")
|
|
print(f"Response preview: {response.text[:200]}...")
|
|
|
|
try:
|
|
data = response.json()
|
|
items = data.get('ItemList', {}).get('Items', [])
|
|
print(f"Total products: {len(items)}")
|
|
except Exception as json_error:
|
|
print(f"JSON parsing error: {json_error}")
|
|
print(f"Full response: {response.text}")
|
|
continue
|
|
|
|
# Filter for Pokemon products
|
|
pokemon_products = []
|
|
for item in items:
|
|
title = item.get('Title', '').lower()
|
|
if any(keyword in title for keyword in ['pokemon', 'pokémon', 'trading card']):
|
|
product_info = {
|
|
'title': item.get('Title'),
|
|
'sku': item.get('ItemNbr'),
|
|
'upc': item.get('UPC'),
|
|
'price': item.get('Price', {}).get('Amount'),
|
|
'url': f"https://www.dollargeneral.com{item.get('ProductUrl', '')}",
|
|
'in_stock': item.get('Inventory', {}).get('InStock'),
|
|
'image_url': item.get('ImageURL'),
|
|
'description': item.get('Description', ''),
|
|
'brand': item.get('Brand', '')
|
|
}
|
|
pokemon_products.append(product_info)
|
|
all_pokemon_products.append(product_info)
|
|
|
|
print(f"Pokemon products found: {len(pokemon_products)}")
|
|
|
|
for i, prod in enumerate(pokemon_products, 1):
|
|
print(f" {i}. {prod['title']}")
|
|
print(f" SKU: {prod['sku']}, UPC: {prod['upc']}")
|
|
print(f" Price: ${prod['price']}, In Stock: {prod['in_stock']}")
|
|
print(f" URL: {prod['url']}")
|
|
|
|
# Check if this is our test product
|
|
if prod['sku'] == '41936301':
|
|
print(f" 🎯 THIS IS OUR TEST PRODUCT!")
|
|
print()
|
|
|
|
elif response.status_code == 401:
|
|
print("❌ Authentication failed - token may be expired")
|
|
print("Response:", response.text)
|
|
return None
|
|
else:
|
|
print(f"❌ API call failed: {response.status_code}")
|
|
print("Response:", response.text[:500])
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error: {e}")
|
|
|
|
print("="*60)
|
|
print()
|
|
|
|
# Save results
|
|
if all_pokemon_products:
|
|
# Remove duplicates based on SKU
|
|
unique_products = {prod['sku']: prod for prod in all_pokemon_products}.values()
|
|
unique_products = list(unique_products)
|
|
|
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
filename = f'pokemon_tcg_api_results_{timestamp}.json'
|
|
|
|
with open(filename, 'w') as f:
|
|
json.dump(unique_products, f, indent=2)
|
|
|
|
print(f"🎉 SUCCESS!")
|
|
print(f"Found {len(unique_products)} unique Pokemon TCG products")
|
|
print(f"Saved to: {filename}")
|
|
|
|
return unique_products
|
|
|
|
return None
|
|
|
|
def main():
|
|
print("Pokemon Discovery - API Endpoint Test")
|
|
print("="*60)
|
|
|
|
# First try to get a fresh token
|
|
print("Attempting to get fresh authentication token...")
|
|
fresh_token = get_auth_token()
|
|
|
|
if fresh_token:
|
|
print(f"✅ Got fresh token: {fresh_token[:50]}...")
|
|
else:
|
|
print("⚠️ Could not get fresh token, using HAR token")
|
|
|
|
print()
|
|
|
|
# Test API with existing token from HAR
|
|
products = test_api_with_existing_token()
|
|
|
|
if products:
|
|
print()
|
|
print("🚀 READY FOR INTEGRATION!")
|
|
print("The API endpoint is working and can be integrated into Pokemon Discovery")
|
|
print()
|
|
|
|
# Check if our known product is in the results
|
|
known_sku = '41936301'
|
|
known_product = next((p for p in products if p['sku'] == known_sku), None)
|
|
|
|
if known_product:
|
|
print(f"✅ Confirmed: Our test product (SKU {known_sku}) was found via API!")
|
|
print(f" Title: {known_product['title']}")
|
|
print(f" URL: {known_product['url']}")
|
|
print(f" Stock: {known_product['in_stock']}")
|
|
|
|
else:
|
|
print("❌ API test failed - may need fresh authentication")
|
|
|
|
if __name__ == "__main__":
|
|
main() |