File size: 3,298 Bytes
d1245ca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | import requests
from bs4 import BeautifulSoup
def scrape_product_info(url: str) -> str:
"""
Scrape product information from a given URL
"""
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
response = requests.get(url, headers=headers, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
# Extract product information from common e-commerce patterns
product_info = {}
# Try to find product title
title_selectors = [
'h1[data-testid="product-title"]', # Amazon
'.product-title',
'.product-name',
'h1.product_title',
'.pdp-product-name', # Flipkart
'[data-automation-id="product-title"]',
'h1'
]
title = None
for selector in title_selectors:
element = soup.select_one(selector)
if element:
title = element.get_text(strip=True)
break
if not title:
# Fallback to page title
title_tag = soup.find('title')
title = title_tag.get_text(strip=True) if title_tag else "Product"
product_info['title'] = title
# Try to find product description
description_selectors = [
'.product-description',
'.product-details',
'[data-testid="product-description"]',
'.product-summary',
'.pdp-product-description-content',
'.feature-bullets ul',
'.a-unordered-list.a-vertical'
]
description_parts = []
for selector in description_selectors:
elements = soup.select(selector)
for element in elements:
text = element.get_text(strip=True)
if text and len(text) > 20: # Filter out short/empty descriptions
description_parts.append(text)
# Try to find specifications
spec_selectors = [
'.product-specifications',
'.tech-specs',
'.product-details-table',
'.specification-table',
'[data-testid="specifications"]'
]
specs = []
for selector in spec_selectors:
elements = soup.select(selector)
for element in elements:
text = element.get_text(strip=True)
if text:
specs.append(text)
# Combine all information
full_description = f"Product: {title}\n\n"
if description_parts:
full_description += "Description: " + " ".join(description_parts[:3]) + "\n\n"
if specs:
full_description += "Specifications: " + " ".join(specs[:2])
return full_description[:2000] # Limit length
except Exception as e:
# st.error(f"Error scraping product information: {str(e)}")
return f"Unable to extract product information from the provided URL. Please enter product description manually."
|