How to Scrape eBay with Python and Proxies: A Step-by-Step Guide

# Create an isolated project folder and virtual environment
mkdir ebay-scraper && cd ebay-scraper
python3 -m venv env
source env/bin/activate        # Windows: env\Scripts\activate

# requests  -> send HTTP requests
# beautifulsoup4 -> parse the returned HTML
# lxml -> fast HTML parser backend for BeautifulSoup
# httpx -> async requests for scaling later
pip install requests beautifulsoup4 lxml httpx









Copy

import requests
from bs4 import BeautifulSoup
from urllib.parse import quote_plus

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                         "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
           "Accept-Language": "en-US,en;q=0.9"}

def scrape_search(query, proxies=None):
    """Return a list of eBay search results for a query."""
    url = f"https://www.ebay.com/sch/i.html?_nkw={quote_plus(query)}"
    html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)
    soup = BeautifulSoup(html.text, "lxml")
    items = []
    for card in soup.select("li.s-item"):
        title = card.select_one(".s-item__title")
        price = card.select_one(".s-item__price")
        link  = card.select_one("a.s-item__link")
        image = card.select_one(".s-item__image-wrapper img")
        if not (title and link):
            continue
        items.append({
            "title": title.get_text(strip=True),
            "price": price.get_text(strip=True) if price else None,
            "url":   link["href"],
            "image": image.get("src") if image else None,
        })
    return items

print(scrape_search("smartphone")[:3])





























Copy

from urllib.parse import quote_plus

def scrape_all_pages(query, max_pages=10, proxies=None):
    """Page through results via the _pgn parameter until empty or max_pages."""
    all_items = []
    for page in range(1, max_pages + 1):
        url = (f"https://www.ebay.com/sch/i.html?_nkw={quote_plus(query)}"
               f"&_pgn={page}")
        html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)
        soup = BeautifulSoup(html.text, "lxml")
        cards = soup.select("li.s-item")
        if not cards:                 # no more results -> stop
            break
        for card in cards:
            title = card.select_one(".s-item__title")
            link  = card.select_one("a.s-item__link")
            if title and link:
                all_items.append({"title": title.get_text(strip=True),
                                  "url": link["href"]})
    return all_items



















Copy

def parse_product(url, proxies=None):
    """Scrape a single eBay listing page."""
    html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)
    soup = BeautifulSoup(html.text, "lxml")

    def text(sel):
        el = soup.select_one(sel)
        return el.get_text(strip=True) if el else None

    # item specifics live in a label/value table
    specifics = {}
    for row in soup.select(".ux-labels-values"):
        label = row.select_one(".ux-labels-values__labels")
        value = row.select_one(".ux-labels-values__values")
        if label and value:
            specifics[label.get_text(strip=True)] = value.get_text(strip=True)

    return {
        "title":     text("h1 span"),
        "price":     text(".x-price-primary"),
        "condition": text(".x-item-condition-text .ux-textspans"),
        "seller":    text(".x-sellercard-atf__info__about-seller"),
        "shipping":  text(".ux-labels-values--shipping .ux-textspans"),
        "item_specifics": specifics,
    }
























Copy

import re, json

def parse_variants(url, proxies=None):
    """eBay keeps multi-SKU variant options (size, colour, storage) in an inline
    JSON blob rather than visible HTML. The exact key and shape change over time,
    so treat the pattern below as a starting point and adjust it to the page you
    inspect in DevTools. This returns the option name/value pairs, not full
    price-per-combination data."""
    html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30).text
    # find the inline state object, then load it with a balanced-brace slice
    start = html.find('"MSKU"')
    if start == -1:
        return []
    brace = html.find("{", start)
    depth, end = 0, brace
    for i in range(brace, len(html)):
        depth += (html[i] == "{") - (html[i] == "}")
        if depth == 0:
            end = i + 1
            break
    try:
        data = json.loads(html[brace:end])
    except json.JSONDecodeError:
        return []
    return [{"name": v.get("displayName"), "value": v.get("valueName")}
            for v in data.get("menuItemMap", {}).values()]

























Copy

def scrape_reviews(item_id, proxies=None):
    """Reviews live on a separate URL keyed by the item ID."""
    url = f"https://www.ebay.com/urw/{item_id}/product-reviews"
    html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)
    soup = BeautifulSoup(html.text, "lxml")
    reviews = []
    for r in soup.select(".ebay-review-section"):
        rating = r.select_one(".star-rating")
        body   = r.select_one(".review-item-content")
        date   = r.select_one(".review-item-date")
        author = r.select_one(".review-item-author")
        reviews.append({
            "rating": rating.get("aria-label") if rating else None,
            "text":   body.get_text(strip=True) if body else None,
            "date":   date.get_text(strip=True) if date else None,
            "author": author.get_text(strip=True) if author else None,
        })
    return reviews

















Copy

import json, csv

results = scrape_search("smartphone")

# Option A - JSON (keeps nested structure)
with open("ebay.json", "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)

# Option B - CSV (flat, opens in Excel/Sheets)
if results:
    with open("ebay.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=results[0].keys())
        writer.writeheader()
        writer.writerows(results)













Copy

HEADERS = {
    # identify as a real desktop browser - eBay returns 403/empty without this
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
    "Accept-Language": "en-US,en;q=0.9",      # match the locale you target
    "Accept-Encoding": "gzip, deflate, br",   # accept compressed responses
    "Referer": "https://www.ebay.com/",       # look like in-site navigation
    "Connection": "keep-alive",
}

# rotate the User-Agent across a list for larger jobs
import random
USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/17.0 Safari/605.1.15",
]
HEADERS["User-Agent"] = random.choice(USER_AGENTS)
















Copy

# Route requests through DataImpulse rotating residential proxies
proxies = {
    "http":  "http://LOGIN:[email protected]:823",
    "https": "http://LOGIN:[email protected]:823",
}
html = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)





Copy

import time, random, asyncio, httpx

# 1) Basic pacing - a random delay between requests
def polite_get(url, **kw):
    time.sleep(random.uniform(1, 3))
    return requests.get(url, headers=HEADERS, timeout=30, **kw)

# 2) Retry with exponential backoff on 429/403
def get_with_retry(url, retries=4, **kw):
    for attempt in range(retries):
        r = requests.get(url, headers=HEADERS, timeout=30, **kw)
        if r.status_code not in (429, 403):
            return r
        time.sleep(2 ** attempt + random.random())   # 1s, 2s, 4s, 8s ...
    return r

# 3) Async collection for high volume (httpx >=0.28 uses proxy=, not proxies=)
async def scrape_many(urls, proxy=None):
    async with httpx.AsyncClient(headers=HEADERS, proxy=proxy, timeout=30) as client:
        tasks = [client.get(u) for u in urls]
        return await asyncio.gather(*tasks)




















Copy