不動産データスクレイピング：プロキシを使って物件情報を収集する方法

# Core libraries for real estate data scraping
# requests -> fetch pages | beautifulsoup4 + lxml -> parse HTML/JSON
pip install requests beautifulsoup4 lxml

# Many listing sites are JavaScript-heavy and well defended (e.g. Zillow).
# For those, add a headless browser:
pip install playwright && playwright install chromium






Copy

import requests

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                         "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
           "Accept-Language": "en-US,en;q=0.9"}

# Route through residential proxies for geo-accurate listings (see Step 5)
proxies = {"http":  "http://LOGIN:[email protected]:823",
           "https": "http://LOGIN:[email protected]:823"}

def get_html(url):
    r = requests.get(url, headers=HEADERS, proxies=proxies, timeout=30)
    r.raise_for_status()
    return r.text

html = get_html("https://example-realestate.com/homes/for_sale/CityName/")















Copy

import json
from bs4 import BeautifulSoup

LISTING_TYPES = ("Residence", "Product", "RealEstateListing", "Offer",
                 "SingleFamilyResidence", "Apartment", "House")

def listings_from_jsonld(html):
    """Where available, sites embed listing data as JSON-LD (schema.org).
    Parsing it is more stable than CSS classes — but not every site has it,
    so fall back to the cards in Step 4 when this returns nothing."""
    soup = BeautifulSoup(html, "lxml")
    nodes, out = [], []
    for tag in soup.find_all("script", type="application/ld+json"):
        try:
            data = json.loads(tag.string or "{}")
        except json.JSONDecodeError:
            continue
        stack = data if isinstance(data, list) else [data]
        while stack:                      # flatten @graph / ItemList / nested
            n = stack.pop()
            if not isinstance(n, dict):
                continue
            if "@graph" in n:
                stack.extend(n["@graph"])
            if n.get("@type") == "ItemList":
                stack.extend(x.get("item", x) for x in n.get("itemListElement", []))
            nodes.append(n)
    for n in nodes:
        types = n.get("@type", "")
        types = types if isinstance(types, list) else [types]
        if any(t in LISTING_TYPES for t in types):
            offers = n.get("offers") or {}
            if isinstance(offers, list):
                offers = offers[0] if offers else {}
            out.append({"name": n.get("name"),
                        "price": offers.get("price") or n.get("price"),
                        "address": n.get("address"),
                        "url": n.get("url")})
    return out

print(listings_from_jsonld(html)[:3])








































Copy

def parse_listing_cards(html):
    """Fallback: parse the search-results cards directly. Selectors differ by
    site and change often — confirm the current ones in DevTools."""
    soup = BeautifulSoup(html, "lxml")
    cards = []
    for card in soup.select("[data-test='property-card']"):
        def t(sel):
            el = card.select_one(sel)
            return el.get_text(strip=True) if el else None
        cards.append({
            "price":   t("[data-test='property-price']"),
            "address": t("[data-test='property-address']"),
            "beds":    t("[data-test='property-beds']"),
            "baths":   t("[data-test='property-baths']"),
            "sqft":    t("[data-test='property-sqft']"),
            "url":     (card.select_one("a[href]") or {}).get("href"),
        })
    return cards

















Copy

import time, random

def scrape_all_pages(base_url, max_pages=10):
    """Page through search results until empty or max_pages."""
    all_listings = []
    for page in range(1, max_pages + 1):
        html = get_html(f"{base_url}?page={page}")
        cards = parse_listing_cards(html)
        if not cards:        # no more results -> stop
            break
        all_listings.extend(cards)
        time.sleep(random.uniform(1, 3))   # pace requests
    return all_listings












Copy

import time, random, json, csv

# Listings differ by region: route through a proxy in the target market so
# prices, availability, and currency match what a local user sees.
proxies = {
    "http":  "http://LOGIN__cr.us;sid.re1:[email protected]:823",
    "https": "http://LOGIN__cr.us;sid.re1:[email protected]:823",
}

listings = scrape_all_pages("https://example-realestate.com/homes/for_sale/CityName/")

with open("listings.json", "w", encoding="utf-8") as f:
    json.dump(listings, f, indent=2, ensure_ascii=False)
if listings:
    with open("listings.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=listings[0].keys())
        w.writeheader(); w.writerows(listings)
















Copy