Glassdoorをスクレイピングする方法：企業レビューと給与データを収集する

# Glassdoor renders content with JavaScript and defends hard, so use a
# real browser (Playwright) rather than plain requests.
pip install playwright beautifulsoup4
playwright install chromium



Copy

from playwright.sync_api import sync_playwright

# Route the browser through a residential proxy (see Step 4)
PROXY = {"server": "http://gw.dataimpulse.com:823",
         "username": "YOUR_DI_LOGIN", "password": "YOUR_DI_PASSWORD"}

def get_html(url):
    with sync_playwright() as p:
        browser = p.chromium.launch(proxy=PROXY, headless=True)
        page = browser.new_page(locale="en-US")
        page.goto(url, wait_until="networkidle", timeout=60000)
        html = page.content()
        browser.close()
        return html

# A PUBLIC company reviews page (do not log in)
html = get_html("https://www.glassdoor.com/Reviews/Example-Company-Reviews-E12345.htm")
















Copy

from bs4 import BeautifulSoup

def parse_reviews(html):
    """Parse the publicly visible reviews. Selectors change often —
    confirm the current ones in DevTools before relying on them."""
    soup = BeautifulSoup(html, "html.parser")
    reviews = []
    for card in soup.select('[data-test="review-details"]'):
        def t(sel):
            el = card.select_one(sel)
            return el.get_text(strip=True) if el else None
        reviews.append({
            "rating": t('[data-test="review-rating"]'),
            "title":  t('[data-test="review-title"]'),
            "pros":   t('[data-test="pros"]'),
            "cons":   t('[data-test="cons"]'),
            "date":   t('[data-test="review-date"]'),
        })
    return reviews

print(parse_reviews(html)[:3])




















Copy

def parse_salaries(html):
    """Parse aggregated, public salary ranges (role-level, not individuals)."""
    soup = BeautifulSoup(html, "html.parser")
    rows = []
    for row in soup.select('[data-test="salary-row"]'):
        def t(sel):
            el = row.select_one(sel)
            return el.get_text(strip=True) if el else None
        rows.append({
            "job_title": t('[data-test="job-title"]'),
            "base_pay":  t('[data-test="median-base"]'),
            "range":     t('[data-test="pay-range"]'),
        })
    return rows

salary_html = get_html("https://www.glassdoor.com/Salaries/example-company-salaries-E12345.htm")
print(parse_salaries(salary_html)[:3])
















Copy

# Rotate residential IPs and look like a real browser.
# Use a fresh session id per run so each crawl gets a clean IP.
PROXY = {"server": "http://gw.dataimpulse.com:823",
         "username": "YOUR_DI_LOGIN__cr.us;sid.run1",
         "password": "YOUR_DI_PASSWORD"}

def fetch(url):
    with sync_playwright() as p:
        browser = p.chromium.launch(proxy=PROXY, headless=True)
        ctx = browser.new_context(
            locale="en-US",
            user_agent=("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                        "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"),
            viewport={"width": 1366, "height": 900},
        )
        page = ctx.new_page()
        page.goto(url, wait_until="networkidle")  # pace requests; Glassdoor rate-limits
        html = page.content()
        browser.close()        # always clean up the browser
        return html



















Copy

import json, csv

reviews = parse_reviews(html)

with open("glassdoor_reviews.json", "w", encoding="utf-8") as f:
    json.dump(reviews, f, indent=2, ensure_ascii=False)

if reviews:
    with open("glassdoor_reviews.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=reviews[0].keys())
        w.writeheader(); w.writerows(reviews)










Copy