PythonでGitHubをスクレイピングする方法

python3 --version 
      
        
      
Copy

python3 -m ensurepip 
      
        
      
Copy

pip3 install requests beautifulsoup4 
      
        
      
Copy

TIMEOUT = 20               # seconds
VERIFY_SSL = True

USER_AGENT = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/118.0.0.0 Safari/537.36"
      
        
   
   
   
   
   
   
      
Copy

import requests
from bs4 import BeautifulSoup

LANGUAGE = "python"      # Programming language. None can be used as "Any" parameter in GitHub Trending
SINCE = "daily"          # Date range: daily / weekly / monthly
PROXY = {
    "host": "DATAIMPULSE HOST",
    "port": "DATAIMPULSE PORT",
    "user": "DATAIMPULSE LOGIN",
    "pass": "DATAIMPULSE PASSWORD",
}

# Proxy configuration
proxy_url = f"http://{PROXY['user']}:{PROXY['pass']}@{PROXY['host']}:{PROXY['port']}"
proxies = {
    "http": proxy_url,
    "https": proxy_url,
}
# Checking proxies
ip = requests.get("https://api.ipify.org", proxies=proxies).text
print("Your IP:", ip)
# Scraping Github Trending section 
url = "https://github.com/trending"
# Adding parameters like Language and Date Range, and make HTTP request to GitHub
params = {"since": SINCE}
if LANGUAGE:
    params["language"] = LANGUAGE
headers = {
    "User-Agent": "Mozilla/5.0"
}

r = requests.get(
    url,
    headers=headers,
    params=params,
    proxies=proxies,
    timeout=20
)

# Scrape and output details
soup = BeautifulSoup(r.text, "html.parser")

print("\n🔥 GitHub Trending:\n")
for count, repo in enumerate(soup.select("article.Box-row"), start=1):
    name = repo.h2.text.strip().replace("\n", "").replace(" ", "")
    link = "https://github.com" + repo.h2.a["href"]
    desc = repo.p.text.strip() if repo.p else "No description"
    stars_today = repo.select_one("span.d-inline-block.float-sm-right").text.strip() if repo.select_one("span.d-inline-block.float-sm-right") else None
    print(f"{count}. {name}")
    print(f"Link: {link}")
    print(f"Stars Today: {stars_today}")
    print(f"Description: {desc}")
    print("-" * 40)
      
        
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
  
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
      
Copy

import sys
import requests
from bs4 import BeautifulSoup
from typing import List, Dict, Optional
# =========================================================
# CONFIG — CHANGE ONLY THIS SECTION
# =========================================================
GITHUB_TRENDING_URL = "https://github.com/trending"
# Proxy configuration (leave PROXY = None to disable proxy)
PROXY = {
    "host": "DATAIMPULSE HOST",
    "port": "DATAIMPULSE PORT",
    "username": "DATAIMPULSE LOGIN",
    "password": "DATAIMPULSE PASSWORD",
}
# PROXY = None  # uncomment to disable proxy completely

LANGUAGE = "python"        # e.g. "python", "javascript", or None
SINCE = "daily"            # "daily", "weekly", "monthly"
TIMEOUT = 20               # seconds
VERIFY_SSL = True

USER_AGENT = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/118.0.0.0 Safari/537.36"
)

# =========================================================
# END CONFIG
# =========================================================
def log(msg: str) -> None:
    print(f"[INFO] {msg}")

def build_proxies(proxy_cfg: Optional[Dict]) -> Optional[Dict[str, str]]:
    if not proxy_cfg:
        return None

    proxy_url = (
        f"http://{proxy_cfg['username']}:{proxy_cfg['password']}"
        f"@{proxy_cfg['host']}:{proxy_cfg['port']}"
    )

    return {
        "http": proxy_url,
        "https": proxy_url,
    }

def test_proxy(proxies: Optional[Dict[str, str]]) -> None:
    log("Testing proxy...")
    r = requests.get(
        "https://api.ipify.org?format=json",
        proxies=proxies,
        timeout=10,
    )
    r.raise_for_status()
    log(f"Proxy OK. Outgoing IP: {r.json()['ip']}")

def scrape_github_trending(
    language: Optional[str],
    since: str,
    proxies: Optional[Dict[str, str]],
    timeout: int,
) -> List[Dict]:
    headers = {"User-Agent": USER_AGENT}

    params = {"since": since}
    if language:
        params["language"] = language

    log("Fetching GitHub Trending page...")
    response = requests.get(
        GITHUB_TRENDING_URL,
        headers=headers,
        params=params,
        proxies=proxies,
        timeout=timeout,
        verify=VERIFY_SSL,
    )
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "lxml")
    repos = []

    for article in soup.select("article.Box-row"):
        name = (
            article.h2.text
            .replace("\n", "")
            .replace(" ", "")
            .strip()
        )
        url = "https://github.com" + article.h2.a["href"]

        description = article.p.text.strip() if article.p else None

        language_el = article.select_one(
            '[itemprop="programmingLanguage"]'
        )
        stars_today_el = article.select_one(
            "span.d-inline-block.float-sm-right"
        )

        repos.append({
            "name": name,
            "url": url,
            "description": description,
            "language": (
                language_el.text.strip()
                if language_el else None
            ),
            "stars_today": (
                stars_today_el.text.strip()
                if stars_today_el else None
            ),
        })

    return repos

def main() -> None:
    proxies = build_proxies(PROXY)

    # 1) Test proxy
    try:
        if proxies:
            test_proxy(proxies)
        else:
            log("Proxy disabled.")
    except Exception as e:
        sys.exit(f"[ERROR] Proxy test failed: {e}")

    # 2) Scrape GitHub Trending
    try:
        repos = scrape_github_trending(
            language=LANGUAGE,
            since=SINCE,
            proxies=proxies,
            timeout=TIMEOUT,
        )
    except Exception as e:
        sys.exit(f"[ERROR] Scraping failed: {e}")

    # 3) Output
    print("\n🔥 GitHub Trending Repositories\n")
    print(f"Language: {LANGUAGE or 'All'} | Since: {SINCE}\n")

    for i, repo in enumerate(repos, 1):
        print(f"{i}. {repo['name']}")
        print(f"   URL: {repo['url']}")
        print(f"   Language: {repo['language']}")
        print(f"   Stars today: {repo['stars_today']}")
        if repo["description"]:
            print(f"   Description: {repo['description']}")
        print()

    log(f"Done. Total repositories: {len(repos)}")

if __name__ == "__main__":
    main()
      
        
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
  
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
      
Copy