Cara melakukan web scraping TripAdvisor dengan Python

python3 --version 
      
        
      
Copy

pip3 install selenium beautifulsoup4 undetected-chromedriver 
      
        
      
Copy

title_from_href(href) 
      
        
      
Copy

m = re.search(r"([\d.,KkMm]+)\s+reviews", text) 
      
        
      
Copy

rating = extract_rating(card)  
      
        
      
Copy

import json
import re
import time
from dataclasses import dataclass
from typing import Any, Optional

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

import undetected_chromedriver as uc
from bs4 import BeautifulSoup
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

SHOW_MORE_CLICKS = 1  # Number of "Show more" clicks should be 1

@dataclass(frozen=True)
class Config:
   url: str = "https://www.tripadvisor.com/Search?q=restaurants+in+new+york"
   output_json: str = "restaurants.json"
   headless: bool = False
   timeout_seconds: int = 30
   sleep_seconds: float = 2.5
   proxy: Optional[str] = "http://gw.dataimpulse.com:823"
   block_geolocation: bool = True

def first_number(text: Optional[str]) -> Optional[float]:
   """Extract first numeric value from text."""
   if not text:
       return None
   m = re.search(r"(\d+(?:[.,]\d+)?)", text)
   return float(m.group(1).replace(",", ".")) if m else None

def parse_reviews_count(text: Optional[str]) -> Optional[int]:
   """Parse review counts like '1,234', '1.2k', '3M'."""
   if not text:
       return None

   t = text.strip().lower()

   m = re.search(r"(\d+(?:[.,]\d+)?)\s*([km])\b", t)
   if m:
       val = first_number(m.group(0))
       if val is None:
           return None
       return int(val * (1_000 if m.group(2) == "k" else 1_000_000))

   m = re.search(r"(\d[\d, ]*)", t)
   if not m:
       return None

   digits = m.group(1).replace(",", "").replace(" ", "")
   return int(digits) if digits.isdigit() else None

def title_from_href(href: Optional[str]) -> Optional[str]:
   """Extract readable title from restaurant URL."""
   if not href:
       return None
   m = re.search(r"/Restaurant_Review-[^?#]*?-Reviews-([^-/?#]+)", href)
   return m.group(1).replace("_", " ").strip() if m else None

def rating_from_bubble_class(class_list: Optional[list[str]]) -> Optional[float]:
   """Extract rating from 'bubble_XX' CSS class."""
   if not class_list:
       return None
   for c in class_list:
       m = re.search(r"\bbubble_(\d{2})\b", c)
       if m:
           return int(m.group(1)) / 10.0
   return None

def extract_rating(card) -> Optional[float]:
   """Extract rating from aria-label, bubble class, or svg title."""
   el = card.select_one('[aria-label*="bubbles"]') or card.select_one('[aria-label*="of 5"]')
   if el:
       v = first_number(el.get("aria-label"))
       if v is not None:
           return v


   bubble = card.select_one(".ui_bubble_rating, [class*='bubble_']")
   if bubble:
       v = rating_from_bubble_class(bubble.get("class"))
       if v is not None:
           return v


   svg_title = card.select_one("svg title")
   if svg_title:
       v = first_number(svg_title.get_text(strip=True))
       if v is not None:
           return v


   return None

def build_driver(cfg: Config) -> uc.Chrome:
   """Create configured Chrome driver."""
   opts = uc.ChromeOptions()

   if cfg.headless:
       opts.add_argument("--headless=new")

   opts.add_argument("--window-size=1280,900")
   opts.add_argument("--disable-blink-features=AutomationControlled")


   if cfg.block_geolocation:
       prefs = {
           "profile.default_content_setting_values.geolocation": 2,
           "profile.default_content_setting_values.notifications": 2,
       }
       opts.add_experimental_option("prefs", prefs)


   if cfg.proxy:
       opts.add_argument(f"--proxy-server={cfg.proxy}")


   return uc.Chrome(options=opts)

def click_first(driver, xpaths: list[str]) -> bool:
   """Click first matching element from xpath list."""
   for xp in xpaths:
       try:
           driver.find_element(By.XPATH, xp).click()
           return True
       except Exception:
           continue
   return False


def get_html(cfg: Config) -> str:
   """Load page and return HTML after interactions."""
   driver: Optional[uc.Chrome] = None

   try:
       driver = build_driver(cfg)
       driver.get(cfg.url)


       WebDriverWait(driver, cfg.timeout_seconds).until(
           lambda d: d.find_elements(By.CSS_SELECTOR, 'a[href*="/Restaurant_Review-"]')
       )


       click_first(driver, [
           '//button[contains(., "Accept")]',
           '//button[contains(., "I accept")]',
           '//button[contains(., "Agree")]',
           '//button[contains(., "OK")]',
       ])


       time.sleep(cfg.sleep_seconds)


       for _ in range(max(0, SHOW_MORE_CLICKS)):
           if not click_first(driver, [
               '//button[contains(., "Show more")]',
               '//a[contains(., "Show more")]',
           ]):
               break
           time.sleep(cfg.sleep_seconds)


       return driver.page_source


   except (TimeoutException, WebDriverException) as e:
       raise RuntimeError(f"Page load failed: {e}") from e


   finally:
       if driver:
           driver.quit()

def parse_restaurants(html: str) -> list[dict[str, Any]]:
   """Parse restaurant cards into structured data."""
   soup = BeautifulSoup(html, "html.parser")


   cards = soup.select('[data-test-attribute="location-results-card"]')
   if not cards:
       cards = [a.parent for a in soup.select('a[href*="/Restaurant_Review-"]') if a.parent]


   results: list[dict[str, Any]] = []
   seen: set[str] = set()


   for card in cards:
       a = card.select_one('a[href*="/Restaurant_Review-"]')
       if not a:
           continue


       href = a.get("href")
       if not href:
           continue


       link = href if href.startswith("http") else f"https://www.tripadvisor.com{href}"
       if link in seen:
           continue
       seen.add(link)


       title = title_from_href(href) or a.get_text(strip=True) or None
       rating = extract_rating(card)


       text = card.get_text(" ", strip=True)
       m = re.search(r"([\d.,KkMm]+)\s+reviews", text)
       reviews = parse_reviews_count(m.group(1)) if m else None


       results.append({
           "title": title,
           "rating": rating,
           "reviews": reviews,
           "link": link
       })


   return results

def save_json(rows: list[dict[str, Any]], path: str) -> None:
   """Save data to JSON file."""
   with open(path, "w", encoding="utf-8") as f:
       json.dump(rows, f, ensure_ascii=False, indent=2)

def main() -> None:
   """Run scraping pipeline."""
   cfg = Config()
   html = get_html(cfg)
   rows = parse_restaurants(html)

   if not rows:
       print("No results found. Try running with headless=False.")
       return


   save_json(rows, cfg.output_json)
   print(f"Saved {len(rows)} restaurants to {cfg.output_json}")

if __name__ == "__main__":
   main()  
      
        
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
            
             
              
                 
         
          
           
      
Copy