Data Extraction Using XPath and lxml

Automated data collection involves simulating user navigation to retrieve and organize information from web pages. The lxml library, combined with XPath, provides a high-performance way to traverse the DOM tree. The following implementation uses a persistent session and targets specific elements within the Douban Top 250 list.

import requests
from lxml import html
import csv
import time

def fetch_movie_rankings():
    endpoint = "https://movie.douban.com/top250"
    browser_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
    }
    
    results = []
    session = requests.Session()
    
    for page_idx in range(10):
        params = {"start": page_idx * 25}
        response = session.get(endpoint, headers=browser_headers, params=params)
        
        if response.status_code == 200:
            tree = html.fromstring(response.content)
            container = tree.xpath('//ol[@class="grid_view"]/li')
            
            for element in container:
                name = element.xpath('.//div[@class="hd"]/a/span[1]/text()')[0]
                score = element.xpath('.//span[@class="rating_num"]/text()')[0]
                quote = element.xpath('.//span[@class="inq"]/text()')
                
                results.append({
                    "title": name,
                    "rating": score,
                    "slogan": quote[0] if quote else ""
                })
        
        time.sleep(0.8)
    return results

if __name__ == "__main__":
    data = fetch_movie_rankings()
    with open("ranking_data.csv", "w", encoding="utf-8-sig", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["title", "rating", "slogan"])
        writer.writeheader()
        writer.writerows(data)

Parsing with BeautifulSoup and CSS Selectors

BeautifulSoup is a flexible library for parsing HTML and XML documents. Its particularly useful for handling malformed markup. This approach demonstrates how to iterate through the list using tag searching and class identifiers.

import requests
from bs4 import BeautifulSoup

def run_soup_scraper():
    headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"}
    base_path = "https://movie.douban.com/top250?start="
    
    for i in range(0, 250, 25):
        content = requests.get(f"{base_path}{i}", headers=headers).text
        soup = BeautifulSoup(content, "html.parser")
        cards = soup.find_all("div", class_="item")
        
        for card in cards:
            main_title = card.find("span", class_="title").get_text()
            rating = card.find("span", class_="rating_num").get_text()
            link = card.find("a")["href"]
            
            print(f"[{rating}] {main_title} - {link}")

run_soup_scraper()

Information Extraction via Regular Expressions

Regular expressions (Regex) can be used for rapid pattern matching when the target data follows a predictable string format. While less robust than tree-based parsers if the HTML structure changes, Regex is often faster for simple extraction tasks from raw strings.

import re
import requests
import pandas as pd

def regex_extraction():
    url = "https://movie.douban.com/top250"
    headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) Chrome/110.0.0.0"}
    
    all_entries = []
    for step in range(0, 250, 25):
        raw_data = requests.get(url, params={"start": step}, headers=headers).text
        
        # Pattern matching for titles and scores
        titles = re.findall(r'<span class="title">([^&\n]+?)</span>', raw_data)
        # Filtering only the primary title
        titles = [t for t in titles if "&nbsp;" not in t]
        
        scores = re.findall(r'<span class="rating_num" property="v:average">(.*?)</span>', raw_data)
        
        for t, s in zip(titles, scores):
            all_entries.append({"Title": t, "Rating": s})
            
    return pd.DataFrame(all_entries)

if __name__ == "__main__":
    df = regex_extraction()
    print(df.head(10))