Many older Python scripts for scraping Taobao model content no longer function due to frequent updates to Taobao’s web pages. This walkthrough uses a refreshed approach to capture profile data and images.

1. Fetch Entry-Level Model Profile Links

Start by retrieving top-list model page content and converting card URLs to profile page URLs. Use Seelnium with a headless driver to handle dynamic content loading, then extract and adjust each link.

#!/usr/bin/env python3
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class TaobaoModelScraper:
    def __init__(self):
        self.base_dir = "taobao_model_data"
        chrome_options = Options()
        chrome_options.add_argument("--headless=new")
        chrome_options.add_argument("--disable-gpu")
        chrome_options.add_argument("--no-sandbox")
        self.list_driver = webdriver.Chrome(options=chrome_options)
        self.detail_driver = webdriver.Chrome(options=chrome_options)

    def run_scraper(self, total_pages):
        for page_num in range(1, total_pages + 1):
            print(f"Processing page {page_num}")
            self.fetch_model_list(page_num)
        self.list_driver.quit()
        self.detail_driver.quit()

    def fetch_model_list(self, page):
        target_url = f"https://mm.taobao.com/json/request_top_list.htm?page={page}"
        self.list_driver.get(target_url)
        WebDriverWait(self.list_driver, 10).until(
            EC.presence_of_all_elements_located((By.CLASS_NAME, "list-item"))
        )
        link_elements = self.list_driver.find_elements(
            By.XPATH, '//div[contains(@class,"list-item")]//p/a'
        )
        profile_links = []
        for elem in link_elements:
            raw_link = elem.get_attribute("href")
            adjusted_link = raw_link.replace("model_card", "model_info")
            profile_links.append(adjusted_link)
            print(adjusted_link)
            self.parse_model_profile(adjusted_link)

if __name__ == "__main__":
    scraper = TaobaoModelScraper()
    scraper.run_scraper(1)

This will output sanitized profile URLs like:

Processing page 1
https://mm.taobao.com/self/model_info.htm?user_id=687471686
https://mm.taobao.com/self/model_info.htm?user_id=405095521

2. Resolve Stale Element Issues

Using a single driver for both list and detail pages causes StaleElementReferenceException because navigating away invalidates cached list elements. The code above already initializes two separate Chrome drivers to avoid this conflict.

3. Extract Model Profile Data

Load each profile page with the dedictaed detail driver, then scrape name, avatar URL, and base bio:

    def parse_model_profile(self, profile_url):
        self.detail_driver.get(profile_url)
        WebDriverWait(self.detail_driver, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, "mm-p-model-info-left-top"))
        )
        
        model_name = self.detail_driver.find_element(
            By.XPATH, '//div[@class="mm-p-model-info-left-top"]/dl/dd/a'
        ).text.strip()
        print(f"Found model: {model_name}, URL: {profile_url}")
        
        avatar_src = self.detail_driver.find_element(
            By.XPATH, '//div[@class="mm-p-model-info-left-top"]/dl/dt/a/img'
        ).get_attribute("src")
        
        bio_items = self.detail_driver.find_elements(
            By.XPATH, '//div[contains(@class,"mm-p-base-info")]/ul/li'
        )
        full_bio = "\n".join([item.text.strip() for item in bio_items])
        
        self.save_profile_assets(model_name, avatar_src, full_bio, profile_url)

4. Save Extracted Data and Avatar

Create nested directories, download the avatar, and write bio/URL to a text file:

import os
import requests

    def save_profile_assets(self, name, avatar_url, bio_text, profile_link):
        safe_name = name.replace("/", "_").replace("\\", "_")
        model_folder = os.path.join(self.base_dir, safe_name)
        os.makedirs(model_folder, exist_ok=True)
        
        avatar_path = os.path.join(model_folder, f"{safe_name}_avatar.jpg")
        avatar_resp = requests.get(avatar_url, timeout=10)
        avatar_resp.raise_for_status()
        with open(avatar_path, "wb") as img_file:
            img_file.write(avatar_resp.content)
        
        bio_path = os.path.join(model_folder, f"{safe_name}_profile.txt")
        with open(bio_path, "w", encoding="utf-8") as bio_file:
            bio_file.write(bio_text + "\n\n")
            bio_file.write(f"Taobao Profile: {profile_link}")
        print(f"Saved assets for {safe_name} to {model_folder}")