This crawler fetches content from a CSDN personal blog (either the main page or a category) and saves each article as a text file. Ensure the target URL includes ?viewmode=contents to retrieev all entries.

Basic Crawler

The basic version extracts article links from a page and saves each article’s content:

# -*- coding: utf-8 -*-
import urllib2
import sys
import time
from bs4 import BeautifulSoup

reload(sys)
sys.setdefaultencoding('utf-8')

base_url = 'http://blog.csdn.net'
target_url = 'http://blog.csdn.net/your_username/article/category/123456?viewmode=contents'

def fetch_page(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
    request = urllib2.Request(url, headers=headers)
    try:
        response = urllib2.urlopen(request)
        return response.read()
    except urllib2.HTTPError as e:
        print(f"Error {e.code}: {e.reason}")
        return None

def extract_article_links(html):
    soup = BeautifulSoup(html, 'lxml')
    articles = soup.find_all('div', class_='list_item list_view')
    links = []
    for article in articles:
        a_tag = article.find('a')
        if a_tag:
            link = base_url + a_tag.get('href')
            links.append(link)
    return links

def save_article_content(html, title):
    soup = BeautifulSoup(html, 'lxml')
    try:
        content_div = soup.find('div', class_='article_content')
        with open(f"{title}.txt", 'w') as f:
            f.write(content_div.text)
        print(f"Saved: {title}")
    except Exception as e:
        print(f"Error saving {title}: {e}")

if __name__ == "__main__":
    print("Downloading CSDN blog content...")
    page_html = fetch_page(target_url)
    if page_html:
        article_links = extract_article_links(page_html)
        for link in article_links:
            article_html = fetch_page(link)
            if article_html:
                soup = BeautifulSoup(article_html, 'lxml')
                try:
                    title = soup.find('div', class_='article_title').find('a').text.strip()
                    save_article_content(article_html, title)
                except Exception as e:
                    print(f"Error processing {link}: {e}")

Enhanced Crawler with Pagination and Sorting

The improved version handles pagination, tracks read/comment counts, sorts articles, and organizes downloads in a folder named after the blogger:

# -*- coding: utf-8 -*-
import urllib2
import time
import os
import datetime
from bs4 import BeautifulSoup
import sys

reload(sys)
sys.setdefaultencoding('utf-8')

def log_message(message):
    timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
    print(f"{timestamp}: {message}")

def get_page_content(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
    request = urllib2.Request(url, headers=headers)
    try:
        response = urllib2.urlopen(request)
        return response.read()
    except urllib2.HTTPError as e:
        print(f"HTTP Error {e.code}: {e.reason}")
        return None

def get_total_pages(html):
    if not html:
        log_message("Page error, stopping.")
        return 1
    soup = BeautifulSoup(html, 'lxml')
    try:
        page_links = soup.find('div', class_='pagelist').find_all('a')
        last_page = page_links[-1].get('href')[-1:]
        log_message(f"Total blog pages: {last_page}")
        return int(last_page)
    except Exception as e:
        return 1

def get_article_items(url):
    page_content = get_page_content(url)
    soup = BeautifulSoup(page_content, 'lxml')
    return soup.find_all('div', class_='list_item list_view')

def process_articles(items, article_list, read_counts):
    for item in items:
        article_data = {}
        title_tag = item.find('a')
        article_url = f"http://blog.csdn.net{title_tag.get('href')}"
        read_text = item.find('span', class_='link_view').text.strip()
        comment_text = item.find('span', class_='link_comments').text.strip()
        
        read_num = int(''.join(filter(str.isdigit, read_text)))
        read_counts.append(read_num)
        
        article_data['read_count'] = read_num
        article_data['title'] = title_tag.text.strip()
        article_data['read_text'] = read_text
        article_data['comment_text'] = comment_text
        article_data['url'] = article_url
        article_list.append(article_data)

def create_directory(path):
    if not os.path.exists(path):
        os.makedirs(path)

def save_article_content(html, directory):
    soup = BeautifulSoup(html, 'lxml')
    try:
        title = soup.find('div', class_='article_title').find('a').text.strip()
    except Exception as e:
        print(f"Title error: {e}")
        return
    try:
        content_div = soup.find('div', class_='article_content')
        file_path = os.path.join(directory, f"{title}.txt")
        with open(file_path, 'w') as f:
            f.write(content_div.text)
        log_message(f"Saved article: {title}")
    except Exception as e:
        print(f"Content error: {e}")

def download_all_articles(article_list, directory):
    log_message("Starting to save all articles...")
    for article in article_list:
        url = article['url']
        log_message(f"Downloading: {url}...")
        article_html = get_page_content(url)
        if article_html:
            save_article_content(article_html, directory)

def get_blogger_name(url):
    if 'viewmode' in url:
        parts = url.split('.net')[1].split('?')[0].split('/')
        return parts[1]
    else:
        parts = url.split('.net')[1].split('/')
        return parts[1]

def main():
    print("""
    *****************************************
    **    CSDN Blog Spider                 **
    **    Created on 2023-01-01            **
    **    @author: ExampleUser             **
    *****************************************
    """)
    target_url = 'http://blog.csdn.net/your_username?viewmode=contents'
    # target_url = 'http://blog.csdn.net/your_username/article/category/123456?viewmode=contents'
    blogger = get_blogger_name(target_url)
    directory = blogger
    create_directory(directory)
    
    main_page = get_page_content(target_url)
    total_pages = get_total_pages(main_page)
    article_list = []
    read_counts = []
    
    for page in range(1, total_pages + 1):
        if 'category' in target_url:
            page_url = f"{target_url}/{page}"
        else:
            page_url = f"{target_url.split('?')[0]}/article/list/{page}?viewmode=contents"
        log_message(f"Processing page {page}: {page_url}")
        items = get_article_items(page_url)
        process_articles(items, article_list, read_counts)
    
    log_message(f"Total articles: {len(article_list)}")
    article_list.sort(key=lambda x: x['read_count'])
    
    with open(os.path.join(directory, "article_list.txt"), 'w') as f:
        for idx, article in enumerate(article_list, 1):
            line = f"Article {idx}|{article['title']}|{article['read_text']}|{article['comment_text']}|{article['url']}"
            print(line)
            f.write(line + "\n")
    
    download_all_articles(article_list, directory)

if __name__ == "__main__":
    main()

Key Features

• Pagination: Atuomatically processes all blog pages (detects total pages from navigation).
• Metrics Tracking: Captures read/comment counts for each article.
• Sorting: Sorts articles by read count (ascendign) for easy analysis.
• Organized Output: Saves articles in a folder named after the blogger, with a detailed list file.

Adjust the target_url in the script to point to your CSDN blog (main page or category) and run the script to download all content.