Novel Data Extraction

Fetching Book Listings

To efficiently extract novel information from recommendation pages, we can parse specific elements without rendering the entire webpage. This approach focuses on retrieving essential book data from the listing section.

request_headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
}
target_url = "https://www.readnovel.com/"
response = request.urlopen(request.Request(target_url, headers=request_headers))
page_content = response.read().decode('utf-8')
document = BeautifulSoup(page_content, 'html.parser')

novel_collection = []
for entry in document.select('#new-book-list li'):
    title_link = entry.select_one('a[data-eid="qd_F24"]')
    description = entry.select_one('p')
    novel_data = {
        'url': title_link['href'],
        'name': title_link.get('title'),
        'summary': description.get_text()
    }
    novel_collection.append(novel_data)
    print(novel_data)

Extracting Novel Details

After obtaining the book list, the next step is to fetch comprehensive details about a selected novel. This includes metadata like title, author, and update information, plus the chapter listing.

detail_headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
}
novel_url = f"https://www.readnovel.com{book_path}#Catalog"
response = request.urlopen(request.Request(novel_url, headers=detail_headers))
content = response.read().decode('utf-8')
page = BeautifulSoup(content, 'html.parser')

metadata = {
    'title': page.find('meta', property='og:title')['content'],
    'description': page.find('meta', property='og:description')['content'],
    'author': page.find('meta', property='og:novel:author')['content'],
    'last_updated': page.find('meta', property='og:novel:update_time')['content'],
    'status': page.find('meta', property='og:novel:status')['content'],
    'latest_chapter': page.find('meta', property='og:novel:latest_chapter_name')['content']
}

catalog_container = page.find('div', id='j-catalogWrap')
chapter_entries = catalog_container.find_all('li', attrs={'data-rid': True})
chapter_list = []

for entry in chapter_entries:
    link_element = entry.find('a')
    chapter_name = link_element.text
    if '第' in chapter_name:
        chapter_list.append({
            'title': chapter_name,
            'link': link_element['href']
        })

for key, value in metadata.items():
    print(f"{key.replace('_', ' ').title()}: {value}")

Reading Chapter Content

For preview functionality, we need to extract individual chapter content. This involves parsing the chapter page to retrieve both the title and full text.

chapter_headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101'
}
chapter_url = f"https://www.readnovel.com{chapter_link}"
response = request.urlopen(request.Request(chapter_url, headers=chapter_headers))
html = response.read().decode('utf-8')
soup = BeautifulSoup(html, 'html.parser')

chapter_header = soup.find('h1', class_='j_chapterName')
chapter_data = {
    'heading': chapter_header.get_text()
}

content_area = soup.find('div', class_='ywskythunderfont')
if content_area:
    paragraphs = content_area.find_all('p')
    chapter_data['body'] = paragraphs[0].get_text()
    print(chapter_data)

Local Storage Implementation

Saving Chapters to Files

Once chapter content is extracted, saving it locally enables offline reading. The file operations handle text encoding and proper formatting.

def persist_chapter(chapter_content):
    safe_filename = f"{chapter_content['heading'].replace('/', '_')}.txt"
    with open(safe_filename, 'w', encoding='utf-8') as output_file:
        formatted_text = chapter_content['body'].replace('\u3000\u3000', '\n\n')
        output_file.write(formatted_text)
    print(f"Successfully saved: {safe_filename}")

Complete Implementation

import request
from bs4 import BeautifulSoup
from random import randint
import colorama
from termcolor import cprint
import readchar

colorama.init()
NOVEL_DB = []
CHAPTER_INDEX = []
BASE_HEADERS = {'User-Agent': 'Mozilla/5.0 (compatible; NovelBot/1.0)'}
COLOR_PALETTE = ['green', 'yellow', 'blue', 'cyan', 'magenta']

def fetch_novels():
    cprint('Acquiring novel catalog...', randint(0, len(COLOR_PALETTE)-1))
    global NOVEL_DB
    NOVEL_DB.clear()
    
    with request.urlopen(request.Request("https://www.readnovel.com/", headers=BASE_HEADERS)) as response:
        html_data = response.read().decode('utf-8')
        parsed_page = BeautifulSoup(html_data, 'html.parser')
        
        for item in parsed_page.select('#new-book-list li'):
            link = item.select_one('a[data-eid="qd_F24"]')
            desc = item.select_one('p')
            novel_entry = {
                'path': link['href'],
                'title': link.get('title'),
                'preview': desc.get_text()
            }
            NOVEL_DB.append(novel_entry)

def get_novel_info(book_path):
    global CHAPTER_INDEX
    CHAPTER_INDEX.clear()
    
    full_url = f"https://www.readnovel.com{book_path}#Catalog"
    with request.urlopen(request.Request(full_url, headers=BASE_HEADERS)) as response:
        content = response.read().decode('utf-8')
        soup = BeautifulSoup(content, 'html.parser')
        
        info = {
            'title': soup.find('meta', property='og:title')['content'],
            'synopsis': soup.find('meta', property='og:description')['content'],
            'writer': soup.find('meta', property='og:novel:author')['content'],
            'modified': soup.find('meta', property='og:novel:update_time')['content'],
            'completion': soup.find('meta', property='og:novel:status')['content'],
            'newest': soup.find('meta', property='og:novel:latest_chapter_name')['content']
        }
        
        catalog = soup.find('div', id='j-catalogWrap')
        for item in catalog.find_all('li', attrs={'data-rid': True}):
            anchor = item.find('a')
            if '第' in anchor.text:
                CHAPTER_INDEX.append({
                    'label': anchor.text,
                    'url': anchor['href']
                })
        
        for attr, val in info.items():
            cprint(f"{attr.title()}: {val}", randint(0, len(COLOR_PALETTE)-1))

def retrieve_chapter(chapter_url):
    target = f"https://www.readnovel.com{chapter_url}"
    with request.urlopen(request.Request(target, headers=BASE_HEADERS)) as response:
        page = response.read().decode('utf-8')
        doc = BeautifulSoup(page, 'html.parser')
        
        header = doc.find('h1', class_='j_chapterName')
        chapter = {'name': header.get_text()}
        
        main_content = doc.find('div', class_='ywskythunderfont')
        if main_content:
            chapter['text'] = main_content.find('p').get_text()
        
        return chapter

def archive_chapter(chapter_data):
    filename = f"{chapter_data['name']}.txt"
    with open(filename, 'w', encoding='utf-8') as storage:
        storage.write(chapter_data['text'].replace('\u3000\u3000', '\n'))
    cprint(f"Archived: {filename}", randint(0, len(COLOR_PALETTE)-1))

def display_catalog():
    for i in range(0, len(NOVEL_DB), 3):
        entries = []
        for j in range(3):
            if i + j < len(NOVEL_DB):
                entries.append(f"{i+j}:{NOVEL_DB[i+j]['title']}")
        cprint(' | '.join(entries), randint(0, len(COLOR_PALETTE)-1))

if __name__ == '__main__':
    fetch_novels()
    cprint('Catalog loaded!', randint(0, len(COLOR_PALETTE)-1))
    
    commands = {
        'q': 'quit',
        'm': 'main',
        'r': 'read',
        'd': 'download',
        'n': 'next',
        'p': 'previous'
    }
    
    current_chapter = 0
    
    while True:
        action = readchar.readkey().lower()
        
        if action == 'q':
            break
        
        if action == 'm':
            display_catalog()
            selection = int(input('Select novel ID: '))
            if selection < len(NOVEL_DB):
                get_novel_info(NOVEL_DB[selection]['path'])
        
        if action == 'r' and CHAPTER_INDEX:
            chapter = retrieve_chapter(CHAPTER_INDEX[current_chapter]['url'])
            cprint(chapter['name'], randint(0, len(COLOR_PALETTE)-1))
            cprint(chapter['text'], randint(0, len(COLOR_PALETTE)-1))
        
        if action == 'd' and CHAPTER_INDEX:
            chapter = retrieve_chapter(CHAPTER_INDEX[current_chapter]['url'])
            archive_chapter(chapter)
        
        if action == 'n' and CHAPTER_INDEX:
            current_chapter = min(current_chapter + 1, len(CHAPTER_INDEX) - 1)
        
        if action == 'p' and CHAPTER_INDEX:
            current_chapter = max(current_chapter - 1, 0)