The target website uses a forum structure with categorized sections. Each category follows the patern http://example.com/forum-XX-1.html where XX represents the category ID. Pagination appears as http://example.com/forum-XX-Y.html with Y indicating the page number.

Implemantation

Fetching Page Content

Dynamic content requires JavaScript rendering. Use Selenium with headless browsing:

from selenium import webdriver
from html import unescape

def fetch_page_source(target_url):
    try:
        driver = webdriver.PhantomJS()
        driver.get(target_url)
        page_content = driver.execute_script("return document.documentElement.outerHTML")
        driver.quit()
        return unescape(page_content).encode('utf-8')
    except Exception as error:
        print(f"Fetch error: {error}")
        return None

Pagination Handling

Determine total pages using PyQuery:

from pyquery import PyQuery as pq

def extract_total_pages(html):
    page_doc = pq(html)
    pagination = page_doc('div.pg')
    
    if pagination.has_class('last'):
        last_link = pagination.find('.last').attr('href')
        return int(last_link.split('-')[1])
    else:
        page_text = pagination.find('span').text()
        return int(''.join(filter(str.isdigit, page_text)))

GIF Extrcation

Parse thumbnail entries and resolve final URLs:

def extract_gif_entries(html):
    page_doc = pq(html)
    return page_doc('div.c.cl').items()

def resolve_gif_url(entry_url):
    page_html = fetch_page_source(entry_url)
    img_doc = pq(page_html)
    return img_doc('td.t_f img').attr('src')

def process_entries(entries, page_index):
    results = []
    for idx, entry in enumerate(entries, 1):
        record = {
            'page': page_index,
            'position': idx,
            'title': entry('a').attr('title'),
            'source_url': f"http://example.com{entry('a').attr('href')}"
        }
        record['gif_url'] = resolve_gif_url(record['source_url'])
        results.append(record)
    return results

Storage Operations

Save files locally and persist metadata:

import os
import urllib.request
import pymongo

def save_gif_media(media_url, filename):
    try:
        urllib.request.urlretrieve(media_url, filename)
    except IOError as e:
        print(f"Storage failed: {e}")

def store_records(records, category):
    client = pymongo.MongoClient('localhost', 27017)
    db = client.GifArchive
    collection = db[category]
    
    for rec in records:
        filepath = f"{category}/{rec['title']}.gif"
        save_gif_media(rec['gif_url'], filepath)
        collection.insert_one(rec)

Database Maintenance

Remove duplicate entries based on title:

def deduplicate_collection(db_name, collection_name):
    client = pymongo.MongoClient()
    coll = client[db_name][collection_name]
    
    for title in coll.distinct('title'):
        duplicates = coll.count_documents({'title': title})
        if duplicates > 1:
            for _ in range(duplicates - 1):
                coll.delete_one({'title': title})

Batch Processing

Reconstruct images from database records:

def batch_export(db_name):
    client = pymongo.MongoClient()
    db = client[db_name]
    
    for coll_name in db.list_collection_names():
        collection = db[coll_name]
        for record in collection.find():
            if 'gif_url' in record:
                save_gif_media(
                    record['gif_url'],
                    f"{coll_name}/{record['title']}.gif"
                )