Getting Started with Scrapy

To build a web crawler using Scrapy, follow these core steps:

1. Define a spider by inheriting from scrapy.Spider or scrapy.CrawlSpider.
2. Set the target URLs in the start_urls list.
3. Implement the parse method to handle response data.
4. Define data structures in the items.py file.
5. Configure custom headers and proxies in middlewares.py.
6. Handle data storage (database, file, FTP) in pipelines.py.

Environment Setup and CLI Commands

Install the necessary packages and configure the environment:

# Scrapy documentation: https://docs.scrapy.org/en/latest/
# pip install scrapy
# pip install pywin32
# pip install twisted
# pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

# Create a new Scrapy project
# scrapy startproject demo_project

# Generate a basic Spider template
# scrapy genspider example_spider example.com

# Generate a CrawlSpider template
# scrapy genspider -t crawl novel_crawler novel.com

# Run the spider
# scrapy crawl example_spider

# If debugging is required, use:
# from scrapy.cmdline import execute
# execute('scrapy crawl example_spider'.split())

# Export results to JSON, CSV, or XML
# scrapy crawl example_spider -o results.json

# Open interactive shell for testing
# scrapy shell www.example.com

Helper script for running the spider directly:

from scrapy.cmdline import execute

def run_spider():
    execute('scrapy crawl example_spider'.split())

if __name__ == '__main__':
    run_spider()

Project Configuration

Modify settings.py to control the behavior of your crawler:

# Disable logging
LOG_ENABLED = False

# Project name used for User-Agent and logs
BOT_NAME = "demo_project"

# Module where spiders are located
SPIDER_MODULES = ["demo_project.spiders"]
NEWSPIDER_MODULE = "demo_project.spiders"

# Default User-Agent header
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'

# Ignore robots.txt rules
ROBOTSTXT_OBEY = False

# Delay between requests (seconds)
DOWNLOAD_DELAY = 3

# Default request headers
DEFAULT_REQUEST_HEADERS = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en",
}

# Enable downloader middlewares
DOWNLOADER_MIDDLEWARES = {
   "demo_project.middlewares.CustomDownloaderMiddleware": 543,
}

# Enable item pipelines (lower number = higher priority)
ITEM_PIPELINES = {
   "demo_project.pipelines.DataSavePipeline": 300,
   "demo_project.pipelines.ImageSavePipeline": 300,
}

# Directory for storing images
IMAGES_STORE = "./images"

Defining Data Items

Use items.py to structure the data scraped from websites:

import scrapy

class ImageData(scrapy.Item):
    image_urls = scrapy.Field()
    images = scrapy.Field()
    file_names = scrapy.Field()

class StoryContent(scrapy.Item):
    title = scrapy.Field()
    chapter = scrapy.Field()
    text = scrapy.Field()

class UserProfile(scrapy.Item):
    username = scrapy.Field()
    years = scrapy.Field()

Pipeline Implementation

Pipelines process the items returned by spiders. Below is an example of saving images and text files:

from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request
from urllib.parse import urlparse
from os.path import basename, splitext

class ImageSavePipeline(ImagesPipeline):
    def file_path(self, request, response=None, info=None, *, item=None):
        parsed = urlparse(request.url)
        name = basename(parsed.path)
        ext = splitext(name)[1]
        return f"{request.meta['file_names'][0]}{ext}"

    def get_media_requests(self, item, info):
        for idx, url in enumerate(item.get('image_urls', [])):
            yield Request(url, meta={'file_names': [item['file_names'][idx]]})

class DataSavePipeline:
    def open_spider(self, spider):
        self.file_handlers = {}

    def close_spider(self, spider):
        for handler in self.file_handlers.values():
            handler.close()

    def process_item(self, item, spider):
        if spider.name == "novel_crawler":
            filename = f"{item['title']}.txt"
            if filename not in self.file_handlers:
                self.file_handlers[filename] = open(filename, 'a', encoding='utf-8')
            self.file_handlers[filename].write(f"{item['chapter']}\n{item['text']}\n")
        return item

Middleware Configuration

Middlewares allow you to modify requests and responses dynamically, such as rotating User-Agents and proxies:

from scrapy import signals
from fake_useragent import UserAgent
from itemadapter import is_item, ItemAdapter

class CustomSpiderMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_opened, signal=signals.spider_opened)
        return middleware

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        yield from result

    def process_spider_exception(self, response, exception, spider):
        pass

    def process_start_requests(self, start_requests, spider):
        yield from start_requests

    def spider_opened(self, spider):
        spider.logger.info(f"Spider initialized: {spider.name}")

class CustomDownloaderMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_opened, signal=signals.spider_opened)
        return middleware

    def process_request(self, request, spider):
        request.headers.setdefault(b'User-Agent', UserAgent().random.encode())
        request.meta['proxy'] = 'http://127.0.0.1:8080'
        return None

    def process_response(self, request, response, spider):
        return response

    def process_exception(self, request, exception, spider):
        pass

    def spider_opened(self, spider):
        spider.logger.info(f"Downloader middleware started for: {spider.name}")

Basic Spider Example

A simple spider using scrapy.Spider to scrape data:

import scrapy

class ExampleSpider(scrapy.Spider):
    name = "example_spider"
    allowed_domains = ["example.com"]
    start_urls = ["https://example.com"]

    def parse(self, response):
        # Available selectors: response.xpath(), response.css(), response.re()
        # Use .extract_first() for single results, .extract() for lists
        print(response.text)

        users = [
            {'username': 'Alice', 'years': 25},
            {'username': 'Bob', 'years': 30}
        ]

        for user in users:
            yield {
                'username': user['username'],
                'years': user['years']
            }

        # Follow next page link
        # next_page = response.css('a.next::attr(href)').get()
        # if next_page:
        #     yield response.follow(next_page, callback=self.parse)

CrawlSpider for Rule-Based Crawling

Use CrawlSpider to follow links automatically based on rules:

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ..items import StoryContent

class NovelCrawler(CrawlSpider):
    name = "novel_crawler"
    allowed_domains = ["novelsite.net"]
    start_urls = ["http://novelsite.net/chapter1.html"]

    rules = (
        Rule(
            LinkExtractor(restrict_xpaths="//div[@class='pagination']/a[contains(text(),'Next')]"),
            callback="parse_chapter",
            follow=True
        ),
    )

    def parse_chapter(self, response):
        item = StoryContent()
        item['title'] = response.xpath("//h1[@class='title']/text()").get()
        item['chapter'] = response.xpath("//h2[@class='chapter']/text()").get()
        item['text'] = " ".join(response.xpath("//div[@id='story']//p/text()").getall())
        return item

Downloading Images

Example of a spider designed to download images from a gallery:

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ..items import ImageData

class GallerySpider(CrawlSpider):
    name = "gallery_spider"
    allowed_domains = ["gallery.example.com"]
    start_urls = ["https://gallery.example.com/album1"]

    rules = (
        Rule(
            LinkExtractor(restrict_xpaths="//div[@class='thumbnails']/a"),
            callback="parse_image",
            follow=True
        ),
    )

    def parse_image(self, response):
        item = ImageData()
        item['image_urls'] = [response.xpath("//img[@id='main-img']/@src").get()]
        item['file_names'] = [response.xpath("//span[@class='img-id']/text()").get()]
        return item