Pipeline Formatting

If you need more data processing, you can use Scrapy's Items to format the data and then uniformly handle it through Pipelines. You can use a Pipeline to open a database connection when the spider starts and close it after the spider finishes.

Usage

a. Write the Pipeline class first:

class XXXPipeline(object):
    def process_item(self, item, spider):
        return item

b. Write the Item class:

class XdbItem(scrapy.Item):
    href = scrapy.Field()
    title = scrapy.Field()

c. Configuration in settings.py:

ITEM_PIPELINES = {
    'myproject.pipelines.MyPipeline': 300,
}

d. In the spider, each timee yield is called with an Item object, process_item will be envoked.

Example spider (spiders/chouti.py):

import scrapy
from myproject.items import XdbItem

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['http://chouti.com/']

    def parse(self, response):
        item_list = response.xpath('//div[@id="content-list"]/div[@class="item"]')
        for item in item_list:
            text = item.xpath('.//a/text()').extract_first()
            href = item.xpath('.//a/@href').extract_first()
            print(text)
            yield XdbItem(title=text, href=href)

Example Pipeline (pipelines.py):

# The built-in Pipeline mechanism:
# 1. Check if the class has a from_crawler method.
#    If yes: obj = MyPipeline.from_crawler(...)
#    If no: obj = MyPipeline()
# 2. obj.open_spider()
# 3. obj.process_item() called for each item
# 4. obj.close_spider()

from scrapy.exceptions import DropItem

class FilePipeline(object):

    def __init__(self, path):
        self.f = None
        self.path = path

    @classmethod
    def from_crawler(cls, crawler):
        """
        Called during initialization to create the pipeline object.
        :param crawler: 
        :return: 
        """
        print('File.from_crawler')
        path = crawler.settings.get('HREF_FILE_PATH')
        return cls(path)

    def open_spider(self, spider):
        """
        Called when the spider starts.
        :param spider: 
        :return: 
        """
        print('File.open_spider')
        self.f = open(self.path, 'a+')

    def process_item(self, item, spider):
        print('File', item['href'])
        self.f.write(item['href'] + '\n')
        return item
    # raise DropItem()  # to drop an item

    def close_spider(self, spider):
        """
        Called when the spider closes.
        :param spider: 
        :return: 
        """
        print('File.close_spider')
        self.f.close()


class DbPipeline(object):
    def __init__(self, path):
        self.f = None
        self.path = path

    @classmethod
    def from_crawler(cls, crawler):
        """
        Called during initialization to create the pipeline object.
        :param crawler: 
        :return: 
        """
        print('DB.from_crawler')
        path = crawler.settings.get('HREF_DB_PATH')
        return cls(path)

    def open_spider(self, spider):
        """
        Called when the spider starts.
        :param spider: 
        :return: 
        """
        print('Db.open_spider')
        self.f = open(self.path, 'a+')

    def process_item(self, item, spider):
        print('Db', item)
        return item

    def close_spider(self, spider):
        """
        Called when the spider closes.
        :param spider: 
        :return: 
        """
        print('Db.close_spider')
        self.f.close()

Settings example (settings.py):

ITEM_PIPELINES = {
   'myproject.pipelines.FilePipeline': 300,
   'myproject.pipelines.DbPipeline': 301,
}

# Custom settings for pipeline
HREF_FILE_PATH = "news.log"
HREF_DB_PATH = "db.log"

Note: Pipelines are shared across all spiders. If you need per-spider customization, use the spider parameter to distinguish logic within process_item.

Custom Deduplication Rules

Scrapy uses scrapy.dupefilter.RFPDupeFilter by default for request deduplication. Related settings:

DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter'
DUPEFILTER_DEBUG = False
JOBDIR = "path for saving crawl state, e.g. /root/"  # final path will be /root/requests.seen

To customize deduplication:

1. Create a custom class
2. Update DUPEFILTER_CLASS to point to your class.

Example custom deduplication file (dupefilters.py):

from scrapy.dupefilter import BaseDupeFilter
from scrapy.utils.request import request_fingerprint

class CustomDupeFilter(BaseDupeFilter):

    def __init__(self):
        self.visited = set()

    @classmethod
    def from_settings(cls, settings):
        return cls()

    def request_seen(self, request):
        # Generate a unique fingerprint for each request URL (similar to MD5)
        fp = request_fingerprint(request=request)
        if fp in self.visited:
            return True
        self.visited.add(fp)
        return False

    def open(self):
        print('Deduplication started')

    def close(self, reason):
        print('Deduplication ended')

Update settings.py:

# Replace default deduplication class
DUPEFILTER_CLASS = 'myproject.dupefilters.CustomDupeFilter'

Use dont_filter=False in requests to apply deduplication (default).

Limiting Crawl Depth

Set DEPTH_LIMIT in settings.py to limit the recursion depth:

DEPTH_LIMIT = 3

Handling Cookies

To retrieve cookies from a response and use them in subsequent requests:

from scrapy.http.cookies import CookieJar

cookie_jar = CookieJar()
cookie_jar.extract_cookies(response, response.request)

# Parse cookies into a dictionary
cookie_dict = {}
for domain, domain_cookies in cookie_jar._cookies.items():
    for path, cookies in domain_cookies.items():
        for name, morsel in cookies.items():
            cookie_dict[name] = morsel.value

Example spider using cookies for login:

import scrapy
from scrapy.http.cookies import CookieJar
from scrapy.http import Request
from urllib.parse import urlencode

class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    allowed_domains = ['chouti.com']
    start_urls = ['https://dig.chouti.com/']

    def __init__(self):
        self.cookie_dict = {}

    def parse(self, response):
        cookie_jar = CookieJar()
        cookie_jar.extract_cookies(response, response.request)
        # Convert cookies to dict
        for domain, domain_cookies in cookie_jar._cookies.items():
            for path, cookies in domain_cookies.items():
                for name, morsel in cookies.items():
                    self.cookie_dict[name] = morsel.value

        yield Request(
            url='https://dig.chouti.com/login',
            method='POST',
            body=urlencode({'phone': '8613121758648', 'password': 'woshiniba', 'oneMonth': 1}),
            cookies=self.cookie_dict,
            headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},
            callback=self.check_login
        )

    def check_login(self, response):
        print(response.text)
        yield Request(
            url='https://dig.chouti.com/all/hot/recent/1',
            cookies=self.cookie_dict,
            callback=self.index
        )

    def index(self, response):
        news_list = response.xpath('//div[@id="content-list"]/div[@class="item"]')
        for news in news_list:
            link_id = news.xpath('.//div[@class="part2"]/@share-linkid').extract_first()
            yield Request(
                url=f'http://dig.chouti.com/link/vote?linksId={link_id}',
                method='POST',
                cookies=self.cookie_dict,
                callback=self.check_result
            )

        # Handle pagination
        page_list = response.xpath('//div[@id="dig_lcpage"]//a/@href').extract()
        for page in page_list:
            page_url = "https://dig.chouti.com" + page
            yield Request(url=page_url, callback=self.index)

    def check_result(self, response):
        print(response.text)