Data Scraping Workflow Review
The standard process for scraping data with requests:
- Specify target URL
- Make request via requests module
- Extract data from response object
- Data parsing (critical step for focused crawlers)
- Persist to storage
Most practical scenarios require focused crawlers that extract specific portions of page data rather than entire page content. This article covers three essential data parsing techniques.
1. Regular Expression Parsing
Common Regex Patterns
Single Character:
. : Any character except newline
[] : Any character in set [aoe] or range [a-w]
\d : Digit [0-9]
\D : Non-digit
\w : Word character (alphanumeric + underscore)
\W : Non-word character
\s : Whitespace (space, tab, newline, etc.)
\S : Non-whitespace
Quantifiers:
* : 0 or more occurrences
+ : 1 or more occurrences
? : 0 or 1 occurrence
{m} : Exactly m times
{m,} : m or more times
{m,n} : Between m and n times
Boundaries:
$ : End with specified pattern
^ : Start with specified pattern
Groups:
(ab) : Capture group
Greedy vs Non-greedy:
.* (greedy)
.*? (non-greedy/lazy)
Flags:
re.I : Case insensitive
re.M : Multiline mode
re.S : Single line/dotall mode
Regex Methods
import re
# Find all matches
re.findall(pattern, string)
# Replace pattern with new value
re.sub(pattern, replacement, string)
# Compile pattern for reuse
pattern = re.compile(r'pattern', re.S)
pattern.findall(string)
Practical Examples
import re
# Extract 'python' from string
key = "javapythonc++php"
result = re.findall('python', key)[0]
# Extract text between tags
key = "<html><h1>hello world<h1></html>"
result = re.findall('<h1>(.*)<h1>', key)[0]
# Extract numbers
string = 'I like girls with height 170'
result = re.findall('\d+', string)
# Extract http/https URLs
key = 'http://www.example.com and https://example.org'
result = re.findall('https?://', key)
# Case-insensitive matching
key = 'lalala<hTml>hello</HtMl>hahah'
result = re.findall('<[Hh][Tt][mM][lL]>(.*)</[Hh][Tt][mM][lL]>', key)
# Match with lazy quantifier
key = 'bobo@hit.edu.com'
result = re.findall('h.*?\.', key)
# Match variations (saas, sas)
key = 'saas and sas and saaas'
result = re.findall('sa{1,2}s', key)
# Match lines starting with specific letter
string = '''fall in love with you
i love you very much
i love she
i love her'''
result = re.findall('^.*', string, re.M)
# Match all lines including newlines
string1 = """<div>静夜思
窗前明月光
疑是地上霜
举头望明月
低头思故乡
</div>"""
result = re.findall('.*', string1, re.S)
Project: Download Images from Joke Website
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests
import re
import os
if __name__ == "__main__":
target_url = 'https://www.qiushibaike.com/pic/%s/'
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
start_page = int(input('Enter start page: '))
end_page = int(input('Enter end page: '))
output_dir = 'downloaded_images'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for page_num in range(start_page, end_page + 1):
print(f'Downloading page {page_num}...')
page_url = target_url % page_num
response = requests.get(url=page_url, headers=headers)
# Parse image URLs from response
image_pattern = '<div class="thumb">.*?<img src="(.*?)".*?>.*?</div>'
compiled_pattern = re.compile(image_pattern, re.S)
image_links = compiled_pattern.findall(response.text)
# Download each image
for img_link in image_links:
full_url = 'https:' + img_link
filename = full_url.split('/')[-1]
filepath = os.path.join(output_dir, filename)
img_data = requests.get(url=full_url, headers=headers).content
with open(filepath, 'wb') as f:
f.write(img_data)
2. XPath Parsing
Installation
pip install lxml
Basic Usage
from lxml import etree
# Parse local file
tree = etree.parse('filename.html')
tree.xpath('xpath_expression')
# Parse network response
tree = etree.HTML(html_string)
tree.xpath('xpath_expression')
Common XPath Expressions
Attribute Selection:
//div[@class="song"]
Hierarchy & Index:
//div[@class="tang"]/ul/li[2]/a
Logical Operations:
//a[@href="" and @class="du"]
Partial Matching:
//div[contains(@class, "ng")]
//div[starts-with(@class, "ta")]
Text Extraction:
/text() - Direct child text only
//text() - All descendant text
//div[@class="song"]/p[1]/text()
//div[@class="tang"]//text()
Attribute Extraction:
//div[@class="tang"]//li[2]/a/@href
Project: Extract Jokes Content
from lxml import etree
import requests
target_url = 'http://www.haoduanzi.com/category-10_2.html'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
}
page_content = requests.get(target_url, headers=headers).text
tree = etree.HTML(page_content)
# Extract titles
title_list = tree.xpath('//div[@class="log cate10 auth1"]/h3/a/text()')
# Extract content
div_elements = tree.xpath('//div[@class="log cate10 auth1"]')
content_list = []
for element in div_elements:
text_items = element.xpath('./div[@class="cont"]//text()')
combined_text = str(text_items)
content_list.append(combined_text)
print(title_list)
print(content_list)
Advanced: Decoding Base64 Encrypted Image URLs
import requests
from lxml import etree
from fake_useragent import UserAgent
import base64
import urllib.request
target_url = 'http://jandan.net/ooxx'
ua = UserAgent(verify_ssl=False, use_cache_server=False).random
headers = {'User-Agent': ua}
page_text = requests.get(url=target_url, headers=headers).text
tree = etree.HTML(page_text)
# Extract encrypted hash values from img-hash elements
encrypted_list = tree.xpath('//span[@class="img-hash"]/text()')
decoded_urls = []
for encoded_url in encrypted_list:
# Decode base64 and prepend protocol
decoded = 'http:' + base64.b64decode(encoded_url).decode()
decoded_urls.append(decoded)
for url in decoded_urls:
filename = url.split('/')[-1]
urllib.request.urlretrieve(url=url, filename=filename)
print(f'{filename} downloaded successfully')
3. BeautifulSoup Parsing
Installation
# Configure pip source (Windows)
# Navigate to %appdata%\pip\ and create pip.ini
[global]
timeout = 6000
index-url = https://mirrors.aliyun.com/pypi/simple/
trusted-host = mirrors.aliyun.com
pip install bs4 lxml
Basic Usage
from bs4 import BeautifulSoup
# Parse local file
soup = BeautifulSoup(open('local_file.html'), 'lxml')
# Parse network/string data
soup = BeautifulSoup(html_string, 'lxml')
Nvaigation Methods
# Find by tag (first match only)
soup.a
# Get attributes
soup.a.attrs # All attributes as dict
soup.a.attrs['href'] # Specific attribute
soup.a['href'] # Shorthand
# Get text content
soup.a.string # None if nested tags exist
soup.a.text # All text combined
soup.a.get_text() # Alternative text extraction
# Find methods
soup.find('a') # First match
soup.find('a', title="xxx") # By attribute
soup.find('a', alt="xxx") # By alt
soup.find('a', class_="xxx") # By class (underscore required)
soup.find('a', id="xxx") # By id
soup.find_all('a') # All matches
soup.find_all(['a', 'b']) # Multiple tags
soup.find_all('a', limit=2) # Limit results
# CSS Selectors
soup.select('#feng') # ID selector
soup.select('.du') # Class selector
soup.select('div a') # Descendant selector
soup.select('div > p > a') # Child selector
Project: Scrape Three Kingdoms Novel Chapters
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
def extract_chapter_content(url):
"""Fetch and extract chapter content"""
response = requests.get(url, headers=headers).text
soup = BeautifulSoup(response, 'lxml')
content_div = soup.find('div', class_='chapter_content')
return content_div.text if content_div else ""
if __name__ == "__main__":
index_url = 'http://www.shicimingju.com/book/sanguoyanyi.html'
index_response = requests.get(url=index_url, headers=headers)
soup = BeautifulSoup(index_response.text, 'lxml')
chapter_links = soup.select('.book-mulu > ul > li > a')
chapter_number = 1
for link in chapter_links:
print(f'Starting chapter {chapter_number}...')
chapter_number += 1
chapter_title = link.string
full_url = 'http://www.shicimingju.com' + link['href']
chapter_text = extract_chapter_content(full_url)
with open('./sanguo_novel.txt', 'a', encoding='utf-8') as f:
f.write(f"{chapter_title}:{chapter_text}\n\n\n\n")
print(f'Completed chapter {chapter_number}')
Method Comparison
| Method | Best For | Pros | Cons |
|---|---|---|---|
| Regex | Pattern matching, fixed formats | Fast, flexible | Complex patterns hard to maintain |
| XPath | XML/HTML structured data | Powerful, readable | Requires library installation |
| BeautifulSoup | HTML parsing, DOM navigation | Easy API, good documentation | Slower on large documants |