Automating Form Submission in Python for Web Crawling Challenges

The challenge requires submitting a form with a username and password (a number under 30) to http://www.heibanke.com/lesson/crawler_ex01/. Here are four implementation approaches:

Method 1: Using urllib

import urllib
import re
import sys

sys.setdefaultencoding('utf-8')
form_data = {'username': 'test_user'}
target_url = 'http://www.heibanke.com/lesson/crawler_ex01/'

for attempt in range(1, 31):
    form_data['password'] = attempt
    encoded_data = urllib.urlencode(form_data)
    response = urllib.urlopen(target_url, encoded_data)
    page_content = response.read()
    if not re.findall('密码错误', page_content):
        print('Success! Next level: http://www.heibanke.com' + 
              re.findall(r'<a href="(.*?)" class', page_content)[0])
        break

Method 2: Using urllib2 with urllib

import urllib2
import urllib
import datetime

start_time = datetime.datetime.now()
credentials = {'username': 'demo_user'}
endpoint = 'http://www.heibanke.com/lesson/crawler_ex01/'

for guess in range(30):
    credentials['password'] = guess
    request = urllib2.Request(endpoint, urllib.urlencode(credentials))
    response = urllib2.urlopen(request)
    if '密码错误' not in response.read():
        print(f'Completed in {datetime.datetime.now()-start_time}')
        break

Method 3: Using requests

import requests

login_url = 'http://www.heibanke.com/lesson/crawler_ex01/'
params = {'username': 'auto_user', 'password': 0}

for trial in range(30):
    params['password'] = trial
    resp = requests.post(login_url, data=params)
    if u'错误' not in resp.text:
        print('Next challenge: http://www.heibanke.com' +
              re.search(r'<a href="(.*?)" class', resp.text).group(1))
        break

Method 4: Browser Automation with Selenium

from selenium import webdriver
import time

driver = webdriver.PhantomJS()
for pwd_attempt in range(30):
    driver.get('http://www.heibanke.com/lesson/crawler_ex01/')
    driver.find_element_by_name('username').send_keys('selenium_user')
    driver.find_element_by_name('password').send_keys(pwd_attempt)
    driver.find_element_by_id('id_submit').click()
    if '错误' not in driver.find_element_by_tag_name('h3').text:
        print(f'Correct password: {pwd_attempt}')
        break
driver.quit()

Tags: python web-scraping automation http-requests Selenium

Posted on Tue, 23 Jun 2026 16:20:41 +0000 by broseph