Automated Web Crawler for URL Discovery

import re
from urllib.parse import urlparse

def extract_protocol(url):
    '''Extract the protocol scheme (e.g., http, https) from a URL.'''
    parsed = urlparse(url)
    return parsed.scheme

def normalize_domain(url, protocol):
    '''Normalize a URL to its domain name for same-site comparison.'''
    domain_part = url.replace(f'{protocol}://', '').replace('://', '')
    if domain_part.startswith('www.'):
        domain_part = domain_part[4:]
    domain = domain_part.split('/')[0]
    return domain

class UrlQueue:
    def __init__(self):
        self.visited = []
        self.unvisited = []
    
    def get_visited(self):
        return self.visited
    
    def get_unvisited(self):
        return self.unvisited
    
    def add_visited(self, url):
        self.visited.append(url)
    
    def add_unvisited(self, url):
        if url and url not in self.visited and url not in self.unvisited:
            self.unvisited.insert(0, url)
    
    def remove_visited(self, url):
        if url in self.visited:
            self.visited.remove(url)
    
    def pop_unvisited(self):
        try:
            return self.unvisited.pop()
        except IndexError:
            return None
    
    def is_unvisited_empty(self):
        return len(self.unvisited) == 0

class WebCrawler:
    def __init__(self, start_url):
        self.queue = UrlQueue()
        self.queue.add_unvisited(start_url)
        self.current_depth = 1
        self.protocol = extract_protocol(start_url)
        self.domain = normalize_domain(start_url, self.protocol)
    
    def extract_links(self, page_url):
        '''Extract all href attributes from anchor tags on a page.'''
        return []
    
    def resolve_links(self, base_url, links):
        '''Convert relative URLs to absolute URLs.'''
        resolved = []
        for link in links:
            if link.startswith('//'):
                resolved.append(f'{self.protocol}:{link}')
            elif link.startswith('http://') or link.startswith('https://'):
                resolved.append(link)
        return resolved
    
    def filter_same_domain(self, urls):
        '''Filter URLs to those within the same domain.'''
        same_domain = []
        for url in urls:
            if self.domain in url:
                same_domain.append(url)
        return same_domain
    
    def filter_exclusions(self, urls):
        '''Exclude URLs from specified domains.'''
        excluded_domains = [
            's.wuage.com', 'static.wuage.com',
            'shop.wuage.com', 'img.wuage.com',
            'medici.wuage.com', 'buyer.wuage.com',
            'item.wuage.com'
        ]
        filtered = []
        for url in urls:
            parsed = urlparse(url)
            domain = parsed.netloc
            if domain not in excluded_domains and url not in filtered:
                filtered.append(url)
        return filtered
    
    def crawl(self, max_depth=1):
        '''Crawl the web starting from the initial URL up to max_depth.'''
        while self.current_depth <= max_depth:
            current_url = self.queue.pop_unvisited()
            if not current_url:
                continue
            
            links = self.extract_links(current_url)
            resolved_links = self.resolve_links(current_url, links)
            same_domain_links = self.filter_same_domain(resolved_links)
            filtered_links = self.filter_exclusions(same_domain_links)
            
            self.queue.add_visited(current_url)
            
            for link in filtered_links:
                self.queue.add_unvisited(link)
            
            self.current_depth += 1
        
        return self.queue.get_visited()

if __name__ == '__main__':
    start_url = 'http://example.com'
    crawler = WebCrawler(start_url)
    visited_urls = crawler.crawl(max_depth=3)
    for url in visited_urls:
        print(url)

Tags: web-crawler python security-testing url-discovery site-mapping

Posted on Mon, 03 Aug 2026 16:13:07 +0000 by seddonym