import re
from urllib.parse import urlparse
def extract_protocol(url):
'''Extract the protocol scheme (e.g., http, https) from a URL.'''
parsed = urlparse(url)
return parsed.scheme
def normalize_domain(url, protocol):
'''Normalize a URL to its domain name for same-site comparison.'''
domain_part = url.replace(f'{protocol}://', '').replace('://', '')
if domain_part.startswith('www.'):
domain_part = domain_part[4:]
domain = domain_part.split('/')[0]
return domain
class UrlQueue:
def __init__(self):
self.visited = []
self.unvisited = []
def get_visited(self):
return self.visited
def get_unvisited(self):
return self.unvisited
def add_visited(self, url):
self.visited.append(url)
def add_unvisited(self, url):
if url and url not in self.visited and url not in self.unvisited:
self.unvisited.insert(0, url)
def remove_visited(self, url):
if url in self.visited:
self.visited.remove(url)
def pop_unvisited(self):
try:
return self.unvisited.pop()
except IndexError:
return None
def is_unvisited_empty(self):
return len(self.unvisited) == 0
class WebCrawler:
def __init__(self, start_url):
self.queue = UrlQueue()
self.queue.add_unvisited(start_url)
self.current_depth = 1
self.protocol = extract_protocol(start_url)
self.domain = normalize_domain(start_url, self.protocol)
def extract_links(self, page_url):
'''Extract all href attributes from anchor tags on a page.'''
return []
def resolve_links(self, base_url, links):
'''Convert relative URLs to absolute URLs.'''
resolved = []
for link in links:
if link.startswith('//'):
resolved.append(f'{self.protocol}:{link}')
elif link.startswith('http://') or link.startswith('https://'):
resolved.append(link)
return resolved
def filter_same_domain(self, urls):
'''Filter URLs to those within the same domain.'''
same_domain = []
for url in urls:
if self.domain in url:
same_domain.append(url)
return same_domain
def filter_exclusions(self, urls):
'''Exclude URLs from specified domains.'''
excluded_domains = [
's.wuage.com', 'static.wuage.com',
'shop.wuage.com', 'img.wuage.com',
'medici.wuage.com', 'buyer.wuage.com',
'item.wuage.com'
]
filtered = []
for url in urls:
parsed = urlparse(url)
domain = parsed.netloc
if domain not in excluded_domains and url not in filtered:
filtered.append(url)
return filtered
def crawl(self, max_depth=1):
'''Crawl the web starting from the initial URL up to max_depth.'''
while self.current_depth <= max_depth:
current_url = self.queue.pop_unvisited()
if not current_url:
continue
links = self.extract_links(current_url)
resolved_links = self.resolve_links(current_url, links)
same_domain_links = self.filter_same_domain(resolved_links)
filtered_links = self.filter_exclusions(same_domain_links)
self.queue.add_visited(current_url)
for link in filtered_links:
self.queue.add_unvisited(link)
self.current_depth += 1
return self.queue.get_visited()
if __name__ == '__main__':
start_url = 'http://example.com'
crawler = WebCrawler(start_url)
visited_urls = crawler.crawl(max_depth=3)
for url in visited_urls:
print(url)
Automated Web Crawler for URL Discovery
Posted on Mon, 03 Aug 2026 16:13:07 +0000 by seddonym