The following implementation demonstrates a multithreaded web scraper designed to extract sequential content from a target website. It utilizes a producer-consumer pattern where a central context manager handles the state of visited and pending URLs, while a thread pool of workers processes the downloading and parsing logic.

Context Manager

This class is responsible for managing the shared state, ensuring thread safety when accessing the queue of pending URLs and the set of processed links.

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.*;

public class CrawlContext {
    private static final int LIMIT_CHAPTERS = 100;
    private static final String STORAGE_LOCATION = "./downloads/";

    private final BlockingQueue<TaskPayload> pendingQueue = new LinkedBlockingQueue<>();
    private final Set<String> visitedSet = Collections.newSetFromMap(new ConcurrentHashMap<>());
    private final Map<String, String> metadataMap = new ConcurrentHashMap<>();

    public boolean scheduleTask(String url, int depth, String bookTitle) {
        if (visitedSet.contains(url)) {
            return false;
        }
        pendingQueue.add(new TaskPayload(url, depth, bookTitle));
        metadataMap.put(url, bookTitle);
        return true;
    }

    public TaskPayload getNextTask() throws InterruptedException {
        return pendingQueue.take();
    }

    public void markCompleted(String url) {
        visitedSet.add(url);
    }

    public int getQueueSize() {
        return pendingQueue.size();
    }

    public static int getDepthLimit() {
        return LIMIT_CHAPTERS;
    }

    public static String getStoragePath() {
        return STORAGE_LOCATION;
    }

    public static class TaskPayload {
        public final String url;
        public final int depth;
        public final String title;

        public TaskPayload(String url, int depth, String title) {
            this.url = url;
            this.depth = depth;
            this.title = title;
        }
    }
}

Worker and Execution

The worker class implements Runnable to handle the network requests and file I/O. It parses the HTML content to identify the main text and the link to the subsequent page.

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

public class ScraperWorker implements Runnable {
    private final CrawlContext context;

    public ScraperWorker(CrawlContext context) {
        this.context = context;
    }

    @Override
    public void run() {
        try {
            while (true) {
                CrawlContext.TaskPayload task = context.getNextTask();
                
                if (task.depth > CrawlContext.getDepthLimit()) {
                    System.out.println("Depth limit reached for " + task.title);
                    context.markCompleted(task.url);
                    continue;
                }

                processPage(task);
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            System.out.println("Worker thread interrupted.");
        }
    }

    private void processPage(CrawlContext.TaskPayload task) {
        try {
            URL target = new URL(task.url);
            URLConnection connection = target.openConnection();
            
            try (BufferedReader reader = new BufferedReader(
                    new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8))) {
                
                StringBuilder htmlBuffer = new StringBuilder();
                String line;
                while ((line = reader.readLine()) != null) {
                    htmlBuffer.append(line);
                }
                
                String rawHtml = htmlBuffer.toString();
                String nextLink = extractNextLink(rawHtml);
                String contentBody = extractContent(rawHtml);

                if (contentBody != null && !contentBody.isEmpty()) {
                    saveContent(task.title, task.depth, contentBody);
                }

                if (nextLink != null) {
                    context.scheduleTask(nextLink, task.depth + 1, task.title);
                }

                context.markCompleted(task.url);
                System.out.printf("Processed: %s (Chapter %d)%n", task.title, task.depth);
            }
        } catch (Exception e) {
            System.err.println("Error processing URL: " + task.url);
            e.printStackTrace();
        }
    }

    private String extractNextLink(String html) {
        // Simplified parsing logic adapted for the target structure
        String anchorTag = "<a id=\"j_chapterNext\" href=\"";
        int startIdx = html.indexOf(anchorTag);
        if (startIdx == -1) return null;
        
        int urlStart = startIdx + anchorTag.length();
        int urlEnd = html.indexOf("\"", urlStart);
        
        if (urlEnd == -1) return null;
        return "https:" + html.substring(urlStart, urlEnd);
    }

    private String extractContent(String html) {
        String contentStartTag = "class=\"read-content j_readContent\"";
        int startIdx = html.indexOf(contentStartTag);
        if (startIdx == -1) return "";
        
        int bodyStart = html.indexOf("<p>", startIdx);
        int endIdx = html.indexOf("<div class=\"admire-wrap\">", bodyStart);
        
        if (endIdx == -1) return "";
        return html.substring(bodyStart, endIdx).replaceAll("<[^>]*>", ""); // Strip tags for clean text
    }

    private void saveContent(String title, int chapter, String data) {
        String fileName = CrawlContext.getStoragePath() + title + "_" + chapter + ".txt";
        try (PrintWriter writer = new PrintWriter(new File(fileName))) {
            writer.println(data);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        CrawlContext manager = new CrawlContext();
        ExecutorService executor = Executors.newFixedThreadPool(4);

        // Seed URLs
        manager.scheduleTask("https://read.qidian.com/chapter/Example1/", 1, "Book A");
        manager.scheduleTask("https://read.qidian.com/chapter/Example2/", 1, "Book B");

        for (int i = 0; i < 4; i++) {
            executor.execute(new ScraperWorker(manager));
        }

        executor.shutdown();
        try {
            executor.awaitTermination(1, TimeUnit.HOURS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}