Как напишешь system design для веб-краулера

from queue import Queue
import requests
from bs4 import BeautifulSoup

class Crawler:
    def __init__(self):
        self.url_queue = Queue()
        self.visited = set()

    def crawl(self, start_url):
        self.url_queue.put(start_url)
        while not self.url_queue.empty():
            url = self.url_queue.get()
            if url not in self.visited:
                try:
                    response = requests.get(url)
                    soup = BeautifulSoup(response.text, 'lxml')
                    # Process content...
                    for link in soup.find_all('a'):
                        self.url_queue.put(link.get('href'))
                except Exception as e:
                    print(f"Error crawling {url}: {e}")
                self.visited.add(url)