How to find all URLs on a domain — multiple methods

import requests
from urllib.parse import urljoin, urlparse
import re

API_KEY = "YOUR_API_KEY"
API_URL = "https://publisher.scrappey.com/api/v1"

def extract_domain(url):
    """Extract domain from URL"""
    return urlparse(url).netloc

def normalize_url(url, base_url):
    """Convert relative URLs to absolute"""
    if url.startswith('http'):
        return url
    return urljoin(base_url, url)

def find_urls_on_page(page_url, domain):
    """Use Scrappey to find all URLs on a page"""
    payload = {
        "cmd": "request.get",
        "url": page_url,
        "requestType": "request",
        "regex": "(?:href|src)="([^"]+)"|(?:href|src)='([^']+)'",
        "filter": ["regex"]
    }
    
    response = requests.post(f"{API_URL}?key={API_KEY}", json=payload)
    data = response.json()
    
    if data.get('solution', {}).get('regex'):
        urls = data['solution']['regex']
        # Filter to same domain
        same_domain = []
        for url in urls:
            normalized = normalize_url(url, page_url)
            if extract_domain(normalized) == domain:
                same_domain.append(normalized)
        return same_domain
    return []

# Start crawling
start_url = "https://example.com"
domain = extract_domain(start_url)
visited = set()
queue = [start_url]

while queue and len(visited) < 200:
    current_url = queue.pop(0)
    if current_url in visited:
        continue
    
    visited.add(current_url)
    print(f"Crawling: {current_url}")
    
    new_urls = find_urls_on_page(current_url, domain)
    for url in new_urls:
        if url not in visited and url not in queue:
            queue.append(url)
    
    print(f"Found {len(new_urls)} URLs, Total: {len(visited)}")