import cloudscraper import sqlite3 import os import difflib import sys import subprocess import logging from bs4 import BeautifulSoup from urllib.parse import urljoin, urlparse from datetime import datetime from dotenv import load_dotenv # --- KONFIGURATION & LOGGING --- load_dotenv() DEBUG_LOG = os.getenv("DEBUG_LOG", "false").lower() == "true" LOG_LEVEL = logging.DEBUG if DEBUG_LOG else logging.INFO BASE_DIR = os.path.dirname(__file__) logging.basicConfig( level=LOG_LEVEL, format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.FileHandler(os.path.join(BASE_DIR, "crawler.log"), encoding='utf-8'), logging.StreamHandler(sys.stdout) ] ) logger = logging.getLogger("crawler") sys.stdout.reconfigure(encoding="utf-8") scraper = cloudscraper.create_scraper( browser={"browser": "chrome", "platform": "windows", "desktop": True} ) # --- DATENBANK SETUP --- DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db") def setup_db(): conn = sqlite3.connect(DB_PATH, timeout=20) cursor = conn.cursor() cursor.execute("CREATE TABLE IF NOT EXISTS seiten_stand (url TEXT PRIMARY KEY, inhalt TEXT, zeit TEXT)") cursor.execute("CREATE TABLE IF NOT EXISTS artikel (artikel_url TEXT PRIMARY KEY, status TEXT)") # Tabelle direkt mit allen Spalten erstellen, ohne ALTER TABLE cursor.execute(""" CREATE TABLE IF NOT EXISTS analysis_queue ( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, raw_text TEXT, source_url TEXT, status TEXT DEFAULT 'PENDING', result_json TEXT, worker_id INTEGER, last_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """) conn.commit() conn.close() def queue_article(url, text, source_url): conn = sqlite3.connect(DB_PATH, timeout=20) cursor = conn.cursor() cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (url,)) if cursor.fetchone() is None: logger.debug(f"Queue: {url}") cursor.execute( "INSERT OR IGNORE INTO analysis_queue (url, raw_text, source_url) VALUES (?, ?, ?)", (url, text[:12000], source_url) ) cursor.execute("INSERT OR IGNORE INTO artikel (artikel_url, status) VALUES (?, ?)", (url, "gequeued")) conn.commit() conn.close() def bereinige_url(url_roh, basis_url): clean = url_roh.replace("”", "").replace('"', "").replace("'", "").strip() if clean.lower().startswith("http"): return clean return urljoin(basis_url, clean) def extrahiere_links(soup, basis_url): links = [] ignore_ext = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip", ".docx") basis_domain = urlparse(basis_url).netloc for a in soup.find_all("a", href=True): full_url = bereinige_url(a["href"], basis_url) if full_url.lower().endswith(ignore_ext): continue if urlparse(full_url).netloc == basis_domain: if len(full_url) > len(basis_url.rstrip("/")) + 1: links.append(full_url) return list(set(links)) def hole_artikel_text(url): try: antwort = scraper.get(url, timeout=10) unter_soup = BeautifulSoup(antwort.text, "html.parser") bereich = unter_soup.find("main") or unter_soup.find("article") or unter_soup.body return bereich.get_text(separator=" ", strip=True) except Exception as e: logger.error(f"Fehler bei {url}: {e}") return f"Fehler: {e}" def finde_link_fuer_text(text_snippet, soup, basis_url): schnipsel = text_snippet.strip().lower() if len(schnipsel) < 10: return None for a in soup.find_all("a", href=True): link_text = a.get_text(separator=" ", strip=True).lower() if schnipsel in link_text or link_text in schnipsel: return bereinige_url(a["href"], basis_url) return None def verarbeite_website(url): logger.info(f"Prüfe: {url}") try: conn = sqlite3.connect(DB_PATH, timeout=20) cursor = conn.cursor() antwort = scraper.get(url, timeout=15) if antwort.status_code != 200: return soup = BeautifulSoup(antwort.text, "html.parser") hauptbereich = soup.find("main") or soup.find("article") or soup.body or soup neuer_text = hauptbereich.get_text(separator=" ", strip=True) cursor.execute("SELECT inhalt FROM seiten_stand WHERE url = ?", (url,)) zeile = cursor.fetchone() alter_text = zeile[0] if zeile else "" aktuelle_links = extrahiere_links(hauptbereich, url) for link in aktuelle_links: cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (link,)) if cursor.fetchone() is None: logger.info(f"Neu: {link}") artikel_inhalt = hole_artikel_text(link) queue_article(link, artikel_inhalt, url) if alter_text != neuer_text: diff = [z[2:] for z in difflib.ndiff(alter_text.splitlines(), neuer_text.splitlines()) if z.startswith("+ ") and len(z[2:].strip()) > 30] for line in diff: passender_link = finde_link_fuer_text(line[:40], hauptbereich, url) target_url = passender_link if passender_link else url cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (target_url,)) if cursor.fetchone() is None: logger.info(f"Änderung: {target_url}") text_zu_analysieren = hole_artikel_text(passender_link) if passender_link else line queue_article(target_url, text_zu_analysieren, url) cursor.execute("INSERT OR REPLACE INTO seiten_stand (url, inhalt, zeit) VALUES (?, ?, ?)", (url, neuer_text, str(datetime.now()))) conn.commit() conn.close() except Exception as e: logger.error(f"Fehler bei {url}: {e}") def ensure_workers_running(): try: max_workers = int(os.getenv("MAX_ANALYZER_WORKERS", "5")) output = subprocess.check_output(["ps", "-ef"]).decode() running_analyzers = output.count("analyzer.py") running_notifiers = output.count("notifier.py") base_dir = os.path.dirname(__file__) if running_analyzers < max_workers: needs = max_workers - running_analyzers logger.info(f"Starte {needs} Analyzer (Ziel: {max_workers})...") for _ in range(needs): subprocess.Popen([sys.executable, os.path.join(base_dir, "analyzer.py")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if running_notifiers < 1: logger.info("Starte Notifier...") subprocess.Popen([sys.executable, os.path.join(base_dir, "notifier.py")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception as e: logger.error(f"Worker-Check Fehler: {e}") # --- HAUPTPROGRAMM --- logger.info("Crawler Start.") setup_db() urls_file = os.path.join(BASE_DIR, "urls.txt") with open(urls_file, "r", encoding="utf-8") as datei: for zeile in datei: adresse = zeile.strip() if not adresse or adresse.startswith("#"): continue verarbeite_website(adresse) ensure_workers_running() logger.info("Crawler Ende.")