From 10cfa491afe3fd7513b8ce8ee20edf219ebcfafc Mon Sep 17 00:00:00 2001 From: Ratatoskr Date: Sun, 12 Apr 2026 17:03:45 +0200 Subject: [PATCH] gemini sessions werden beim start bereinigt --- crawler.py | 151 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 116 insertions(+), 35 deletions(-) diff --git a/crawler.py b/crawler.py index b4d3a28..fea8267 100644 --- a/crawler.py +++ b/crawler.py @@ -4,6 +4,8 @@ import os import difflib import sys import subprocess +import shutil +from pathlib import Path import logging import requests from bs4 import BeautifulSoup @@ -19,11 +21,11 @@ BASE_DIR = os.path.dirname(__file__) logging.basicConfig( level=LOG_LEVEL, - format='%(asctime)s [%(levelname)s] %(message)s', + format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ - logging.FileHandler(os.path.join(BASE_DIR, "crawler.log"), encoding='utf-8'), - logging.StreamHandler(sys.stdout) - ] + logging.FileHandler(os.path.join(BASE_DIR, "crawler.log"), encoding="utf-8"), + logging.StreamHandler(sys.stdout), + ], ) logger = logging.getLogger("crawler") sys.stdout.reconfigure(encoding="utf-8") @@ -35,6 +37,7 @@ scraper = cloudscraper.create_scraper( # --- DATENBANK SETUP --- DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db") + def sende_admin(titel, nachricht): admin_url = f"{os.getenv('NTFY_BASE')}/polit-scraper-admin" try: @@ -43,16 +46,22 @@ def sende_admin(titel, nachricht): data=nachricht.encode("utf-8"), headers={"Title": titel.encode("utf-8"), "Priority": "4"}, auth=(os.getenv("ADMIN_USER"), os.getenv("ADMIN_PW")), - timeout=5 + timeout=5, ) - except: pass + except: + pass + def setup_db(): conn = sqlite3.connect(DB_PATH, timeout=20) cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS seiten_stand (url TEXT PRIMARY KEY, inhalt TEXT, zeit TEXT)") - cursor.execute("CREATE TABLE IF NOT EXISTS artikel (artikel_url TEXT PRIMARY KEY, status TEXT)") - + cursor.execute( + "CREATE TABLE IF NOT EXISTS seiten_stand (url TEXT PRIMARY KEY, inhalt TEXT, zeit TEXT)" + ) + cursor.execute( + "CREATE TABLE IF NOT EXISTS artikel (artikel_url TEXT PRIMARY KEY, status TEXT)" + ) + # Tabelle direkt mit allen Spalten erstellen, ohne ALTER TABLE cursor.execute(""" CREATE TABLE IF NOT EXISTS analysis_queue ( @@ -70,16 +79,27 @@ def setup_db(): conn.commit() conn.close() + def ist_sinnvoller_text(text): if len(text.strip()) < 300: return False - nav_keywords = ["startseite", "lokales", "aktuelles", "veranstaltungen", "kontakt", "impressum", "baustelle", "meldungen"] + nav_keywords = [ + "startseite", + "lokales", + "aktuelles", + "veranstaltungen", + "kontakt", + "impressum", + "baustelle", + "meldungen", + ] text_lower = text.lower() match_count = sum(text_lower.count(k) for k in nav_keywords) if match_count > 5: return False return True + def queue_article(url, text, source_url): if not ist_sinnvoller_text(text): logger.debug(f"Überspringe Müll-Text: {url}") @@ -92,52 +112,68 @@ def queue_article(url, text, source_url): logger.debug(f"Queue: {url}") cursor.execute( "INSERT OR IGNORE INTO analysis_queue (url, raw_text, source_url) VALUES (?, ?, ?)", - (url, text[:12000], source_url) + (url, text[:12000], source_url), + ) + cursor.execute( + "INSERT OR IGNORE INTO artikel (artikel_url, status) VALUES (?, ?)", + (url, "gequeued"), ) - cursor.execute("INSERT OR IGNORE INTO artikel (artikel_url, status) VALUES (?, ?)", (url, "gequeued")) conn.commit() conn.close() + def bereinige_url(url_roh, basis_url): clean = url_roh.replace("”", "").replace('"', "").replace("'", "").strip() - if clean.lower().startswith("http"): return clean + if clean.lower().startswith("http"): + return clean return urljoin(basis_url, clean) + def extrahiere_links(soup, basis_url): links = [] ignore_ext = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip", ".docx") basis_domain = urlparse(basis_url).netloc for a in soup.find_all("a", href=True): full_url = bereinige_url(a["href"], basis_url) - if full_url.lower().endswith(ignore_ext): continue + if full_url.lower().endswith(ignore_ext): + continue if urlparse(full_url).netloc == basis_domain: if len(full_url) > len(basis_url.rstrip("/")) + 1: links.append(full_url) return list(set(links)) + def hole_artikel_text(url): try: antwort = scraper.get(url, timeout=10) soup = BeautifulSoup(antwort.text, "html.parser") for element in soup(["header", "footer", "nav", "script", "style", "aside"]): element.decompose() - bereich = soup.find("main") or soup.find("article") or soup.find("div", class_="content") - if not bereich: return None + bereich = ( + soup.find("main") + or soup.find("article") + or soup.find("div", class_="content") + ) + if not bereich: + return None text = bereich.get_text(separator=" ", strip=True) return text[:12000] except Exception as e: logger.error(f"Fehler bei {url}: {e}") return None + def finde_link_fuer_text(text_snippet, soup, basis_url): schnipsel = text_snippet.strip().lower() - if len(schnipsel) < 10: return None + if len(schnipsel) < 10: + return None for a in soup.find_all("a", href=True): link_text = a.get_text(separator=" ", strip=True).lower() if schnipsel in link_text or link_text in schnipsel: return bereinige_url(a["href"], basis_url) return None + def verarbeite_website(url): logger.info(f"Prüfe: {url}") try: @@ -146,23 +182,29 @@ def verarbeite_website(url): antwort = scraper.get(url, timeout=15) if antwort.status_code != 200: logger.error(f"Website Fehler: {url} Status {antwort.status_code}") - sende_admin("Crawler Fehler", f"URL {url} lieferte Status {antwort.status_code}.") + sende_admin( + "Crawler Fehler", f"URL {url} lieferte Status {antwort.status_code}." + ) return - + soup = BeautifulSoup(antwort.text, "html.parser") for element in soup(["header", "footer", "nav", "script", "style", "aside"]): element.decompose() - - hauptbereich = soup.find("main") or soup.find("article") or soup.find("div", class_="content") + + hauptbereich = ( + soup.find("main") + or soup.find("article") + or soup.find("div", class_="content") + ) if not hauptbereich: logger.debug(f"Überspringe Seite ohne Inhalts-Struktur: {url}") return - + neuer_text = hauptbereich.get_text(separator=" ", strip=True) cursor.execute("SELECT inhalt FROM seiten_stand WHERE url = ?", (url,)) zeile = cursor.fetchone() alter_text = zeile[0] if zeile else "" - + aktuelle_links = extrahiere_links(soup, url) for link in aktuelle_links: cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (link,)) @@ -171,54 +213,93 @@ def verarbeite_website(url): artikel_inhalt = hole_artikel_text(link) if artikel_inhalt: queue_article(link, artikel_inhalt, url) - + if alter_text != neuer_text: - diff = [z[2:] for z in difflib.ndiff(alter_text.splitlines(), neuer_text.splitlines()) if z.startswith("+ ") and len(z[2:].strip()) > 30] + diff = [ + z[2:] + for z in difflib.ndiff(alter_text.splitlines(), neuer_text.splitlines()) + if z.startswith("+ ") and len(z[2:].strip()) > 30 + ] for line in diff: passender_link = finde_link_fuer_text(line[:40], soup, url) target_url = passender_link if passender_link else url - cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (target_url,)) + cursor.execute( + "SELECT 1 FROM artikel WHERE artikel_url = ?", (target_url,) + ) if cursor.fetchone() is None: logger.info(f"Änderung: {target_url}") - text_zu_analysieren = hole_artikel_text(passender_link) if passender_link else line + text_zu_analysieren = ( + hole_artikel_text(passender_link) if passender_link else line + ) if text_zu_analysieren: queue_article(target_url, text_zu_analysieren, url) - - cursor.execute("INSERT OR REPLACE INTO seiten_stand (url, inhalt, zeit) VALUES (?, ?, ?)", (url, neuer_text, str(datetime.now()))) + + cursor.execute( + "INSERT OR REPLACE INTO seiten_stand (url, inhalt, zeit) VALUES (?, ?, ?)", + (url, neuer_text, str(datetime.now())), + ) conn.commit() conn.close() except Exception as e: logger.error(f"Fehler bei {url}: {e}") + def ensure_workers_running(): try: max_workers = int(os.getenv("MAX_ANALYZER_WORKERS", "5")) output = subprocess.check_output(["ps", "-ef"]).decode() running_analyzers = output.count("analyzer.py") running_notifiers = output.count("notifier.py") - + base_dir = os.path.dirname(__file__) if running_analyzers < max_workers: needs = max_workers - running_analyzers logger.info(f"Starte {needs} Analyzer (Ziel: {max_workers})...") for _ in range(needs): - subprocess.Popen([sys.executable, os.path.join(base_dir, "analyzer.py")], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.Popen( + [sys.executable, os.path.join(base_dir, "analyzer.py")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) if running_notifiers < 1: logger.info("Starte Notifier...") - subprocess.Popen([sys.executable, os.path.join(base_dir, "notifier.py")], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.Popen( + [sys.executable, os.path.join(base_dir, "notifier.py")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) except Exception as e: logger.error(f"Worker-Check Fehler: {e}") + # --- HAUPTPROGRAMM --- logger.info("Crawler Start.") setup_db() + +# Lösche Gemini CLI Daten auf dem Server +# Die CLI speichert Sitzungen in history und tmp Verzeichnissen +gemini_paths = [ + Path("/root/.gemini/history"), + Path("/root/.gemini/tmp") +] + +for path in gemini_paths: + if path.exists(): + try: + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + logger.info(f"Erfolgreich geleert: {path}") + except Exception as e: + logger.error(f"Fehler beim Leeren von {path}: {e}") + else: + logger.info(f"Pfad existiert nicht: {path}") + urls_file = os.path.join(BASE_DIR, "urls.txt") with open(urls_file, "r", encoding="utf-8") as datei: for zeile in datei: adresse = zeile.strip() - if not adresse or adresse.startswith("#"): continue + if not adresse or adresse.startswith("#"): + continue verarbeite_website(adresse) ensure_workers_running()