Ratatoskr/polit-scraper
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gemini sessions werden beim start bereinigt

Browse Source
This commit is contained in:
Ratatoskr
2026-04-12 17:03:45 +02:00
parent 35c06e384c
commit 10cfa491af
1 changed files with 116 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
crawler.py
View File

@@ -4,6 +4,8 @@ import os
import difflib import difflib
import sys import sys
import subprocess import subprocess
import shutil
from pathlib import Path
import logging import logging
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -19,11 +21,11 @@ BASE_DIR = os.path.dirname(__file__)
logging.basicConfig( logging.basicConfig(
level=LOG_LEVEL, level=LOG_LEVEL,
format='%(asctime)s [%(levelname)s] %(message)s', format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[
logging.FileHandler(os.path.join(BASE_DIR, "crawler.log"), encoding='utf-8'), logging.FileHandler(os.path.join(BASE_DIR, "crawler.log"), encoding="utf-8"),
logging.StreamHandler(sys.stdout) logging.StreamHandler(sys.stdout),
] ],
) )
logger = logging.getLogger("crawler") logger = logging.getLogger("crawler")
sys.stdout.reconfigure(encoding="utf-8") sys.stdout.reconfigure(encoding="utf-8")
@@ -35,6 +37,7 @@ scraper = cloudscraper.create_scraper(
# --- DATENBANK SETUP --- # --- DATENBANK SETUP ---
DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db") DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db")
def sende_admin(titel, nachricht): def sende_admin(titel, nachricht):
admin_url = f"{os.getenv('NTFY_BASE')}/polit-scraper-admin" admin_url = f"{os.getenv('NTFY_BASE')}/polit-scraper-admin"
try: try:
@@ -43,16 +46,22 @@ def sende_admin(titel, nachricht):
data=nachricht.encode("utf-8"), data=nachricht.encode("utf-8"),
headers={"Title": titel.encode("utf-8"), "Priority": "4"}, headers={"Title": titel.encode("utf-8"), "Priority": "4"},
auth=(os.getenv("ADMIN_USER"), os.getenv("ADMIN_PW")), auth=(os.getenv("ADMIN_USER"), os.getenv("ADMIN_PW")),
timeout=5 timeout=5,
) )
except: pass except:
pass
def setup_db(): def setup_db():
conn = sqlite3.connect(DB_PATH, timeout=20) conn = sqlite3.connect(DB_PATH, timeout=20)
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS seiten_stand (url TEXT PRIMARY KEY, inhalt TEXT, zeit TEXT)") cursor.execute(
cursor.execute("CREATE TABLE IF NOT EXISTS artikel (artikel_url TEXT PRIMARY KEY, status TEXT)") "CREATE TABLE IF NOT EXISTS seiten_stand (url TEXT PRIMARY KEY, inhalt TEXT, zeit TEXT)"
)
cursor.execute(
"CREATE TABLE IF NOT EXISTS artikel (artikel_url TEXT PRIMARY KEY, status TEXT)"
)
# Tabelle direkt mit allen Spalten erstellen, ohne ALTER TABLE # Tabelle direkt mit allen Spalten erstellen, ohne ALTER TABLE
cursor.execute(""" cursor.execute("""
CREATE TABLE IF NOT EXISTS analysis_queue ( CREATE TABLE IF NOT EXISTS analysis_queue (
@@ -70,16 +79,27 @@ def setup_db():
conn.commit() conn.commit()
conn.close() conn.close()
def ist_sinnvoller_text(text): def ist_sinnvoller_text(text):
if len(text.strip()) < 300: if len(text.strip()) < 300:
return False return False
nav_keywords = ["startseite", "lokales", "aktuelles", "veranstaltungen", "kontakt", "impressum", "baustelle", "meldungen"] nav_keywords = [
"startseite",
"lokales",
"aktuelles",
"veranstaltungen",
"kontakt",
"impressum",
"baustelle",
"meldungen",
]
text_lower = text.lower() text_lower = text.lower()
match_count = sum(text_lower.count(k) for k in nav_keywords) match_count = sum(text_lower.count(k) for k in nav_keywords)
if match_count > 5: if match_count > 5:
return False return False
return True return True
def queue_article(url, text, source_url): def queue_article(url, text, source_url):
if not ist_sinnvoller_text(text): if not ist_sinnvoller_text(text):
logger.debug(f"Überspringe Müll-Text: {url}") logger.debug(f"Überspringe Müll-Text: {url}")
@@ -92,52 +112,68 @@ def queue_article(url, text, source_url):
logger.debug(f"Queue: {url}") logger.debug(f"Queue: {url}")
cursor.execute( cursor.execute(
"INSERT OR IGNORE INTO analysis_queue (url, raw_text, source_url) VALUES (?, ?, ?)", "INSERT OR IGNORE INTO analysis_queue (url, raw_text, source_url) VALUES (?, ?, ?)",
(url, text[:12000], source_url) (url, text[:12000], source_url),
)
cursor.execute(
"INSERT OR IGNORE INTO artikel (artikel_url, status) VALUES (?, ?)",
(url, "gequeued"),
) )
cursor.execute("INSERT OR IGNORE INTO artikel (artikel_url, status) VALUES (?, ?)", (url, "gequeued"))
conn.commit() conn.commit()
conn.close() conn.close()
def bereinige_url(url_roh, basis_url): def bereinige_url(url_roh, basis_url):
clean = url_roh.replace("”", "").replace('"', "").replace("'", "").strip() clean = url_roh.replace("”", "").replace('"', "").replace("'", "").strip()
if clean.lower().startswith("http"): return clean if clean.lower().startswith("http"):
return clean
return urljoin(basis_url, clean) return urljoin(basis_url, clean)
def extrahiere_links(soup, basis_url): def extrahiere_links(soup, basis_url):
links = [] links = []
ignore_ext = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip", ".docx") ignore_ext = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip", ".docx")
basis_domain = urlparse(basis_url).netloc basis_domain = urlparse(basis_url).netloc
for a in soup.find_all("a", href=True): for a in soup.find_all("a", href=True):
full_url = bereinige_url(a["href"], basis_url) full_url = bereinige_url(a["href"], basis_url)
if full_url.lower().endswith(ignore_ext): continue if full_url.lower().endswith(ignore_ext):
continue
if urlparse(full_url).netloc == basis_domain: if urlparse(full_url).netloc == basis_domain:
if len(full_url) > len(basis_url.rstrip("/")) + 1: if len(full_url) > len(basis_url.rstrip("/")) + 1:
links.append(full_url) links.append(full_url)
return list(set(links)) return list(set(links))
def hole_artikel_text(url): def hole_artikel_text(url):
try: try:
antwort = scraper.get(url, timeout=10) antwort = scraper.get(url, timeout=10)
soup = BeautifulSoup(antwort.text, "html.parser") soup = BeautifulSoup(antwort.text, "html.parser")
for element in soup(["header", "footer", "nav", "script", "style", "aside"]): for element in soup(["header", "footer", "nav", "script", "style", "aside"]):
element.decompose() element.decompose()
bereich = soup.find("main") or soup.find("article") or soup.find("div", class_="content") bereich = (
if not bereich: return None soup.find("main")
or soup.find("article")
or soup.find("div", class_="content")
)
if not bereich:
return None
text = bereich.get_text(separator=" ", strip=True) text = bereich.get_text(separator=" ", strip=True)
return text[:12000] return text[:12000]
except Exception as e: except Exception as e:
logger.error(f"Fehler bei {url}: {e}") logger.error(f"Fehler bei {url}: {e}")
return None return None
def finde_link_fuer_text(text_snippet, soup, basis_url): def finde_link_fuer_text(text_snippet, soup, basis_url):
schnipsel = text_snippet.strip().lower() schnipsel = text_snippet.strip().lower()
if len(schnipsel) < 10: return None if len(schnipsel) < 10:
return None
for a in soup.find_all("a", href=True): for a in soup.find_all("a", href=True):
link_text = a.get_text(separator=" ", strip=True).lower() link_text = a.get_text(separator=" ", strip=True).lower()
if schnipsel in link_text or link_text in schnipsel: if schnipsel in link_text or link_text in schnipsel:
return bereinige_url(a["href"], basis_url) return bereinige_url(a["href"], basis_url)
return None return None
def verarbeite_website(url): def verarbeite_website(url):
logger.info(f"Prüfe: {url}") logger.info(f"Prüfe: {url}")
try: try:
@@ -146,23 +182,29 @@ def verarbeite_website(url):
antwort = scraper.get(url, timeout=15) antwort = scraper.get(url, timeout=15)
if antwort.status_code != 200: if antwort.status_code != 200:
logger.error(f"Website Fehler: {url} Status {antwort.status_code}") logger.error(f"Website Fehler: {url} Status {antwort.status_code}")
sende_admin("Crawler Fehler", f"URL {url} lieferte Status {antwort.status_code}.") sende_admin(
"Crawler Fehler", f"URL {url} lieferte Status {antwort.status_code}."
)
return return
soup = BeautifulSoup(antwort.text, "html.parser") soup = BeautifulSoup(antwort.text, "html.parser")
for element in soup(["header", "footer", "nav", "script", "style", "aside"]): for element in soup(["header", "footer", "nav", "script", "style", "aside"]):
element.decompose() element.decompose()
hauptbereich = soup.find("main") or soup.find("article") or soup.find("div", class_="content") hauptbereich = (
soup.find("main")
or soup.find("article")
or soup.find("div", class_="content")
)
if not hauptbereich: if not hauptbereich:
logger.debug(f"Überspringe Seite ohne Inhalts-Struktur: {url}") logger.debug(f"Überspringe Seite ohne Inhalts-Struktur: {url}")
return return
neuer_text = hauptbereich.get_text(separator=" ", strip=True) neuer_text = hauptbereich.get_text(separator=" ", strip=True)
cursor.execute("SELECT inhalt FROM seiten_stand WHERE url = ?", (url,)) cursor.execute("SELECT inhalt FROM seiten_stand WHERE url = ?", (url,))
zeile = cursor.fetchone() zeile = cursor.fetchone()
alter_text = zeile[0] if zeile else "" alter_text = zeile[0] if zeile else ""
aktuelle_links = extrahiere_links(soup, url) aktuelle_links = extrahiere_links(soup, url)
for link in aktuelle_links: for link in aktuelle_links:
cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (link,)) cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (link,))
@@ -171,54 +213,93 @@ def verarbeite_website(url):
artikel_inhalt = hole_artikel_text(link) artikel_inhalt = hole_artikel_text(link)
if artikel_inhalt: if artikel_inhalt:
queue_article(link, artikel_inhalt, url) queue_article(link, artikel_inhalt, url)
if alter_text != neuer_text: if alter_text != neuer_text:
diff = [z[2:] for z in difflib.ndiff(alter_text.splitlines(), neuer_text.splitlines()) if z.startswith("+ ") and len(z[2:].strip()) > 30] diff = [
z[2:]
for z in difflib.ndiff(alter_text.splitlines(), neuer_text.splitlines())
if z.startswith("+ ") and len(z[2:].strip()) > 30
]
for line in diff: for line in diff:
passender_link = finde_link_fuer_text(line[:40], soup, url) passender_link = finde_link_fuer_text(line[:40], soup, url)
target_url = passender_link if passender_link else url target_url = passender_link if passender_link else url
cursor.execute("SELECT 1 FROM artikel WHERE artikel_url = ?", (target_url,)) cursor.execute(
"SELECT 1 FROM artikel WHERE artikel_url = ?", (target_url,)
)
if cursor.fetchone() is None: if cursor.fetchone() is None:
logger.info(f"Änderung: {target_url}") logger.info(f"Änderung: {target_url}")
text_zu_analysieren = hole_artikel_text(passender_link) if passender_link else line text_zu_analysieren = (
hole_artikel_text(passender_link) if passender_link else line
)
if text_zu_analysieren: if text_zu_analysieren:
queue_article(target_url, text_zu_analysieren, url) queue_article(target_url, text_zu_analysieren, url)
cursor.execute("INSERT OR REPLACE INTO seiten_stand (url, inhalt, zeit) VALUES (?, ?, ?)", (url, neuer_text, str(datetime.now()))) cursor.execute(
"INSERT OR REPLACE INTO seiten_stand (url, inhalt, zeit) VALUES (?, ?, ?)",
(url, neuer_text, str(datetime.now())),
)
conn.commit() conn.commit()
conn.close() conn.close()
except Exception as e: except Exception as e:
logger.error(f"Fehler bei {url}: {e}") logger.error(f"Fehler bei {url}: {e}")
def ensure_workers_running(): def ensure_workers_running():
try: try:
max_workers = int(os.getenv("MAX_ANALYZER_WORKERS", "5")) max_workers = int(os.getenv("MAX_ANALYZER_WORKERS", "5"))
output = subprocess.check_output(["ps", "-ef"]).decode() output = subprocess.check_output(["ps", "-ef"]).decode()
running_analyzers = output.count("analyzer.py") running_analyzers = output.count("analyzer.py")
running_notifiers = output.count("notifier.py") running_notifiers = output.count("notifier.py")
base_dir = os.path.dirname(__file__) base_dir = os.path.dirname(__file__)
if running_analyzers < max_workers: if running_analyzers < max_workers:
needs = max_workers - running_analyzers needs = max_workers - running_analyzers
logger.info(f"Starte {needs} Analyzer (Ziel: {max_workers})...") logger.info(f"Starte {needs} Analyzer (Ziel: {max_workers})...")
for _ in range(needs): for _ in range(needs):
subprocess.Popen([sys.executable, os.path.join(base_dir, "analyzer.py")], subprocess.Popen(
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) [sys.executable, os.path.join(base_dir, "analyzer.py")],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
if running_notifiers < 1: if running_notifiers < 1:
logger.info("Starte Notifier...") logger.info("Starte Notifier...")
subprocess.Popen([sys.executable, os.path.join(base_dir, "notifier.py")], subprocess.Popen(
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) [sys.executable, os.path.join(base_dir, "notifier.py")],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except Exception as e: except Exception as e:
logger.error(f"Worker-Check Fehler: {e}") logger.error(f"Worker-Check Fehler: {e}")
# --- HAUPTPROGRAMM --- # --- HAUPTPROGRAMM ---
logger.info("Crawler Start.") logger.info("Crawler Start.")
setup_db() setup_db()
# Lösche Gemini CLI Daten auf dem Server
# Die CLI speichert Sitzungen in history und tmp Verzeichnissen
gemini_paths = [
Path("/root/.gemini/history"),
Path("/root/.gemini/tmp")
]
for path in gemini_paths:
if path.exists():
try:
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
logger.info(f"Erfolgreich geleert: {path}")
except Exception as e:
logger.error(f"Fehler beim Leeren von {path}: {e}")
else:
logger.info(f"Pfad existiert nicht: {path}")
urls_file = os.path.join(BASE_DIR, "urls.txt") urls_file = os.path.join(BASE_DIR, "urls.txt")
with open(urls_file, "r", encoding="utf-8") as datei: with open(urls_file, "r", encoding="utf-8") as datei:
for zeile in datei: for zeile in datei:
adresse = zeile.strip() adresse = zeile.strip()
if not adresse or adresse.startswith("#"): continue if not adresse or adresse.startswith("#"):
continue
verarbeite_website(adresse) verarbeite_website(adresse)
ensure_workers_running() ensure_workers_running()