die alte web_check-py ist nun modular in analyzer.py crawler.py und notifier.py aufgeteilt. monitor.py bietet status über analyzer

analyzer.py

@@ -0,0 +1,132 @@
+import sqlite3
+import json
+import os
+import subprocess
+import time
+import sys
+import logging
+import re
+from datetime import datetime
+from dotenv import load_dotenv
+
+# --- KONFIGURATION & LOGGING ---
+load_dotenv()
+BASE_DIR = os.path.dirname(__file__)
+DEBUG_LOG = os.getenv("DEBUG_LOG", "false").lower() == "true"
+LOG_LEVEL = logging.DEBUG if DEBUG_LOG else logging.INFO
+MY_PID = os.getpid()
+
+logging.basicConfig(
+    level=LOG_LEVEL,
+    format='%(asctime)s [%(levelname)s] [%(process)d] %(message)s',
+    handlers=[
+        logging.FileHandler(os.path.join(BASE_DIR, "analyzer.log"), encoding='utf-8'),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(f"analyzer_{MY_PID}")
+
+GEMINI_MODEL = "gemini-2.5-flash-lite"
+DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db")
+INSTRUCTION_PATH = os.path.join(BASE_DIR, "gemini_instructions.md")
+
+def extract_json(text):
+    start = text.find("{")
+    ende = text.rfind("}") + 1
+    if start != -1 and ende > start:
+        json_str = text[start:ende]
+        json_str = re.sub(r'```json\s*', '', json_str)
+        json_str = re.sub(r'\s*```', '', json_str)
+        return json_str.strip()
+    return None
+
+def analysiere_mit_gemini(text):
+    sicherer_text = text[:8000]
+    anweisung = f"Analysiere diesen Text.\n\nTEXT:\n{sicherer_text}"
+    
+    env = os.environ.copy()
+    env["GEMINI_SYSTEM_MD"] = INSTRUCTION_PATH
+    env["NODE_ENV"] = "production"
+    env["CI"] = "true"
+    env["FORCE_COLOR"] = "0"
+
+    befehl = ["gemini", "-m", GEMINI_MODEL, "-p", anweisung]
+    
+    try:
+        prozess = subprocess.run(
+            befehl, 
+            capture_output=True, 
+            text=True, 
+            encoding="utf-8", 
+            timeout=300, 
+            env=env,
+            stdin=subprocess.DEVNULL
+        )
+        stdout = prozess.stdout.strip()
+        stderr = prozess.stderr.strip()
+
+        if DEBUG_LOG:
+            logger.debug(f"DEBUG_STDOUT: {stdout}")
+            if stderr: logger.debug(f"DEBUG_STDERR: {stderr}")
+
+        if "Error: setRawMode EIO" in stderr:
+            logger.error(f"TTY_ERROR detected in stderr: {stderr}")
+            return "TTY_ERROR"
+
+        if "exhausted your capacity" in stderr or "exhausted your capacity" in stdout:
+            return "RATE_LIMIT"
+
+        json_str = extract_json(stdout)
+        if json_str:
+            return json_str
+        
+        logger.error(f"PARSING_FAILED. Roh-Output war: {stdout}")
+        return None
+    except Exception as e:
+        logger.error(f"SUBPROCESS_CRASH: {e}")
+        return None
+
+def main():
+    logger.info(f"Worker {MY_PID} bereit.")
+    while True:
+        try:
+            conn = sqlite3.connect(DB_PATH, timeout=60)
+            cursor = conn.cursor()
+            
+            cursor.execute("""
+                UPDATE analysis_queue 
+                SET status='PROCESSING', worker_id=?, last_update=CURRENT_TIMESTAMP 
+                WHERE id=(SELECT id FROM analysis_queue WHERE status='PENDING' ORDER BY id ASC LIMIT 1)
+            """, (MY_PID,))
+            conn.commit()
+            
+            cursor.execute("SELECT id, url, raw_text FROM analysis_queue WHERE status='PROCESSING' AND worker_id=?", (MY_PID,))
+            job = cursor.fetchone()
+            
+            if job:
+                job_id, url, text = job
+                logger.info(f"Analysiere Job {job_id}: {url}")
+                ergebnis = analysiere_mit_gemini(text)
+                
+                if ergebnis == "RATE_LIMIT" or ergebnis == "TTY_ERROR":
+                    logger.warning(f"Job {job_id} ({url}) returned {ergebnis}. Re-queueing and waiting.")
+                    cursor.execute("UPDATE analysis_queue SET status='PENDING', worker_id=NULL WHERE id=?", (job_id,))
+                    conn.commit()
+                    time.sleep(60)
+                elif ergebnis:
+                    cursor.execute("UPDATE analysis_queue SET status='ANALYZED', result_json=?, worker_id=NULL WHERE id=?", (ergebnis, job_id))
+                    logger.info(f"Job {job_id} ANALYZED.")
+                    conn.commit()
+                else:
+                    cursor.execute("UPDATE analysis_queue SET status='FAILED', worker_id=NULL WHERE id=?", (job_id,))
+                    logger.error(f"Job {job_id} FAILED (Parser returned None).")
+                    conn.commit()
+            conn.close()
+            time.sleep(1)
+        except sqlite3.OperationalError:
+            time.sleep(5)
+        except Exception as e:
+            logger.error(f"Main Loop Fehler: {e}")
+            time.sleep(10)
+
+if __name__ == "__main__": main()