die alte web_check-py ist nun modular in analyzer.py crawler.py und notifier.py aufgeteilt. monitor.py bietet status über analyzer

2026-04-04 14:32:24 +02:00
parent ee625f40d5
commit 86fc7817cc
4 changed files with 499 additions and 0 deletions
--- a/monitor.py
+++ b/monitor.py
@@ -0,0 +1,70 @@
+import sqlite3
+import os
+import time
+from datetime import datetime
+
+# Wir suchen die DB im selben Ordner wie das Skript
+DB_PATH = os.path.join(os.path.dirname(__file__), "polit_scraper.db")
+
+def get_stats():
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        
+        # Statistiken
+        cursor.execute("SELECT status, count(*) FROM analysis_queue GROUP BY status")
+        stats = dict(cursor.fetchall())
+        
+        # Laufende Jobs (PROCESSING)
+        cursor.execute("SELECT id, url, worker_id, last_update FROM analysis_queue WHERE status='PROCESSING' ORDER BY last_update DESC")
+        processing = cursor.fetchall()
+        
+        # Letzte 10 Einträge (allgemein)
+        cursor.execute("SELECT id, status, url, created_at FROM analysis_queue ORDER BY id DESC LIMIT 10")
+        recent = cursor.fetchall()
+        
+        conn.close()
+        return stats, processing, recent
+    except Exception as e:
+        return {}, [], []
+
+def main():
+    while True:
+        try:
+            stats, processing, recent = get_stats()
+            os.system('clear')
+            print(f"--- Polit-Scraper Live Monitor --- {datetime.now().strftime('%H:%M:%S')}")
+            print("-" * 80)
+            
+            # Stats Anzeige
+            all_statuses = ['PENDING', 'PROCESSING', 'ANALYZED', 'COMPLETED', 'FAILED']
+            stat_line = " | ".join([f"{s}: {stats.get(s, 0)}" for s in all_statuses])
+            print(stat_line)
+            print("-" * 80)
+            
+            # PROCESSING Jobs (aktiv)
+            print(f"{'AKTIV (PROCESSING)':<80}")
+            print(f"{'ID':<4} | {'Worker':<8} | {'URL (Beginn)'}")
+            print("-" * 80)
+            for r_id, url, worker_id, ts in processing:
+                url_display = (url[:60] + '..') if len(url) > 60 else url
+                print(f"{r_id:<4} | {str(worker_id):<8} | {url_display}")
+            print("-" * 80)
+            
+            # LETZTE 10 Jobs
+            print(f"{'LETZTE 10 JOB-STATUS':<80}")
+            print(f"{'ID':<4} | {'Status':<11} | {'URL (Ende)'}")
+            print("-" * 80)
+            for r_id, status, url, ts in recent:
+                url_display = (url[:60] + '..') if len(url) > 60 else url
+                print(f"{r_id:<4} | {status:<11} | {url_display}")
+            
+            time.sleep(2)
+        except KeyboardInterrupt:
+            break
+        except Exception as e:
+            print(f"Fehler: {e}")
+            time.sleep(5)
+
+if __name__ == "__main__":
+    main()