Ratatoskr/polit-scraper
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fehlende bibliothek hinzugefügt, bei der ich mir fast sicher bin, das ich sie drin hatte

Browse Source
This commit is contained in:
Ratatoskr
2026-04-11 10:08:42 +02:00
parent c127908584
commit 35c06e384c
1 changed files with 69 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
notifier.py
View File

@@ -6,6 +6,7 @@ import time
import requests import requests
import logging import logging
import sys import sys
import re
from dotenv import load_dotenv from dotenv import load_dotenv
# --- KONFIGURATION & LOGGING --- # --- KONFIGURATION & LOGGING ---
@@ -27,14 +28,15 @@ DB_PATH = os.path.join(BASE_DIR, "polit_scraper.db")
logging.basicConfig( logging.basicConfig(
level=logging.DEBUG if DEBUG_LOG else logging.INFO, level=logging.DEBUG if DEBUG_LOG else logging.INFO,
format='%(asctime)s [%(levelname)s] %(message)s', format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[
logging.FileHandler(os.path.join(BASE_DIR, "notifier.log"), encoding='utf-8'), logging.FileHandler(os.path.join(BASE_DIR, "notifier.log"), encoding="utf-8"),
logging.StreamHandler(sys.stdout) logging.StreamHandler(sys.stdout),
] ],
) )
logger = logging.getLogger("notifier") logger = logging.getLogger("notifier")
def ensure_tables(): def ensure_tables():
conn = sqlite3.connect(DB_PATH, timeout=60) conn = sqlite3.connect(DB_PATH, timeout=60)
cursor = conn.cursor() cursor = conn.cursor()
@@ -54,27 +56,50 @@ def ensure_tables():
conn.commit() conn.commit()
conn.close() conn.close()
def sende_signal(titel, nachricht, klick_url): def sende_signal(titel, nachricht, klick_url):
if not SIGNAL_ENABLED: return if not SIGNAL_ENABLED:
return
message_content = f"{titel}\n{nachricht}\n{klick_url}" message_content = f"{titel}\n{nachricht}\n{klick_url}"
cmd = [ cmd = [
"signal-cli", "--config", "/root/.local/share/signal-cli", "/opt/signal-cli-0.13.21/bin/signal-cli",
"-u", SIGNAL_NUMBER, "send", "-m", message_content, "-g", SIGNAL_GROUPE_ID, "--config",
"/root/.local/share/signal-cli",
"-u",
SIGNAL_NUMBER,
"send",
"-m",
message_content,
"-g",
SIGNAL_GROUPE_ID,
] ]
try: try:
subprocess.run(cmd, capture_output=True, text=True, timeout=30) subprocess.run(cmd, capture_output=True, text=True, timeout=30, env=os.environ)
except Exception as e: except Exception as e:
logger.error(f"Signal-Fehler: {e}") logger.error(f"Signal-Fehler: {e}")
def sende_ntfy(titel, nachricht, klick_url, priority=3): def sende_ntfy(titel, nachricht, klick_url, priority=3):
if not NTFY_ENABLED or not NTFY_BASE: return if not NTFY_ENABLED or not NTFY_BASE:
return
url = f"{NTFY_BASE}/{TOPIC_PUBLIC}" url = f"{NTFY_BASE}/{TOPIC_PUBLIC}"
headers = {"Title": titel.encode("utf-8"), "Click": klick_url, "Priority": str(priority)} headers = {
"Title": titel.encode("utf-8"),
"Click": klick_url,
"Priority": str(priority),
}
try: try:
requests.post(url, data=nachricht.encode("utf-8"), headers=headers, auth=(ADMIN_USER, ADMIN_PW), timeout=10) requests.post(
url,
data=nachricht.encode("utf-8"),
headers=headers,
auth=(ADMIN_USER, ADMIN_PW),
timeout=10,
)
except Exception as e: except Exception as e:
logger.error(f"NTFY-Fehler: {e}") logger.error(f"NTFY-Fehler: {e}")
def main(): def main():
logger.info("Notifier startet.") logger.info("Notifier startet.")
ensure_tables() ensure_tables()
@@ -84,25 +109,44 @@ def main():
conn.execute("PRAGMA journal_mode=WAL") # Parallele Zugriffe erlauben conn.execute("PRAGMA journal_mode=WAL") # Parallele Zugriffe erlauben
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute("SELECT id, url, result_json FROM analysis_queue WHERE status = 'ANALYZED' ORDER BY id ASC LIMIT 1") cursor.execute(
"SELECT id, url, result_json FROM analysis_queue WHERE status = 'ANALYZED' ORDER BY id ASC LIMIT 1"
)
job = cursor.fetchone() job = cursor.fetchone()
if job: if job:
job_id, url, result_json = job job_id, url, result_json = job
try: try:
analyse = json.loads(result_json) # Entferne ungültige Steuerzeichen und versuche das JSON zu parsen
clean_json = re.sub(r"[\x00-\x1f\x7f]", "", result_json)
analyse = json.loads(clean_json)
score = analyse.get("relevanz_score", 0) score = analyse.get("relevanz_score", 0)
if score >= MIN_RELEVANZ_SCORE: if score >= MIN_RELEVANZ_SCORE:
sende_ntfy(analyse.get("titel", "Polit-Update"), analyse.get("nachricht", ""), url, analyse.get("priority", 3)) sende_ntfy(
sende_signal(analyse.get("titel", "Polit-Update"), analyse.get("nachricht", ""), url) analyse.get("titel", "Polit-Update"),
analyse.get("nachricht", ""),
url,
analyse.get("priority", 3),
)
sende_signal(
analyse.get("titel", "Polit-Update"),
analyse.get("nachricht", ""),
url,
)
cursor.execute("UPDATE analysis_queue SET status = 'COMPLETED' WHERE id = ?", (job_id,)) cursor.execute(
"UPDATE analysis_queue SET status = 'COMPLETED' WHERE id = ?",
(job_id,),
)
conn.commit() conn.commit()
logger.info(f"Job {job_id} verarbeitet.") logger.info(f"Job {job_id} verarbeitet.")
except Exception as e: except Exception as e:
logger.error(f"Fehler Job {job_id}: {e}") logger.error(f"Fehler Job {job_id}: {e}")
cursor.execute("UPDATE analysis_queue SET status = 'FAILED' WHERE id = ?", (job_id,)) cursor.execute(
"UPDATE analysis_queue SET status = 'FAILED' WHERE id = ?",
(job_id,),
)
conn.commit() conn.commit()
conn.close() conn.close()
@@ -113,4 +157,6 @@ def main():
logger.error(f"Notifier Loop: {e}") logger.error(f"Notifier Loop: {e}")
time.sleep(10) time.sleep(10)
if __name__ == "__main__": main()
if __name__ == "__main__":
main()