diff --git a/crawler.py b/crawler.py index ba8d51e..2e9123b 100644 --- a/crawler.py +++ b/crawler.py @@ -143,8 +143,28 @@ def verarbeite_website(url): try: conn = sqlite3.connect(DB_PATH, timeout=20) cursor = conn.cursor() +import requests +from urllib.parse import urljoin, urlparse + +# ... +def sende_admin(titel, nachricht): + admin_url = f"{os.getenv('NTFY_BASE')}/polit-scraper-admin" + try: + requests.post( + admin_url, + data=nachricht.encode("utf-8"), + headers={"Title": titel.encode("utf-8"), "Priority": "4"}, + auth=(os.getenv("ADMIN_USER"), os.getenv("ADMIN_PW")), + timeout=5 + ) + except: pass + +# ... in verarbeite_website ... antwort = scraper.get(url, timeout=15) - if antwort.status_code != 200: return + if antwort.status_code != 200: + logger.error(f"Website Fehler: {url} Status {antwort.status_code}") + sende_admin("Crawler Fehler", f"URL {url} lieferte Status {antwort.status_code}.") + return soup = BeautifulSoup(antwort.text, "html.parser") # Müll entfernen