diff --git a/searcher/id.py b/searcher/id.py index a90e4a0..37dacdd 100644 --- a/searcher/id.py +++ b/searcher/id.py @@ -26,11 +26,14 @@ class ID: logger.exception("An error occurred while attempting to open the id.json file.") return ids - def add(self, id: str) -> bool: - if not id in self._ids: - self._ids.append(id) + def contains(self, id_: str) -> bool: + return id_ in self._ids + + def add(self, id_: str) -> bool: + if not id_ in self._ids: + self._ids.append(id_) with open("id.json", "w") as f: json.dump(self._ids[-MAX_ID:], f, indent=3) self._ids = self._ids[-MAX_ID:] return True - return False \ No newline at end of file + return False diff --git a/searcher/searcher.py b/searcher/searcher.py index 4f3da01..642b7fe 100644 --- a/searcher/searcher.py +++ b/searcher/searcher.py @@ -8,11 +8,34 @@ import threading from typing import List, Union class Searcher: + _HANDLER_MAX_ATTEMPTS: int = 3 + _HANDLER_INITIAL_BACKOFF: float = 2.0 + def __init__(self, searches: Union[List[Search], Search], request_verify: bool = True): self._searches: List[Search] = searches if isinstance(searches, list) else [searches] self._request_verify = request_verify self._id = ID() + def _handle_with_retry(self, search: Search, ad) -> bool: + for attempt in range(1, self._HANDLER_MAX_ATTEMPTS + 1): + try: + search.handler(ad, search.name) + return True + except Exception: + if attempt == self._HANDLER_MAX_ATTEMPTS: + logger.exception( + f"[{search.name}] Handler failed for ad {ad.id} after {attempt} attempts." + ) + return False + + delay = self._HANDLER_INITIAL_BACKOFF * (2 ** (attempt - 1)) + logger.warning( + f"[{search.name}] Handler failed for ad {ad.id}. " + f"Retrying in {delay:.0f}s ({attempt}/{self._HANDLER_MAX_ATTEMPTS})." + ) + time.sleep(delay) + return False + def _search(self, search: Search) -> None: client = Client(proxy=search.proxy, request_verify=self._request_verify) while True: @@ -20,12 +43,20 @@ class Searcher: try: response = client.search(**search.parameters._kwargs, sort=Sort.NEWEST) logger.debug(f"Successfully found {response.total} ad{'s' if response.total > 1 else ''}.") - ads = [ad for ad in response.ads if self._id.add(ad.id)] + ads = [ad for ad in response.ads if not self._id.contains(ad.id)] if len(ads): logger.info(f"Successfully found {len(ads)} new ad{'s' if len(ads) > 1 else ''}!") + + notified = 0 for ad in ads: - search.handler(ad, search.name) - except: + if self._handle_with_retry(search, ad) and self._id.add(ad.id): + notified += 1 + + if len(ads) and notified != len(ads): + logger.warning( + f"[{search.name}] {len(ads) - notified} ad{'s were' if len(ads) - notified > 1 else ' was'} not marked as seen and will be retried." + ) + except Exception: logger.exception(f"An error occured.") time.sleep(search.delay - (time.time() - before) if search.delay - (time.time() - before) > 0 else 0) @@ -37,4 +68,4 @@ class Searcher: for search in self._searches: threading.Thread(target=self._search, args=(search,), name=search.name).start() time.sleep(5) # Add latency between each thread to prevent spam - return True \ No newline at end of file + return True