chore(changelog): update changelog for v1.1.0

Changed: 'models' folder has been renamed to 'model'

.dockerignore

@@ -0,0 +1,3 @@
+__pycache__
+*.pyc
+data/

CHANGELOG.md

@@ -1,2 +1,20 @@
+## v1.1.0
+
+### Added
+- Docker support 🎉
+
+### Changed
+- Project source code moved to `/lbc-finder`
+- `id.json` and `logs` are now stored in `/data` for persistent storage (useful for Docker)
+
+## v1.0.1
+
+### Added
+- Retry-based error handling for ad handler calls (#3)
+- `contains` method in `ID` class (#3)
+
+### Changed
+- Bumped [lbc](https://github.com/etienne-hd/lbc) to v1.1.2
+
 ## v1.0.0
-* Initial release
+- Initial release

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY lbc-finder /app
+
+CMD ["python", "main.py"]

README.md

@@ -4,7 +4,7 @@
 **Stay notified when new ads appear on Leboncoin**
 
 ```python
-from models import Search, Parameters
+from model import Search, Parameters
 import lbc
 
 def handle(ad: lbc.Ad, search_name: str):
@@ -51,7 +51,6 @@ This project uses [lbc](https://github.com/etienne-hd/lbc), an unofficial librar
 * Easy integration with notifications (Discord, Telegram, email…) via handler
 
 ## Installation
-Docker support will be added soon.
 
 Required **Python 3.9+**
 1. **Clone the repository**
@@ -64,6 +63,53 @@ Required **Python 3.9+**
     pip install -r requirements.txt
     ```
 
+## Docker
+
+You can run **lbc-finder** using Docker without installing Python locally.
+
+### Pull the image
+
+The easiest way is to use the prebuilt image from Docker Hub:
+
+```bash
+docker pull etiennehode/lbc-finder:latest
+```
+
+### Build locally
+
+If you prefer to build the image yourself:
+
+```bash
+docker build -t lbc-finder .
+```
+
+### Run the container
+
+```bash
+docker run -d \
+  --name lbc-finder \
+  -v lbc_data:/app/data \
+  -v $(pwd)/config.py:/app/config.py \
+  etiennehode/lbc-finder:latest
+```
+
+### Volumes
+
+Two volumes are used:
+
+| Volume           | Description                         |
+| ---------------- | ----------------------------------- |
+| `/app/config.py` | Your search configuration file      |
+| `/app/data`      | Persistent storage for detected ads |
+
+Example:
+
+```bash
+-v $(pwd)/config.py:/app/config.py
+```
+
+This mounts your local `config.py` file into the container so you can easily edit your searches without rebuilding the image.
+
 ## Configuration
 A [config.py](config.py) file is provided by default in the project, it contains a basic configuration.
 
@@ -73,7 +119,7 @@ Each `Search` object should be configured with the rules for the ads you want to
 
 For example, if you want to track ads for a **Porsche 944** priced between 0€ and 25,000€ anywhere in France:
 ```python
-from models import Search, Parameters
+from model import Search, Parameters
 
 Search(
     name="Porsche 944",
@@ -120,7 +166,7 @@ You can configure a proxy, here is an example:
 
 ```python
 from lbc import Proxy
-from models import Search
+from model import Search
 
 proxy = Proxy(
     host="127.0.0.1",
@@ -152,4 +198,4 @@ This project is licensed under the MIT License.
 
 <a href="https://www.buymeacoffee.com/etienneh" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" ></a>
 
-You can contact me via [Telegram](https://t.me/etienne_hd) or [Discord](https://discord.com/users/1153975318990827552) if you need help with scraping services or want to write a library.
+You can reach out to me on [Telegram](https://t.me/etienne_hd) or [Discord](https://discord.com/users/1153975318990827552) if you're looking for custom scraping services.

lbc-finder/config.py

@@ -1,4 +1,4 @@
-from models import Search, Parameters
+from model import Search, Parameters
 import lbc
 
 def handle(ad: lbc.Ad, search_name: str):

lbc-finder/model/search.py

@@ -1,7 +1,7 @@
 from lbc import Proxy, Ad
 from .parameters import Parameters
 from dataclasses import dataclass
-from typing import Callable
+from typing import Callable, Optional
 
 @dataclass
 class Search:
@@ -9,4 +9,4 @@ class Search:
     parameters: Parameters
     delay: float
     handler: Callable[[Ad, str], None]
-    proxy: Proxy = None
+    proxy: Optional[Proxy] = None

lbc-finder/searcher/id.py

@@ -16,21 +16,26 @@ class ID:
 
     def _get_ids(self) -> List[str]:
         ids: List[str] = []
-        if os.path.exists("id.json"):
-            with open("id.json", "r") as f:
+        id_path = os.path.join("data", "id.json")
+        if os.path.exists(id_path):
+            with open(id_path, "r") as f:
                 try:
                     ids = json.load(f)
                 except json.JSONDecodeError:
-                    os.remove("id.json")
+                    os.remove(id_path)
                 except:
                     logger.exception("An error occurred while attempting to open the id.json file.")
         return ids
 
-    def add(self, id: str) -> bool:
-        if not id in self._ids:
-            self._ids.append(id)
-            with open("id.json", "w") as f:
+    def contains(self, id_: str) -> bool:
+        return id_ in self._ids
+
+    def add(self, id_: str) -> bool:
+        id_path = os.path.join("data", "id.json")
+        if not id_ in self._ids:
+            self._ids.append(id_)
+            with open(id_path, "w") as f:
                 json.dump(self._ids[-MAX_ID:], f, indent=3)
             self._ids = self._ids[-MAX_ID:]
             return True
-        return False
+        return False

lbc-finder/searcher/logger.py

@@ -4,8 +4,8 @@ from datetime import datetime
 
 # File management
 timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-file_path: str = os.path.join("logs", f"log_{timestamp}.log")
-os.makedirs("logs", exist_ok=True)
+file_path: str = os.path.join("data", "logs", f"log_{timestamp}.log")
+os.makedirs(os.path.join("data", "logs"), exist_ok=True)
 
 # Config logging
 logger = logging.getLogger("lbc-finder")

lbc-finder/searcher/searcher.py

@@ -0,0 +1,71 @@
+from model import Search
+from lbc import Client, Sort
+from .id import ID
+from .logger import logger
+
+import time
+import threading
+from typing import List, Union
+
+class Searcher:
+    def __init__(self, searches: Union[List[Search], Search], request_verify: bool = True,
+            handler_max_attempts: int = 3, handler_initial_backoff: float = 2.0):
+        self._searches: List[Search] = searches if isinstance(searches, list) else [searches]
+        self._request_verify = request_verify
+        self._handler_max_attempts = handler_max_attempts
+        self._handler_initial_backoff = handler_initial_backoff
+        self._id = ID()
+
+    def _handle_with_retry(self, search: Search, ad) -> bool:
+        for attempt in range(1, self._handler_max_attempts + 1):
+            try:
+                search.handler(ad, search.name)
+                return True
+            except Exception:
+                if attempt == self._handler_max_attempts:
+                    logger.exception(
+                        f"[{search.name}] Handler failed for ad {ad.id} after {attempt} attempts."
+                    )
+                    return False
+
+                delay = self._handler_initial_backoff * (2 ** (attempt - 1))
+                logger.warning(
+                    f"[{search.name}] Handler failed for ad {ad.id}. "
+                    f"Retrying in {delay:.0f}s ({attempt}/{self._handler_max_attempts})."
+                )
+                time.sleep(delay)
+        return False
+
+    def _search(self, search: Search) -> None:
+        client = Client(proxy=search.proxy, request_verify=self._request_verify)
+        while True:
+            before = time.time()
+            try:
+                response = client.search(**search.parameters._kwargs, sort=Sort.NEWEST)
+                logger.debug(f"Successfully found {response.total} ad{'s' if response.total > 1 else ''}.")
+                ads = [ad for ad in response.ads if not self._id.contains(ad.id)]
+                if len(ads):
+                    logger.info(f"Successfully found {len(ads)} new ad{'s' if len(ads) > 1 else ''}!")
+
+                notified = 0
+                for ad in ads:
+                    if self._handle_with_retry(search, ad) and self._id.add(ad.id):
+                        notified += 1
+
+                if len(ads) and notified != len(ads):
+                    logger.warning(
+                        f"[{search.name}] {len(ads) - notified} ad{'s were' if len(ads) - notified > 1 else ' was'} not marked as seen and will be retried."
+                    )
+            except Exception:
+                logger.exception(f"An error occured.")
+            time.sleep(search.delay - (time.time() - before) if search.delay - (time.time() - before) > 0 else 0)
+
+    def start(self) -> bool:
+        if not len(self._searches):
+            logger.warning("No search rules have been set. Please create search rules in config.py (see example in README.md).")
+            return False
+
+        for search in self._searches:
+            threading.Thread(target=self._search, args=(search,), name=search.name).start()
+            time.sleep(5) # Add latency between each thread to prevent spam
+        return True

requirements.txt

@@ -1 +1 @@
-lbc==1.0.9
+lbc==1.1.2

searcher/searcher.py

@@ -1,40 +0,0 @@
-from models import Search
-from lbc import Client, Sort
-from .id import ID
-from .logger import logger
-
-import time
-import threading
-from typing import List, Union
-
-class Searcher:
-    def __init__(self, searches: Union[List[Search], Search], request_verify: bool = True):
-        self._searches: List[Search] = searches if isinstance(searches, list) else [searches]
-        self._request_verify = request_verify
-        self._id = ID()
-
-    def _search(self, search: Search) -> None:
-        client = Client(proxy=search.proxy, request_verify=self._request_verify)
-        while True:
-            before = time.time()
-            try:
-                response = client.search(**search.parameters._kwargs, sort=Sort.NEWEST)
-                logger.debug(f"Successfully found {response.total} ad{'s' if response.total > 1 else ''}.")
-                ads = [ad for ad in response.ads if self._id.add(ad.id)]
-                if len(ads):
-                    logger.info(f"Successfully found {len(ads)} new ad{'s' if len(ads) > 1 else ''}!")
-                for ad in ads:
-                    search.handler(ad, search.name)
-            except:
-                logger.exception(f"An error occured.")
-            time.sleep(search.delay - (time.time() - before) if search.delay - (time.time() - before) > 0 else 0)
-
-    def start(self) -> bool:
-        if not len(self._searches):
-            logger.warning("No search rules have been set. Please create search rules in config.py (see example in README.md).")
-            return False
-
-        for search in self._searches:
-            threading.Thread(target=self._search, args=(search,), name=search.name).start()
-            time.sleep(5) # Add latency between each thread to prevent spam
-        return True