mirror of
https://github.com/etienne-hd/lbc-finder.git
synced 2026-04-29 12:25:35 +02:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6dc50b436 | ||
|
|
d48349b271 | ||
|
|
31682fabbf | ||
|
|
b2d5d400da | ||
|
|
e60ad25035 | ||
|
|
f2a63ec7ff | ||
|
|
d4e76c75af | ||
|
|
af4264270b | ||
|
|
c1b4437e8d | ||
|
|
37ec53c56a | ||
|
|
4806146200 | ||
|
|
8f3f39c0fc | ||
|
|
b4946264ff | ||
|
|
fee4101455 | ||
|
|
6764ebf631 |
3
.dockerignore
Normal file
3
.dockerignore
Normal file
@@ -0,0 +1,3 @@
|
||||
__pycache__
|
||||
*.pyc
|
||||
data/
|
||||
31
CHANGELOG.md
31
CHANGELOG.md
@@ -1,2 +1,31 @@
|
||||
## v1.1.1
|
||||
|
||||
### Added
|
||||
- Automatic installation of Python libraries from `config/requirements.txt` when running the Docker container.
|
||||
|
||||
### Changed
|
||||
- Moved `config.py` to `config/__init__.py` to allow multiple configuration files in Docker.
|
||||
|
||||
### Updated
|
||||
- Discord example updated to reflect the new configuration structure.
|
||||
|
||||
## v1.1.0
|
||||
|
||||
### Added
|
||||
- Docker support 🎉
|
||||
|
||||
### Changed
|
||||
- Project source code moved to `/lbc-finder`
|
||||
- `id.json` and `logs` are now stored in `/data` for persistent storage (useful for Docker)
|
||||
|
||||
## v1.0.1
|
||||
|
||||
### Added
|
||||
- Retry-based error handling for ad handler calls (#3)
|
||||
- `contains` method in `ID` class (#3)
|
||||
|
||||
### Changed
|
||||
- Bumped [lbc](https://github.com/etienne-hd/lbc) to v1.1.2
|
||||
|
||||
## v1.0.0
|
||||
* Initial release
|
||||
- Initial release
|
||||
14
Dockerfile
Normal file
14
Dockerfile
Normal file
@@ -0,0 +1,14 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt && rm requirements.txt
|
||||
|
||||
COPY lbc-finder /app
|
||||
|
||||
COPY docker-entrypoint.sh /
|
||||
RUN chmod +x /docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["python", "main.py"]
|
||||
67
README.md
67
README.md
@@ -51,7 +51,6 @@ This project uses [lbc](https://github.com/etienne-hd/lbc), an unofficial librar
|
||||
* Easy integration with notifications (Discord, Telegram, email…) via handler
|
||||
|
||||
## Installation
|
||||
Docker support will be added soon.
|
||||
|
||||
Required **Python 3.9+**
|
||||
1. **Clone the repository**
|
||||
@@ -64,8 +63,70 @@ Required **Python 3.9+**
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Docker
|
||||
|
||||
You can run **lbc-finder** using Docker without installing Python locally.
|
||||
|
||||
### Pull the image
|
||||
|
||||
The easiest way is to use the prebuilt image from Docker Hub:
|
||||
|
||||
```bash
|
||||
docker pull etiennehode/lbc-finder:latest
|
||||
````
|
||||
|
||||
### Build locally
|
||||
|
||||
If you prefer to build the image yourself:
|
||||
|
||||
```bash
|
||||
docker build -t lbc-finder .
|
||||
```
|
||||
|
||||
### Run the container
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name lbc-finder \
|
||||
-v lbc_data:/app/data \
|
||||
-v $(pwd)/config:/app/config \
|
||||
etiennehode/lbc-finder:latest
|
||||
```
|
||||
|
||||
### Volumes
|
||||
|
||||
| Path in container | Description |
|
||||
| ----------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| `/app/config` | Your search configuration files **and optional `requirements.txt`** for additional Python libraries |
|
||||
| `/app/data` | Persistent storage for detected ads |
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
-v $(pwd)/config:/app/config
|
||||
```
|
||||
|
||||
This mounts your local `config/` folder into the container so you can easily edit your searches.
|
||||
|
||||
> **Extra Python libraries:**
|
||||
> If your configuration requires additional Python packages, create a `requirements.txt` file inside your `config/` folder.
|
||||
> The container startup script will automatically install all listed packages before running **lbc-finder**:
|
||||
|
||||
```sh
|
||||
#!/bin/sh
|
||||
if [ -e config/requirements.txt ]
|
||||
then
|
||||
pip install --no-cache-dir -r config/requirements.txt
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
```
|
||||
|
||||
This way, you can extend the container with any Python library you need without modifying the Docker image itself.
|
||||
|
||||
|
||||
## Configuration
|
||||
A [config.py](config.py) file is provided by default in the project, it contains a basic configuration.
|
||||
A [config](lbc-finder/config/__init__.py) file is provided by default in the project, it contains a basic configuration.
|
||||
|
||||
Inside this file, you must define a `CONFIG` variable, which is an list of `Search` objects.
|
||||
|
||||
@@ -152,4 +213,4 @@ This project is licensed under the MIT License.
|
||||
|
||||
<a href="https://www.buymeacoffee.com/etienneh" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" ></a>
|
||||
|
||||
You can contact me via [Telegram](https://t.me/etienne_hd) or [Discord](https://discord.com/users/1153975318990827552) if you need help with scraping services or want to write a library.
|
||||
You can reach out to me on [Telegram](https://t.me/etienne_hd) or [Discord](https://discord.com/users/1153975318990827552) if you're looking for custom scraping services.
|
||||
7
docker-entrypoint.sh
Normal file
7
docker-entrypoint.sh
Normal file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
if [ -e config/requirements.txt ]
|
||||
then
|
||||
pip install --no-cache-dir -r config/requirements.txt
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
@@ -1,12 +1,7 @@
|
||||
from model import Search, Parameters
|
||||
import lbc
|
||||
|
||||
def handle(ad: lbc.Ad, search_name: str):
|
||||
print(f"[{search_name}] New ads!")
|
||||
print(f"Title : {ad.subject}")
|
||||
print(f"Price : {ad.price} €")
|
||||
print(f"URL : {ad.url}")
|
||||
print("-" * 40)
|
||||
from .handler import handle
|
||||
|
||||
location = lbc.City(
|
||||
lat=48.85994982004764,
|
||||
@@ -13,7 +13,7 @@ def handle(ad: lbc.Ad, search_name: str) -> None:
|
||||
"embeds": [
|
||||
{
|
||||
"title": ad.title,
|
||||
"description": f"```{ad.body}```",
|
||||
"description": f"```{ad.body[:4087]}...```" if len(ad.body) >= 4090 else f"```{ad.body[:4090]}```",
|
||||
"url": ad.url,
|
||||
"color": 14381568,
|
||||
"author": {
|
||||
@@ -46,4 +46,5 @@ def handle(ad: lbc.Ad, search_name: str) -> None:
|
||||
"attachments": []
|
||||
}
|
||||
|
||||
requests.post(WEBHOOK_URL, json=payload)
|
||||
response = requests.post(WEBHOOK_URL, json=payload)
|
||||
response.raise_for_status()
|
||||
1
examples/discord/config/requirements.txt
Normal file
1
examples/discord/config/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
requests==2.32.3
|
||||
26
lbc-finder/config/__init__.py
Normal file
26
lbc-finder/config/__init__.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from model import Search, Parameters
|
||||
import lbc
|
||||
|
||||
from .handler import handle
|
||||
|
||||
location = lbc.City(
|
||||
lat=48.85994982004764,
|
||||
lng=2.33801967847424,
|
||||
radius=10_000, # 10 km
|
||||
city="Paris"
|
||||
)
|
||||
|
||||
CONFIG = [
|
||||
Search(
|
||||
name="Location Paris",
|
||||
parameters=Parameters(
|
||||
text="maison",
|
||||
locations=[location],
|
||||
category=lbc.Category.IMMOBILIER,
|
||||
square=[200, 400],
|
||||
price=[300_000, 700_000]
|
||||
),
|
||||
delay=60 * 5, # Check every 5 minutes
|
||||
handler=handle
|
||||
),
|
||||
]
|
||||
9
lbc-finder/config/handler.py
Normal file
9
lbc-finder/config/handler.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import lbc
|
||||
|
||||
def handle(ad: lbc.Ad, search_name: str):
|
||||
print(f"[{search_name}] New ads!")
|
||||
print(f"Title : {ad.subject}")
|
||||
print(f"Price : {ad.price} €")
|
||||
print(f"URL : {ad.url}")
|
||||
print("-" * 40)
|
||||
|
||||
@@ -16,12 +16,13 @@ class ID:
|
||||
|
||||
def _get_ids(self) -> List[str]:
|
||||
ids: List[str] = []
|
||||
if os.path.exists("id.json"):
|
||||
with open("id.json", "r") as f:
|
||||
id_path = os.path.join("data", "id.json")
|
||||
if os.path.exists(id_path):
|
||||
with open(id_path, "r") as f:
|
||||
try:
|
||||
ids = json.load(f)
|
||||
except json.JSONDecodeError:
|
||||
os.remove("id.json")
|
||||
os.remove(id_path)
|
||||
except:
|
||||
logger.exception("An error occurred while attempting to open the id.json file.")
|
||||
return ids
|
||||
@@ -30,9 +31,10 @@ class ID:
|
||||
return id_ in self._ids
|
||||
|
||||
def add(self, id_: str) -> bool:
|
||||
id_path = os.path.join("data", "id.json")
|
||||
if not id_ in self._ids:
|
||||
self._ids.append(id_)
|
||||
with open("id.json", "w") as f:
|
||||
with open(id_path, "w") as f:
|
||||
json.dump(self._ids[-MAX_ID:], f, indent=3)
|
||||
self._ids = self._ids[-MAX_ID:]
|
||||
return True
|
||||
@@ -4,8 +4,8 @@ from datetime import datetime
|
||||
|
||||
# File management
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
file_path: str = os.path.join("logs", f"log_{timestamp}.log")
|
||||
os.makedirs("logs", exist_ok=True)
|
||||
file_path: str = os.path.join("data", "logs", f"log_{timestamp}.log")
|
||||
os.makedirs(os.path.join("data", "logs"), exist_ok=True)
|
||||
|
||||
# Config logging
|
||||
logger = logging.getLogger("lbc-finder")
|
||||
Reference in New Issue
Block a user