mirror of
https://github.com/etienne-hd/lbc-finder.git
synced 2026-07-11 15:26:18 +02:00
style: format lbc-finder with ruff
This commit is contained in:
@@ -3,11 +3,11 @@ import lbc
|
|||||||
|
|
||||||
from .handler import handle
|
from .handler import handle
|
||||||
|
|
||||||
location = lbc.City(
|
location = lbc.City(
|
||||||
lat=48.85994982004764,
|
lat=48.85994982004764,
|
||||||
lng=2.33801967847424,
|
lng=2.33801967847424,
|
||||||
radius=10_000, # 10 km
|
radius=10_000, # 10 km
|
||||||
city="Paris"
|
city="Paris",
|
||||||
)
|
)
|
||||||
|
|
||||||
CONFIG = [
|
CONFIG = [
|
||||||
@@ -18,9 +18,9 @@ CONFIG = [
|
|||||||
locations=[location],
|
locations=[location],
|
||||||
category=lbc.Category.IMMOBILIER,
|
category=lbc.Category.IMMOBILIER,
|
||||||
square=[200, 400],
|
square=[200, 400],
|
||||||
price=[300_000, 700_000]
|
price=[300_000, 700_000],
|
||||||
),
|
),
|
||||||
delay=60 * 5, # Check every 5 minutes
|
delay=60 * 5, # Check every 5 minutes
|
||||||
handler=handle
|
handler=handle,
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,50 +1,43 @@
|
|||||||
import lbc
|
import lbc
|
||||||
import requests
|
import requests
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Final
|
|
||||||
|
|
||||||
WEBHOOK_URL: Final[str] = ...
|
WEBHOOK_URL: str = ...
|
||||||
|
|
||||||
|
|
||||||
def handle(ad: lbc.Ad, search_name: str) -> None:
|
def handle(ad: lbc.Ad, search_name: str) -> None:
|
||||||
timestamp = datetime.strptime(ad.index_date, "%Y-%m-%d %H:%M:%S").timestamp()
|
timestamp = datetime.strptime(ad.index_date, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"content": None,
|
"content": None,
|
||||||
"embeds": [
|
"embeds": [
|
||||||
{
|
{
|
||||||
"title": ad.title,
|
"title": ad.title,
|
||||||
"description": f"```{ad.body[:4087]}...```" if len(ad.body) >= 4090 else f"```{ad.body[:4090]}```",
|
"description": f"```{ad.body[:4087]}...```"
|
||||||
|
if len(ad.body) >= 4090
|
||||||
|
else f"```{ad.body[:4090]}```",
|
||||||
"url": ad.url,
|
"url": ad.url,
|
||||||
"color": 14381568,
|
"color": 14381568,
|
||||||
"author": {
|
"author": {"name": ad.user.name, "icon_url": ad.user.profile_picture},
|
||||||
"name": ad.user.name,
|
"image": {"url": ad.images[0] if ad.images else None},
|
||||||
"icon_url": ad.user.profile_picture
|
|
||||||
},
|
|
||||||
"image": {
|
|
||||||
"url": ad.images[0] if ad.images else None
|
|
||||||
},
|
|
||||||
"fields": [
|
"fields": [
|
||||||
{
|
{
|
||||||
"name": "🕒 Publication",
|
"name": "🕒 Publication",
|
||||||
"value": f"<t:{int(timestamp)}:R>",
|
"value": f"<t:{int(timestamp)}:R>",
|
||||||
"inline": True
|
"inline": True,
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "💰 Price",
|
|
||||||
"value": f"`{ad.price}€`",
|
|
||||||
"inline": True
|
|
||||||
},
|
},
|
||||||
|
{"name": "💰 Price", "value": f"`{ad.price}€`", "inline": True},
|
||||||
{
|
{
|
||||||
"name": "📍 Location",
|
"name": "📍 Location",
|
||||||
"value": f"`{ad.location.city_label}`",
|
"value": f"`{ad.location.city_label}`",
|
||||||
"inline": True
|
"inline": True,
|
||||||
}
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"username": search_name,
|
"username": search_name,
|
||||||
"attachments": []
|
"attachments": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
response = requests.post(WEBHOOK_URL, json=payload)
|
response = requests.post(WEBHOOK_URL, json=payload)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|||||||
@@ -3,11 +3,11 @@ import lbc
|
|||||||
|
|
||||||
from .handler import handle
|
from .handler import handle
|
||||||
|
|
||||||
location = lbc.City(
|
location = lbc.City(
|
||||||
lat=48.85994982004764,
|
lat=48.85994982004764,
|
||||||
lng=2.33801967847424,
|
lng=2.33801967847424,
|
||||||
radius=10_000, # 10 km
|
radius=10_000, # 10 km
|
||||||
city="Paris"
|
city="Paris",
|
||||||
)
|
)
|
||||||
|
|
||||||
CONFIG = [
|
CONFIG = [
|
||||||
@@ -18,9 +18,9 @@ CONFIG = [
|
|||||||
locations=[location],
|
locations=[location],
|
||||||
category=lbc.Category.IMMOBILIER,
|
category=lbc.Category.IMMOBILIER,
|
||||||
square=[200, 400],
|
square=[200, 400],
|
||||||
price=[300_000, 700_000]
|
price=[300_000, 700_000],
|
||||||
),
|
),
|
||||||
delay=60 * 5, # Check every 5 minutes
|
delay=60 * 5, # Check every 5 minutes
|
||||||
handler=handle
|
handler=handle,
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import lbc
|
import lbc
|
||||||
|
|
||||||
|
|
||||||
def handle(ad: lbc.Ad, search_name: str):
|
def handle(ad: lbc.Ad, search_name: str):
|
||||||
print(f"[{search_name}] New ads!")
|
print(f"[{search_name}] New ads!")
|
||||||
print(f"Title : {ad.subject}")
|
print(f"Title : {ad.subject}")
|
||||||
print(f"Price : {ad.price} €")
|
print(f"Price : {ad.price} €")
|
||||||
print(f"URL : {ad.url}")
|
print(f"URL : {ad.url}")
|
||||||
print("-" * 40)
|
print("-" * 40)
|
||||||
|
|
||||||
|
|||||||
+3
-1
@@ -1,9 +1,11 @@
|
|||||||
from searcher import Searcher
|
from searcher import Searcher
|
||||||
from config import CONFIG
|
from config import CONFIG
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
searcher = Searcher(searches=CONFIG)
|
searcher = Searcher(searches=CONFIG)
|
||||||
searcher.start()
|
searcher.start()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -1,2 +1,4 @@
|
|||||||
from .search import Search
|
from .search import Search
|
||||||
from .parameters import Parameters
|
from .parameters import Parameters
|
||||||
|
|
||||||
|
__all__ = ["Search", "Parameters"]
|
||||||
|
|||||||
@@ -1,24 +1,28 @@
|
|||||||
from typing import Optional, Union, List
|
|
||||||
from lbc import Category, Region, Department, City, OwnerType
|
from lbc import Category, Region, Department, City, OwnerType
|
||||||
|
|
||||||
from typing import overload
|
from typing import overload
|
||||||
|
|
||||||
|
|
||||||
class Parameters:
|
class Parameters:
|
||||||
@overload
|
@overload
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
url: Optional[str] = None,
|
url: str | None = None,
|
||||||
text: Optional[str] = None,
|
text: str | None = None,
|
||||||
category: Category = Category.TOUTES_CATEGORIES,
|
category: Category = Category.TOUTES_CATEGORIES,
|
||||||
locations: Optional[Union[List[Union[Region, Department, City]], Union[Region, Department, City]]] = None,
|
locations: list[Region | Department | City]
|
||||||
|
| Region
|
||||||
|
| Department
|
||||||
|
| City
|
||||||
|
| None = None,
|
||||||
limit: int = 35,
|
limit: int = 35,
|
||||||
limit_alu: int = 3,
|
limit_alu: int = 3,
|
||||||
page: int = 1,
|
page: int = 1,
|
||||||
owner_type: Optional[OwnerType] = None,
|
owner_type: OwnerType | None = None,
|
||||||
shippable: Optional[bool] = None,
|
shippable: bool | None = None,
|
||||||
search_in_title_only: bool = False,
|
search_in_title_only: bool = False,
|
||||||
**kwargs
|
**kwargs,
|
||||||
): ...
|
): ...
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
self._kwargs = kwargs
|
self._kwargs = kwargs
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from lbc import Proxy, Ad
|
from lbc import Proxy, Ad
|
||||||
from .parameters import Parameters
|
from .parameters import Parameters
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Optional
|
from typing import Callable
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Search:
|
class Search:
|
||||||
@@ -9,4 +10,4 @@ class Search:
|
|||||||
parameters: Parameters
|
parameters: Parameters
|
||||||
delay: float
|
delay: float
|
||||||
handler: Callable[[Ad, str], None]
|
handler: Callable[[Ad, str], None]
|
||||||
proxy: Optional[Proxy] = None
|
proxy: Proxy | None = None
|
||||||
|
|||||||
@@ -1,2 +1,4 @@
|
|||||||
from .searcher import Searcher
|
from .searcher import Searcher
|
||||||
from .logger import logger
|
from .logger import logger
|
||||||
|
|
||||||
|
__all__ = ["Searcher", "logger"]
|
||||||
|
|||||||
@@ -1,21 +1,21 @@
|
|||||||
from .logger import logger
|
from .logger import logger
|
||||||
|
|
||||||
from typing import List, Final
|
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
|
|
||||||
MAX_ID: Final[int] = 10_000
|
MAX_ID: int = 10_000
|
||||||
|
|
||||||
|
|
||||||
class ID:
|
class ID:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._ids: List[str] = self._get_ids()
|
self._ids: list[str] = self._get_ids()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ids(self) -> List[str]:
|
def ids(self) -> list[str]:
|
||||||
return self._ids
|
return self._ids
|
||||||
|
|
||||||
def _get_ids(self) -> List[str]:
|
def _get_ids(self) -> list[str]:
|
||||||
ids: List[str] = []
|
ids: list[str] = []
|
||||||
id_path = os.path.join("data", "id.json")
|
id_path = os.path.join("data", "id.json")
|
||||||
if os.path.exists(id_path):
|
if os.path.exists(id_path):
|
||||||
with open(id_path, "r") as f:
|
with open(id_path, "r") as f:
|
||||||
@@ -23,8 +23,10 @@ class ID:
|
|||||||
ids = json.load(f)
|
ids = json.load(f)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
os.remove(id_path)
|
os.remove(id_path)
|
||||||
except:
|
except Exception:
|
||||||
logger.exception("An error occurred while attempting to open the id.json file.")
|
logger.exception(
|
||||||
|
"An error occurred while attempting to open the id.json file."
|
||||||
|
)
|
||||||
return ids
|
return ids
|
||||||
|
|
||||||
def contains(self, id_: str) -> bool:
|
def contains(self, id_: str) -> bool:
|
||||||
@@ -32,7 +34,7 @@ class ID:
|
|||||||
|
|
||||||
def add(self, id_: str) -> bool:
|
def add(self, id_: str) -> bool:
|
||||||
id_path = os.path.join("data", "id.json")
|
id_path = os.path.join("data", "id.json")
|
||||||
if not id_ in self._ids:
|
if id_ not in self._ids:
|
||||||
self._ids.append(id_)
|
self._ids.append(id_)
|
||||||
with open(id_path, "w") as f:
|
with open(id_path, "w") as f:
|
||||||
json.dump(self._ids[-MAX_ID:], f, indent=3)
|
json.dump(self._ids[-MAX_ID:], f, indent=3)
|
||||||
|
|||||||
@@ -10,14 +10,17 @@ os.makedirs(os.path.join("data", "logs"), exist_ok=True)
|
|||||||
# Config logging
|
# Config logging
|
||||||
logger = logging.getLogger("lbc-finder")
|
logger = logging.getLogger("lbc-finder")
|
||||||
|
|
||||||
formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(threadName)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
formatter = logging.Formatter(
|
||||||
|
"[%(asctime)s] [%(levelname)s] [%(threadName)s] %(message)s",
|
||||||
|
datefmt="%Y-%m-%d %H:%M:%S",
|
||||||
|
)
|
||||||
stream_handler = logging.StreamHandler()
|
stream_handler = logging.StreamHandler()
|
||||||
stream_handler.setFormatter(formatter)
|
stream_handler.setFormatter(formatter)
|
||||||
logger.addHandler(stream_handler)
|
logger.addHandler(stream_handler)
|
||||||
logger.setLevel(logging.INFO)
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
# Log File
|
# Log File
|
||||||
file_handler = logging.FileHandler(file_path, mode='w', encoding='utf-8')
|
file_handler = logging.FileHandler(file_path, mode="w", encoding="utf-8")
|
||||||
file_handler.setLevel(logging.WARNING)
|
file_handler.setLevel(logging.WARNING)
|
||||||
file_handler.setFormatter(formatter)
|
file_handler.setFormatter(formatter)
|
||||||
logger.addHandler(file_handler)
|
logger.addHandler(file_handler)
|
||||||
|
|||||||
@@ -5,12 +5,19 @@ from .logger import logger
|
|||||||
|
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
from typing import List, Union
|
|
||||||
|
|
||||||
class Searcher:
|
class Searcher:
|
||||||
def __init__(self, searches: Union[List[Search], Search], request_verify: bool = True,
|
def __init__(
|
||||||
handler_max_attempts: int = 3, handler_initial_backoff: float = 2.0):
|
self,
|
||||||
self._searches: List[Search] = searches if isinstance(searches, list) else [searches]
|
searches: list[Search] | Search,
|
||||||
|
request_verify: bool = True,
|
||||||
|
handler_max_attempts: int = 3,
|
||||||
|
handler_initial_backoff: float = 2.0,
|
||||||
|
):
|
||||||
|
self._searches: list[Search] = (
|
||||||
|
searches if isinstance(searches, list) else [searches]
|
||||||
|
)
|
||||||
self._request_verify = request_verify
|
self._request_verify = request_verify
|
||||||
self._handler_max_attempts = handler_max_attempts
|
self._handler_max_attempts = handler_max_attempts
|
||||||
self._handler_initial_backoff = handler_initial_backoff
|
self._handler_initial_backoff = handler_initial_backoff
|
||||||
@@ -42,10 +49,14 @@ class Searcher:
|
|||||||
before = time.time()
|
before = time.time()
|
||||||
try:
|
try:
|
||||||
response = client.search(**search.parameters._kwargs, sort=Sort.NEWEST)
|
response = client.search(**search.parameters._kwargs, sort=Sort.NEWEST)
|
||||||
logger.debug(f"Successfully found {response.total} ad{'s' if response.total > 1 else ''}.")
|
logger.debug(
|
||||||
|
f"Successfully found {response.total} ad{'s' if response.total > 1 else ''}."
|
||||||
|
)
|
||||||
ads = [ad for ad in response.ads if not self._id.contains(ad.id)]
|
ads = [ad for ad in response.ads if not self._id.contains(ad.id)]
|
||||||
if len(ads):
|
if len(ads):
|
||||||
logger.info(f"Successfully found {len(ads)} new ad{'s' if len(ads) > 1 else ''}!")
|
logger.info(
|
||||||
|
f"Successfully found {len(ads)} new ad{'s' if len(ads) > 1 else ''}!"
|
||||||
|
)
|
||||||
|
|
||||||
notified = 0
|
notified = 0
|
||||||
for ad in ads:
|
for ad in ads:
|
||||||
@@ -57,15 +68,23 @@ class Searcher:
|
|||||||
f"[{search.name}] {len(ads) - notified} ad{'s were' if len(ads) - notified > 1 else ' was'} not marked as seen and will be retried."
|
f"[{search.name}] {len(ads) - notified} ad{'s were' if len(ads) - notified > 1 else ' was'} not marked as seen and will be retried."
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(f"An error occured.")
|
logger.exception("An error occured.")
|
||||||
time.sleep(search.delay - (time.time() - before) if search.delay - (time.time() - before) > 0 else 0)
|
time.sleep(
|
||||||
|
search.delay - (time.time() - before)
|
||||||
|
if search.delay - (time.time() - before) > 0
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
|
||||||
def start(self) -> bool:
|
def start(self) -> bool:
|
||||||
if not len(self._searches):
|
if not len(self._searches):
|
||||||
logger.warning("No search rules have been set. Please create search rules in config.py (see example in README.md).")
|
logger.warning(
|
||||||
|
"No search rules have been set. Please create search rules in config.py (see example in README.md)."
|
||||||
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
for search in self._searches:
|
for search in self._searches:
|
||||||
threading.Thread(target=self._search, args=(search,), name=search.name).start()
|
threading.Thread(
|
||||||
time.sleep(5) # Add latency between each thread to prevent spam
|
target=self._search, args=(search,), name=search.name
|
||||||
|
).start()
|
||||||
|
time.sleep(5) # Add latency between each thread to prevent spam
|
||||||
return True
|
return True
|
||||||
|
|||||||
Reference in New Issue
Block a user