diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py index b5758c2..1f78893 100644 --- a/homeharvest/core/scrapers/models.py +++ b/homeharvest/core/scrapers/models.py @@ -121,7 +121,8 @@ class Property: neighborhoods: Optional[str] = None county: Optional[str] = None fips_code: Optional[str] = None - agents: list[Agent] = None + agents: list[Agent] | None = None + brokers: list[Broker] | None = None nearby_schools: list[str] = None assessed_value: int | None = None estimated_value: int | None = None diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index ed7cb27..b6bd95b 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -10,7 +10,7 @@ from typing import Dict, Union, Optional from .. import Scraper -from ..models import Property, Address, ListingType, Description, PropertyType, Agent +from ..models import Property, Address, ListingType, Description, PropertyType, Agent, Broker class RealtorScraper(Scraper): @@ -180,6 +180,7 @@ def handle_listing(self, listing_id: str) -> list[Property]: ), days_on_mls=days_on_mls, agents=prop_details.get("agents"), + brokers=prop_details.get("brokers"), nearby_schools=prop_details.get("schools"), assessed_value=prop_details.get("assessed_value"), estimated_value=prop_details.get("estimated_value"), @@ -295,6 +296,7 @@ def handle_address(self, property_id: str) -> list[Property]: address=self._parse_address(property_info, search_type="handle_address"), description=self._parse_description(property_info), agents=prop_details.get("agents"), + brokers=prop_details.get("brokers"), nearby_schools=prop_details.get("schools"), assessed_value=prop_details.get("assessed_value"), estimated_value=prop_details.get("estimated_value"), @@ -553,6 +555,7 @@ def process_property(result: dict) -> Property | None: fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None, days_on_mls=self.calculate_days_on_mls(result), agents=prop_details.get("agents"), + brokers=prop_details.get("brokers"), nearby_schools=prop_details.get("schools"), assessed_value=prop_details.get("assessed_value"), estimated_value=prop_details.get("estimated_value"), @@ -665,7 +668,13 @@ def get_prop_details(self, property_id: str) -> dict: email phones { number type ext primary } } - + + consumer_advertisers { + name + phone + href + type + } nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) { __typename schools { district { __typename id name } } @@ -700,7 +709,9 @@ def get_key(keys: list): except (KeyError, TypeError, IndexError): return {} - ads = get_key(["data", "home", "advertisers"]) + agents = get_key(["data", "home", "advertisers"]) + advertisers = get_key(["data", "home", "consumer_advertisers"]) + schools = get_key(["data", "home", "nearbySchools", "schools"]) assessed_value = get_key(["data", "home", "taxHistory", 0, "assessment", "total"]) estimated_value = get_key(["data", "home", "estimates", "currentValues", 0, "estimate"]) @@ -709,11 +720,18 @@ def get_key(keys: list): name=ad["name"], email=ad["email"], phones=ad["phones"] - ) for ad in ads] + ) for ad in agents] + + brokers = [Broker( + name=ad["name"], + phone=ad["phone"], + website=ad["href"] + ) for ad in advertisers if ad.get("type") != "Agent"] schools = [school["district"]["name"] for school in schools if school['district'].get('name')] return { "agents": agents if agents else None, + "brokers": brokers if brokers else None, "schools": schools if schools else None, "assessed_value": assessed_value if assessed_value else None, "estimated_value": estimated_value if estimated_value else None, diff --git a/homeharvest/utils.py b/homeharvest/utils.py index da57ca9..55eb7a3 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -40,6 +40,9 @@ "agent", "agent_email", "agent_phones", + "broker", + "broker_phone", + "broker_website", "nearby_schools", "primary_photo", "alt_photos", @@ -65,6 +68,13 @@ def process_result(result: Property) -> pd.DataFrame: prop_data["agent_email"] = agents[0].email prop_data["agent_phones"] = agents[0].phones + if "brokers" in prop_data: + brokers = prop_data["brokers"] + if brokers: + prop_data["broker"] = brokers[0].name + prop_data["broker_phone"] = brokers[0].phone + prop_data["broker_website"] = brokers[0].website + prop_data["price_per_sqft"] = prop_data["prc_sqft"] prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None diff --git a/pyproject.toml b/pyproject.toml index 2406b30..fb3d0c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "homeharvest" -version = "0.3.21" +version = "0.3.22" description = "Real estate scraping library" authors = ["Zachary Hampton ", "Cullen Watson "] homepage = "https://github.com/Bunsly/HomeHarvest"