diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py index 0ab0f19..da24829 100644 --- a/homeharvest/core/scrapers/models.py +++ b/homeharvest/core/scrapers/models.py @@ -73,6 +73,7 @@ class Description: year_built: int | None = None garage: float | None = None stories: int | None = None + text: str | None = None @dataclass @@ -101,6 +102,7 @@ class Property: latitude: float | None = None longitude: float | None = None neighborhoods: Optional[str] = None - + county: Optional[str] = None + fips_code: Optional[str] = None agents: list[Agent] = None nearby_schools: list[str] = None diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index febd942..a72ab6d 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -176,6 +176,7 @@ def handle_listing(self, listing_id: str) -> list[Property]: year_built=property_info["details"].get("year_built"), garage=property_info["details"].get("garage"), stories=property_info["details"].get("stories"), + text=property_info["description"].get("text"), ), days_on_mls=days_on_mls, agents=agents_schools["agents"], @@ -330,6 +331,7 @@ def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[i type name stories + text } source { id @@ -353,10 +355,17 @@ def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[i lat } } + county { + name + fips_code + } neighborhoods { name } } + tax_record { + public_record_id + } primary_photo { href } @@ -536,6 +545,9 @@ def process_property(result: dict) -> Property | None: longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None, address=self._parse_address(result, search_type="general_search"), description=self._parse_description(result), + neighborhoods=self._parse_neighborhoods(result), + county=result["location"]["county"].get("name"), + fips_code=result["location"]["county"].get("fips_code"), days_on_mls=self.calculate_days_on_mls(result), agents=agents_schools["agents"], nearby_schools=agents_schools["schools"], @@ -725,6 +737,7 @@ def _parse_description(result: dict) -> Description: year_built=description_data.get("year_built"), garage=description_data.get("garage"), stories=description_data.get("stories"), + text=description_data.get("text"), ) @staticmethod diff --git a/homeharvest/utils.py b/homeharvest/utils.py index 64a52d2..3915344 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -8,6 +8,7 @@ "mls", "mls_id", "status", + "text", "style", "street", "unit", @@ -28,6 +29,9 @@ "price_per_sqft", "latitude", "longitude", + "neighborhoods", + "county", + "fips_code", "stories", "hoa_fee", "parking_garage", @@ -77,6 +81,8 @@ def process_result(result: Property) -> pd.DataFrame: prop_data["year_built"] = description.year_built prop_data["parking_garage"] = description.garage prop_data["stories"] = description.stories + prop_data["text"] = description.text + properties_df = pd.DataFrame([prop_data]) properties_df = properties_df.reindex(columns=ordered_properties)