added county, fips, and text desciption fields (#72)

Bunsly · Apr 20, 2024 · 7a525ca · 7a525ca
1 parent 7246703
commit 7a525ca
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 1 deletion.
diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py
@@ -73,6 +73,7 @@ class Description:
     year_built: int | None = None
     garage: float | None = None
     stories: int | None = None
+    text: str | None = None
 
 
 @dataclass
@@ -101,6 +102,7 @@ class Property:
     latitude: float | None = None
     longitude: float | None = None
     neighborhoods: Optional[str] = None
-
+    county: Optional[str] = None
+    fips_code: Optional[str] = None
     agents: list[Agent] = None
     nearby_schools: list[str] = None
diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py
@@ -176,6 +176,7 @@ def handle_listing(self, listing_id: str) -> list[Property]:
                 year_built=property_info["details"].get("year_built"),
                 garage=property_info["details"].get("garage"),
                 stories=property_info["details"].get("stories"),
+                text=property_info["description"].get("text"),
             ),
             days_on_mls=days_on_mls,
             agents=agents_schools["agents"],
@@ -330,6 +331,7 @@ def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[i
                                     type
                                     name
                                     stories
+                                    text
                                 }
                                 source {
                                     id
@@ -353,10 +355,17 @@ def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[i
                                             lat
                                         }
                                     }
+                                    county {
+                                        name
+                                        fips_code
+                                    }
                                     neighborhoods {
                                         name
                                     }
                                 }
+                                tax_record {
+                                    public_record_id
+                                }
                                 primary_photo {
                                     href
                                 }
@@ -536,6 +545,9 @@ def process_property(result: dict) -> Property | None:
                 longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
                 address=self._parse_address(result, search_type="general_search"),
                 description=self._parse_description(result),
+                neighborhoods=self._parse_neighborhoods(result),
+                county=result["location"]["county"].get("name"),
+                fips_code=result["location"]["county"].get("fips_code"),
                 days_on_mls=self.calculate_days_on_mls(result),
                 agents=agents_schools["agents"],
                 nearby_schools=agents_schools["schools"],
@@ -725,6 +737,7 @@ def _parse_description(result: dict) -> Description:
             year_built=description_data.get("year_built"),
             garage=description_data.get("garage"),
             stories=description_data.get("stories"),
+            text=description_data.get("text"),
         )
 
     @staticmethod

diff --git a/homeharvest/utils.py b/homeharvest/utils.py
@@ -8,6 +8,7 @@
     "mls",
     "mls_id",
     "status",
+    "text",
     "style",
     "street",
     "unit",
@@ -28,6 +29,9 @@
     "price_per_sqft",
     "latitude",
     "longitude",
+    "neighborhoods",
+    "county",
+    "fips_code",
     "stories",
     "hoa_fee",
     "parking_garage",
@@ -77,6 +81,8 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["year_built"] = description.year_built
     prop_data["parking_garage"] = description.garage
     prop_data["stories"] = description.stories
+    prop_data["text"] = description.text
+
 
     properties_df = pd.DataFrame([prop_data])
     properties_df = properties_df.reindex(columns=ordered_properties)