Merge branch 'master' into add_agent

.github/workflows/publish-to-pypi.yml

@@ -30,4 +30,4 @@ jobs:
       if: startsWith(github.ref, 'refs/tags')
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -4,4 +4,4 @@
 **/.pytest_cache/
 *.pyc
 /.ipynb_checkpoints/
-*.csv
+*.csv

README.md

@@ -21,7 +21,7 @@
 ```bash
 pip install -U homeharvest
 ```
-  _Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_ 
+  _Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
 
 ## Usage
 
@@ -39,11 +39,11 @@ properties = scrape_property(
   location="San Diego, CA",
   listing_type="sold",  # or (for_sale, for_rent, pending)
   past_days=30,  # sold in last 30 days - listed in last 30 days if (for_sale, for_rent)
-  
-  # date_from="2023-05-01", # alternative to past_days 
-  # date_to="2023-05-28", 
+
+  # date_from="2023-05-01", # alternative to past_days
+  # date_to="2023-05-28",
   # foreclosure=True
-  
+
   # mls_only=True,  # only fetch MLS listings
 )
 print(f"Number of properties: {len(properties)}")
@@ -84,7 +84,7 @@ Optional
 │
 ├── date_from, date_to (string): Start and end dates to filter properties listed or sold, both dates are required.
 |    (use this to get properties in chunks as there's a 10k result limit)
-│    Format for both must be "YYYY-MM-DD". 
+│    Format for both must be "YYYY-MM-DD".
 │    Example: "2023-05-01", "2023-05-15" (fetches properties listed/sold between these dates)
 │
 ├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings)
@@ -128,19 +128,21 @@ Property
 │ ├── sold_price
 │ ├── last_sold_date
 │ ├── price_per_sqft
+│ ├── parking_garage
 │ └── hoa_fee
 
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude
 
-└── Parking Details:
-    └── parking_garage
+├── Agent Info:
+│ ├── agent
+│ ├── broker
+│ └── broker_phone
 ```
 
 ### Exceptions
 The following exceptions may be raised when using HomeHarvest:
 
 - `InvalidListingType` - valid options: `for_sale`, `for_rent`, `sold`
 - `InvalidDate` - date_from or date_to is not in the format YYYY-MM-DD
-

homeharvest/cli.py

@@ -5,9 +5,7 @@
 
 def main():
     parser = argparse.ArgumentParser(description="Home Harvest Property Scraper")
-    parser.add_argument(
-        "location", type=str, help="Location to scrape (e.g., San Francisco, CA)"
-    )
+    parser.add_argument("location", type=str, help="Location to scrape (e.g., San Francisco, CA)")
 
     parser.add_argument(
         "-l",
@@ -35,9 +33,7 @@ def main():
         help="Name of the output file (without extension)",
     )
 
-    parser.add_argument(
-        "-p", "--proxy", type=str, default=None, help="Proxy to use for scraping"
-    )
+    parser.add_argument("-p", "--proxy", type=str, default=None, help="Proxy to use for scraping")
     parser.add_argument(
         "-d",
         "--days",

homeharvest/core/scrapers/__init__.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 import requests
+import uuid
 from .models import Property, ListingType, SiteName
 
 
@@ -27,6 +28,12 @@ def __init__(
 
         if not session:
             self.session = requests.Session()
+            self.session.headers.update(
+                {
+                    "auth": f"Bearer {self.get_access_token()}",
+                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                }
+            )
         else:
             self.session = session
 
@@ -43,12 +50,26 @@ def __init__(
         self.date_to = scraper_input.date_to
         self.foreclosure = scraper_input.foreclosure
 
-    def search(self) -> list[Property]:
-        ...
+    def search(self) -> list[Property]: ...
 
     @staticmethod
-    def _parse_home(home) -> Property:
-        ...
+    def _parse_home(home) -> Property: ...
 
-    def handle_location(self):
-        ...
+    def handle_location(self): ...
+
+    def get_access_token(self):
+        url = "https://graph.realtor.com/auth/token"
+
+        payload = f'{{"client_app_id":"rdc_mobile_native,24.20.4.149916,iphone","device_id":"{str(uuid.uuid4()).upper()}","grant_type":"device_mobile"}}'
+        headers = {
+            "Host": "graph.realtor.com",
+            "x-client-version": "24.20.4.149916",
+            "accept": "*/*",
+            "content-type": "Application/json",
+            "user-agent": "Realtor.com/24.20.4.149916 CFNetwork/1410.0.3 Darwin/22.6.0",
+            "accept-language": "en-US,en;q=0.9",
+        }
+        response = requests.post(url, headers=headers, data=payload)
+
+        data = response.json()
+        return data["access_token"]

homeharvest/core/scrapers/models.py

@@ -69,6 +69,12 @@ class Description:
     stories: int | None = None
 
 
+@dataclass
+class Agent:
+    name: str | None = None
+    phone: str | None = None
+
+
 @dataclass
 class Property:
     property_url: str
@@ -89,3 +95,5 @@ class Property:
     latitude: float | None = None
     longitude: float | None = None
     neighborhoods: Optional[str] = None
+
+    agents: list[Agent] = None

homeharvest/exceptions.py

@@ -1,5 +1,6 @@
 class InvalidListingType(Exception):
     """Raised when a provided listing type is does not exist."""
 
+
 class InvalidDate(Exception):
-    """Raised when only one of date_from or date_to is provided or not in the correct format. ex: 2023-10-23 """
+    """Raised when only one of date_from or date_to is provided or not in the correct format. ex: 2023-10-23"""

homeharvest/utils.py

@@ -31,6 +31,9 @@
     "stories",
     "hoa_fee",
     "parking_garage",
+    "agent",
+    "broker",
+    "broker_phone",
     "primary_photo",
     "alt_photos",
 ]
@@ -48,6 +51,14 @@ def process_result(result: Property) -> pd.DataFrame:
         prop_data["state"] = address_data.state
         prop_data["zip_code"] = address_data.zip
 
+    if "agents" in prop_data:
+        agents = prop_data["agents"]
+        if agents:
+            prop_data["agent"] = agents[0].name
+            if len(agents) > 1:
+                prop_data["broker"] = agents[1].name
+                prop_data["broker_phone"] = agents[1].phone
+
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
 
     description = result.description
@@ -72,9 +83,7 @@ def process_result(result: Property) -> pd.DataFrame:
 
 def validate_input(listing_type: str) -> None:
     if listing_type.upper() not in ListingType.__members__:
-        raise InvalidListingType(
-            f"Provided listing type, '{listing_type}', does not exist."
-        )
+        raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.")
 
 
 def validate_dates(date_from: str | None, date_to: str | None) -> None: