Skip to content

Commit

Permalink
- cullen merge
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZacharyHampton
ZacharyHampton committed Oct 4, 2023
1 parent 088088a commit 29664e4
Show file tree
Hide file tree
Showing 10 changed files with 269 additions and 1,007 deletions.
98 changes: 6 additions & 92 deletions homeharvest/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,14 @@
from concurrent.futures import ThreadPoolExecutor

from .core.scrapers import ScraperInput
from .core.scrapers.redfin import RedfinScraper
from .utils import process_result, ordered_properties
from .core.scrapers.realtor import RealtorScraper
from .core.scrapers.zillow import ZillowScraper
from .core.scrapers.models import ListingType, Property, SiteName
from .exceptions import InvalidSite, InvalidListingType


_scrapers = {
"redfin": RedfinScraper,
"realtor.com": RealtorScraper,
"zillow": ZillowScraper,
}


Expand All @@ -26,86 +23,6 @@ def _validate_input(site_name: str, listing_type: str) -> None:
raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.")


def _get_ordered_properties(result: Property) -> list[str]:
return [
"property_url",
"site_name",
"listing_type",
"property_type",
"status_text",
"baths_min",
"baths_max",
"beds_min",
"beds_max",
"sqft_min",
"sqft_max",
"price_min",
"price_max",
"unit_count",
"tax_assessed_value",
"price_per_sqft",
"lot_area_value",
"lot_area_unit",
"address_one",
"address_two",
"city",
"state",
"zip_code",
"posted_time",
"area_min",
"bldg_name",
"stories",
"year_built",
"agent_name",
"agent_phone",
"agent_email",
"days_on_market",
"sold_date",
"mls_id",
"img_src",
"latitude",
"longitude",
"description",
]


def _process_result(result: Property) -> pd.DataFrame:
prop_data = result.__dict__

prop_data["site_name"] = prop_data["site_name"].value
prop_data["listing_type"] = prop_data["listing_type"].value.lower()
if "property_type" in prop_data and prop_data["property_type"] is not None:
prop_data["property_type"] = prop_data["property_type"].value.lower()
else:
prop_data["property_type"] = None
if "address" in prop_data:
address_data = prop_data["address"]
prop_data["address_one"] = address_data.address_one
prop_data["address_two"] = address_data.address_two
prop_data["city"] = address_data.city
prop_data["state"] = address_data.state
prop_data["zip_code"] = address_data.zip_code

del prop_data["address"]

if "agent" in prop_data and prop_data["agent"] is not None:
agent_data = prop_data["agent"]
prop_data["agent_name"] = agent_data.name
prop_data["agent_phone"] = agent_data.phone
prop_data["agent_email"] = agent_data.email

del prop_data["agent"]
else:
prop_data["agent_name"] = None
prop_data["agent_phone"] = None
prop_data["agent_email"] = None

properties_df = pd.DataFrame([prop_data])
properties_df = properties_df[_get_ordered_properties(result)]

return properties_df


def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None, sold_last_x_days: int = None) -> pd.DataFrame:
"""
Helper function to scrape a single site.
Expand All @@ -124,22 +41,20 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius
site = _scrapers[site_name.lower()](scraper_input)
results = site.search()

properties_dfs = [_process_result(result) for result in results]
properties_dfs = [df.dropna(axis=1, how="all") for df in properties_dfs if not df.empty]
properties_dfs = [process_result(result) for result in results]
if not properties_dfs:
return pd.DataFrame()

return pd.concat(properties_dfs, ignore_index=True)
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties]


def scrape_property(
location: str,
site_name: Union[str, list[str]] = "realtor.com",
#: site_name: Union[str, list[str]] = "realtor.com",
listing_type: str = "for_sale",
radius: float = None,
sold_last_x_days: int = None,
proxy: str = None,
keep_duplicates: bool = False
) -> pd.DataFrame:
"""
Scrape property from various sites from a given location and listing type.
Expand All @@ -153,6 +68,7 @@ def scrape_property(
:param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold')
:returns: pd.DataFrame containing properties
"""
site_name = "realtor.com"

if site_name is None:
site_name = list(_scrapers.keys())
Expand Down Expand Up @@ -183,13 +99,11 @@ def scrape_property(

final_df = pd.concat(results, ignore_index=True)

columns_to_track = ["address_one", "address_two", "city"]
columns_to_track = ["Street", "Unit", "Zip"]

#: validate they exist, otherwise create them
for col in columns_to_track:
if col not in final_df.columns:
final_df[col] = None

if not keep_duplicates:
final_df = final_df.drop_duplicates(subset=columns_to_track, keep="first")
return final_df
106 changes: 28 additions & 78 deletions homeharvest/core/scrapers/models.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from dataclasses import dataclass
from enum import Enum
from typing import Tuple
from datetime import datetime
from typing import Optional


class SiteName(Enum):
Expand All @@ -23,46 +22,13 @@ class ListingType(Enum):
SOLD = "SOLD"


class PropertyType(Enum):
HOUSE = "HOUSE"
BUILDING = "BUILDING"
CONDO = "CONDO"
TOWNHOUSE = "TOWNHOUSE"
SINGLE_FAMILY = "SINGLE_FAMILY"
MULTI_FAMILY = "MULTI_FAMILY"
MANUFACTURED = "MANUFACTURED"
NEW_CONSTRUCTION = "NEW_CONSTRUCTION"
APARTMENT = "APARTMENT"
APARTMENTS = "APARTMENTS"
LAND = "LAND"
LOT = "LOT"
OTHER = "OTHER"

BLANK = "BLANK"

@classmethod
def from_int_code(cls, code):
mapping = {
1: cls.HOUSE,
2: cls.CONDO,
3: cls.TOWNHOUSE,
4: cls.MULTI_FAMILY,
5: cls.LAND,
6: cls.OTHER,
8: cls.SINGLE_FAMILY,
13: cls.SINGLE_FAMILY,
}

return mapping.get(code, cls.BLANK)


@dataclass
class Address:
address_one: str | None = None
address_two: str | None = "#"
street: str | None = None
unit: str | None = None
city: str | None = None
state: str | None = None
zip_code: str | None = None
zip: str | None = None


@dataclass
Expand All @@ -74,47 +40,31 @@ class Agent:

@dataclass
class Property:
property_url: str
site_name: SiteName
listing_type: ListingType
address: Address
property_type: PropertyType | None = None

# house for sale
tax_assessed_value: int | None = None
lot_area_value: float | None = None
lot_area_unit: str | None = None
stories: int | None = None
year_built: int | None = None
price_per_sqft: int | None = None
property_url: str | None = None
mls: str | None = None
mls_id: str | None = None

agent: Agent | None = None
img_src: str | None = None
description: str | None = None
status_text: str | None = None
posted_time: datetime | None = None

# building for sale
bldg_name: str | None = None
area_min: int | None = None

beds_min: int | None = None
beds_max: int | None = None

baths_min: float | None = None
baths_max: float | None = None

sqft_min: int | None = None
sqft_max: int | None = None

price_min: int | None = None
price_max: int | None = None

unit_count: int | None = None

status: str | None = None
style: str | None = None

beds: int | None = None
baths_full: int | None = None
baths_half: int | None = None
list_price: int | None = None
list_date: str | None = None
sold_price: int | None = None
last_sold_date: str | None = None
prc_sqft: float | None = None
est_sf: int | None = None
lot_sf: int | None = None
hoa_fee: int | None = None

address: Address | None = None

yr_blt: int | None = None
latitude: float | None = None
longitude: float | None = None

sold_date: datetime | None = None
days_on_market: int | None = None
stories: int | None = None
prkg_gar: float | None = None

neighborhoods: Optional[str] = None
Loading

0 comments on commit 29664e4

Please sign in to comment.