-
Notifications
You must be signed in to change notification settings - Fork 66
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
9 changed files
with
218 additions
and
199 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| from homeharvest import scrape_property | ||
| from datetime import datetime | ||
|
|
||
| # Generate filename based on current timestamp | ||
| current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | ||
| filename = f"output/{current_timestamp}.csv" | ||
|
|
||
| properties = scrape_property( | ||
| location="San Diego, CA", | ||
| listing_type="sold", # for_sale, for_rent | ||
| last_x_days=30, # sold/listed in last 30 days | ||
| mls_only=True, # only fetch MLS listings | ||
| ) | ||
| print(f"Number of properties: {len(properties)}") | ||
|
|
||
| # Export to csv | ||
| properties.to_csv(filename, index=False) | ||
| print(properties.head()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,103 +1,41 @@ | ||
| import warnings | ||
| import pandas as pd | ||
| import concurrent.futures | ||
| from concurrent.futures import ThreadPoolExecutor | ||
|
|
||
| from .core.scrapers import ScraperInput | ||
| from .utils import process_result, ordered_properties | ||
| from .utils import process_result, ordered_properties, validate_input | ||
| from .core.scrapers.realtor import RealtorScraper | ||
| from .core.scrapers.models import ListingType, Property, SiteName | ||
| from .exceptions import InvalidListingType | ||
|
|
||
|
|
||
| _scrapers = { | ||
| "realtor.com": RealtorScraper, | ||
| } | ||
|
|
||
|
|
||
| def _validate_input(listing_type: str) -> None: | ||
| if listing_type.upper() not in ListingType.__members__: | ||
| raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") | ||
| from .core.scrapers.models import ListingType | ||
| from .exceptions import InvalidListingType, NoResultsFound | ||
|
|
||
|
|
||
| def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None, sold_last_x_days: int = None) -> pd.DataFrame: | ||
| def scrape_property( | ||
| location: str, | ||
| listing_type: str = "for_sale", | ||
| radius: float = None, | ||
| mls_only: bool = False, | ||
| last_x_days: int = None, | ||
| proxy: str = None, | ||
| ) -> pd.DataFrame: | ||
| """ | ||
| Helper function to scrape a single site. | ||
| Scrape properties from Realtor.com based on a given location and listing type. | ||
| """ | ||
| _validate_input(listing_type) | ||
| validate_input(listing_type) | ||
|
|
||
| scraper_input = ScraperInput( | ||
| location=location, | ||
| listing_type=ListingType[listing_type.upper()], | ||
| site_name=SiteName.get_by_value(site_name.lower()), | ||
| proxy=proxy, | ||
| radius=radius, | ||
| sold_last_x_days=sold_last_x_days | ||
| mls_only=mls_only, | ||
| last_x_days=last_x_days, | ||
| ) | ||
|
|
||
| site = _scrapers[site_name.lower()](scraper_input) | ||
| site = RealtorScraper(scraper_input) | ||
| results = site.search() | ||
| print(f"found {len(results)}") | ||
|
|
||
| properties_dfs = [process_result(result) for result in results] | ||
| if not properties_dfs: | ||
| return pd.DataFrame() | ||
|
|
||
| return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] | ||
|
|
||
|
|
||
| def scrape_property( | ||
| location: str, | ||
| listing_type: str = "for_sale", | ||
| radius: float = None, | ||
| sold_last_x_days: int = None, | ||
| proxy: str = None, | ||
| ) -> pd.DataFrame: | ||
| """ | ||
| Scrape properties from Realtor.com based on a given location and listing type. | ||
| :param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way') | ||
| :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold'). Default is 'for_sale'. | ||
| :param radius: Radius in miles to find comparable properties on individual addresses. Optional. | ||
| :param sold_last_x_days: Number of past days to filter sold properties. Optional. | ||
| :param proxy: Proxy IP address to be used for scraping. Optional. | ||
| :returns: pd.DataFrame containing properties | ||
| """ | ||
| site_name = "realtor.com" | ||
|
|
||
| if site_name is None: | ||
| site_name = list(_scrapers.keys()) | ||
|
|
||
| if not isinstance(site_name, list): | ||
| site_name = [site_name] | ||
|
|
||
| results = [] | ||
|
|
||
| if len(site_name) == 1: | ||
| final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy, sold_last_x_days) | ||
| results.append(final_df) | ||
| else: | ||
| with ThreadPoolExecutor() as executor: | ||
| futures = { | ||
| executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy, sold_last_x_days): s_name | ||
| for s_name in site_name | ||
| } | ||
|
|
||
| for future in concurrent.futures.as_completed(futures): | ||
| result = future.result() | ||
| results.append(result) | ||
|
|
||
| results = [df for df in results if not df.empty and not df.isna().all().all()] | ||
|
|
||
| if not results: | ||
| return pd.DataFrame() | ||
|
|
||
| final_df = pd.concat(results, ignore_index=True) | ||
|
|
||
| columns_to_track = ["Street", "Unit", "Zip"] | ||
|
|
||
| #: validate they exist, otherwise create them | ||
| for col in columns_to_track: | ||
| if col not in final_df.columns: | ||
| final_df[col] = None | ||
| raise NoResultsFound("no results found for the query") | ||
|
|
||
| return final_df | ||
| with warnings.catch_warnings(): | ||
| warnings.simplefilter("ignore", category=FutureWarning) | ||
| return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.