-
Notifications
You must be signed in to change notification settings - Fork 66
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
bd33c3b
commit ff95ca0
Showing
5 changed files
with
573 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,228 @@ | ||
""" | ||
homeharvest.redfin.__init__ | ||
~~~~~~~~~~~~ | ||
This module implements the scraper for redfin.com | ||
""" | ||
import json | ||
from typing import Any | ||
from .. import Scraper | ||
from ..models import Property, Address, Status | ||
from ....exceptions import NoResultsFound, SearchTooBroad | ||
from datetime import datetime | ||
|
||
|
||
class RedfinScraper(Scraper): | ||
def __init__(self, scraper_input): | ||
super().__init__(scraper_input) | ||
self.listing_type = scraper_input.status | ||
|
||
def _handle_location(self): | ||
url = "https://www.redfin.com/stingray/do/location-autocomplete?v=2&al=1&location={}".format( | ||
self.location | ||
) | ||
|
||
response = self.session.get(url) | ||
response_json = json.loads(response.text.replace("{}&&", "")) | ||
|
||
def get_region_type(match_type: str): | ||
if match_type == "4": | ||
return "2" #: zip | ||
elif match_type == "2": | ||
return "6" #: city | ||
elif match_type == "1": | ||
return "address" #: address, needs to be handled differently | ||
elif match_type == "11": | ||
return "state" | ||
|
||
if "exactMatch" not in response_json["payload"]: | ||
raise NoResultsFound( | ||
"No results found for location: {}".format(self.location) | ||
) | ||
|
||
if response_json["payload"]["exactMatch"] is not None: | ||
target = response_json["payload"]["exactMatch"] | ||
else: | ||
target = response_json["payload"]["sections"][0]["rows"][0] | ||
|
||
return target["id"].split("_")[1], get_region_type(target["type"]) | ||
|
||
def _parse_home(self, home: dict, single_search: bool = False) -> Property: | ||
def get_value(key: str) -> Any | None: | ||
if key in home and "value" in home[key]: | ||
return home[key]["value"] | ||
|
||
if not single_search: | ||
address = Address( | ||
street=get_value("streetLine"), | ||
city=home.get("city"), | ||
state=home.get("state"), | ||
zip=home.get("zip"), | ||
) | ||
else: | ||
address_info = home.get("streetAddress") | ||
|
||
address = Address( | ||
street=address_info.get("assembledAddress"), | ||
city=home.get("city"), | ||
state=home.get("state"), | ||
zip=home.get("zip"), | ||
) | ||
|
||
url = "https://www.redfin.com{}".format(home["url"]) | ||
lot_size_data = home.get("lotSize") | ||
|
||
if not isinstance(lot_size_data, int): | ||
lot_size = ( | ||
lot_size_data.get("value", None) | ||
if isinstance(lot_size_data, dict) | ||
else None | ||
) | ||
else: | ||
lot_size = lot_size_data | ||
|
||
lat_long = get_value("latLong") | ||
|
||
return Property( | ||
status=self.listing_type, | ||
address=address, | ||
property_url=url, | ||
beds=home["beds"] if "beds" in home else None, | ||
baths_full=home["baths"] if "baths" in home else None, | ||
list_price=get_value("price"), | ||
est_sf=get_value("sqFt"), | ||
stories=home["stories"] if "stories" in home else None, | ||
yr_blt=get_value("yearBuilt") | ||
if not single_search | ||
else home.get("yearBuilt"), | ||
lot_sf=lot_size, | ||
prc_sqft=get_value("pricePerSqFt") | ||
if type(home.get("pricePerSqFt")) != int | ||
else home.get("pricePerSqFt"), | ||
mls_id=get_value("mlsId"), | ||
latitude=lat_long.get("latitude") if lat_long else None, | ||
longitude=lat_long.get("longitude") if lat_long else None, | ||
last_sold_date=datetime.fromtimestamp(home["soldDate"] / 1000) | ||
if "soldDate" in home | ||
else None, | ||
) | ||
|
||
def _handle_rentals(self, region_id, region_type): | ||
url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true®ion_id={region_id}®ion_type={region_type}&num_homes=100000" | ||
|
||
response = self.session.get(url) | ||
response.raise_for_status() | ||
homes = response.json() | ||
|
||
properties_list = [] | ||
|
||
for home in homes["homes"]: | ||
home_data = home["homeData"] | ||
rental_data = home["rentalExtension"] | ||
|
||
property_url = f"https://www.redfin.com{home_data.get('url', '')}" | ||
address_info = home_data.get("addressInfo", {}) | ||
centroid = address_info.get("centroid", {}).get("centroid", {}) | ||
address = Address( | ||
street=address_info.get("formattedStreetLine"), | ||
city=address_info.get("city"), | ||
state=address_info.get("state"), | ||
zip=address_info.get("zip"), | ||
) | ||
|
||
price_range = rental_data.get("rentPriceRange", {"min": None, "max": None}) | ||
bed_range = rental_data.get("bedRange", {"min": None, "max": None}) | ||
bath_range = rental_data.get("bathRange", {"min": None, "max": None}) | ||
sqft_range = rental_data.get("sqftRange", {"min": None, "max": None}) | ||
|
||
property_ = Property( | ||
property_url=property_url, | ||
status=Status.FOR_RENT.value, | ||
address=address, | ||
latitude=centroid.get("latitude"), | ||
longitude=centroid.get("longitude"), | ||
baths_full=bath_range.get("min"), | ||
beds=bed_range.get("min"), | ||
list_price=price_range.get("min"), | ||
est_sf=sqft_range.get("min"), | ||
) | ||
|
||
properties_list.append(property_) | ||
|
||
if not properties_list: | ||
raise NoResultsFound("No rentals found for the given location.") | ||
|
||
return properties_list | ||
|
||
def _parse_building(self, building: dict) -> Property: | ||
street_address = " ".join( | ||
[ | ||
building["address"]["streetNumber"], | ||
building["address"]["directionalPrefix"], | ||
building["address"]["streetName"], | ||
building["address"]["streetType"], | ||
] | ||
) | ||
return Property( | ||
status=self.status, | ||
address=Address( | ||
street=street_address, | ||
city=building["address"]["city"], | ||
state=building["address"]["stateOrProvinceCode"], | ||
zip=building["address"]["postalCode"], | ||
), | ||
property_url="https://www.redfin.com{}".format(building["url"]), | ||
) | ||
|
||
def handle_address(self, home_id: str): | ||
""" | ||
EPs: | ||
https://www.redfin.com/stingray/api/home/details/initialInfo?al=1&path=/TX/Austin/70-Rainey-St-78701/unit-1608/home/147337694 | ||
https://www.redfin.com/stingray/api/home/details/mainHouseInfoPanelInfo?propertyId=147337694&accessLevel=3 | ||
https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=147337694&accessLevel=3 | ||
https://www.redfin.com/stingray/api/home/details/belowTheFold?propertyId=147337694&accessLevel=3 | ||
""" | ||
url = "https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId={}&accessLevel=3".format( | ||
home_id | ||
) | ||
|
||
response = self.session.get(url) | ||
response_json = json.loads(response.text.replace("{}&&", "")) | ||
|
||
parsed_home = self._parse_home( | ||
response_json["payload"]["addressSectionInfo"], single_search=True | ||
) | ||
return [parsed_home] | ||
|
||
def search(self): | ||
region_id, region_type = self._handle_location() | ||
|
||
if region_type == "state": | ||
raise SearchTooBroad( | ||
"State searches are not supported, please use a more specific location." | ||
) | ||
|
||
if region_type == "address": | ||
home_id = region_id | ||
return self.handle_address(home_id) | ||
|
||
if self.listing_type == Status.FOR_RENT: | ||
return self._handle_rentals(region_id, region_type) | ||
else: | ||
if self.listing_type == Status.FOR_SALE: | ||
url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&num_homes=100000" | ||
else: | ||
url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&sold_within_days=30&num_homes=100000" | ||
response = self.session.get(url) | ||
response_json = json.loads(response.text.replace("{}&&", "")) | ||
|
||
if "payload" in response_json: | ||
homes_list = response_json["payload"].get("homes", []) | ||
buildings_list = response_json["payload"].get("buildings", {}).values() | ||
|
||
homes = [self._parse_home(home) for home in homes_list] + [ | ||
self._parse_building(building) for building in buildings_list | ||
] | ||
return homes | ||
else: | ||
return [] |
Oops, something went wrong.