refactor: move proxy to session

Bunsly · Sep 19, 2023 · 1f0feb8 · 1f0feb8
1 parent 5f31bed
commit 1f0feb8
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 11 deletions.
diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py
@@ -16,7 +16,7 @@ def __init__(self, scraper_input: ScraperInput):
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
 
-        self.session = requests.Session()
+        self.session = requests.Session(proxies=scraper_input.proxy)
         self.listing_type = scraper_input.listing_type
         self.site_name = scraper_input.site_name
 

diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py
@@ -39,7 +39,6 @@ def handle_location(self):
             "https://parser-external.geo.moveaws.com/suggest",
             params=params,
             headers=headers,
-            proxies=self.proxy,
         )
         response_json = response.json()
 
@@ -105,7 +104,7 @@ def handle_address(self, property_id: str) -> list[Property]:
             "variables": variables,
         }
 
-        response = self.session.post(self.search_url, json=payload, proxies=self.proxy)
+        response = self.session.post(self.search_url, json=payload)
         response_json = response.json()
 
         property_info = response_json["data"]["property"]
@@ -218,7 +217,7 @@ def handle_area(
             "variables": variables,
         }
 
-        response = self.session.post(self.search_url, json=payload, proxies=self.proxy)
+        response = self.session.post(self.search_url, json=payload)
         response.raise_for_status()
         response_json = response.json()
 

diff --git a/homeharvest/core/scrapers/redfin/__init__.py b/homeharvest/core/scrapers/redfin/__init__.py
@@ -16,7 +16,7 @@ def _handle_location(self):
             self.location
         )
 
-        response = self.session.get(url, proxies=self.proxy)
+        response = self.session.get(url)
         response_json = json.loads(response.text.replace("{}&&", ""))
 
         def get_region_type(match_type: str):
@@ -111,7 +111,7 @@ def get_value(key: str) -> Any | None:
     def _handle_rentals(self, region_id, region_type):
         url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true&region_id={region_id}&region_type={region_type}&num_homes=100000"
 
-        response = self.session.get(url, proxies=self.proxy)
+        response = self.session.get(url)
         response.raise_for_status()
         homes = response.json()
 
@@ -211,7 +211,7 @@ def handle_address(self, home_id: str):
             home_id
         )
 
-        response = self.session.get(url, proxies=self.proxy)
+        response = self.session.get(url)
         response_json = json.loads(response.text.replace("{}&&", ""))
 
         parsed_home = self._parse_home(
@@ -233,7 +233,7 @@ def search(self):
                 url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&num_homes=100000"
             else:
                 url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&sold_within_days=30&num_homes=100000"
-            response = self.session.get(url, proxies=self.proxy)
+            response = self.session.get(url)
             response_json = json.loads(response.text.replace("{}&&", ""))
             homes = [
                 self._parse_home(home) for home in response_json["payload"]["homes"]

diff --git a/homeharvest/core/scrapers/zillow/__init__.py b/homeharvest/core/scrapers/zillow/__init__.py
@@ -26,13 +26,13 @@ def is_plausible_location(self, location: str) -> bool:
             "}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render"
         ).format(location)
 
-        response = self.session.get(url, proxies=self.proxy)
+        response = self.session.get(url)
 
         return response.json()["results"] != []
 
     def search(self):
         resp = self.session.get(
-            self.url, headers=self._get_headers(), proxies=self.proxy
+            self.url, headers=self._get_headers()
         )
         resp.raise_for_status()
         content = resp.text
@@ -131,7 +131,7 @@ def _fetch_properties_backend(self, coords):
             "isDebugRequest": False,
         }
         resp = self.session.put(
-            url, headers=self._get_headers(), json=payload, proxies=self.proxy
+            url, headers=self._get_headers(), json=payload
         )
         resp.raise_for_status()
         a = resp.json()