Merge pull request #199 from dod-advana/anthony/slowdown_crawls

implemented new crawler settings and add oneoff placeholder
dod-advana · Oct 12, 2023 · a7affa8 · a7affa8
2 parents aec1397 + 147f9fe
commit a7affa8
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/dataPipelines/gc_scrapy/gc_scrapy/runspider_settings.py b/dataPipelines/gc_scrapy/gc_scrapy/runspider_settings.py
@@ -16,8 +16,14 @@
     "ROBOTSTXT_OBEY": False,
     "LOG_LEVEL": "INFO",
     "DOWNLOAD_FAIL_ON_DATALOSS": False,
+
+    # Slow down crawler
+    "DOWNLOAD_DELAY": 0.1,  # Delay between requests in seconds
+    "DOWNLOAD_TIMEOUT": 3.5,  # Time till skip
+    "RETRY_ENABLE": True,
+    "RETRY_TIMES": 2,
+    "CONCURRENT_REQUESTS": 10,
 }
-
 selenium_settings = {
     "SELENIUM_DRIVER_NAME": "chrome",
     "SELENIUM_DRIVER_EXECUTABLE_PATH": "/usr/local/bin/chromedriver",
@@ -32,7 +38,6 @@
     ],
     "DOWNLOADER_MIDDLEWARES": {
         **general_settings["DOWNLOADER_MIDDLEWARES"],
-        #                                                                              make sure the values are not clashing
         "dataPipelines.gc_scrapy.gc_scrapy.downloader_middlewares.SeleniumMiddleware": max(
             general_settings["DOWNLOADER_MIDDLEWARES"].values()
         )

diff --git a/paasJobs/crawler_schedule/oneoff.txt b/paasJobs/crawler_schedule/oneoff.txt