Feature - Add support for scrapy

README.rst

@@ -173,6 +173,21 @@ Attaching raw HTTP request data
 
 If you are in a web server environment and have HTTP request details available, you can pass these and the headers through in a dictionary (see :code:`sample.py`).
 
+Scrapy
+++++++
+
+To configure scrapy to automatically send all exceptions raised in spiders and item pipelines to Raygun:
+
+settings.py
+
+.. code:: python
+
+  RAYGUN_API_KEY = 'paste_your_api_key_here'
+
+  EXTENSIONS = {
+      'raygun4py.extension.scrapy.Provider': 400
+  }
+
 Code running on Google App Engine should now be supported - you can test this locally, and has been reported working once deployed (the latter currently requires a paid account due to needed SSL support).
 
 Documentation

python3/raygun4py/extension/scrapy.py

@@ -0,0 +1,29 @@
+from raygun4py import raygunprovider
+from scrapy import signals
+from scrapy.exceptions import NotConfigured
+
+
+class Provider(object):
+    def __init__(self, api_key):
+        self.sender = raygunprovider.RaygunSender(api_key)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        api_key = crawler.settings.get("RAYGUN_API_KEY")
+        if not api_key:
+            raise NotConfigured
+
+        extension = cls(api_key)
+        crawler.signals.connect(extension.spider_error, signals.spider_error)
+        crawler.signals.connect(extension.item_error, signals.item_error)
+
+        return extension
+
+    def _handle_exception(self, failure):
+        self.sender.send_exception(exc_info=(failure.type, failure.value, failure.tb))
+
+    def item_error(self, item, response, spider, failure):
+        self._handle_exception(failure)
+
+    def spider_error(self, failure, response, spider):
+        self._handle_exception(failure)

python3/tests/extension/test_scrapy.py

@@ -0,0 +1,51 @@
+import unittest
+
+from unittest.mock import MagicMock, call
+from twisted.python import failure
+from raygun4py.extension.scrapy import Provider
+from scrapy.http import Response
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from scrapy.item import Item
+from scrapy.signals import item_error, spider_error
+from scrapy.exceptions import NotConfigured
+
+
+class TestProvider(unittest.TestCase):
+    def _getDivisionFailure(self):
+        try:
+            1 / 0
+        except:
+            f = failure.Failure()
+        return f
+
+    def test__handler_exception_called(self):
+        provider = Provider(api_key="test")
+        crawler = get_crawler(Spider)
+        spider = crawler._create_spider("scrapytest.org")
+        item = Item()
+        response = Response("scrapytest.org", status=400)
+
+        provider._handle_exception = MagicMock()
+        failure = self._getDivisionFailure()
+
+        provider.item_error(item, response, spider, failure)
+
+        provider._handle_exception.assert_called_once_with(failure)
+
+    def test_from_crawler_not_configured(self):
+        crawler = get_crawler(Spider, settings_dict=None)
+        self.assertRaises(NotConfigured, Provider.from_crawler, crawler)
+
+    def test_from_crawler_configured(self):
+        crawler = get_crawler(Spider, settings_dict={"RAYGUN_API_KEY": "test"})
+        crawler.signals.connect = MagicMock()
+        provider = Provider.from_crawler(crawler)
+
+        crawler.signals.connect.assert_has_calls(
+            [
+                call(provider.spider_error, spider_error,),
+                call(provider.item_error, item_error,),
+            ],
+            any_order=True,
+        )

setup.py

@@ -1,7 +1,7 @@
 import sys
 from setuptools import setup
 
-packages = ['raygun4py', 'raygun4py.middleware']
+packages = ['raygun4py', 'raygun4py.middleware', 'raygun4py.extension']
 
 base_dir = 'python2'
 if sys.version_info[0] == 3:
@@ -20,7 +20,8 @@
     'mock >= 2.0.0',
     'django == 1.8.8',
     'flask >= 0.10',
-    'WebTest >= 2.0.32'
+    'WebTest >= 2.0.32',
+    'scrapy >= 1.8.0'
 ]
 
 setup(