Stack overflow (max recursion depth) on client timeout

[robsdedude]

My server looks something like this

ws_clients = 0

@socketio.on('connect')
def ws_connect():
    global ws_clients
    ws_clients += 1
    ws_user_count_update()

@socketio.on('disconnect')
def ws_disconnect():
    global ws_clients
    ws_clients -= 1
    ws_user_count_update()

def ws_user_count_update():
    socketio.emit('user_count', json.dumps(ws_clients + db.user_count()))

Now when a client times out and ws_user_count_update is called (e.g. by an ordinary disconnecting client) before the server notices the timeout an infinite recursion is started. Easiest way to archive this is by stopping the server somewhere in ws_disconnect for a while.

Traceback (most recent call last):
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/eventlet/wsgi.py", line 481, in handle_one_response
    result = self.application(self.environ, start_response)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__
    return self.wsgi_app(environ, start_response)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 41, in __call__
    start_response)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/middleware.py", line 47, in __call__
    return self.engineio_app.handle_request(environ, start_response)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 334, in handle_request
    return self.eio.handle_request(environ, start_response)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 230, in handle_request
    transport, b64)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 328, in _handle_connect
    if self._trigger_event('connect', sid, environ, async=False) is False:
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 357, in _trigger_event
    return self.handlers[event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 472, in _handle_eio_connect
    return self._handle_connect(sid, '/')
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 395, in _handle_connect
    self.environ[sid]) is False:
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 462, in _trigger_event
    return self.handlers[namespace][event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 213, in _handler
    *args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 572, in _handle_event
    ret = handler()
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 503, in ws_connect
    ws_user_count_update()
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 514, in ws_user_count_update
    socketio.emit('user_count', json.dumps(ws_clients + db.user_count()))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 330, in emit
    callback=kwargs.get('callback'))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 207, in emit
    self.manager.emit(event, data, namespace, room, skip_sid, callback)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/base_manager.py", line 127, in emit
    self.server._emit_internal(sid, event, data, namespace, id)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 377, in _emit_internal
    binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 388, in _send_packet
    self.eio.send(sid, encoded_packet, binary=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 171, in send
    socket.send(packet.Packet(packet.MESSAGE, data=data, binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 62, in send
    self.close(wait=False, abort=True)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 102, in close
    self.server._trigger_event('disconnect', self.sid, async=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 357, in _trigger_event
    return self.handlers[event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 504, in _handle_eio_disconnect
    self._handle_disconnect(sid, '/')
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 416, in _handle_disconnect
    self._trigger_event('disconnect', '/', sid)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 462, in _trigger_event
    return self.handlers[namespace][event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 213, in _handler
    *args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 574, in _handle_event
    ret = handler(*args)
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 510, in ws_disconnect
    ws_user_count_update()
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 514, in ws_user_count_update
    socketio.emit('user_count', json.dumps(ws_clients + db.user_count()))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 330, in emit
    callback=kwargs.get('callback'))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 207, in emit
    self.manager.emit(event, data, namespace, room, skip_sid, callback)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/base_manager.py", line 127, in emit
    self.server._emit_internal(sid, event, data, namespace, id)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 377, in _emit_internal
    binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 388, in _send_packet
    self.eio.send(sid, encoded_packet, binary=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 171, in send
    socket.send(packet.Packet(packet.MESSAGE, data=data, binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 62, in send
    self.close(wait=False, abort=True)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 102, in close
    self.server._trigger_event('disconnect', self.sid, async=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 357, in _trigger_event
    return self.handlers[event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 504, in _handle_eio_disconnect
    self._handle_disconnect(sid, '/')
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 416, in _handle_disconnect
    self._trigger_event('disconnect', '/', sid)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 462, in _trigger_event
    return self.handlers[namespace][event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 213, in _handler
    *args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 574, in _handle_event
    ret = handler(*args)
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 510, in ws_disconnect
    ws_user_count_update()
  File "/home/rouven/D_Data/DFKI/tags/server.py", line 514, in ws_user_count_update
    socketio.emit('user_count', json.dumps(ws_clients + db.user_count()))

...

  File "/home/rouven/D_Data/DFKI/tags/server.py", line 514, in ws_user_count_update
    socketio.emit('user_count', json.dumps(ws_clients + db.user_count()))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 330, in emit
    callback=kwargs.get('callback'))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 207, in emit
    self.manager.emit(event, data, namespace, room, skip_sid, callback)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/base_manager.py", line 127, in emit
    self.server._emit_internal(sid, event, data, namespace, id)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 377, in _emit_internal
    binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 388, in _send_packet
    self.eio.send(sid, encoded_packet, binary=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 171, in send
    socket.send(packet.Packet(packet.MESSAGE, data=data, binary=binary))
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 62, in send
    self.close(wait=False, abort=True)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/socket.py", line 102, in close
    self.server._trigger_event('disconnect', self.sid, async=False)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/engineio/server.py", line 357, in _trigger_event
    return self.handlers[event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 504, in _handle_eio_disconnect
    self._handle_disconnect(sid, '/')
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 416, in _handle_disconnect
    self._trigger_event('disconnect', '/', sid)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/socketio/server.py", line 462, in _trigger_event
    return self.handlers[namespace][event](*args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 213, in _handler
    *args)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask_socketio/__init__.py", line 562, in _handle_event
    with app.request_context(self.server.environ[sid]):
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask/app.py", line 1773, in request_context
    return RequestContext(self, environ)
  File "/home/rouven/D_Data/DFKI/tags/venv/local/lib/python2.7/site-packages/flask/ctx.py", line 222, in __init__
    request = app.request_class(environ)
RuntimeError: maximum recursion depth exceeded while calling a Python object

I assume that the issue lies within engineio/socket.py:Socket.close where

    self.server._trigger_event('disconnect', self.sid, async=False)

is called before flagging the socket as closed.

    self.closed = True

[miguelgrinberg]

Yes, I see the problem. I need to think about this, because there are some use cases in which it is required that the disconnected client is still available during the disconnect handler (for example, to check what rooms that client was in).

[robsdedude]

Easiest solution that comes to my mind is introducing a before_disconnect and an after_disconnect event. Even though this is a not really socketIO-ish way to go...

[miguelgrinberg]

@roba91 I was thinking more in splitting the closed state into closing (acts as if closed, but preserves all data structures such as rooms, etc., and does not trigger disconnect events) and closed (proper closed, cleans everything up, triggers disconnect events).

[Nixellion]

I'll +1 to the issue, just stumbled upon this problem myself. I'm using socketio and flask to write home assisting AI. And this errors always happens if someone connects from the mobile phone over the unstable network connection, or if the phone's browser is in the background. So I'd say it reliably brings the server down.

I hope this will be resolved soon ;)

[miguelgrinberg]

I think this is fixed in release 1.1.0 of package python-engineio. Please let me know if this is still a problem.

[robsdedude]

Indeed this seems to be fixed. Thanks for your great work! Keep it up =)