diff --git a/plugins/janus_sip.c b/plugins/janus_sip.c
index 03d9be6807..dd5381efa0 100644
--- a/plugins/janus_sip.c
+++ b/plugins/janus_sip.c
@@ -2528,7 +2528,12 @@ static void janus_sip_hangup_media_internal(janus_plugin_session *handle) {
 		session->media.ready = FALSE;
 		session->media.on_hold = FALSE;
 		janus_sip_call_update_status(session, janus_sip_call_status_closing);
-		nua_bye(session->stack->s_nh_i, TAG_END());
+
+		if(g_atomic_int_get(&session->established))
+			nua_bye(session->stack->s_nh_i, TAG_END());
+		else
+			nua_respond(session->stack->s_nh_i, 480, sip_status_phrase(480), TAG_END());
+
 		/* Notify the operation */
 		json_t *event = json_object();
 		json_object_set_new(event, "sip", json_string("event"));