Dump stacks for all running threads and fix engine status dumped by StackTraceDump extension

scrapy/contrib/debug.py

@@ -1,12 +1,14 @@
 """
-Extensions for debugging Scrapy 
+Extensions for debugging Scrapy
 
 See documentation in docs/topics/extensions.rst
 """
 
 import os
+import sys
 import signal
 import traceback
+import threading
 from pdb import Pdb
 
 from scrapy.utils.engine import format_engine_status
@@ -14,7 +16,9 @@ from scrapy import log
 
 
 class StackTraceDump(object):
-    def __init__(self):
+
+    def __init__(self, crawler=None):
+        self.crawler = crawler
         try:
             signal.signal(signal.SIGUSR2, self.dump_stacktrace)
             signal.signal(signal.SIGQUIT, self.dump_stacktrace)
@@ -22,13 +26,27 @@ class StackTraceDump(object):
             # win32 platforms don't support SIGUSR signals
             pass
 
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
     def dump_stacktrace(self, signum, frame):
-        msg = "Dumping stack trace and engine status" + os.linesep
-        msg += "".join(traceback.format_stack(frame))
-        msg += os.linesep
-        msg += format_engine_status()
+        stackdumps = self._thread_stacks()
+        enginestatus = format_engine_status(self.crawler.engine)
+        msg = "Dumping stack trace and engine status" \
+            "\n{0}\n{1}".format(enginestatus, stackdumps)
         log.msg(msg)
 
+    def _thread_stacks(self):
+        id2name = dict((th.ident, th.name) for th in threading.enumerate())
+        dumps = ''
+        for id_, frame in sys._current_frames().items():
+            name = id2name.get(id_, '')
+            dump = ''.join(traceback.format_stack(frame))
+            dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
+        return dumps
+
 
 class Debugger(object):
     def __init__(self):