Merge pull request #2857 from redapple/scrapy-components-logs-startup

Log versions information at startup

scrapy/commands/version.py

@@ -1,12 +1,8 @@
 from __future__ import print_function
-import sys
-import platform
-
-import twisted
-import OpenSSL
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.utils.versions import scrapy_components_versions
 
 
 class Command(ScrapyCommand):
@@ -27,38 +23,8 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         if opts.verbose:
-            import cssselect
+            for name, version in scrapy_components_versions():
-            import parsel
+                print("%-9s : %s" % (name, version))
-            import lxml.etree
-            import w3lib
-
-            lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
-            libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
-            try:
-                w3lib_version = w3lib.__version__
-            except AttributeError:
-                w3lib_version = "<1.14.3"
-
-            print("Scrapy    : %s" % scrapy.__version__)
-            print("lxml      : %s" % lxml_version)
-            print("libxml2   : %s" % libxml2_version)
-            print("cssselect : %s" % cssselect.__version__)
-            print("parsel    : %s" % parsel.__version__)
-            print("w3lib     : %s" % w3lib_version)
-            print("Twisted   : %s" % twisted.version.short())
-            print("Python    : %s" % sys.version.replace("\n", "- "))
-            print("pyOpenSSL : %s" % self._get_openssl_version())
-            print("Platform  : %s" % platform.platform())
         else:
             print("Scrapy %s" % scrapy.__version__)
 
-    def _get_openssl_version(self):
-        try:
-            openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
-                .decode('ascii', errors='replace')
-        # pyOpenSSL 0.12 does not expose openssl version
-        except AttributeError:
-            openssl = 'Unknown OpenSSL version'
-
-        return '{} ({})'.format(OpenSSL.version.__version__, openssl)

scrapy/utils/log.py

@@ -11,6 +11,8 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import overridden_settings, Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.versions import scrapy_components_versions
+
 
 logger = logging.getLogger(__name__)
 
@@ -142,7 +144,10 @@ def _get_handler(settings):
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
-
+    logger.info("Versions: %(versions)s}",
+                {'versions': ", ".join("%s %s" % (name, version)
+                    for name, version in scrapy_components_versions()
+                    if name != "Scrapy")})
     d = dict(overridden_settings(settings))
     logger.info("Overridden settings: %(settings)r", {'settings': d})
 

scrapy/utils/versions.py

@@ -0,0 +1,44 @@
+import platform
+import sys
+
+import cssselect
+import lxml.etree
+import parsel
+import twisted
+import w3lib
+
+import scrapy
+
+
+def scrapy_components_versions():
+    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
+    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
+    try:
+        w3lib_version = w3lib.__version__
+    except AttributeError:
+        w3lib_version = "<1.14.3"
+
+    return [
+        ("Scrapy", scrapy.__version__),
+        ("lxml", lxml_version),
+        ("libxml2", libxml2_version),
+        ("cssselect", cssselect.__version__),
+        ("parsel", parsel.__version__),
+        ("w3lib", w3lib_version),
+        ("Twisted", twisted.version.short()),
+        ("Python", sys.version.replace("\n", "- ")),
+        ("pyOpenSSL", _get_openssl_version()),
+        ("Platform",  platform.platform()),
+    ]
+
+
+def _get_openssl_version():
+    try:
+        import OpenSSL
+        openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
+            .decode('ascii', errors='replace')
+    # pyOpenSSL 0.12 does not expose openssl version
+    except AttributeError:
+        openssl = 'Unknown OpenSSL version'
+
+    return '{} ({})'.format(OpenSSL.version.__version__, openssl)