mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-26 16:03:49 +00:00
048044c1f8
* improved detection of inside-project environments * make list command faster (by only instantiating the spider manger) * print a warning when extensions (middlewares, etc) are disabled with a message on NotConfigured exception * assert that scrapy configuration hasn't been loaded in scrapyd.runner * simplified IgnoreRequest exception, to avoid loading settings when importing scrapy.exceptions * added test to make sure certain modules don't cause scrapy.conf module to be loaded, to ensure the scrapyd runner bootstraping performs properly
42 lines
1.0 KiB
Python
42 lines
1.0 KiB
Python
from __future__ import with_statement
|
|
|
|
import sys
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from contextlib import contextmanager
|
|
|
|
from scrapyd import get_application
|
|
from scrapyd.interfaces import IEggStorage
|
|
from scrapyd.eggutils import activate_egg
|
|
|
|
@contextmanager
|
|
def project_environment(project):
|
|
app = get_application()
|
|
eggstorage = app.getComponent(IEggStorage)
|
|
version, eggfile = eggstorage.get(project)
|
|
if eggfile:
|
|
prefix = '%s-%s-' % (project, version)
|
|
fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')
|
|
lf = os.fdopen(fd, 'wb')
|
|
shutil.copyfileobj(eggfile, lf)
|
|
lf.close()
|
|
activate_egg(eggpath)
|
|
else:
|
|
eggpath = None
|
|
try:
|
|
assert 'scrapy.conf' not in sys.modules, "Scrapy settings already loaded"
|
|
yield
|
|
finally:
|
|
if eggpath:
|
|
os.remove(eggpath)
|
|
|
|
def main():
|
|
project = os.environ['SCRAPY_PROJECT']
|
|
with project_environment(project):
|
|
from scrapy.cmdline import execute
|
|
execute()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|