mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-27 15:24:12 +00:00
31 lines
914 B
Python
31 lines
914 B
Python
import os
|
|
from ConfigParser import NoSectionError
|
|
|
|
from scrapy.spiderqueue import SqliteSpiderQueue
|
|
|
|
def get_spider_queues(config):
|
|
"""Return a dict of Spider Quees keyed by project name"""
|
|
dbsdir = config.get('dbs_dir', 'dbs')
|
|
if not os.path.exists(dbsdir):
|
|
os.makedirs(dbsdir)
|
|
d = {}
|
|
for project in get_project_list(config):
|
|
dbpath = os.path.join(dbsdir, '%s.db' % project)
|
|
d[project] = SqliteSpiderQueue(dbpath)
|
|
return d
|
|
|
|
def get_project_list(config):
|
|
"""Get list of projects by inspecting the eggs dir and the ones defined in
|
|
the scrapyd.conf [settings] section
|
|
"""
|
|
eggs_dir = config.get('eggs_dir', 'eggs')
|
|
if os.path.exists(eggs_dir):
|
|
projects = os.listdir(eggs_dir)
|
|
else:
|
|
projects = []
|
|
try:
|
|
projects += [x[0] for x in config.cp.items('settings')]
|
|
except NoSectionError:
|
|
pass
|
|
return projects
|