#!/bin/bash # # This script is a quick system test for Scrapyd that: # # 1. runs scrapyd # 2. creates a new project and deploys it on scrapyd # 3. schedules a spider on scrapyd and waits for it to finish # 4. check the spider scraped the expected data # set -e export PATH=$PATH:$(pwd)/bin export PYTHONPATH=$PYTHONPATH:$(pwd) scrapyd_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d) scrapyd_log=$scrapyd_dir/scrapyd.log scrapy_dir=$(mktemp /tmp/test-scrapy.XXXXXXX -d) echo "scrapyd dir: $scrapyd_dir" echo "scrapy dir : $scrapy_dir" twistd -ny extras/scrapyd.tac -d $scrapyd_dir -l $scrapyd_log & cd $scrapy_dir scrapy startproject testproj cd testproj cat > testproj/spiders/insophia.py < scrapy.cfg <