BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-26 12:43:48 +00:00
Code Releases Wiki Activity

added gzip support for logs

Browse Source 
--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40917
This commit is contained in:
samus_ 2009-02-25 05:48:01 +00:00
parent 82dc57e59b
commit de1b9e5ec9
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py
View File

@ -1,4 +1,5 @@
import sys, os, time, datetime, pickle
from __future__ import with_statement
import sys, os, time, datetime, pickle, gzip
from twisted.internet import protocol, reactor
from twisted.spread import pb
@ -38,6 +39,15 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
self.procman.update_master(self.domain, "running")
def processEnded(self, reason):
if settings.getbool('CLUSTER_WORKER_GZIP_LOGS'):
try:
with open(self.logfile) as f_in:
with gzip.open("%s.gz" % self.logfile, "wb") as f_out:
f_out.writelines(f_in)
os.remove(self.logfile)
self.logfile = "%s.gz" % self.logfile
except e:
log.msg("failed to compress %s exception=%s (domain=%s, pid=%s)" % (self.logfile, self.domain, self.pid))
log.msg("ClusterWorker: finished domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile))
log.msg("Reason type: %s. value: %s" % (reason.type, reason.value) )
del self.procman.running[self.domain]