From de1b9e5ec9abb0ab38395d40ab3ae238dd6471da Mon Sep 17 00:00:00 2001 From: samus_ Date: Wed, 25 Feb 2009 05:48:01 +0000 Subject: [PATCH] added gzip support for logs --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40917 --- .../trunk/scrapy/contrib/pbcluster/worker/manager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py index 3a7956eb4..a30a0a2a3 100644 --- a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py @@ -1,4 +1,5 @@ -import sys, os, time, datetime, pickle +from __future__ import with_statement +import sys, os, time, datetime, pickle, gzip from twisted.internet import protocol, reactor from twisted.spread import pb @@ -38,6 +39,15 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.procman.update_master(self.domain, "running") def processEnded(self, reason): + if settings.getbool('CLUSTER_WORKER_GZIP_LOGS'): + try: + with open(self.logfile) as f_in: + with gzip.open("%s.gz" % self.logfile, "wb") as f_out: + f_out.writelines(f_in) + os.remove(self.logfile) + self.logfile = "%s.gz" % self.logfile + except e: + log.msg("failed to compress %s exception=%s (domain=%s, pid=%s)" % (self.logfile, self.domain, self.pid)) log.msg("ClusterWorker: finished domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile)) log.msg("Reason type: %s. value: %s" % (reason.type, reason.value) ) del self.procman.running[self.domain]