profiling: improve list+deque+cache performance

results:

 == With 1 priorities (/tmp/pq-13882-1-50000) ==

 pushpops = 50000, times = 30
 heapq implementation: 12.6956150532
 dict+deque implementation: 5.3080239296
 deque+heapq implementation: 3.11057305336
 deque+defaultdict+deque implementation: 3.06583619118
 list+deque implementation: 4.85028195381
 list+deque+cache implementation: 4.86092495918

 == With 3 priorities (/tmp/pq-13882-3-50000) ==

 pushpops = 50000, times = 30
 heapq implementation: 13.9048631191
 dict+deque implementation: 6.526501894
 deque+heapq implementation: 9.95749187469
 deque+defaultdict+deque implementation: 4.94318699837
 list+deque implementation: 5.48832702637
 list+deque+cache implementation: 4.77395009995

 == With 5 priorities (/tmp/pq-13882-5-50000) ==

 pushpops = 50000, times = 30
 heapq implementation: 14.1862449646
 dict+deque implementation: 7.45535206795
 deque+heapq implementation: 11.7175529003
 deque+defaultdict+deque implementation: 5.40972518921
 list+deque implementation: 5.87488412857
 list+deque+cache implementation: 4.73579287529

 == With 10 priorities (/tmp/pq-13882-10-50000) ==

 pushpops = 50000, times = 30
 heapq implementation: 14.2052979469
 dict+deque implementation: 9.94834208488
 deque+heapq implementation: 13.0460109711
 deque+defaultdict+deque implementation: 5.79300785065
 list+deque implementation: 6.9981739521
 list+deque+cache implementation: 4.81988596916

 == With 100 priorities (/tmp/pq-13882-100-50000) ==

 pushpops = 50000, times = 30
 heapq implementation: 14.9574189186
 dict+deque implementation: 55.6348400116
 deque+heapq implementation: 14.9515259266
 deque+defaultdict+deque implementation: 10.6776599884
 list+deque implementation: 25.9212520123
 list+deque+cache implementation: 4.77596998215

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40996

scrapy/trunk/profiling/priorityqueue/pq_classes.py

@@ -216,20 +216,19 @@ class PriorityQueue6(object):
         self.priolist = [deque() for _ in range(size)]
 
     def push(self, item, priority=0):
-        final = priority + self.zero
-        if self.index > final:
-            self.index = final
-        self.priolist[final].appendleft(item)
+        i = priority + self.zero
+        self.priolist[i].appendleft(item)
+        self.index = min(self.index, i)
 
     def pop(self):
         cached = self.priolist[self.index]
-        if len(cached):
+        if cached:
             return (cached.pop(), self.index - self.zero)
 
         for prio, queue in enumerate(self.priolist[self.index:]):
-            if len(queue):
-                final = prio + self.index - self.zero
-                return (queue.pop(), final)
+            if queue:
+                self.index += prio
+                return (queue.pop(), self.index - self.zero)
 
         raise IndexError("pop from an empty queue")