simpledb collector: moved to_sdb_value function to utils.simpledb, and added unittests

scrapy/stats/collector/simpledb.py

@@ -4,16 +4,14 @@ A Stats collector for persisting stats to Amazon SimpleDB.
 Requires the boto library: http://code.google.com/p/boto/
 """
 
-from datetime import datetime
-
 from boto import connect_sdb
 from twisted.internet import threads
 
+from scrapy.utils.simpledb import to_sdb_value
 from scrapy.stats.collector import StatsCollector
 from scrapy import log
 from scrapy.conf import settings
 
-
 class SimpledbStatsCollector(StatsCollector):
 
     def __init__(self):
@@ -45,17 +43,9 @@ class SimpledbStatsCollector(StatsCollector):
     def _get_timestamp(self, domain):
         return datetime.utcnow()
 
-    def _to_sdb_value(self, obj, ref=None):
-        if isinstance(obj, bool):
-            return u'%d' % obj
-        elif isinstance(obj, (int, long)):
-            return "%016d" % obj
-        elif isinstance(obj, datetime):
-            return obj.isoformat()
-        elif isinstance(obj, basestring):
-            return obj
-        elif obj is None:
-            return u''
-        else:
-            raise TypeError("%s unsupported type '%s' referenced as '%s'" % \
-                (type(self).__name__, type(obj).__name__, ref))
+    def _to_sdb_value(self, obj, key=None):
+        try:
+            return to_sdb_value(obj)
+        except TypeError:
+            raise TypeError("%s unsupported type %r used in key %r" % \
+                (type(self).__name__, type(obj).__name__, key))

scrapy/tests/test_utils_simpledb.py

@@ -0,0 +1,21 @@
+import unittest
+from datetime import datetime
+
+from scrapy.utils.simpledb import to_sdb_value
+
+class SimpleddbUtilsTest(unittest.TestCase):
+
+    def test_to_sdb_value(self):
+        self.assertEqual(to_sdb_value(123), u'0000000000000123')
+        self.assertEqual(to_sdb_value(123L), u'0000000000000123')
+        self.assertEqual(to_sdb_value(True), u'1')
+        self.assertEqual(to_sdb_value(False), u'0')
+        self.assertEqual(to_sdb_value(None), u'')
+        self.assertEqual(to_sdb_value(datetime(2009, 01, 01, 10, 10, 10)), \
+            u'2009-01-01T10:10:10')
+        self.assertEqual(to_sdb_value('test'), 'test')
+        self.assertEqual(to_sdb_value(u'test'), u'test')
+        self.assertRaises(TypeError, to_sdb_value, object())
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/utils/simpledb.py

@@ -0,0 +1,19 @@
+"""Helper functions for Amazon SimpleDB"""
+
+from datetime import datetime
+
+def to_sdb_value(obj):
+    """Convert the given object to proper value to store in Amazon SimpleDB"""
+    if isinstance(obj, bool):
+        return u'%d' % obj
+    elif isinstance(obj, (int, long)):
+        return "%016d" % obj
+    elif isinstance(obj, datetime):
+        return obj.isoformat()
+    elif isinstance(obj, basestring):
+        return obj
+    elif obj is None:
+        return u''
+    else:
+        raise TypeError("Unsupported Type: %s" % type(obj).__name__)
+