Homework 5: Recommender System for del.icio.us

Due: Sunday May 18th at 5pm. Send to email removed

Note: This homework should be done individually, but feel free to discuss questions about Python programming.

• heavier weights for users who have more similar tagging history to yours.
• lower weights for users who have not been using del.icio.us for some time;

• relative to the user having tagged weigend.com (i.e., to you want to weight URLs that came afterwards higher than those that came before?), and
• relative to the current time (should a URL tagged yesterday be weighted more than a URL tagged a year ago?)

    # go to your homework directory
    > curl "http://effbot.org/media/downloads/elementtree-1.2.6-20050316.zip" > elementtree-1.2.6-20050316.zip
    > unzip elementtree-1.2.6-20050316.zip
    > cd elementtree-1.2.6-20050316/
    > python setup.py build;
    > python setup.py install;

Playing Nice with Delicious

# The first attempt:
#   Try to play nice with Delicious by using a memoizing decorator and a
#   rate limiting decorator to follow their rules.
#
# The hack:
#   While the Delicious API says that you should wait at least 1 second, it
#   will still throttle/ban you if you do a posts/all type call that often.
#   To handle the large amount of data that need to be gathered and possibly
#   from different hosts, we implement a SQLite based cache.
#
 
from datetime import datetime, timedelta
import time
import pydelicious as deli
import sqlite3 as sqlite
import pickle
 
class CacheMissError(Exception):
    pass
 
class DBCache(object):
    def __init__(self, dbfile, debug = False):
        self.__state = {}
        self.__connection = sqlite.connect(dbfile)
        self.__connection.text_factory = str
        self.__cursor = self.__connection.cursor()
        self.__debug = debug
        try:
            self.__cursor.execute('CREATE TABLE `cache` (id INTEGER PRIMARY KEY, `func` varchar(255), `args` blob, `result` blob);');
        except:
            pass
 
    def save(self, func, args, result):
        self.__cursor.execute('INSERT INTO `cache` (`func`, `args`, `result`) VALUES (?, ?, ?)', (func, pickle.dumps(args), pickle.dumps(result)))
        self.__connection.commit()
        if self.__debug:
            print 'Saving %s(%s) -> %s!' % (func, args, result)
 
    def load(self, func, args):
        self.__cursor.execute('SELECT `result` FROM `cache` WHERE `func` = ? AND `args` = ?;', (func, pickle.dumps(args)))
        result = self.__cursor.fetchone()
        if result:
            value = pickle.loads(result[0])
            if self.__debug:
                print 'Loaded %s(%s) -> %s!' % (func, args, value)
            return value
        else:
            raise CacheMissError
 
class memoized(object):
    """Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """
    def __init__(self, func):
        self.func = func
        self.cache = {}
    def __call__(self, *args):
        try:
            return self.cache[args]
        except KeyError:
            self.cache[args] = value = self.func(*args)
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)
 
class cached(object):
    """Use DBCache to cache
    """
    def __init__(self, cache):
        self.cache = cache
 
    def __call__(self, f):
        def wrapper(*args):
            try:
                return self.cache.load(self.func.__name__, args)
            except CacheMissError:
                value = self.func(*args)
                self.cache.save(self.func.__name__, args, value)
                return value
            except TypeError:
                # uncachable -- for instance, passing a list as an argument.
                # Better to not cache than to blow up entirely.
                return self.func(*args)
 
        self.func = f
        wrapper.__name__ = f.__name__
        wrapper.__dict__.update(f.__dict__)
        wrapper.__doc__ = f.__doc__
        return wrapper
 
class rate_limited(object):
    """Decorator that limits the rate for a given resource.  Wait at least
    min_delay in seconds before consecutive calls to resource"""
    def __init__(self, resource, min_delay, debug = False):
        self.resource = resource
        self.min_delay = timedelta(seconds=min_delay)
        self.last_run = None
        self.debug = debug
 
    def __call__(self, f):
        def wrapper(*args, **kw):
            if self.last_run:
                now = datetime.now()
                delta = self.min_delay - (now - self.last_run)
                if self.debug:
                    print 'Now %s : Last %s' % (now, self.last_run)
                if delta > timedelta():
                    if self.debug:
                        print 'Sleeping for %s' % (delta.seconds + float(delta.microseconds) / 1000000)
                    time.sleep(delta.seconds + float(delta.microseconds) / 1000000)
            self.last_run = datetime.now()
            return self.func(*args, **kw)
 
        self.func = f
        wrapper.__name__ = f.__name__
        wrapper.__dict__.update(f.__dict__)
        wrapper.__doc__ = f.__doc__
        return wrapper
 
rate = 5
debug = False
cache = DBCache('tasty.db', debug)
 
@cached(cache)
@rate_limited('delicious', rate, debug)
def get_userposts(user):
    return deli.get_userposts(user)
 
@cached(cache)
@rate_limited('delicious', rate, debug)
def get_urlposts(url):
    return deli.get_urlposts(url)
 
def add_list_to_set(aset, alist):
    [aset.add(x) for x in alist]
 
def main():
    target_user = u'aweigend'
    user_set = set()
    url_set = set()
 
    get_userposts(target_user)