PythonWise: June 2009

Friday, June 12, 2009

A Twitter Trends / Google News meshup

This tried to automatically explain why things are trending on Twitter.

The Python Server (trends.py)

#!/usr/bin/env python
'''A twitter trends/google news mesh

loader_thread get news trends every 1min and set _TRENDS_HTML
The web server serves _TRENDS_HTML with is refresed via JavaScript every 30sec
'''

from urllib import urlopen, urlencode
import feedparser
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from os.path import dirname, join, splitext
import json
from threading import Thread
from time import sleep

def ascii_clean(text):
    return text.encode("ascii", "ignore") # Pure ACII

def trend_news(trend):
    query = {
        "q" : ascii_clean(trend),
        "output" : "rss"
    }
    url = "http://news.google.com/news?" + urlencode(query)
    return feedparser.parse(url).entries

def current_trends():
    url = "http://search.twitter.com/trends.json"
    return json.load(urlopen(url))["trends"]

def news_html(news):
    html = '<li><a href="%(link)s">%(title)s</a></li>'
    chunks = map(lambda e: html % e, news)
    return "\n".join(["<ul>"] + chunks + ["</ul>"])

def trend_html(trend):
   thtml = '<a href="%(url)s">%(name)s' % trend
   nhtml = news_html(trend_news(trend["name"]))
   return '<tr><td>%s</td><td>%s</td><tr>' % (thtml, nhtml)

def table_html(trends):
    return ("<table>" + 
            "<tr><th>Trend</th><th>Related News</th></tr>" +
            "".join(map(trend_html, trends)) +
            "</table>"
           )

_TRENDS_HTML = ""
def loader_thread():
    global _TRENDS_HTML

    while 1:
        _TRENDS_HTML = ascii_clean(table_html(current_trends()))
        sleep(60)

def run_loader_thread():
    t = Thread(target=loader_thread)
    t.daemon = 1
    t.start()

def trends_html():
    while not _TRENDS_HTML:
        sleep(0.1)

    return _TRENDS_HTML

def index_html():
    return open(join(dirname(__file__), "index.html")).read()

class RequestHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/":
            self.wfile.write(index_html())
        elif self.path.startswith("/trends"):
            self.wfile.write(trends_html())
        elif splitext(self.path)[1] in (".js", ".css"):
            self.wfile.write(open(".%s" % self.path).read())
        else:
            self.send_error(404, "Not Found")

if __name__ == "__main__":
    run_loader_thread()
    server = HTTPServer(("", 8888), RequestHandler)
    server.serve_forever()

The Auto-Refreshing Web Client (index.html)

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
    <head>
        <title>Trends</title>
        <link rel="stylesheet" href="trends.css" />
        <style>
            body {
                font-family: Serif, Helvetica;
            }

            table {
                width: 100%;
                border: 2px solid gray;
            }

            td {
                vertical-align: top;
                border-bottom: 1px solid black;
            }

            h1 {
                text-align: center;
                font-variant: small-caps;
            }

            a {
                text-decoration: none;
                color: black;
            }

            a:hover {
                background: silver;
            }

            ul {
                margin: 0px;
            }
        </style>
    </head>
    <body>
        <h1>Trends</h1>
        <div id="trends">
            Loading ...
        </div>
    </body>
    <script src="jquery.js"></script>
    <script>
        function load() {
            $('#trends').load('/trends');
        }
        $(document).ready(function() {
            load();
            setInterval(load, 30 * 1000);
        });
    </script>
</html>

Wednesday, June 10, 2009

Show "digits only" version of phone number

#!/bin/bash
# Convert phone numbers to numeric (1-800-T-MOBILE -> 1-800-8-662453)

if [ $# -ne 1 ]; then
    echo "usage: $(basename $0) PHONE-NUMBER"
    exit 1
fi

echo $1 | \
    tr '[:lower:]' '[:upper:]' | \
    tr ABCDEFGHIJKLMNOPQRSTUVWXYZ 22233344455566677778889999

Saturday, June 06, 2009

JSON Pretiffier

#!/usr/bin/env python
'''JSON prettifier'''

def shift(array):
   try:
       return array.pop(0)
   except IndexError:
       return None

def fopen(name, mode, default):
   if name in (None, "-"):
       return default
   return open(name, mode)

def main(argv=None):
   import sys
   from optparse import OptionParser

   argv = argv or sys.argv

   indent = 4
   parser = OptionParser("%prog [INFILE [OUTFILE]]")
   parser.add_option("-i", "--indent", help="indent size (%s)" % indent,
                     dest="indent", default=indent, type="int")
   opts, args = parser.parse_args(argv[1:])
   if len(args) not in (0, 1, 2):
       parser.error("wrong number of arguments") # Will exit

   try:
       info = fopen(shift(args), "r", sys.stdin)
       outfo = fopen(shift(args), "w", sys.stdout)
   except IOError, e:
       raise SystemExit("error: %s" % e)

   import json
   json.dump(json.load(info), outfo, indent=opts.indent)
      
if __name__ == "__main__":
   main()

Update (Aug 14, 2009): Just found out about python -mjson.tool, oh well ...

Thursday, June 04, 2009

strftime for JavaScript

/* strftime for JavaScript 

   Field description (taken from http://tinyurl.com/65s2qw)

    %a  Locale’s abbreviated weekday name.   
    %A  Locale’s full weekday name.   
    %b  Locale’s abbreviated month name.   
    %B  Locale’s full month name.   
    %c  Locale’s appropriate date and time representation.   
    %d  Day of the month as a decimal number [01,31].   
    %H  Hour (24-hour clock) as a decimal number [00,23].   
    %I  Hour (12-hour clock) as a decimal number [01,12].   
    %j  Day of the year as a decimal number [001,366].   
    %m  Month as a decimal number [01,12].   
    %M  Minute as a decimal number [00,59].   
    %p  Locale’s equivalent of either AM or PM.
    %S  Second as a decimal number [00,61].
    %U  Week number of the year (Sunday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Sunday are considered to be in week 0.
    %w  Weekday as a decimal number [0(Sunday),6].   
    %W  Week number of the year (Monday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Monday are considered to be in week 0.
    %x  Locale’s appropriate date representation.   
    %X  Locale’s appropriate time representation.   
    %y  Year without century as a decimal number [00,99].   
    %Y  Year with century as a decimal number.   
    %Z  Time zone name (no characters if no time zone exists).   
    %%  A literal '%' character.
*/

var days = [ 
    'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
    'Saturday' 
];

var months = [
    'January', 'February', 'March', 'April', 'May', 'June', 'July',
    'August', 'September', 'October', 'November', 'December'
];

function shortname(name) {
    return name.substr(0, 3);
}

function zeropad(n, size) {
    n = '' + n; /* Make sure it's a string */
    size = size || 2;
    while (n.length < size) {
        n = '0' + n;
    }
    return n;
}

function twelve(n) {
    return (n <= 12) ? n : 24 - n;
}

function strftime(format, date) {
    date = date || new Date();
    var fields = {
        a: shortname(days[date.getDay()]),
        A: days[date.getDay()],
        b: shortname(months[date.getMonth()]),
        B: months[date.getMonth()],
        c: date.toString(),
        d: zeropad(date.getDate()),
        H: zeropad(date.getHours()),
        I: zeropad(twelve(date.getHours())),
        /* FIXME: j: */
        m: zeropad(date.getMonth() + 1),
        M: zeropad(date.getMinutes()),
        p: (date.getHours() >= 12) ? 'PM' : 'AM',
        S: zeropad(date.getSeconds()),
        w: zeropad(date.getDay() + 1),
        /* FIXME: W: */
        x: date.toLocaleDateString(),
        X: date.toLocaleTimeString(),
        y: ('' + date.getFullYear()).substr(2, 4),
        Y: '' + date.getFullYear(),
        /* FIXME: Z: */
        '%' : '%'
    };

    var result = '', i = 0;
    while (i < format.length) {
        if (format[i] === '%') {
            result = result + fields[format[i + 1]];
            ++i;
        }
        else {
            result = result + format[i];
        }
        ++i;
    }
    return result;
}

Quick search using xapian "omega"

A little hack to search current directory using Xapian's omega.

#!/bin/bash

# Minimal search using Xpians Omega (http://xapian.org/docs/omega/overview.html)
# Use "omsearch -i" to index current directoy, then use "omsearch QUERY" to
# search


index() {
    mkdir .omega
    cat >> .omega/omega.conf << EOF
    database_dir $PWD/.omega
    template_dir /var/lib/xapian-omega/templates
    log_dir /tmp
EOF
    omindex --db .omega/default .
}

search() {
    omega=/usr/lib/cgi-bin/omega/omega
    config=$PWD/.omega/omega.conf
    results=.omega/results.html
    OMEGA_CONFIG_FILE=$config $omega "P=$1" "HITSPERPAGE=1000" > $results
    egrep -o '<B><A HREF=".*"' $results | cut -d\" -f2 | \
        xargs -i printf ".%s\n" {}

}

USAGE="usage: $(basename $0) [-i | QUERY]"

index=no
while getopts "ih" opt
do
    case $opt in
        h | H ) echo $USAGE; exit;;
        i ) index=yes;;
        * ) echo "error: unknown option" >&2; exit 1;;
    esac
done

shift $(($OPTIND - 1))

if [ "$index" == "yes" ]; then
    if [ $# -gt 0 ]; then
        echo "error: -i don't take any parameters"
        exit 1
    fi
    index
    exit
fi

if [ $# -ne 1 ]; then
    echo $USAGE
    exit 1
fi

if [ ! -d .omega ]; then
    echo "error: can't find .omega, run with -i to index first"
    exit 1
fi
search $1

PythonWise