If it won't be simple, it simply won't be. [Hire me, source code] by Miki Tebeka, CEO, 353Solutions

Friday, June 12, 2009

A Twitter Trends / Google News meshup

This tried to automatically explain why things are trending on Twitter.

The Python Server (trends.py)


#!/usr/bin/env python
'''A twitter trends/google news mesh

loader_thread get news trends every 1min and set _TRENDS_HTML
The web server serves _TRENDS_HTML with is refresed via JavaScript every 30sec
'''

from urllib import urlopen, urlencode
import feedparser
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from os.path import dirname, join, splitext
import json
from threading import Thread
from time import sleep

def ascii_clean(text):
return text.encode("ascii", "ignore") # Pure ACII

def trend_news(trend):
query = {
"q" : ascii_clean(trend),
"output" : "rss"
}
url = "http://news.google.com/news?" + urlencode(query)
return feedparser.parse(url).entries

def current_trends():
url = "http://search.twitter.com/trends.json"
return json.load(urlopen(url))["trends"]

def news_html(news):
html = '<li><a href="%(link)s">%(title)s</a></li>'
chunks = map(lambda e: html % e, news)
return "\n".join(["<ul>"] + chunks + ["</ul>"])

def trend_html(trend):
thtml = '<a href="%(url)s">%(name)s' % trend
nhtml = news_html(trend_news(trend["name"]))
return '<tr><td>%s</td><td>%s</td><tr>' % (thtml, nhtml)

def table_html(trends):
return ("<table>" +
"<tr><th>Trend</th><th>Related News</th></tr>" +
"".join(map(trend_html, trends)) +
"</table>"
)

_TRENDS_HTML = ""
def loader_thread():
global _TRENDS_HTML

while 1:
_TRENDS_HTML = ascii_clean(table_html(current_trends()))
sleep(60)

def run_loader_thread():
t = Thread(target=loader_thread)
t.daemon = 1
t.start()

def trends_html():
while not _TRENDS_HTML:
sleep(0.1)

return _TRENDS_HTML

def index_html():
return open(join(dirname(__file__), "index.html")).read()

class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/":
self.wfile.write(index_html())
elif self.path.startswith("/trends"):
self.wfile.write(trends_html())
elif splitext(self.path)[1] in (".js", ".css"):
self.wfile.write(open(".%s" % self.path).read())
else:
self.send_error(404, "Not Found")

if __name__ == "__main__":
run_loader_thread()
server = HTTPServer(("", 8888), RequestHandler)
server.serve_forever()


The Auto-Refreshing Web Client (index.html)


<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>Trends</title>
<link rel="stylesheet" href="trends.css" />
<style>
body {
font-family: Serif, Helvetica;
}

table {
width: 100%;
border: 2px solid gray;
}

td {
vertical-align: top;
border-bottom: 1px solid black;
}

h1 {
text-align: center;
font-variant: small-caps;
}

a {
text-decoration: none;
color: black;
}

a:hover {
background: silver;
}

ul {
margin: 0px;
}
</style>
</head>
<body>
<h1>Trends</h1>
<div id="trends">
Loading ...
</div>
</body>
<script src="jquery.js"></script>
<script>
function load() {
$('#trends').load('/trends');
}
$(document).ready(function() {
load();
setInterval(load, 30 * 1000);
});
</script>
</html>

Wednesday, June 10, 2009

Show "digits only" version of phone number

#!/bin/bash
# Convert phone numbers to numeric (1-800-T-MOBILE -> 1-800-8-662453)

if [ $# -ne 1 ]; then
echo "usage: $(basename $0) PHONE-NUMBER"
exit 1
fi

echo $1 | \
tr '[:lower:]' '[:upper:]' | \
tr ABCDEFGHIJKLMNOPQRSTUVWXYZ 22233344455566677778889999

Saturday, June 06, 2009

JSON Pretiffier

#!/usr/bin/env python
'''JSON prettifier'''

def shift(array):
try:
return array.pop(0)
except IndexError:
return None

def fopen(name, mode, default):
if name in (None, "-"):
return default
return open(name, mode)

def main(argv=None):
import sys
from optparse import OptionParser

argv = argv or sys.argv

indent = 4
parser = OptionParser("%prog [INFILE [OUTFILE]]")
parser.add_option("-i", "--indent", help="indent size (%s)" % indent,
dest="indent", default=indent, type="int")
opts, args = parser.parse_args(argv[1:])
if len(args) not in (0, 1, 2):
parser.error("wrong number of arguments") # Will exit

try:
info = fopen(shift(args), "r", sys.stdin)
outfo = fopen(shift(args), "w", sys.stdout)
except IOError, e:
raise SystemExit("error: %s" % e)

import json
json.dump(json.load(info), outfo, indent=opts.indent)

if __name__ == "__main__":
main()


Update (Aug 14, 2009): Just found out about python -mjson.tool, oh well ...

Thursday, June 04, 2009

strftime for JavaScript

/* strftime for JavaScript 

   Field description (taken from http://tinyurl.com/65s2qw)

    %a  Locale’s abbreviated weekday name.   
    %A  Locale’s full weekday name.   
    %b  Locale’s abbreviated month name.   
    %B  Locale’s full month name.   
    %c  Locale’s appropriate date and time representation.   
    %d  Day of the month as a decimal number [01,31].   
    %H  Hour (24-hour clock) as a decimal number [00,23].   
    %I  Hour (12-hour clock) as a decimal number [01,12].   
    %j  Day of the year as a decimal number [001,366].   
    %m  Month as a decimal number [01,12].   
    %M  Minute as a decimal number [00,59].   
    %p  Locale’s equivalent of either AM or PM.
    %S  Second as a decimal number [00,61].
    %U  Week number of the year (Sunday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Sunday are considered to be in week 0.
    %w  Weekday as a decimal number [0(Sunday),6].   
    %W  Week number of the year (Monday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Monday are considered to be in week 0.
    %x  Locale’s appropriate date representation.   
    %X  Locale’s appropriate time representation.   
    %y  Year without century as a decimal number [00,99].   
    %Y  Year with century as a decimal number.   
    %Z  Time zone name (no characters if no time zone exists).   
    %%  A literal '%' character.
*/

var days = [ 
    'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
    'Saturday' 
];

var months = [
    'January', 'February', 'March', 'April', 'May', 'June', 'July',
    'August', 'September', 'October', 'November', 'December'
];

function shortname(name) {
    return name.substr(0, 3);
}

function zeropad(n, size) {
    n = '' + n; /* Make sure it's a string */
    size = size || 2;
    while (n.length < size) {
        n = '0' + n;
    }
    return n;
}

function twelve(n) {
    return (n <= 12) ? n : 24 - n;
}

function strftime(format, date) {
    date = date || new Date();
    var fields = {
        a: shortname(days[date.getDay()]),
        A: days[date.getDay()],
        b: shortname(months[date.getMonth()]),
        B: months[date.getMonth()],
        c: date.toString(),
        d: zeropad(date.getDate()),
        H: zeropad(date.getHours()),
        I: zeropad(twelve(date.getHours())),
        /* FIXME: j: */
        m: zeropad(date.getMonth() + 1),
        M: zeropad(date.getMinutes()),
        p: (date.getHours() >= 12) ? 'PM' : 'AM',
        S: zeropad(date.getSeconds()),
        w: zeropad(date.getDay() + 1),
        /* FIXME: W: */
        x: date.toLocaleDateString(),
        X: date.toLocaleTimeString(),
        y: ('' + date.getFullYear()).substr(2, 4),
        Y: '' + date.getFullYear(),
        /* FIXME: Z: */
        '%' : '%'
    };

    var result = '', i = 0;
    while (i < format.length) {
        if (format[i] === '%') {
            result = result + fields[format[i + 1]];
            ++i;
        }
        else {
            result = result + format[i];
        }
        ++i;
    }
    return result;
}

Quick search using xapian "omega"

A little hack to search current directory using Xapian's omega.

#!/bin/bash

# Minimal search using Xpians Omega (http://xapian.org/docs/omega/overview.html)
# Use "omsearch -i" to index current directoy, then use "omsearch QUERY" to
# search


index() {
mkdir .omega
cat >> .omega/omega.conf << EOF
database_dir $PWD/.omega
template_dir /var/lib/xapian-omega/templates
log_dir /tmp
EOF
omindex --db .omega/default .
}

search() {
omega=/usr/lib/cgi-bin/omega/omega
config=$PWD/.omega/omega.conf
results=.omega/results.html
OMEGA_CONFIG_FILE=$config $omega "P=$1" "HITSPERPAGE=1000" > $results
egrep -o '<B><A HREF=".*"' $results | cut -d\" -f2 | \
xargs -i printf ".%s\n" {}

}

USAGE="usage: $(basename $0) [-i | QUERY]"

index=no
while getopts "ih" opt
do
case $opt in
h | H ) echo $USAGE; exit;;
i ) index=yes;;
* ) echo "error: unknown option" >&2; exit 1;;
esac
done

shift $(($OPTIND - 1))

if [ "$index" == "yes" ]; then
if [ $# -gt 0 ]; then
echo "error: -i don't take any parameters"
exit 1
fi
index
exit
fi

if [ $# -ne 1 ]; then
echo $USAGE
exit 1
fi

if [ ! -d .omega ]; then
echo "error: can't find .omega, run with -i to index first"
exit 1
fi
search $1

Monday, May 11, 2009

Avoiding "peek" with "itertools.chain"

Sometimes you need to filter out some header information. However the only clear anchor you have is the start of real data - you need a "peek" function.

It's easy to avoid writing the "peek" logic, just use itertools.chain.

#!/usr/bin/env python
'''Avoiding the need for "peek" with itertools.chain'''

from itertools import chain

def is_first_data(line):
return line.startswith("Name:")

def skip_header(data):
data = iter(data)
for line in data:
if is_first_data(line):
return chain([line], data)

# FIXME: We might want to raise something here
return []


if __name__ == "__main__":
data = [
"this is the header",
"it might change every time",
"and you'll never find a good regexp for it",
"The first line of data is easy to know",
"Name: Duffy",
"Type: Duck",
"Anger: 10",
"",
"Name: Bugs",
"Type: Bunny",
"Anger: 0",
]

data = skip_header(data)
for line in data:
print line

Friday, May 01, 2009

Subversion IRC Bot

#!/usr/bin/env python
'''Push changes in subversion repository to IRC channel'''

from twisted.words.protocols import irc
from twisted.internet.protocol import ReconnectingClientFactory
import re
from subprocess import Popen, PIPE
from xml.etree.cElementTree import parse as xmlparse
from cStringIO import StringIO

class IRCClient(irc.IRCClient):
nickname = "svnbot"
realname = "Subversion Bot"
channel = "#dev"

instance = None # Running instance

def signedOn(self):
IRCClient.instance = self
self.join(self.channel)

def svn(self, revision, author, comment):
comment = (comment[:57] + "...") if len(comment) > 60 else comment
message = "SVN revision %s by %s: %s" % (revision, author, comment)
self.say(self.channel, message)


class SVNPoller:
def __init__(self, root, user, password):
self.pre = ["svn", "--xml", "--username", user, "--password", password]
self.root = root
self.last_revision = self.get_last_revision()

def check(self):
if not IRCClient.instance:
return

try:
last_revision = self.get_last_revision()
if (not last_revision) or (last_revision == self.last_revision):
return

for rev in range(self.last_revision + 1, last_revision + 1):
author, comment = self.revision_info(rev)
IRCClient.instance.svn(rev, author, comment)
self.last_revision = last_revision
except Exception, e:
print "ERROR: %s" % e

def svn(self, *cmd):
pipe = Popen(self.pre + list(cmd) + [self.root], stdout=PIPE)
try:
data = pipe.communicate()[0]
except IOError:
data = ""
return xmlparse(StringIO(data))

def get_last_revision(self):
tree = self.svn("info")
revision = tree.find("//commit").get("revision")
return int(revision)

def revision_info(self, revision):
tree = self.svn("log", "-r", str(revision))
author = tree.find("//author").text
comment = tree.find("//msg").text

return author, comment

if __name__ == "__main__":
from twisted.internet import reactor
from twisted.internet.task import LoopingCall

factory = ReconnectingClientFactory()
factory.protocol = IRCClient
reactor.connectTCP("irc.mycompany.com", 6667, factory)

poller = SVNPoller("http://svn.mycompany.com", "bugs", "carrot")
task = LoopingCall(poller.check)
task.start(1)

reactor.run()

Wednesday, April 22, 2009

Solving Euler Question 24

I'm learning Clojure by solving project Euler. Here is a python version of one of the solutions.

#!/usr/bin/env python
''' Solving http://projecteuler.net/index.php?section=problems&id=24

Note that Python's 2.6 itertools.permutations return the permutation in order so
we can just write:
from itertools import islice, permutations
print islice(permutations(range(10)), 999999, None).next()

And it'll work much faster :)
'''

from itertools import islice, ifilter

def is_last_permutation(n):
return n == sorted(n, reverse=1)

def next_head(n):
'''Find next number to be 'head'.

It is smallest number if the tail that is bigger than the head.
In the case of (2 4 3 1) it will pick 3 to get the next permutation
of (3 1 2 4)
'''
return sorted(filter(lambda i: i > n[0], n[1:]))[0]

def remove(element, items):
return filter(lambda i: i != element, items)

def next_permutation(n):
if is_last_permutation(n):
return None

sub = next_permutation(n[1:])
if sub:
return [n[0]] + sub

head = next_head(n)
return [head] + sorted([n[0]] + remove(head, n[1:]))

def nth(it, n):
'''Return the n'th element of an iterator'''
return islice(it, n, None).next()

def iterate(func, n):
'''iterate(func, n) -> n, func(n), func(func(n)) ...'''
while 1:
yield n
n = func(n)

def permutations(n):
return ifilter(None, iterate(next_permutation, n))

if __name__ == "__main__":
n = range(10)
m = 1000000
print "Calculateing the %d permutation of %s" % (m, n)
print nth(permutations(n), m-1)

Friday, April 03, 2009

pmap

#!/usr/bin/env python
'''Parallel map (Unix only)'''

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

# The big advantage of this implementation is that "fork" is very fast on
# copying data, so if you pass big arrays as arguments and return small values
# this is a win

# FIXME
# * For too many items, we get "Too many open files"
# * Handle child exceptions

from os import fork, pipe, fdopen, waitpid, P_WAIT
from marshal import dump, load
from itertools import takewhile, count

def spawn(func, data):
read_fo, write_fo = map(fdopen, pipe(), ("rb", "wb"))
pid = fork()
if pid: # Parent
return pid, read_fo

# Child
dump(func(data), write_fo)
write_fo.close()
raise SystemExit

def wait(child):
pid, fo = child
waitpid(pid, P_WAIT)
value = load(fo)
fo.close()
return value

def pmap(func, items):
'''
>>> pmap(lambda x: x * 2, range(5))
[0, 2, 4, 6, 8]
'''
children = map(lambda item: spawn(func, item), items)
return map(wait, children)

if __name__ == "__main__":
def fib(n):
a, b = 1, 1
while n > 1:
a, b = b, a + b
n -= 1
return b

items = range(10)
print "pmap(fib, %s)" % str(items)
print pmap(fib, items)

Tuesday, March 17, 2009

__subclasses__

New style classes have a __subclasses__ method which I find useful the problem of "find me all instances to run" - without any metaclass black voodoo :)

A trivial example:
#!/usr/bin/env python

class Animal(object):
pass

class Dog(Animal):
def talk(self):
return "whoof whoof"

class Cat(Animal):
def talk(self):
return "miao"

class Pig(Animal):
def talk(self):
return "oink oink"

def all_animals():
return Animal.__subclasses__()

if __name__ == "__main__":
for animal in all_animals():
print "%s says: %s" % (animal.__name__, animal().talk())


This way, if we add a new animal all_animals will work without any change from us.

Wednesday, March 11, 2009

glue


Sometime you need to produce a quick prototype for a site that is an aggregation of several other sites.

One quick way is to glue the parts you want from each site into the page.
We'll use a more relaxed version of trampoline to overcome cross site scripting.

redirect.cgi

#!/usr/bin/env python
'''Get content of "foreign" web pages, enable cross-site AJAX'''

from urllib2 import urlopen, URLError
from cgi import FieldStorage

if __name__ == "__main__":
import cgitb; cgitb.enable()

form = FieldStorage()
url = form.getvalue("url", "")
if not url:
raise SystemExit("no url")

try:
o = urlopen(url)
except (URLError, ValueError), e:
raise SystemExit("error: %s" % e)

print "Content-type: %(content-type)s\n" % o.headers
print o.read()
And then in our page we'll glue the parts. I'm using the same part from Weather Underground but with different area. (Ignoring the fact they have a good API :)

index.html

<html>
<head>
<title></title>
<style>
h1.title {
text-align: center;
}
.header {
background: blue;
color: white;
font-size: 1.3em;
font-weight: bold;
font-variant: small-caps;
}
div.left {
float: left;
width: 50%;
height: 400px;
overflow: auto;
}
div.right {
float: right;
width: 50%;
height: 400px;
overflow: auto;
}
</style>
</head>
<body>
<h1 class="title">Weather Meshup</h1>

<div>
<div class="left">
<div class="header">Beverly Hills</div>
<div id="bh">Loading ...</div>
</div>

<div class="right">
<div class="header">New York</div>
<div id="ny">Loading ...</div>
</div>
</div>
<div>
<div class="left">
<div class="header">Chicago</div>
<div id="ch">Loading ...</div>
</div>

<div class="right">
<div class="header">Washington</div>
<div id="wa">Loading ...</div>
</div>
</div>
</body>
<script src="jquery.js"></script>
<script>
function weather_url(zipcode) {
return 'http://www.wund.com/cgi-bin/findweather/getForecast' +
'?query=' + zipcode;
}

function load_foreign(id, url, selector) {
$('#' + id).load('redirect.cgi ' + selector, {url: url});
}

function load_weather(id, zipcode) {
var url = weather_url(zipcode);
load_foreign(id, url, 'div#conditions');
}


function on_ready()
{
load_weather('bh', '90210');
load_weather('ny', '10001');
load_weather('ch', '60290');
load_weather('wa', '20001');
}

$(document).ready(on_ready);
</script>
</html>

Thursday, February 26, 2009

@reboot

cron has a nice special rule called @reboot which will run every time the machine is rebooted (doh!).

I use it for two main things:
  1. Notify me when one of my servers was rebooted (sometimes IT don't tell me in advance)
  2. Run my services. This is easier than writing an /etc/init.d scripts and I store my servers crontab in the source control, so deploying a new machine is easier.
An example:
# Notify that machine was rebooted
@reboot /path/to/mail -s "Machine $HOSTNANE rebooted" me@somewhere.com < /dev/null
# Run my server
@reboot (cd /path/to/awesome/server/directory && ./run_server)


Where run_server is usually something like:
#!/bin/bash

nohup ./server.py&

Tuesday, February 17, 2009

calc


A very simple command line/GUI calculator. The main trick here is to export all of math to the global namespace and then just eval the expression.

Thursday, February 12, 2009

twiver


A Twitter search client the updates every 10min.

#!/usr/bin/env python

from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from urlparse import urlparse
from urllib2 import urlopen
from operator import itemgetter
from rfc822 import parsedate_tz
from time import strftime
import re
from functools import partial

try:
import json
except ImportError:
import simplejson as json

HTML = '''
<html>
<head>
<title>Twiver - Refreshing Twitter Search</title>
<style>
h2 {
font-variant: small-caps;
}
table#results {
border: 1px solid black;
width: 100%;
}
table#results tr:hover {
background: silver;
}
span#updated {
font-family: Monospace;
}
</style>
</head>
<body>
<h2>Twiver - Refreshing Twitter Search</h2>
Query: <input id="query" size="60" /> <button id="run">Go</button>
<table id="results">
</table>
<span id="updated"></span>
<hr />
By <a href="mailto:miki.tebeka@gmail.com">Miki</a>
</body>
<script src="jquery.js"></script>
<script>
var running = 0;

function handle_result(data) {
if (!running) {
return;
}

var table = $('#results');
table.empty();
$.each(data, function (i, text) {
var tr = $('<tr><td>' + text + '</td></tr>');
table.append(tr);
});
$('#updated').html('Updated: ' + new Date());

setTimeout(update, 10 * 1000);
}

function update() {
var query = $.trim($('#query').val());
url = '/search?q=' + query;
$.getJSON(url, handle_result);
}

function run() {
var button = $('#run');
if (button.text() == "Go") {
var query = $.trim($('#query').val());
/* FIXME: The best way will be to disable the button until there
is text
*/
if (query.length == 0) {
alert("Please enter *something*");
return;
}
button.text("Stop");
running = 1;
update();
}
else {
running = 0;
button.text("Go");
}
}

function on_ready()
{
$('#run').click(run);
}
$(document).ready(on_ready);
</script>
</html>
'''

def format_time(time):
# Wed, 11 Feb 2009 00:10:36 +0000
time = parsedate_tz(time)
return strftime("%m/%d/%Y %H:%M", time[:9])

# 'http://mikitebeka.com' ->
# '<a href="http://mikitebeka.com">http://mikitebeka.com</a>'
inject_links = partial(
re.compile("(http://[^ ]+)").sub,
"<a target=\"_new\" href=\"\\1\">\\1</a>")

def format_result(result):
time = format_time(result["created_at"])
text = inject_links(result["text"])
return "[%s] %s" % (time, text)

class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/":
self.wfile.write(HTML)
elif self.path.startswith("/search"):
self.updates()
elif self.path.endswith(".js"):
self.wfile.write(open(".%s" % self.path).read())
else:
self.send_error(404, "Not Found")

def updates(self):
o = urlparse(self.path)
url = "http://search.twitter.com/search.json?" + o.query
data = urlopen(url).read()
obj = json.loads(data)
results = map(format_result, obj["results"])
self.wfile.write(json.dumps(results))

if __name__ == "__main__":
server = HTTPServer(("", 8888), RequestHandler)
server.serve_forever()

Blog Archive