Show "digits only" version of phone number

#!/bin/bash
# Convert phone numbers to numeric (1-800-T-MOBILE -> 1-800-8-662453)

if [ $# -ne 1 ]; then
    echo "usage: $(basename $0) PHONE-NUMBER"
    exit 1
fi

echo $1 | \
    tr '[:lower:]' '[:upper:]' | \
    tr ABCDEFGHIJKLMNOPQRSTUVWXYZ 22233344455566677778889999

JSON Pretiffier

#!/usr/bin/env python
'''JSON prettifier'''

def shift(array):
   try:
       return array.pop(0)
   except IndexError:
       return None

def fopen(name, mode, default):
   if name in (None, "-"):
       return default
   return open(name, mode)

def main(argv=None):
   import sys
   from optparse import OptionParser

   argv = argv or sys.argv

   indent = 4
   parser = OptionParser("%prog [INFILE [OUTFILE]]")
   parser.add_option("-i", "--indent", help="indent size (%s)" % indent,
                     dest="indent", default=indent, type="int")
   opts, args = parser.parse_args(argv[1:])
   if len(args) not in (0, 1, 2):
       parser.error("wrong number of arguments") # Will exit

   try:
       info = fopen(shift(args), "r", sys.stdin)
       outfo = fopen(shift(args), "w", sys.stdout)
   except IOError, e:
       raise SystemExit("error: %s" % e)

   import json
   json.dump(json.load(info), outfo, indent=opts.indent)
      
if __name__ == "__main__":
   main()

Update (Aug 14, 2009): Just found out about python -mjson.tool, oh well ...

strftime for JavaScript

/* strftime for JavaScript 

   Field description (taken from http://tinyurl.com/65s2qw)

    %a  Locale’s abbreviated weekday name.   
    %A  Locale’s full weekday name.   
    %b  Locale’s abbreviated month name.   
    %B  Locale’s full month name.   
    %c  Locale’s appropriate date and time representation.   
    %d  Day of the month as a decimal number [01,31].   
    %H  Hour (24-hour clock) as a decimal number [00,23].   
    %I  Hour (12-hour clock) as a decimal number [01,12].   
    %j  Day of the year as a decimal number [001,366].   
    %m  Month as a decimal number [01,12].   
    %M  Minute as a decimal number [00,59].   
    %p  Locale’s equivalent of either AM or PM.
    %S  Second as a decimal number [00,61].
    %U  Week number of the year (Sunday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Sunday are considered to be in week 0.
    %w  Weekday as a decimal number [0(Sunday),6].   
    %W  Week number of the year (Monday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Monday are considered to be in week 0.
    %x  Locale’s appropriate date representation.   
    %X  Locale’s appropriate time representation.   
    %y  Year without century as a decimal number [00,99].   
    %Y  Year with century as a decimal number.   
    %Z  Time zone name (no characters if no time zone exists).   
    %%  A literal '%' character.
*/

var days = [ 
    'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
    'Saturday' 
];

var months = [
    'January', 'February', 'March', 'April', 'May', 'June', 'July',
    'August', 'September', 'October', 'November', 'December'
];

function shortname(name) {
    return name.substr(0, 3);
}

function zeropad(n, size) {
    n = '' + n; /* Make sure it's a string */
    size = size || 2;
    while (n.length < size) {
        n = '0' + n;
    }
    return n;
}

function twelve(n) {
    return (n <= 12) ? n : 24 - n;
}

function strftime(format, date) {
    date = date || new Date();
    var fields = {
        a: shortname(days[date.getDay()]),
        A: days[date.getDay()],
        b: shortname(months[date.getMonth()]),
        B: months[date.getMonth()],
        c: date.toString(),
        d: zeropad(date.getDate()),
        H: zeropad(date.getHours()),
        I: zeropad(twelve(date.getHours())),
        /* FIXME: j: */
        m: zeropad(date.getMonth() + 1),
        M: zeropad(date.getMinutes()),
        p: (date.getHours() >= 12) ? 'PM' : 'AM',
        S: zeropad(date.getSeconds()),
        w: zeropad(date.getDay() + 1),
        /* FIXME: W: */
        x: date.toLocaleDateString(),
        X: date.toLocaleTimeString(),
        y: ('' + date.getFullYear()).substr(2, 4),
        Y: '' + date.getFullYear(),
        /* FIXME: Z: */
        '%' : '%'
    };

    var result = '', i = 0;
    while (i < format.length) {
        if (format[i] === '%') {
            result = result + fields[format[i + 1]];
            ++i;
        }
        else {
            result = result + format[i];
        }
        ++i;
    }
    return result;
}

Quick search using xapian "omega"

A little hack to search current directory using Xapian's omega.

#!/bin/bash

# Minimal search using Xpians Omega (http://xapian.org/docs/omega/overview.html)
# Use "omsearch -i" to index current directoy, then use "omsearch QUERY" to
# search


index() {
    mkdir .omega
    cat >> .omega/omega.conf << EOF
    database_dir $PWD/.omega
    template_dir /var/lib/xapian-omega/templates
    log_dir /tmp
EOF
    omindex --db .omega/default .
}

search() {
    omega=/usr/lib/cgi-bin/omega/omega
    config=$PWD/.omega/omega.conf
    results=.omega/results.html
    OMEGA_CONFIG_FILE=$config $omega "P=$1" "HITSPERPAGE=1000" > $results
    egrep -o '<B><A HREF=".*"' $results | cut -d\" -f2 | \
        xargs -i printf ".%s\n" {}

}

USAGE="usage: $(basename $0) [-i | QUERY]"

index=no
while getopts "ih" opt
do
    case $opt in
        h | H ) echo $USAGE; exit;;
        i ) index=yes;;
        * ) echo "error: unknown option" >&2; exit 1;;
    esac
done

shift $(($OPTIND - 1))

if [ "$index" == "yes" ]; then
    if [ $# -gt 0 ]; then
        echo "error: -i don't take any parameters"
        exit 1
    fi
    index
    exit
fi

if [ $# -ne 1 ]; then
    echo $USAGE
    exit 1
fi

if [ ! -d .omega ]; then
    echo "error: can't find .omega, run with -i to index first"
    exit 1
fi
search $1

Avoiding "peek" with "itertools.chain"

Sometimes you need to filter out some header information. However the only clear anchor you have is the start of real data - you need a "peek" function.

It's easy to avoid writing the "peek" logic, just use itertools.chain.

#!/usr/bin/env python
'''Avoiding the need for "peek" with itertools.chain'''

from itertools import chain

def is_first_data(line):
    return line.startswith("Name:")

def skip_header(data):
    data = iter(data)
    for line in data:
        if is_first_data(line):
            return chain([line], data)

    # FIXME: We might want to raise something here
    return []


if __name__ == "__main__":
    data = [
        "this is the header",
        "it might change every time",
        "and you'll never find a good regexp for it",
        "The first line of data is easy to know",
        "Name: Duffy",
        "Type: Duck",
        "Anger: 10",
        "",
        "Name: Bugs",
        "Type: Bunny",
        "Anger: 0",
    ]

    data = skip_header(data)
    for line in data:
        print line

Subversion IRC Bot

#!/usr/bin/env python
'''Push changes in subversion repository to IRC channel'''

from twisted.words.protocols import irc
from twisted.internet.protocol import ReconnectingClientFactory
import re
from subprocess import Popen, PIPE
from xml.etree.cElementTree import parse as xmlparse
from cStringIO import StringIO

class IRCClient(irc.IRCClient):
    nickname = "svnbot"
    realname = "Subversion Bot"
    channel = "#dev"

    instance = None # Running instance

    def signedOn(self):
        IRCClient.instance = self
        self.join(self.channel)

    def svn(self, revision, author, comment):
        comment = (comment[:57] + "...") if  len(comment) > 60 else comment
        message = "SVN revision %s by %s: %s" % (revision, author, comment)
        self.say(self.channel, message)


class SVNPoller:
    def __init__(self, root, user, password):
        self.pre = ["svn", "--xml", "--username", user, "--password", password]
        self.root = root
        self.last_revision = self.get_last_revision()

    def check(self):
        if not IRCClient.instance:
            return

        try:
            last_revision = self.get_last_revision()
            if (not last_revision) or (last_revision == self.last_revision):
                return

            for rev in range(self.last_revision + 1, last_revision + 1):
                author, comment = self.revision_info(rev)
                IRCClient.instance.svn(rev, author, comment)
            self.last_revision = last_revision
        except Exception, e:
            print "ERROR: %s" % e

    def svn(self, *cmd):
        pipe = Popen(self.pre +  list(cmd) + [self.root], stdout=PIPE)
        try:
            data = pipe.communicate()[0]
        except IOError:
            data = ""
        return xmlparse(StringIO(data))

    def get_last_revision(self):
        tree = self.svn("info")
        revision = tree.find("//commit").get("revision")
        return int(revision)

    def revision_info(self, revision):
        tree = self.svn("log", "-r", str(revision))
        author = tree.find("//author").text
        comment = tree.find("//msg").text

        return author, comment

if __name__ == "__main__":
    from twisted.internet import reactor
    from twisted.internet.task import LoopingCall

    factory = ReconnectingClientFactory()
    factory.protocol = IRCClient
    reactor.connectTCP("irc.mycompany.com", 6667, factory)

    poller = SVNPoller("http://svn.mycompany.com", "bugs", "carrot")
    task = LoopingCall(poller.check)
    task.start(1)

    reactor.run()

Solving Euler Question 24

I'm learning Clojure by solving project Euler. Here is a python version of one of the solutions.

#!/usr/bin/env python
''' Solving http://projecteuler.net/index.php?section=problems&id=24

Note that Python's 2.6 itertools.permutations return the permutation in order so
we can just write:
    from itertools import islice, permutations
    print islice(permutations(range(10)), 999999, None).next()

And it'll work much faster :)
'''

from itertools import islice, ifilter

def is_last_permutation(n):
    return n == sorted(n, reverse=1)

def next_head(n):
    '''Find next number to be 'head'. 

    It is smallest number if the tail that is bigger than the head.
    In the case of (2 4 3 1) it will pick 3 to get the next permutation
    of (3 1 2 4)
    '''
    return sorted(filter(lambda i: i > n[0], n[1:]))[0]

def remove(element, items):
    return filter(lambda i: i != element, items)

def next_permutation(n):
    if is_last_permutation(n):
        return None

    sub = next_permutation(n[1:])
    if sub:
        return [n[0]] + sub

    head = next_head(n)
    return [head] + sorted([n[0]] + remove(head, n[1:]))

def nth(it, n):
    '''Return the n'th element of an iterator'''
    return islice(it, n, None).next()

def iterate(func, n):
    '''iterate(func, n) -> n, func(n), func(func(n)) ...'''
    while 1:
        yield n
        n = func(n)

def permutations(n):
    return ifilter(None, iterate(next_permutation, n))

if __name__ == "__main__":
    n = range(10)
    m = 1000000
    print "Calculateing the %d permutation of %s" % (m, n)
    print nth(permutations(n), m-1)

pmap

#!/usr/bin/env python
'''Parallel map (Unix only)'''

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

# The big advantage of this implementation is that "fork" is very fast on
# copying data, so if you pass big arrays as arguments and return small values
# this is a win

# FIXME 
# * For too many items, we get "Too many open files"
# * Handle child exceptions

from os import fork, pipe, fdopen, waitpid, P_WAIT
from marshal import dump, load
from itertools import takewhile, count

def spawn(func, data):
    read_fo, write_fo = map(fdopen, pipe(), ("rb", "wb"))
    pid = fork()
    if pid: # Parent
        return pid, read_fo

    # Child
    dump(func(data), write_fo)
    write_fo.close()
    raise SystemExit

def wait(child):
    pid, fo = child
    waitpid(pid, P_WAIT)
    value = load(fo)
    fo.close()
    return value

def pmap(func, items):
    '''
    >>> pmap(lambda x: x * 2, range(5))
    [0, 2, 4, 6, 8]
    '''
    children = map(lambda item: spawn(func, item), items)
    return map(wait, children)

if __name__ == "__main__":
    def fib(n):
        a, b = 1, 1
        while n > 1:
            a, b = b, a + b
            n -= 1
        return b

    items = range(10)
    print "pmap(fib, %s)" % str(items)
    print pmap(fib, items)

__subclasses__

New style classes have a __subclasses__ method which I find useful the problem of "find me all instances to run" - without any metaclass black voodoo :)

A trivial example:

#!/usr/bin/env python

class Animal(object):
    pass

class Dog(Animal):
    def talk(self):
        return "whoof whoof"

class Cat(Animal):
    def talk(self):
        return "miao"

class Pig(Animal):
    def talk(self):
        return "oink oink"

def all_animals():
    return Animal.__subclasses__()

if __name__ == "__main__":
    for animal in all_animals():
        print "%s says: %s" % (animal.__name__, animal().talk())

This way, if we add a new animal all_animals will work without any change from us.

glue

Sometime you need to produce a quick prototype for a site that is an aggregation of several other sites.

One quick way is to glue the parts you want from each site into the page.
We'll use a more relaxed version of trampoline to overcome cross site scripting.

redirect.cgi

#!/usr/bin/env python
'''Get content of "foreign" web pages, enable cross-site AJAX'''

from urllib2 import urlopen, URLError
from cgi import FieldStorage

if __name__ == "__main__":
   import cgitb; cgitb.enable()

   form = FieldStorage()
   url = form.getvalue("url", "")
   if not url:
       raise SystemExit("no url")

   try:
       o = urlopen(url)
   except (URLError, ValueError), e:
       raise SystemExit("error: %s" % e)

   print "Content-type: %(content-type)s\n" % o.headers
   print o.read()

And then in our page we'll glue the parts. I'm using the same part from Weather Underground but with different area. (Ignoring the fact they have a good API :)

index.html

<html>
   <head>
       <title></title>
       <style>
           h1.title {
               text-align: center;
           }
           .header {
               background: blue;
               color: white;
               font-size: 1.3em;
               font-weight: bold;
               font-variant: small-caps;
           }
           div.left {
               float: left;
               width: 50%;
               height: 400px;
               overflow: auto;
           }
           div.right {
               float: right;
               width: 50%;
               height: 400px;
               overflow: auto;
           }
       </style>
   </head>
   <body>
       <h1 class="title">Weather Meshup</h1>

       <div>
           <div class="left">
               <div class="header">Beverly Hills</div>
               <div id="bh">Loading ...</div>
           </div>

           <div class="right">
               <div class="header">New York</div>
               <div id="ny">Loading ...</div>
           </div>
       </div>
       <div>
           <div class="left">
               <div class="header">Chicago</div>
               <div id="ch">Loading ...</div>
           </div>

           <div class="right">
               <div class="header">Washington</div>
               <div id="wa">Loading ...</div>
           </div>
       </div>
   </body>
   <script src="jquery.js"></script>
   <script>
       function weather_url(zipcode) {
           return 'http://www.wund.com/cgi-bin/findweather/getForecast' +
                  '?query=' + zipcode;
       }

       function load_foreign(id, url, selector) {
           $('#' + id).load('redirect.cgi ' + selector, {url: url});
       }

       function load_weather(id, zipcode) {
           var url = weather_url(zipcode);
           load_foreign(id, url, 'div#conditions');
       }


       function on_ready()
       {
           load_weather('bh', '90210');
           load_weather('ny', '10001');
           load_weather('ch', '60290');
           load_weather('wa', '20001');
       }

       $(document).ready(on_ready);
   </script>
</html>

@reboot

cron has a nice special rule called @reboot which will run every time the machine is rebooted (doh!).

I use it for two main things:

1. Notify me when one of my servers was rebooted (sometimes IT don't tell me in advance)
2. Run my services. This is easier than writing an /etc/init.d scripts and I store my servers crontab in the source control, so deploying a new machine is easier.
   

An example:

# Notify that machine was rebooted
@reboot /path/to/mail -s "Machine $HOSTNANE rebooted" me@somewhere.com < /dev/null
# Run my server
@reboot (cd /path/to/awesome/server/directory && ./run_server)

Where run_server is usually something like:

#!/bin/bash

nohup ./server.py&

calc

A very simple command line/GUI calculator. The main trick here is to export all of math to the global namespace and then just eval the expression.

twiver

A Twitter search client the updates every 10min.

#!/usr/bin/env python

from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from urlparse import urlparse
from urllib2 import urlopen
from operator import itemgetter
from rfc822 import parsedate_tz
from time import strftime
import re
from functools import partial

try:
  import json
except ImportError:
  import simplejson as json

HTML = '''
<html>
    <head>
        <title>Twiver - Refreshing Twitter Search</title>
        <style>
            h2 {
                font-variant: small-caps;
            }
            table#results {
                border: 1px solid black;
                width: 100%;
            }
            table#results tr:hover {
                background: silver;
            }
            span#updated {
                font-family: Monospace;
            }
        </style>
    </head>
    <body>
        <h2>Twiver - Refreshing Twitter Search</h2>
        Query: <input id="query" size="60" /> <button id="run">Go</button>
        <table id="results">
        </table>
        <span id="updated"></span>
        <hr />
        By <a href="mailto:miki.tebeka@gmail.com">Miki</a>
    </body>
    <script src="jquery.js"></script>
    <script>
        var running = 0;

        function handle_result(data) {
            if (!running) {
                return;
            }

            var table = $('#results');
            table.empty();
            $.each(data, function (i, text) {
                var tr = $('<tr><td>' + text + '</td></tr>');
                table.append(tr);
            });
            $('#updated').html('Updated: ' + new Date());

            setTimeout(update, 10 * 1000);
        }

        function update() {
            var query = $.trim($('#query').val());
            url = '/search?q=' + query;
            $.getJSON(url, handle_result);
        }

        function run() {
            var button = $('#run');
            if (button.text() == "Go") {
                var query = $.trim($('#query').val());
                /* FIXME: The best way will be to disable the button until there
                  is text
                */
                if (query.length == 0) {
                    alert("Please enter *something*");
                    return;
                }
                button.text("Stop");
                running = 1;
                update();
            }
            else {
                running = 0;
                button.text("Go");
            }
        }

        function on_ready()
        {
            $('#run').click(run);
        }
        $(document).ready(on_ready);
    </script>
</html>
'''

def format_time(time):
  # Wed, 11 Feb 2009 00:10:36 +0000
  time = parsedate_tz(time)
  return strftime("%m/%d/%Y %H:%M", time[:9])

# 'http://mikitebeka.com' -> 
# '<a href="http://mikitebeka.com">http://mikitebeka.com</a>'
inject_links = partial(
  re.compile("(http://[^ ]+)").sub,
  "<a target=\"_new\" href=\"\\1\">\\1</a>")

def format_result(result):
  time = format_time(result["created_at"])
  text = inject_links(result["text"])
  return "[%s] %s" % (time, text)

class RequestHandler(BaseHTTPRequestHandler):
  def do_GET(self):
      if self.path == "/":
          self.wfile.write(HTML)
      elif self.path.startswith("/search"):
          self.updates()
      elif self.path.endswith(".js"):
          self.wfile.write(open(".%s" % self.path).read())
      else:
          self.send_error(404, "Not Found")

  def updates(self):
      o = urlparse(self.path)
      url = "http://search.twitter.com/search.json?" + o.query
      data = urlopen(url).read()
      obj = json.loads(data)
      results = map(format_result, obj["results"])
      self.wfile.write(json.dumps(results))

if __name__ == "__main__":
  server = HTTPServer(("", 8888), RequestHandler)
  server.serve_forever()

"Game of Life" in JavaScript

<html>
    <head>
        <title>Game Of Life</title>
        <style>
            table {
                border: 5px solid black;
            }
            td {
                width: 15px;
                height: 15px;
            }
            td.dead {
                background: green;
            }
            td.alive {
                background: red;
            }
        </style>
    </head>
    <body>
        <b>Game of Life<b>
        <table id="board">
        </table>
        <button id="run">Run</button>
        <button id="step">Step</button>
        <button id="clear">Clear</button>

    </body>
    <script src="jquery.js"></script>
    <script>
        var NUM_ROWS = 30;
        var NUM_COLS = 30;
        var DEAD = 'dead';
        var ALIVE = 'alive';
        var RUNNING = 0;
        /* We keep this for fast access since jQuery selectors
        $('#board tr:nth-child(' + row + 1 + ') td:nth-child(' + col + 1 + ')'
        are slow
        */
        var BOARD = [];

        function copy_board() {
            var board = [];
            for (var row = 0; row < NUM_ROWS; ++row) {
                var cells = [];
                for (col = 0; col < NUM_COLS; ++col) {
                    var state = get_cell(row, col).hasClass(ALIVE) ?
                                ALIVE : DEAD;
                    cells.push(state);
                }
                board.push(cells);
            }

            return board;
        }

        function cell_neighbours(row, col) {
            var uprow = (row + 1) % NUM_ROWS;
            var downrow = (row - 1) % NUM_ROWS;
            var leftcol = (col - 1) % NUM_COLS;
            var rightcol = (col + 1) % NUM_COLS;

            /* FIXME: Find a better way (since -1 % 10 => -1) */
            downrow = (downrow < 0) ? NUM_ROWS - 1 : downrow;
            leftcol = (leftcol < 0) ? NUM_COLS - 1 : leftcol;

            return [
                [uprow, leftcol], [uprow, col], [uprow, rightcol],
                [row, leftcol], [row, rightcol],
                [downrow, leftcol], [downrow, col], [downrow, rightcol]
            ];
        }

        /* http://en.wikipedia.org/wiki/Conway%27s_Game_of_Life#Rules */
        function calc_new_state(current_state, num_alive) {
            if (current_state == ALIVE) {
                if (num_alive < 2) {
                    return DEAD;
                }
                else if (num_alive > 3) {
                    return DEAD;
                }
                else {
                    return ALIVE;
                }
            }
            else {
                if (num_alive == 3) {
                    return ALIVE;
                }
                else {
                    return DEAD;
                }
            }
        }


        function on_step() {
            /* Copy old board since we're going to change the current */
            var board = copy_board();

            function is_alive(cell) {
                return board[cell[0]][cell[1]] == ALIVE;
            }

            for (var row = 0; row < NUM_ROWS; ++row) {
                for (var col = 0; col < NUM_COLS; ++col) {
                    var neighbours = cell_neighbours(row, col);
                    var num_alive = $.grep(neighbours, is_alive).length;
                    var current_state = board[row][col];
                    var new_state = calc_new_state(current_state, num_alive);
                    if (new_state != current_state) {
                        toggle(row, col);
                    }
                }
            }
        }

        function run() {
            if (!RUNNING) {
                return;
            }
            on_step();
            setTimeout(run, 200);
        }

        function on_run() {
            var button = $('#run');
            if (button.text() == 'Run') {
                button.text('Stop');
                RUNNING = 1;
                run();
            }
            else {
                button.text('Run');
                RUNNING = 0;
            }
        }

        function get_cell(row, col) {
            return BOARD[row][col];
        }

        function toggle(row, col) {
            var cell = get_cell(row, col);

            if (cell.hasClass(ALIVE)) {
                var add = DEAD;
                var remove = ALIVE;
            }
            else {
                var add = ALIVE;
                var remove = DEAD;
            }

            cell.removeClass(remove).addClass(add);
        }

        function make_handler(row, col) {
            return function() {
                toggle(row, col);
            }
        }

        function initiaize_board() {
            var table = $('#board');
            for (var row = 0; row < NUM_ROWS; ++row) {
                var tr = $('<tr />');
                var cells = [];
                for (var col = 0; col < NUM_COLS; ++col) {
                    var td = $('<td />');
                    td.addClass(DEAD);
                    td.click(make_handler(row, col));
                    tr.append(td);
                    cells.push(td);
                }
                table.append(tr);
                BOARD.push(cells);
            }
        }

        function on_clear() {
            $('.' + ALIVE).removeClass(ALIVE).addClass(DEAD);
        }

        function on_ready()
        {
            initiaize_board();
            $('#step').click(on_step);
            $('#run').click(on_run);
            $('#clear').click(on_clear);
        }

        $(document).ready(on_ready);
    </script>
</html>