flatten

#!/usr/bin/env python

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

def flatten(items):
    '''Flatten a nested list.

    >>> a = [[1], 2, [[[3]], 4]]
    >>> list(flatten(a))
    [1, 2, 3, 4]
    >>>
    '''
    for item in items:
        if getattr(item, "__iter__", None):
            for subitem in flatten(item):
                yield subitem
        else:
            yield item

if __name__ == "__main__":
    from doctest import testmod
    testmod()

pipe

printobj

'''Quick and dirty object "repr"'''

__author__ = "Miki Tebeka "
# FIXME: Find how to make doctest play with "regular" class definition

def printobj(obj):
    '''
    Quick and dirty object "repr"

    >>> class Point: pass
    >>> p = Point()
    >>> p.x, p.y = 1, 2
    >>> printobj(p)
    ('y', 2)
    ('x', 1)
    >>>
    '''
    print "\n".join(map(str, obj.__dict__.items()))

if __name__ == "__main__":
    from doctest import testmod
    testmod()

CGI trampoline for cross site AJAX

Most (all?) browsers won't let you do cross site AJAX calls.

One solution is JSONP (which is supported by jQuery). However not all servers support it.

The other solution is to create a "trampoline" in your site that returns the data from the remote site:

whichscm

#!/usr/bin/env python
'''Find under which SCM directory is'''

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

from os import sep
from os.path import join, isdir, abspath
from itertools import ifilter, imap

def updirs(path):
    parts = path.split(sep)
    if not parts[0]:
        parts[0] = sep # FIXME: Windows
    while parts:
        yield join(*parts)
        parts.pop()

def scmdirs(path, scms):
    for scmext in scms:
        yield join(path, scmext)

def scm(dirname):
    return dirname[-3:].lower()

def scms(path, scms):
    return imap(scm, ifilter(isdir, scmdirs(path, scms)))


def whichscm(path):
    path = abspath(path)

    for scm in scms(path, (".svn", "CVS")):
        return scm

    scmdirs = (".bzr", ".hg", ".git")
    for dirname in updirs(path):
        for scm in scms(dirname, (".bzr", ".hg", ".git")):
            return scm

def main(argv=None):
    if argv is None:
        import sys
        argv = sys.argv

    from optparse import OptionParser

    parser = OptionParser("usage: %prog [DIRNAME]")

    opts, args = parser.parse_args(argv[1:])
    if len(args) not in (0, 1):
        parser.error("wrong number of arguments") # Will exit

    dirname = args[0] if args else "."
    if not isdir(dirname):
        raise SystemExit("error: %s is not a directory" % dirname)

    scm = whichscm(dirname)
    if not scm:
        raise SystemExit("error: can't find scm for %s" % dirname)

    print scm

if __name__ == "__main__":
    main()

wholistens

#!/usr/bin/env python
'''Find out who is listening on a port'''

from os import popen
from os.path import isdir
import re

is_int = re.compile("\d+").match

def find_pid(port):
    for line in popen("netstat -nlp 2>&1"):
        match = re.search(":(%s)\\s+" % port, line)
        if not match:
            continue

        pidname = line.split()[-1].strip()
        return pidname.split("/")[0]

    return None

def find_cmdline(pid):
    cmd = open("/proc/%s/cmdline" % pid, "rb").read()

    return " ".join(cmd.split(chr(0)))

def find_pwd(pid):
    data = open("/proc/%s/environ" % pid, "rb").read()
    for line in data.split(chr(0)):
        if line.startswith("PWD"):
            return line.split("=")[1]

    return None

def main(argv=None):
    if argv is None:
        import sys
        argv = sys.argv

    from optparse import OptionParser

    parser = OptionParser("usage: %prog PORT")
    opts, args = parser.parse_args(argv[1:])
    if len(args) != 1:
        parser.error("wrong number of arguments") # Will exit

    port = args[0]
    pid = find_pid(port)
    if not (pid and is_int(pid)):
        raise SystemExit(
            "error: can't find who listens on port %s"
            " [try again with sudo?] " % port)

    if not isdir("/proc/%s" % pid):
        raise SystemExit("error: can't find information on pid %s" % pid)

    pwd = find_pwd(pid) or "<unknown>"
    print "%s (pid=%s, pwd=%s)" % (find_cmdline(pid), pid, pwd)

if __name__ == "__main__":
    main()

Note: This does not work on OSX (no /proc and different netstat api)

Code in googlecode

I'll post all the code shown here in http://pythonwise.googlecode.com/.

I've uploaded most of the code from 2008 to 2006, will add the other stuff bit by bit.

Computer Load - The AJAX Way

Show computer load using jquery, flot and Python's BaseHTTPServer (all is less that 70 lines of code).

#!/usr/bin/env python
'''Server to show computer load'''

import re
from os import popen
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from socket import gethostname

def load():
    '''Very fancy computer load :)'''
    output = popen("uptime").read()
    match = re.search("load average(s)?:\\s+(\\d+\\.\\d+)", output)
    return float(match.groups()[1]) * 100

HTML = '''
<html>
    <head>
        <script src="jquery.js"></script>
        <script src="jquery.flot.js"></script>
        <title>%s load</title>
    </head>
    <body>
        <center>
            <h1>%s load</h1>
            <div id="chart" style="width:600px;height:400px;">
            Loading ...
            </div>
        </center>
    </body>
    <script>
    var samples = [];
    var options = {
        yaxis: {
            min: 0,
            max: 100
        },
        xaxis: {
            ticks: []
        }
    };

    function get_data() {
        $.getJSON("/data", function(data) {
            samples.push(data);
            if (samples.length > 120) {
                samples.shift();
            }

            var xy = [];
            for (var i = 0; i < samples.length; ++i) {
                xy.push([i, samples[i]]);
            }
            $.plot($('#chart'), [xy], options);
        });
    }

    $(document).ready(function() {
        setInterval(get_data, 1000);
    });
    </script>
</html>
''' % (gethostname(), gethostname())

class RequestHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/":
            self.wfile.write(HTML)
        elif self.path.endswith(".js"):
            self.wfile.write(open(".%s" % self.path).read())
        else:
            self.wfile.write("%.2f" % load())

if __name__ == "__main__":
    server = HTTPServer(("", 8888), RequestHandler)
    server.serve_forever()

bazaar is slow - who cares?

BS: git is faster than mercurial is faster than bazaar!
ME: Frankly dear, I don't give a damn.
BS: But speed is important!
ME: It is. However when you choose a source control system (if you have the
privilege of doing so), there are many more things to consider:

• Does it fit my work model?
• Is it stable?
• Will it stay for long?
• What's the community like?
• Is development active?
• ...
• Is it fast enough?
  

BS: But bazaar is the slowest
ME: For many, many projects, it's fast enough
BS: So who too choose?
ME: You do your own math. I chose `bazaar` because it has two features that
the others (to my knowledge) don't have:

• It knows about directories (I like to check-in empty logs directory - it simplifies the logging code)
• You can check-in files from another directory (see here)
  

And, it's fast enough for me (about 1sec for bzr st on ~200K LOC)

BS: OK! ... but git storage is better than mercurial is better than bazaar!
ME: <sigh> Why do I even bother? </sigh> ...

Next week - LISP is faster than Python ;)

-------
BS = blogosphere
ME = me

UPDATE (2009/01/27)
Bazaar slowness started to annoy me too much. I felt that every "bzr st" was taking way to much time (not to mention the updated). So I switched to mercurial.

The difference in timing is even noticeable in the most trivial operations:

[09:19] $time hg --version > /dev/null

real 0m0.060s
user 0m0.048s
sys 0m0.012s
[09:20] fattoc $time bzr --version > /dev/null

real 0m0.191s
user 0m0.144s
sys 0m0.048s
[09:21] $

You feel this 0.13 seconds. It seems that hg --version return immediately but bzr --version takes it's time.

Sometimes speed *does* matter.

smokejs

SmokeJS is a discovery based unittest framework for JavaScript (like nose and py.test)

You can run tests either in the command line (with SpiderMonkey or Rhino) or in the browser.

Go, check it out and fill in bugs...

Better "start"

Updated start to support multiple desktop managers in Linux.

enum

Yet another enum implementation.

mikistools

I've placed some of the utilities I use daily at http://code.google.com/p/mikistools/.

next_n

Suppose you want to find the next n elements of a stream the matches a predicate.

(I just used it in web scraping with BeautifulSoup to get the next 5 sibling "tr" for a table).

#!/usr/bin/env python

from itertools import ifilter, islice

def next_n(items, pred, count):
 return islice(ifilter(pred, items), count)

if __name__ == "__main__":
 from gmpy import is_prime
 from itertools import count
 for prime in next_n(count(1), is_prime, 10):
     print prime

Will print

2
3
5
7
11
13
17
19
23
29

(Using gmpy for is_prime)

Tagcloud

Generating tagcloud (using Mako here, but you can use any other templating system)

tagcloud.cgi

tagcloud.mako

Fading Div

Add a new fading (background) div to your document:

<html>
    <body>
    </body>
    <script>
        function set_color(elem) {
            var colorstr = elem.fade_color.toString(16).toUpperCase();
            /* Pad to 6 digits */
            while (colorstr.length < 6) {
                colorstr = "0" + colorstr;
            }
            elem.style.background = "#" + colorstr;
        }

        function fade(elem, color) {
            if (typeof(color) != "undefined") {
                elem.fade_color = color;
            }
            else {
                elem.fade_color += 0x001111;
            }
            set_color(elem);

            if (elem.fade_color < 0xFFFFFF) {
                setTimeout(function() { fade(elem); }, 200);
            }
        }

        function initialize()
        {
            var div = document.createElement("div");
            div.innerHTML = "I'm Fading";

            document.body.appendChild(div);
            fade(div, 0xFF0000); /* Red */
        }

        window.onload = initialize;
    </script>
</html>

XML RPC File Server

#!/usr/bin/env python
'''Simple file client/server using XML RPC'''

from SimpleXMLRPCServer import SimpleXMLRPCServer
from xmlrpclib import ServerProxy, Error as XMLRPCError
import socket

def get_file(filename):
  fo = open(filename, "rb")
  try: # When will "with" be here?
      return fo.read()
  finally:
      fo.close()

def main(argv=None):
  if argv is None:
      import sys
      argv = sys.argv

  default_port = "3030"
  from optparse import OptionParser

  parser = OptionParser("usage: %prog [options] [[HOST:]PORT]")
  parser.add_option("--get", help="get file", dest="filename",
          action="store", default="")

  opts, args = parser.parse_args(argv[1:])
  if len(args) not in (0, 1):
      parser.error("wrong number of arguments") # Will exit

  if args:
      port = args[0]
  else:
      port = default_port

  if ":" in port:
      host, port = port.split(":")
  else:
      host = "localhost"

  try:
      port = int(port)
  except ValueError:
      raise SystemExit("error: bad port - %s" % port)

  if opts.filename:
      try:
          proxy = ServerProxy("http://%s:%s" % (host, port))
          print proxy.get_file(opts.filename)
          raise SystemExit
      except XMLRPCError, e:
          error = "error: can't get %s (%s)" % (opts.filename, e.faultString)
          raise SystemExit(error)
      except socket.error, e:
          raise SystemExit("error: can't connect (%s)" % e)

  server = SimpleXMLRPCServer(("localhost", port))
  server.register_function(get_file)
  print "Serving files on port %d" % port
  server.serve_forever()

if __name__ == "__main__":
  main()

This is a huge security hole, use at your own risk.

web-install

#!/bin/bash
# Do the `./configure && make && sudo make install` dance, given a download URL

if [ $# -ne 1 ]; then
    echo "usage: `basename $0` URL"
    exit 1
fi

set -e  # Fail on errors

url=$1

wget --no-check-certificate $url
archive=`basename $url`

if echo $archive | grep -q .tar.bz2; then
    tar -xjf $archive
else
    tar -xzf $archive
fi

cd ${archive/.tar*}

if [ -f setup.py ]; then
    sudo python setup.py install
else
    ./configure && make && sudo make install
fi

cd ..

Registering URL clicks

Some sites (such as Google), gives you a "trampoline" URL so they can register what you have clicked on. I find it highly annoying since you can't tell where you are going just by hovering above the URL and you can't "copy link location" to a document.

The problem is that these people are just lazy:

<html>
   <body>
       <a href="http://pythonwise.blogspot.com"
           onclick="jump(this, 1);">Pythonwise</a> knows.
   </body>
   <script src="jquery.js"></script>
   <script>
       function jump(url, value)
       {
           $.post("jump.cgi", {
               url: url,
               value: value
           });

           return true;
       }
   </script>
</html>

Notes:

• Using jQuery
• "value" can be anything you want to identify this specific click. I'd use a UUID and some table for registering who is the user, what is the url, the time ...

num_checkins

#!/bin/bash
# How many checking I did today?
# Without arguments will default to current directory

svn log -r"{`date +%Y%m%d`}:HEAD" $1 | grep "| $USER |" | wc -l