extract-audio

OK, not Python - but sometime bash is a better tool.

#!/bin/bash
# Extract audio from video files
# Uses ffmpeg and lame

# Miki Tebeka <miki.tebeka@gmail.com>

if [ $# -ne 2 ]; then
    echo "usage: `basename $0` INPUT_VIDEO OUTPUT_MP3"
    exit 1
fi

infile=$1
outfile=$2

if [ ! -f $infile ]; then
    echo "error: can't find $infile"
    exit 1
fi

if [ -f $outfile ]; then
    echo "error: $outfile exists"
    exit 1
fi

fifoname=/tmp/encode.$$
mkfifo $fifoname
mplayer -vc null -vo null -ao pcm:fast -ao pcm:file=$fifoname $1&
lame $fifoname $outfile
rm $fifoname

pfilter

#!/usr/bin/env python
'''Path filter, to be used in pipes to filter out paths.

* Unix test commands (such as -f can be specified as well)
* {} replaces file name

Examples:
    # List only files in current directory
    ls -a | pfilter -f

    # Find files not versioned in svn 
    # (why, oh why, does svn *always* return 0?)
    find . | pfilter 'test -n "`svn info {} 2>&1 | grep Not`"'
'''

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

from os import system

def pfilter(path, command):
    '''Filter path according to command'''

    if "{}" in command:
        command = command.replace("{}", path)
    else:
        command = "%s %s" % (command, path)

    if command.startswith("-"):
        command = "test %s" %  command

    # FIXME: win32 support
    command += " 2>&1 > /dev/null"

    return system(command) == 0

def main(argv=None):
    if argv is None:
        import sys
        argv = sys.argv

    from sys import stdin
    from itertools import imap, ifilter
    from string import strip
    from functools import partial

    if len(argv) != 2:
        from os.path import basename
        from sys import stderr
        print >> stderr, "usage: %s COMMAND" % basename(argv[0])
        print >> stderr
        print >> stderr, __doc__
        raise SystemExit(1)

    command = argv[1]
    # Don't you love functional programming?
    for path in ifilter(partial(pfilter, command=command), imap(strip, stdin)):
        print path

if __name__ == "__main__":
    main()

Opening File according to mime type

Most of the modern desktops already have a command line utility to open file according to their mime type (GNOME/gnome-open, OSX/open, Windows/start, XFCE/exo-open, KDE/kfmclient ...)

However, most (all?) of them rely on the file extension, where I needed something to view attachments from mutt. Which passes the file data in stdin.

So, here we go (I call this attview):

#!/usr/bin/env python
'''View attachment with right application'''

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

from os import popen, system
from os.path import isfile
import re

class ViewError(Exception):
   pass

def view_attachment(data):
    # In the .destop file, the file name is %u or %U
    u_sub = re.compile("%u", re.I).sub

    FILENAME = "/tmp/attview"
    fo = open(FILENAME, "wb")
    fo.write(data)
    fo.close()

    mime_type = popen("file -ib %s" % FILENAME).read().strip()
    if ";" in mime_type:
        mime_type = mime_type[:mime_type.find(";")]
    if mime_type == "application/x-not-regular-file":
        raise ViewError("can't guess mime type")

    APPS_DIR = "/usr/share/applications"
    for line in open("%s/defaults.list" % APPS_DIR):
        if line.startswith(mime_type):
            mime, appfile = line.strip().split("=")
            break
    else:
        raise ViewError("can't find how to open %s" % mime_type)

    appfile = "%s/%s" % (APPS_DIR, appfile)
    if not isfile(appfile):
        raise ViewError("can't find %s" % appfile)
    for line in open(appfile):
        line = line.strip()
        if line.startswith("Exec"):
            key, cmd = line.split("=")
            fullcmd = u_sub(FILENAME, cmd)
            if fullcmd == cmd:
                fullcmd += " %s" % FILENAME
            system(fullcmd + "&")
            break
    else:
        raise ViewError("can't find Exec in %s" % appfile)


def main(argv=None):
    from sys import stdin
    if argv is None:
        import sys
        argv = sys.argv

    from optparse import OptionParser

    parser = OptionParser("usage: %prog [FILENAME]")

    opts, args = parser.parse_args(argv[1:])
    if len(args) not in (0, 1):
        parser.error("wrong number of arguments") # Will exit

    filename = args[0] if args else "-"

    if filename == "-":
        data = stdin.read()
    else:
        try:
            data = open(filename, "rb").read()
        except IOError, e:
            raise SystemExit("error: %s" % e.strerror)

    try:
        view_attachment(data)
    except ViewError, e:
        raise SystemExit("error: %s" % e)

if __name__ == "__main__":
    main()

Playing with bits

def mask(size):
    '''Mask for `size' bits

    >>> mask(3)
    7
    '''
    return (1L << size) - 1

def num2bits(num, width=32):
    '''String represntation (in bits) of a number

    >>> num2bits(3, 5) 
    '00011'
    '''
    s = ""
    for bit in range(width - 1, -1, -1):
        if num & (1L << bit):
            s += "1"
        else:
            s += "0"
    return s

def get_bit(value, bit):
    '''Get value of bit

    >>> num2bits(5, 5)
    '00101'
    >>> get_bit(5, 0)
    1
    >>> get_bit(5, 1)
    0
    '''
    return (value >> bit) & 1

def get_range(value, start, end):
    '''Get range of bits

    >>> num2bits(5, 5)
    '00101'
    >>> get_range(5, 0, 1)
    1
    >>> get_range(5, 1, 2)
    2
    '''
    return (value >> start) & mask(end - start + 1)

def set_bit(num, bit, value):
    '''Set bit `bit' in num to `value' 

    >>> i = 5
    >>> set_bit(i, 1, 1)
    7
    >>> set_bit(i, 0, 0)
    4
    '''
    if value:
        return num | (1L << bit)
    else:
        return num & (~(1L << bit))

def sign_extend(num, size):
    '''Sign exten number who is `size' bits wide
    
    >>> sign_extend(5, 2)
    1
    >>> sign_extend(5, 3)
    -3
    '''
    m = mask(size - 1)
    res = num & m
    # Positive
    if (num & (1L << (size - 1))) == 0:
        return res

    # Negative, 2's complement
    res = ~res
    res &= m
    res += 1
    return -res

rotate and stretch

from operator import itemgetter
from itertools import imap, chain, repeat

def rotate(matrix):
  '''Rotate matrix 90 degrees'''
  def row(row_num):
      return map(itemgetter(row_num), matrix)

  return map(row, range(len(matrix[0])))

def stretch(items, times):
  '''stretch([1,2], 3) -> [1,1,1,2,2,2]'''
  return reduce(add, map(lambda item: [item] * times, items), [])

def istretch(items, count):
    '''istretch([1,2], 3) -> [1,1,1,2,2,2] (generator)'''
    return chain(*imap(lambda i: repeat(i, count), items))

num2eng

Just found this on the web ...

svnfind

#!/usr/bin/env python
# Find paths matching directories in subversion repository

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"

# TODO:
# * Limit search depth
# * Add option to case [in]sensitive
# * Handling of svn errors
# * Support more of "find" predicates (-type, -and, -mtime ...)
# * Another porject: Pre index (using swish-e ...) and update only from
#   changelog

from os import popen

def join(path1, path2):
   if not path1.endswith("/"):
       path1 += "/"
   return "%s%s" % (path1, path2)

def svn_walk(root):
   command = "svn ls '%s'" % root
   for path in popen(command):
       path = join(root, path.strip())
       yield path
       if path.endswith("/"): # A directory
           for subpath in svn_walk(path):
                   yield subpath

def main(argv=None):
   if argv is None:
       import sys
       argv = sys.argv

   import re
   from itertools import ifilter
   from optparse import OptionParser

   parser = OptionParser("usage: %prog PATH EXPR")

   opts, args = parser.parse_args(argv[1:])
   if len(args) != 2:
       parser.error("wrong number of arguments") # Will exit

   path, expr = args
   try:
       pred = re.compile(expr, re.I).search
   except re.error:
       raise SystemExit("error: bad search expression: %s" % expr)

   found = 0
   for path in ifilter(pred, svn_walk(path)):
       found = 1
       print path

   if not found:
       raise SystemError("error: nothing matched %s" % expr)

if __name__ == "__main__":
   main()

Simple Text Summarizer

Comments:

• About 50 lines of code
• Gives reasonable results (try it out)
• tokenize need to be improved much more (better detection, stop words ...)
• split_to_sentences need to be improved much more (handle 3.2, Mr. Smith ...)
• In real life you'll need to "clean" the text (Ads, credits, ...)
  

attrgetter is fast

#!/usr/bin/env python

from operator import attrgetter
from random import shuffle

class Point:
    def __init__(self, x, y):
        self.x, self.y = x, y

def sort1(points):
    points.sort(key = lambda p: p.x)

def sort2(points):
    points.sort(key = attrgetter("x"))

if __name__ == "__main__":
    from timeit import Timer

    points1 = [Point(x, 2 * x) for x in range(100)]
    points2 = points1[:]

    num_times = 10000

    t1 = Timer("sort1(points1)", "from __main__ import sort1, points1")
    print t1.timeit(num_times)

    t2 = Timer("sort2(points2)", "from __main__ import sort2, points2")
    print t2.timeit(num_times)

$ ./attr.py
0.492087125778
0.29891705513
$

Faster and Shorter "dot" using itertools

Let's calculate the dot product of two vectors:

from itertools import starmap, izip
from operator import mul

def dot1(v1, v2):
  result = 0
  for i, value in enumerate(v1):
      result += value * v2[i]
  return result

def dot2(v1, v2):
  return sum(starmap(mul, izip(v1, v2)))

if __name__ == "__main__":
  from timeit import Timer

  num_times = 1000
  v1 = range(100)
  v2 = range(100)

  t1 = Timer("dot1(%s, %s)" % (v1, v2), "from __main__ import dot1")
  print t1.timeit(num_times) # 0.038722038269

  t2 = Timer("dot2(%s, %s)" % (v1, v2), "from __main__ import dot2")
  print t2.timeit(num_times) # 0.0260770320892

dot2 is faster and shorter, however dot1 is more readable - my vote goes to dot2.

Going Scheme Style

Lets count the words our code, omitting comments:

#!/usr/bin/env python

from functools import partial
import re

filter_comment = partial(re.compile("#.*$").sub, "")
words = re.compile("[^ \t\n\r]+").findall

def num_words(text):
  '''Return the number of words in a code segment, ignoring comments

  >>> num_words("")
  0
  >>> num_words("1 + 1")
  3
  >>> num_words("1 + 1 # add 1 to 1")
  3
  '''
  return sum(map(len,
                 map(words,
                     map(filter_comment,
                         text.splitlines()))))


if __name__ == "__main__":
  import doctest
  doctest.testmod()

Word Reduction

A little solution to http://ddj.com/cpp/202806370?pgno=3:

#!/usr/bin/env python

DICTIONRAY = set()

def load_dictionary(filename):
  DICTIONRAY.add("a")
  DICTIONRAY.add("i")
  for line in open(filename):
      DICTIONRAY.add(line.strip())

def _reduction(word):
  if word not in DICTIONRAY:
      return []
  if len(word) == 1:
      return [word]

  for i in range(len(word)):
      subword = "%s%s" % (word[:i], word[i+1:])
      if subword not in DICTIONRAY:
          continue
      path = reduction(subword)
      if path:
          return [word] + path
  return []

CACHE = {}
def reduction(word):
  if word not in CACHE:
      CACHE[word] = _reduction(word)

  return CACHE[word]

def main(argv=None):
  if argv is None:
      import sys
      argv = sys.argv

  from os.path import isfile
  from optparse import OptionParser

  parser = OptionParser("usage: %prog DICTIONRAY")

  opts, args = parser.parse_args(argv[1:])
  if len(args) != 1:
      parser.error("wrong number of arguments") # Will exit

  dictfile = args[0]
  if not isfile(dictfile):
      raise SystemExit("error: can't find %s" % dictfile)

  load_dictionary(dictfile)
  for word in sorted(DICTIONRAY, key=lambda w: len(w), reverse=1):
      path = reduction(word)
      if path:
          print "\n".join(path)
          break

if __name__ == "__main__":
  main()

Works fast enough as well (running on SIGNLE.TXT):

mtebeka@bugs:word-reduction - 08:43 $ time ./word_reduction.py dictionaries/SINGLE.TXT
restraint's
restraints
restrains
retrains
retains
retain
retin
rein
rin
in
n

real    0m4.088s
user    0m4.023s
sys     0m0.065s
mtebeka@bugs:word-reduction - 08:43 $

Quick Web Searches

A little script to search the documents in Journal of Machine Learning.

import webbrowser
from urllib import urlencode

def jmlr(words):
 query = "site:http://jmlr.csail.mit.edu filetype:pdf " + " ".join(words)
 url = "http://www.google.com/search?" + urlencode([("q", query)])

 webbrowser.open(url)

if __name__ == "__main__":
 from optparse import OptionParser

 parser = OptionParser("usage: %prog WORD1 [WORD2 ...]")

 opts, args = parser.parse_args()
 if not args:
     parser.error("wrong number of arguments") # Will exit

 jmlr(args)

I have many more is these for Google search, Wikipedia search, Acronym Search ...
The trick is to do a search using the web interface and then look at the URL of the results page.

Minimal testing

I'm using py.test as our test suite. And found out that even the most minimal tests give me great benefits:

def test_joke():
 pass

One I have that in a file call test_joke.py,
py.test will pick it up and try to run it.

The good thing to have only this minimal code is that the test will fail if you happen to introduce a syntax error or some error in the module initialization.

Of course, when I have more time I beef up the tests ;)

What make it even more useful is a continuous integration system that run the tests every time someone checks in the code.

Persistant ID generator

Let's say you want to map key->number, and every time you get a new key you give it a different number.
(Useful for mapping words to vector index in IR)

The easy way is:

from collections import defaultdict
from itertools import count

vector_index = defaultdict(count(0).next)
print vector_index["a"] # 0
print vector_index["a"] # 0
print vector_index["b"] # 1

Python Load Time Is Slow

I'm very fond of CGI, the main reason is that if a CGI crashes - it's only this process for this user. Not the whole system.

However, Python is *slow* starting up - a problem in CGI world. I still use it but wish it'll load faster.

Here are some number for comparison (all programs just print "Hello there" and exit):

hw          real: 0.001 user: 0.001 sys: 0.000 
hwcc        real: 0.002 user: 0.001 sys: 0.001 
hw.lua      real: 0.004 user: 0.001 sys: 0.004 
hw.pl       real: 0.004 user: 0.002 sys: 0.001 
hw.rb       real: 0.006 user: 0.004 sys: 0.002 
hw.php      real: 0.017 user: 0.013 sys: 0.005 
hw.py       real: 0.019 user: 0.011 sys: 0.008 
hw.lsp      real: 0.023 user: 0.011 sys: 0.011 
hw.scm      real: 0.027 user: 0.019 sys: 0.008 
hwcs.exe    real: 0.036 user: 0.029 sys: 0.007 
HT.class    real: 0.084 user: 0.027 sys: 0.01 

See the makefile below for which is each program.
As you can see Python is somewhere in the middle, not as bad as Java and C#, but about 5 times slower than Perl.

Makefile:

SCRIPTS = hw.py hw.rb hw.pl hw.lsp hw.scm hw.lua hw.php
GENERATED = hw hwcc hwcs.exe
JAVA = HT.class
PROGRAMS = $(SCRIPTS) $(GENERATED) $(JAVA)

all: $(PROGRAMS) times
@echo DONE

hw: hw.c
gcc -o $@ -O3 $<

hwcc: hw.cc
g++ -o $@ -O3 $<

HT.class: hw.java
javac $<

and timeit is:

#!/bin/bash

TIMEFORMAT='real: %3R user: %3U sys: %3S'
SPACES="                               "
MAXLEN=10


for program in $*;
do
name=${program/.class/}
if [ ${program/.exe/} != $program ]; then
    timestr=`(time mono $program > /dev/null) 2>&1`
elif [ $name == $program ]; then
    timestr=`(time ./$program > /dev/null) 2>&1`
else
    timestr=`(time java -client $name > /dev/null) 2>&1`
fi
padlen=$(($MAXLEN - ${#program}))
echo "${program}${SPACES:0:$padlen}${timestr}"
done

Simple launcher for XFCE

I'm using xfce4 as a window manager, it has a nice launcher called xfrun4, however it only tries to execute applications.

I've written a little launcher that uses exo-open to open it's argument. This way I can open PDF files, directories etc.
(It's not as fancy as Mac's QuickSilver or Windows SlickRun, but it does the job)

In order to get quick access, open the keyboard settings, create a new theme (you can't change the default) and add the launcher there. I usually go with "CTRL-SHIFT-K" to activate.

#!/usr/bin/env python
'''Simple lanucher'''

from Tkinter import Tk, Label, Entry
from tkFont import Font
from tkMessageBox import showerror

from os.path import exists, expanduser
from os import environ, popen

def launch(name):
   name = expanduser(name)
   if not exists(name):
       fullname = popen("which %s 2>/dev/null" % name).read().strip()
       if not fullname:
           raise ValueError("can't find %s" % name)
       name = fullname

   popen("/usr/bin/exo-open \"%s\"" % name).read()


USER_CANCEL = 0
ROOT = None
COMMAND = None

def quit(event):
   global USER_CANCEL

   USER_CANCEL = 1
   ROOT.quit()

def build_ui():
   global ROOT, COMMAND

   ROOT = Tk()
   ROOT.title("Launchie")
   ROOT.bind("<Escape>", quit)
   COMMAND = Entry(width=80, font=Font(size=14))
   COMMAND.pack()
   COMMAND.bind("<Return>", lambda e: ROOT.quit())

def show_ui():
   global USER_CANCEL

   USER_CANCEL = 0
   COMMAND.focus()
   ROOT.mainloop()

   return COMMAND.get().strip()


def main(argv=None):
   if argv is None:
       import sys
       argv = sys.argv

   from optparse import OptionParser

   parser = OptionParser("usage: %prog")

   opts, args = parser.parse_args(argv[1:])
   if len(args) != 0:
       parser.error("wrong number of arguments") # Will exit


   build_ui()

   while 1:
       try:
           command = show_ui()
           if USER_CANCEL:
               raise SystemExit

           if not command:
               showerror("Launchie Error", "Please enter *something*")
               continue

           launch(command)
           break
       except ValueError:
           showerror("Lanuchie Error", "Can't launch %s" % command)

if __name__ == "__main__":
   main()

The reason I chose Tkinter is that it's already installed with Python and it's good enough for simple stuff like this.

start

Many times, I open files from the command line.
However each OS has it's own utility for opening file, so I have this little script called start (yes, I started my life on windows ;).

#!/bin/bash
# Open a file from command line, multi OS

# Miki Tebeka <miki.tebeka@gmail.com>

if [ $# -ne 1 ]; then
   echo "usage: `basename $0` PATH"
   exit 1
fi

if [ ! -e $1 ]; then
   echo "error: can't find $1" 1>&2
   exit 1
fi

case `uname` in
   Linux) open=exo-open;;
   Darwin) open=open;;
   CYGWIN*) open=cygstart;;
   MINGW32*) open=start;;
   *) echo "error: no start program for `uname` platform" 1>&2; exit 1;;
esac

$open "$1"

Supporting Search Query Syntax

It's very easy to add support for search-engine like syntax in your program.
The idea is to convert the query to a Python expression and then evaluate it.

We'll support the following syntax:

word1 word2       - word1 and word2
word1 AND word2   - word1 and word2
word1 word2       - word1 and word2
word1 OR word2    - word1 ord word2
NOT word          - Not containing word

The code is very simple:

def is_operator(token):
  return token in set(["and", "not", "or", "(", ")"])

def should_insert_and(expr, token):
  if not expr:
      return 0

  if is_operator(expr[-1]):
      return 0

  if is_operator(token):
      return 0

  return 1

def match(query, text):
  words = set(text.lower().split())

  expr = []
  for token in query.lower().split():

      if should_insert_and(expr, token):
          expr.append("and")

      if is_operator(token):
          expr.append(token)
      else:
          expr.append(token in words)

  py_expr = " ".join(map(str, expr))
  return eval(py_expr)

def test():
  assert match("a", "a"), "a --- a"
  assert not match("a", ""), " --- a"
  assert match("a AND b", "a c b"), "a c b --- a AND b"
  assert not match("a AND b", "a c"), "a c --- a AND b"
  assert match("NOT ( a OR b )", "z"), "z --- NOT ( a OR b )"
  assert match("a OR b", "b"), "b --- a OR b"

Notes:
1. We don't do any fancy tokenization (text and query), but in most cases this should be enough.
2. We place an AND where it's missing.

Cheetah Templates

Currently doing some web development and found Cheetah very useful.
I like Cheetah since it's syntax is very similar to Python and I can use my existing Python objects with it.

I have one master template that set the site general site look and feel (with the master CSS of course).

#from time import ctime

#attr NAME = "???"

#def head
#end def

#def body
OOOPS, head will roll...
#end def

<html>
<head>
<link rel="stylesheet" type="text/css" href="style.css" />
$head
</head>
<body>
<div class="header">My Wonderful Site - $NAME</div>
$body

<hr />
<div class="footer">
Generated $ctime()
</div>
</body>
</html>

$head and $body are place holders that the specific pages will fill.
Pages also define $NAME which will be shown in the header.

The an specific page (index.tmpl) can be:

#include "master.tmpl"

#attr NAME = "INDEX"

#def body
This is my site index page, see also <a href="other.cgi">other page</a>. <br />

Oh, and also random = $random;
#end def

And the CGI script:

#!/usr/local/bin/python

from Cheetah.Template import Template

from random import randint


def main():
   random = randint(0, 100)

   print "Content-Type: text/html"
   print

   page = Template(file="index.tmpl", searchList=[locals()])
   print page.respond()

if __name__ == "__main__":
   main()

Note that I pass locals() as the search list. This frees me from creating a mapping dictionary (exporting random to the template).

That's about it, you can use the master template and the site main CSS to have a uniform looking site and let each page implement just the $body and $head if it needs to.