EnSalvage

I wrote a little utility to help a fried whose Entourage database got corrupted.

If anybody cares, here it is, you can even download a .app for Mac :)

binary

#!/bin/bash
# Show binary representation of a number

if [ $# -ne 1 ]; then
    echo "usage: `basename $0` NUMBER" 1>&2
    exit 1
fi

echo "obase=2; $1" | bc -l

strftime

#!/usr/bin/env python
'''Convert seconds since epoch to human readable time format.
   Useful in cases where logs dump time information in epoch.

    $ echo 1255099899 | strftime
    10/09/2009 07:51
    $
'''

from time import strftime, localtime
import re

def main(argv=None):
    import sys
    from optparse import OptionParser

    argv = argv or sys.argv

    default_format = "%m/%d/%Y %H:%M"
    parser = OptionParser("%prog [options] [FILENAME]")
    parser.add_option("--format", dest="format",
        help="time format [%s]" % default_format, default=default_format)
    parser.add_option("--doc", help="show strftime documentation",
        dest="doc", action="store_true", default=0)
    parser.add_option("--quote", help="quote result (e.g. '10/2/2009')",
        dest="quote", default="")

    opts, args = parser.parse_args(argv[1:])
    if len(args) not in (0, 1):
        parser.error("wrong number of arguments") # Will exit

    if opts.doc:
        import webbrowser
        url = "http://docs.python.org/library/time.html#time.strftime"
        webbrowser.open(url)
        raise SystemExit

    infile = args[0] if args else "-"
    if infile == "-":
        info = sys.stdin
    else:
        try:
            info = open(infile)
        except IOError, e:
            raise SystemExit("error: can't open %s - %s" % (infile, e))

    def callback(match):
        t = localtime(float(match.group()))
        human = strftime(opts.format, t)
        if opts.quote:
            human = human.replace(opts.quote, r"\\%s" % opts.quote)
            human = opts.quote + human + opts.quote

        return human

    # We assume 1XXXXXXXXX (2001 +) dates, so we won't convert *everything*
    print re.sub("1\d{9}(\.\d+)?", callback, info.read())


if __name__ == "__main__":
    main()

Uncomment Your Code

process

A combination of pgrep and pkill.

#!/usr/bin/env python
'''Find processes by regexp'''

# Miki Tebeka <miki.tebeka@gmail.com>

import re
from os import popen, getpid, kill
import signal
from sys import platform

def ask_user():
    answer = ""
    while not answer:
        answer = raw_input("kill? [y/n]").strip()
        if not answer:
            continue
        if answer.lower() in ("y", "yes"):
            return 1
        elif answer.lower() in ("n", "no"):
            return 0
        else:
            print "Please answer yes or no"

def find_signum(signal_name):
    if signal_name[:3] == "SIG":
        signum = getattr(signal, signal_name, None)
    else:
        try:
            signum = int(signal_name)
        except ValueError:
            signum = None

    return signum

def main(argv=None):
    if argv is None:
        import sys
        argv = sys.argv

    from optparse import OptionParser

    parser = OptionParser("usage: %prog [options] [REGEX]")
    parser.add_option("-k", "--kill", help="kill processes",
            dest="kill", action="store_true", default=0)
    parser.add_option("-i", "--interactive", 
        help="interactive killing (ask before each one)",
        dest="interactive", action="store_true", default=0)
    parser.add_option("-a", "--annotate", help="annotate matching lines",
        dest="annotate", action="store_true", default=0)
    parser.add_option("-s", help="signal to send", default="SIGTERM",
            action="store", dest="signal")
    parser.add_option("-9", help="send SIGKILL", dest="signal", 
            action="store_const", const="SIGKILL")
    parser.add_option("-g", "--gvim", help="kill also gvim sessions",
            action="store_true", default=0, dest="gvim")

    opts, args = parser.parse_args(argv[1:])
    if len(args) not in (0, 1):
        parser.error("wrong number of arguments") # Will exit

    regexp = args[0] if args else "."

    try:
        is_interesting = re.compile(regexp).search
    except re.error:
        raise SystemExit("error: bad regular expression - %s" % args[0])

    signal = find_signum(opts.signal)
    if signal is None:
        raise SystemExit("error: bad signal - %s" % opts.signal)

    found = 0
    first = 1
    if platform == "darwin":
        cmd = "ps ax"
        index = 0
    else:
        cmd = "ps -eff"
        index = 1

    for line in popen(cmd):
        line = line.rstrip()
        # Print header line
        if first:
            first = 0
            print line
            continue

        if (not opts.annotate) and (not is_interesting(line)):
            continue

        fields = line.split()
        pid = int(fields[index])
        if pid == getpid():
            continue

        # Color lines
        if is_interesting(line):
            found = 1
            if opts.annotate:
                print "\033[32m%s\033[0m" % line
            else:
                print line
        elif opts.annotate:
            print line

        # Kill only intersting lines
        if not is_interesting(line):
            continue
        do_kill = 0
        if opts.kill and opts.interactive:
            do_kill = ask_user()
        elif opts.kill and not opts.gvim:
            do_kill = "vim" not in line.lower()
        elif opts.kill:
            do_kill = 1

        if do_kill:
            try:
                kill(pid, signal)
                print "%d killed" % pid
            except OSError, e:
                print "error: %s" % e
        elif opts.kill:
            print "not killing %s" % pid

    if not found:
        error = "error: can't find any process matching `%s'" % args[0]
        raise SystemExit(error)

if __name__ == "__main__":
    main()

pyindent

Indent C files according to PEP 7.
(uses indent)

#!/bin/sh

# Indent C files according to http://www.python.org/dev/peps/pep-0007/
# Miki Tebeka <miki.tebeka@gmail.com>

indent -cli4 -npcs -l78 -bad -bap -br -ce -nbc -di1 -psl -i4 -brs -lp -cdw -ut -ts4 $@

timeat

Find local time at various locations, this is a small mashup between Google map and EarthTools.

Dynamicall Generating Static Content

Sometimes you want to generate a resource (say a chart) once and then keep it. This way you don't need to keep track of all the objects you've created and can easily clean (since if someone want the resource again it will be generated again).

One way is to do it with lighttpd 404 handlers (view complete code).

lighttpd.conf

# lighttpd configuration file
server.modules  = ( "mod_cgi" )

# This is generated by by web-up
include "lighttpd.inc"

server.document-root = var.root
server.pid-file = var.root + "/lighttpd.pid"

# files to check for if .../ is requested
index-file.names            = ( "index.cgi" )

## set the event-handler (read the performance section in the manual)
# server.event-handler = "freebsd-kqueue" # needed on OS X

# mimetype mapping
mimetype.assign             = (
 ".pdf"          =>      "application/pdf",
 ".sig"          =>      "application/pgp-signature",
 ".spl"          =>      "application/futuresplash",
 ".class"        =>      "application/octet-stream",
 ".ps"           =>      "application/postscript",
 ".torrent"      =>      "application/x-bittorrent",
 ".dvi"          =>      "application/x-dvi",
 ".gz"           =>      "application/x-gzip",
 ".pac"          =>      "application/x-ns-proxy-autoconfig",
 ".swf"          =>      "application/x-shockwave-flash",
 ".tar.gz"       =>      "application/x-tgz",
 ".tgz"          =>      "application/x-tgz",
 ".tar"          =>      "application/x-tar",
 ".zip"          =>      "application/zip",
 ".mp3"          =>      "audio/mpeg",
 ".m3u"          =>      "audio/x-mpegurl",
 ".wma"          =>      "audio/x-ms-wma",
 ".wax"          =>      "audio/x-ms-wax",
 ".ogg"          =>      "application/ogg",
 ".wav"          =>      "audio/x-wav",
 ".gif"          =>      "image/gif",
 ".jpg"          =>      "image/jpeg",
 ".jpeg"         =>      "image/jpeg",
 ".png"          =>      "image/png",
 ".xbm"          =>      "image/x-xbitmap",
 ".xpm"          =>      "image/x-xpixmap",
 ".xwd"          =>      "image/x-xwindowdump",
 ".css"          =>      "text/css",
 ".html"         =>      "text/html",
 ".htm"          =>      "text/html",
 ".js"           =>      "text/javascript",
 ".asc"          =>      "text/plain",
 ".c"            =>      "text/plain",
 ".cpp"          =>      "text/plain",
 ".log"          =>      "text/plain",
 ".conf"         =>      "text/plain",
 ".text"         =>      "text/plain",
 ".txt"          =>      "text/plain",
 ".dtd"          =>      "text/xml",
 ".xml"          =>      "text/xml",
 ".mpeg"         =>      "video/mpeg",
 ".mpg"          =>      "video/mpeg",
 ".mov"          =>      "video/quicktime",
 ".qt"           =>      "video/quicktime",
 ".avi"          =>      "video/x-msvideo",
 ".asf"          =>      "video/x-ms-asf",
 ".asx"          =>      "video/x-ms-asf",
 ".wmv"          =>      "video/x-ms-wmv",
 ".bz2"          =>      "application/x-bzip",
 ".tbz"          =>      "application/x-bzip-compressed-tar",
 ".tar.bz2"      =>      "application/x-bzip-compressed-tar"
)


$HTTP["url"] =~ "\.pdf$" {
 server.range-requests = "disable"
}

# which extensions should not be handle via static-file transfer
static-file.exclude-extensions = ( ".cgi" )

# This is where the magic happens :)
$HTTP["url"] =~ "/charts/.*\.png$" {
   server.error-handler-404 = "/chart.cgi"
}

chart.cgi

#!/usr/bin/env python
'''This CGI script is called by lighttpd 404 error handler when it tries to find
/charts/XXX.png.

This way the image is generated once and then we let lighttpd serve it as static
content.
'''

# The below is needed to tell pylab to work without a screen
import matplotlib; matplotlib.use("Agg")
from pylab import hist, randn, title, savefig

from os import environ
from os.path import basename
from urlparse import urlparse

def generate_chart(name, outfile):
   # Generate a random chart
   hist(randn(1000), 100)
   title("Chart for \"%s\"" % name)
   savefig(outfile)

def main():
   import cgitb; cgitb.enable()

   o = urlparse(environ.get("REQUEST_URI", ""))
   if not o.path:
       print "Content-type: text/plain\n"
       print "ERROR: Can't find path"
       raise SystemExit

   outfile = "charts/%s" % basename(o.path)
   name = basename(o.path).replace(".png", "")
   if not name:
       print "Content-type: text/plain\n"
       print "ERROR: 'name' not specified"
       raise SystemExit

   generate_chart(name, outfile)
   image = open(outfile, "rb").read()

   print "Content-type: image/png\n"
   print image

if __name__ == "__main__":
   main()

Enabling Flash on Google Chrom (OSX)

The current release of Google Chrom For Mac has support for Flash, but you need to enable it with the --enable-plugins command line switch.

Here's how to make it persistent (well, until the next update):

[17:31] ~ $cd /Applications/Google\ Chrome.app/Contents/MacOS/
[17:32] MacOS $sudo mv Google\ Chrome _chrome
[17:32] MacOS $sudo gvim Google\ Chrome
[17:33] MacOS $sudo chmod +x Google\ Chrome 

And the content of the new Google Chrome is:

#!/bin/bash


"/Applications/Google Chrome.app/Contents/MacOS/_chrome" --enable-plugins $*

Happy surfing ...

VIm command to open a Python module

pyhelp

#!/usr/bin/env python

# Show help for Python

__author__ = "Miki Tebeka <miki.tebeka@gmail.com>"
__license__ = "BSD"

def main(argv=None):
    import sys
    from optparse import OptionParser
    import webbrowser

    argv = argv or sys.argv

    parser = OptionParser("%prog [MODULE[.FUNCITON]]")
    opts, args = parser.parse_args(argv[1:])

    if len(args) not in (0, 1):
        parser.error("wrong number of arguments") # Will exit

    root = "http://docs.python.org"

    if not args:
        url = "%s/%s" % (root, "modindex.html")
    elif "." in args[0]:
        module, function = args[0].split(".")
        url = "%s/library/%s.html#%s.%s" % (root, module, module, function)
    else:
        url = "%s/library/%s.html" % (root, args[0])

    webbrowser.open(url)

if __name__ == "__main__":
    main()

Two gnupg Wrappers

encrypt

#!/bin/bash

# Encrypt input using gnupg and shred it
# THIS DELETES THE ORIGINAL FILE

err=0
for file in $@
do
    if [ $file != ${file/.gpg/} ]; then
        echo "$file already encrypted"
        continue
    fi
    if [ -d $file ]; then
        fname=$file.tar.bz2
        tar -cjf $fname $file
        find $file -type f -exec shred -u {} \;
        rm -r $file
    elif [ -f $file ]; then
        fname=$file
    else
        echo "ERROR: Unknown file type $file"
        err=1
        continue
    fi

    gpg -r miki -e $fname
    if [ $? -ne 0 ]; then
        echo "ERROR: can't encrypt"
        err=1
        continue
    fi

    shred -u $fname
    if [ $? -ne 0 ]; then
        echo "ERROR: can't shred"
        err=1
        continue
    fi
done

exit $err

decrypt

#!/bin/bash

# Decrypt a file encrypted with gpg


# Check command line
if [ $# -lt 1 ]; then
 echo "usage: `basename $0` FILE[s]"
 exit 1
fi

err=0
for file in $@
do
 if [ ! -f $file ]; then
  echo "$file: Not found"
  err=1
  continue
 fi

 # Output to current directory
 bfile=`basename $file`
    ext=`echo $file | sed -e 's/.*\.\([!.]*\)/\1/'`
    if [ "$ext" != "gpg" ] && [ "$ext" != "pgp" ]; then
  echo "Cowardly refusing to decrypt a file without gpg/pgp extension"
        err=1
        continue
 fi

 outfile=${bfile%.$ext}

    echo $file '->' $outfile
 gpg -d -o $outfile $file
 if [ $? -ne 0 ]; then
        echo "error decrypting $file"
        err=1
 fi
done

exit $err

What's My IP?

#!/usr/bin/env python
'''Find local machine IP (cross platform)'''

from subprocess import Popen, PIPE
from sys import platform
import re

COMMANDS = {
    "darwin" : "/sbin/ifconfig",
    "linux" : "/sbin/ifconfig",
    "linux2" : "/sbin/ifconfig",
    "win32" : "ipconfig",
}

def my_ip():
    command = COMMANDS.get(platform, "")
    assert command, "don't know how to get IP for current platform"

    pipe = Popen([command], stdout=PIPE)
    pipe.wait()
    output = pipe.stdout.read()

    for ip in re.findall("(\d+\.\d+\.\d+\.\d+)", output):
        if ip.startswith("127.0.0"):
            continue
        return ip

if __name__ == "__main__":
    print my_ip()

Little Genetics

#!/usr/bin/env python
'''Playing with genetic algorithms, see 
http://en.wikipedia.org/wiki/Genetic_programming.

The main idea that the "chromosome" represents variables in our algorithm and we
have a fitness function to check how good is it. For each generation we keep the
best and then mutate and crossover some of them.

Since the best chromosomes move from generation to generation, we cache the
fitness function results.

I'm pretty sure I got the basis for this from somewhere on the net, just don't
remeber where :)
'''

from itertools import starmap
from random import random, randint, choice
from sys import stdout

MUTATE_PROBABILITY = 0.1

def mutate_gene(n, range):
    if random() > MUTATE_PROBABILITY:
        return n

    while 1:
        # Make sure we mutated something
        new = randint(range[0], range[1])
        if new != n:
            return new

def mutate(chromosome, ranges):
    def mutate(gene, range):
        return mutate_gene(gene, range)

    while 1:
        new = tuple(starmap(mutate, zip(chromosome, ranges)))
        if new != chromosome:
            return new

def crossover(chromosome1, chromosome2):
    return tuple(map(choice, zip(chromosome1, chromosome2)))

def make_chromosome(ranges):
    return tuple(starmap(randint, ranges))

def breed(population, size, ranges):
    new = population[:]
    while len(new) < size:
        new.append(crossover(choice(population), choice(population)))
        new.append(mutate(choice(population), ranges))

    return new[:size]

def evaluate(fitness, chromosome, data, cache):
    if chromosome not in cache:
        cache[chromosome] = fitness(chromosome, data)

    return cache[chromosome]

def update_score_cache(population, fitness, data, cache):
    for chromosome in population:
        if chromosome in cache:
            continue
        cache[chromosome] = fitness(chromosome, data)

def find_solution(fitness, data, ranges, popsize, nruns, verbose=0):
    score_cache = {}
    population = [make_chromosome(ranges) for i in range(popsize)]
    for generation in xrange(nruns):
        update_score_cache(population, fitness, data, score_cache)
        population.sort(key=score_cache.get, reverse=1)
        if verbose:
            best = population[0]
            err = score_cache[best]
            print "%s: a=%s, b=%s, err=%s" % (generation, best[0], best[1], err)

        base = population[:popsize/4]
        population = breed(base, popsize, ranges)

    population.sort(key=score_cache.get, reverse=1)
    return population[0], score_cache[population[0]]

def test(show_graph=1):
    '''Try to find a linear equation a*x + b that is closest to log(x)'''
    from math import log
    xs = range(100)
    data = map(lambda i: log(i+1) * 100, xs)
    def fitness(chromosome, data):
        '''Calculate average error'''
        a, b = chromosome
        def f(x):
            return a * x + b
        values = map(f, xs)
        diffs = map(lambda i: abs(values[i] - data[i]), xs)

        # We want minimal error so return 1/error
        return 1 / (sum(diffs) / len(diffs))

    # Show a nice plot
    (a, b), err = find_solution(fitness, data, ((0, 100), (0, 100)), 10, 100, 1)
    print "best: a=%s, b=%s (error=%s)" % (a, b, err)

    data2 = map(lambda x: a * x + b, range(100))
    if not show_graph:
        return

    import pylab
    l1, l2 = pylab.plot(xs, data, xs, data2)
    pylab.legend((l1, l2), ("log(x+1)", "%s * x + %s" % (a, b)))
    pylab.show()

if __name__ == "__main__":
    test()

A Twitter Trends / Google News meshup

This tried to automatically explain why things are trending on Twitter.

The Python Server (trends.py)

#!/usr/bin/env python
'''A twitter trends/google news mesh

loader_thread get news trends every 1min and set _TRENDS_HTML
The web server serves _TRENDS_HTML with is refresed via JavaScript every 30sec
'''

from urllib import urlopen, urlencode
import feedparser
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from os.path import dirname, join, splitext
import json
from threading import Thread
from time import sleep

def ascii_clean(text):
    return text.encode("ascii", "ignore") # Pure ACII

def trend_news(trend):
    query = {
        "q" : ascii_clean(trend),
        "output" : "rss"
    }
    url = "http://news.google.com/news?" + urlencode(query)
    return feedparser.parse(url).entries

def current_trends():
    url = "http://search.twitter.com/trends.json"
    return json.load(urlopen(url))["trends"]

def news_html(news):
    html = '<li><a href="%(link)s">%(title)s</a></li>'
    chunks = map(lambda e: html % e, news)
    return "\n".join(["<ul>"] + chunks + ["</ul>"])

def trend_html(trend):
   thtml = '<a href="%(url)s">%(name)s' % trend
   nhtml = news_html(trend_news(trend["name"]))
   return '<tr><td>%s</td><td>%s</td><tr>' % (thtml, nhtml)

def table_html(trends):
    return ("<table>" + 
            "<tr><th>Trend</th><th>Related News</th></tr>" +
            "".join(map(trend_html, trends)) +
            "</table>"
           )

_TRENDS_HTML = ""
def loader_thread():
    global _TRENDS_HTML

    while 1:
        _TRENDS_HTML = ascii_clean(table_html(current_trends()))
        sleep(60)

def run_loader_thread():
    t = Thread(target=loader_thread)
    t.daemon = 1
    t.start()

def trends_html():
    while not _TRENDS_HTML:
        sleep(0.1)

    return _TRENDS_HTML

def index_html():
    return open(join(dirname(__file__), "index.html")).read()

class RequestHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/":
            self.wfile.write(index_html())
        elif self.path.startswith("/trends"):
            self.wfile.write(trends_html())
        elif splitext(self.path)[1] in (".js", ".css"):
            self.wfile.write(open(".%s" % self.path).read())
        else:
            self.send_error(404, "Not Found")

if __name__ == "__main__":
    run_loader_thread()
    server = HTTPServer(("", 8888), RequestHandler)
    server.serve_forever()

The Auto-Refreshing Web Client (index.html)

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
    <head>
        <title>Trends</title>
        <link rel="stylesheet" href="trends.css" />
        <style>
            body {
                font-family: Serif, Helvetica;
            }

            table {
                width: 100%;
                border: 2px solid gray;
            }

            td {
                vertical-align: top;
                border-bottom: 1px solid black;
            }

            h1 {
                text-align: center;
                font-variant: small-caps;
            }

            a {
                text-decoration: none;
                color: black;
            }

            a:hover {
                background: silver;
            }

            ul {
                margin: 0px;
            }
        </style>
    </head>
    <body>
        <h1>Trends</h1>
        <div id="trends">
            Loading ...
        </div>
    </body>
    <script src="jquery.js"></script>
    <script>
        function load() {
            $('#trends').load('/trends');
        }
        $(document).ready(function() {
            load();
            setInterval(load, 30 * 1000);
        });
    </script>
</html>

Show "digits only" version of phone number

#!/bin/bash
# Convert phone numbers to numeric (1-800-T-MOBILE -> 1-800-8-662453)

if [ $# -ne 1 ]; then
    echo "usage: $(basename $0) PHONE-NUMBER"
    exit 1
fi

echo $1 | \
    tr '[:lower:]' '[:upper:]' | \
    tr ABCDEFGHIJKLMNOPQRSTUVWXYZ 22233344455566677778889999

JSON Pretiffier

#!/usr/bin/env python
'''JSON prettifier'''

def shift(array):
   try:
       return array.pop(0)
   except IndexError:
       return None

def fopen(name, mode, default):
   if name in (None, "-"):
       return default
   return open(name, mode)

def main(argv=None):
   import sys
   from optparse import OptionParser

   argv = argv or sys.argv

   indent = 4
   parser = OptionParser("%prog [INFILE [OUTFILE]]")
   parser.add_option("-i", "--indent", help="indent size (%s)" % indent,
                     dest="indent", default=indent, type="int")
   opts, args = parser.parse_args(argv[1:])
   if len(args) not in (0, 1, 2):
       parser.error("wrong number of arguments") # Will exit

   try:
       info = fopen(shift(args), "r", sys.stdin)
       outfo = fopen(shift(args), "w", sys.stdout)
   except IOError, e:
       raise SystemExit("error: %s" % e)

   import json
   json.dump(json.load(info), outfo, indent=opts.indent)
      
if __name__ == "__main__":
   main()

Update (Aug 14, 2009): Just found out about python -mjson.tool, oh well ...