Je t’embrasse Salutations from Silicon Valley, California

20May/110

commands to subprocess transition

The Python Documentation goes a long way in describing how to transition old code, but for some reason, fails to mention how to transition the legacy "commands.getoutput()" and "commands.getstatusoutput()" functionality into the new Python 2.6+ model of using the subprocess module.

Here is the code required (direct pull/modification of original 2.5 commands module code):

# Get the output from a shell command into a string.
# The exit status is ignored; a trailing newline is stripped.
# Assume the command will work with '{ ... ; } 2>&1' around it..
def getoutput(cmd):
    return getstatusoutput(cmd)[1]

# Ditto but preserving the exit status.
# Returns a pair (sts, output)
def getstatusoutput(cmd):
    import subprocess, os
    p = subprocess.Popen('{ ' + cmd + '; } 2>&1',
                         shell=True, stdout=subprocess.PIPE)
    sts = os.waitpid(p.pid, 0)[1]
    text = p.stdout.read()
    p.stdout.close()
    if text[-1:] == '\n': text = text[:-1]
    return sts, text
Filed under: Python No Comments
13Apr/110

Pretty-Print for Python Dict/List/Tuple

I wrote this a while ago to give me a better view into complex nested structures Dict-of-List-of-Dict, etc.
The benefit here is that these functions can be wrapped as a Python module, and imported to all your projects for debug.

Enjoy.

import sys

def newline():
    sys.stdout.write('\n')
    sys.stdout.flush()

def __type_quote(this):
    if type(this) == type(1): return str(this)
    else: return "'"+str(this)+"'"

def __has_children(parent):
    answer = False
    valid = [type([1,2]), type((1,2)), type({1:2})]
    if type(parent) == type([1,2]):
        for each in parent:
            if type(each) in valid: answer = True
    elif type(parent) == type((1,2)):
        answer = __has_children(list(parent))
    elif type(parent) == type({1:2}):
        for (key, value) in parent.items():
            if type(value) in valid: answer = True
    return answer

def __print_list(list, n=0, shrink=False, opener="[", closer="]"):
    assert type(list) == type([])
    sp = "".join([" "]*n)
    valid = [type([1,2]), type((1,2)), type({1:2})]

    # find the max index (for spacing of the format string)
    size = len(list)
    fstring = "%s  %-"+str(size)+"s: "

    # start printing stuff
    sys.stdout.write(opener)
    sys.stdout.flush()
    list.sort()
    for index in range(len(list)):
        newline()
        sys.stdout.write(fstring%(sp, index))
        sys.stdout.flush()
        this = list[index]
        if not __has_children(this) and type(this) in valid:
            if type(this) == type([1,2]) and len(this) > 10 and shrink == True:
                sys.stdout.write("['%s', ... ,'%s']"%(this[0], this[-1]))
                sys.stdout.flush()
            else:
                sys.stdout.write(str(this))
                sys.stdout.flush()
        elif type(this) == type([1,2]): __print_list(this, n+4+size)
        elif type(this) == type({1:2}): __print_dict(this, n+4+size)
        elif type(this) == type((1,2)): __print_tuple(this, n+4+size)
        else:
            sys.stdout.write(str(this))
            sys.stdout.flush()
    if len(list) > 0:
        newline()
        sys.stdout.write("%s%s"%(sp, closer))
    else: sys.stdout.write("%s"%closer)
    sys.stdout.flush()

def __print_dict(dict, n=0, shrink=False):
    assert type(dict) == type({1:2})
    sp = "".join([" "]*n)
    valid = [type([1,2]), type((1,2)), type({1:2})]

    # find the max key-size (for spacing of the format string)
    size = 0
    for key in dict.keys():
        if len(str(key)) > size: size = len(str(key))
    fstring = "%s  %-"+str(size)+"s: "

    # start printing stuff
    sys.stdout.write("{")
    sys.stdout.flush()
    keys = dict.keys()
    keys.sort()
    for key in keys:
        newline()
        sys.stdout.write(fstring%(sp, key))
        sys.stdout.flush()
        this = dict[key]
        if not __has_children(this) and type(this) in valid:
            if type(this) == type([1,2]) and len(this) > 10 and shrink == True:
                sys.stdout.write("[%s, ... ,%s]"%(__type_quote(this[0]),
                                                  __type_quote(this[-1])))
                sys.stdout.flush()
            else:
                sys.stdout.write(str(this))
                sys.stdout.flush()
        elif type(this) == type([1,2]): __print_list(this, n+4+size)
        elif type(this) == type({1:2}): __print_dict(this, n+4+size)
        elif type(this) == type((1,2)): __print_tuple(this, n+4+size)
        else:
            sys.stdout.write(__type_quote(this))
            sys.stdout.flush()
    if len(dict.keys()) > 0:
        newline()
        sys.stdout.write("%s}"%sp)
    else: sys.stdout.write("}")
    sys.stdout.flush()

def __print_tuple(tup, n=0, shrink=False):
    assert type(tup) == type((1,2))
    __print_list(list(tup), n, shrink, opener="(", closer=")")

def print_dict(d):
    __print_dict(d, shrink=True)
    newline()

def print_list(l):
    __print_list(l, shrink=True)
    newline()

def print_tuple(t):
    __print_tuple(t, shrink=True)
    newline()
Filed under: Python No Comments
20Jun/100

Not-so-common uses of ‘import’

As they say, Python is a language that does not force you to do things. Instead, it assumes the programmer to be a consenting adult who, knowingly or unknowingly has the power to bend and mold existing paradigms to fit whatever task is at hand. While this easy-peasy mindset sometimes can be an absolute asset, it will inevitably (hopefully not more than once) lead you down the winding path of destruction. Personally I feel that nothing in Python is quite as capable at dragging people down like the "import" statement.

Standard Imports:

import os
from os.path import isfile as isFile
from time import *

Hopefully this is not the first time you've heard this, but you should avoid the asterisk on imports. Unless you own the import (like a file with a bunch of common functions in it) then you can never be sure that a function defined within the import has not just blown away a function with the same name in your application.

User-specific Imports:
But what about the following... (Linux only I believe)

import os
import sys
sys.path.append(os.path.join(os.environ['HOME'], 'scripts'))
import userScript

This can be really helpful to provide some user-specific functionality. Basically the username determines the home directory, and the home directory then provides you with a specific subdirectory of Python scripts. If you log in as "root" you will get one set of functionality, whereas if you log in as "user" you will get a different set of functionality.

Watch out though, as environmental variables are very inaccurate. Anybody and anything can change these variables, and can leave you depending on incorrect or invalid values. One of the easiest ways to see this is by having a proper-daemon C/C++ application that calls a Python script. When you log on as any user and execute the binary, you will see the expected results. If you (via socket or RPC or whatever) call it remotely, the user defaults to the system, which unfortunately does not have certain environmental variables (like a "HOME" directory)... in other words, remote call = failure.

Application-relative (not path-relative) Imports:
Here is another one that I have seen, but have rarely used. Similarly to the previous condition, it requires appending to the system path, but this time we are working with a relative directory.

import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import someClass

There are two special things happening here. First, we are extracting the direct relative path to the file containing this code. If we were to run this through the interpreter, we would get an error because there is no "__file__". Second, because "__file__" is always the relative path to the Python file, it doesn't matter where you execute from, absolute path or not, you will always be referring to ".." from the Python file. This is EXTREMELY useful if you need to create an application that can be run via both relative and absolute path.

Path-agnostic Import Best Practices:
Although I have given multiple examples of how to get the system path to not care where you are executing from, I have for the most part found that the best practice with import paths is to use a configuration file. Why did I just complicate the import process even more? Well, lets take a look at the code:

import os
import sys
lines = open(os.path.join(os.path.dirname(__file__), "config.cfg")).splitlines()
paths = [line.split("=")[1] for line in lines if "path" is line.split("=")[0]]
for path in paths: sys.path.append(path)
import customModule

The benefit here is that now when you want to swap out "customModule" with a different version for testing, you dont need to move any directories, or rename any imports. All you do is change the config file. And yes, I did go a little overboard here, and allow you to add as many paths to the system path list as you want. The other big benefit here is that since you are always specifying an absolute path in the configuration file, you are always guaranteed that the import will succeed.

Filed under: Python No Comments
28May/100

Using a Property to get a Class name into an Attribute

For one reason or another, I found myself in a slight predicament. I needed to get the name of my class into an attribute.

Thanks to Google + KyLev I managed to procure a workable solution:

class Problem(object):
    moduleIndex = 0
    def __init__(self):
        self.__class__.moduleIndex = self.__class__.moduleIndex + 1

class Solution(object):
    moduleIndex = 0
    moduleName = property(fget=lambda self: self.__class__.__name__)
    def __init__(self):
        self.__class__.moduleIndex = self.__class__.moduleIndex + 1
        print "[%s] %s"%(self.moduleIndex, self.moduleName)

if __name__ == "__main__": t = Solution()
Filed under: Python No Comments
25Nov/090

Calculating Network Addresses

I have frequently run up against the problem of generating network addresses (strictly speaking IPv4) but I am positive that the principles demonstrated below can carry over into IPv6.

The biggest issue is that in calculating addresses, the best way to find the "Nth address from here" is to convert to decimal and add. Well it's not so difficult to convert 4 octets into a 32-bit binary number, and crunch that into a single decimal number, add "N", convert back to binary, divide into octets, and convert back to decimal... but it takes forever. So why take forever when you don't need to. The trick? Divide & Conquer.

After walking through the theory, masks, addresses, ranges, compliments... I finally put together the following as a helper. Feel free to comment.

Filed under: Python Continue reading
2Aug/090

RegEx – Lists & Hashes with Escape Characters

I recently ran across a need to match escape characters within a regular expression. The key here is that you need to make use of a fun little RegEx feature called "Zero-Width, Negative, Look-Back Assertion". Its a loaded title, but crams a LOT of functionality into a little space... None. This all stems from a post I made on Python-Forum...

Just FYI, so that nobody hassles me later, you need to include the Python Regular Expression module in order to use any of these examples.

import re

Problem #1: How do you represent an "escaped character" in a Regular Expression?
Solution #1: Use positive look-back assertion on the existence of a single backslash prior to any character.

text=r'\f\o\o'
regex = r'(.(?:(?<=\\).))'
i = re.finditer(regex, text)
for m in i: print m.groups()

Problem #2: It is not possible to put the previous regex inside a character set. So how then do you create a set of "escaped characters + non-terminators"?
Solution #2: Use the "?" on each "character" in the set.

text='{foo:bar}{a:\/foo\.bar}'
regex = r'(\{(?:(?:(?<=\\).)?[^\{\}\[\]\/\.]?)+\})'
i = re.finditer(regex, text)
for m in i: print m.groups()

Problem #3: How do you keep multiple occurrences from matching?
Solution #3: Explicit match using anchors. In other words, use "^" and "$" to bound your matching appropriately.

text='{a:\/foo\.bar}'
regex = r'^(\{(?:(?:(?<=\\).)?[^\{\}\[\]\/\.]?)+\})$'
i = re.finditer(regex, text)
for m in i: print m.groups()

Problem #4: How do you match practically any character that could occur within 2 bounding characters (like /regex/ in a sed-esque fashion)?
Solution #4: Exactly like before, except now, the only character we cant have in the middle is one of the plain (non-escaped) bounding characters.

text='/g[@#$%\/{foo:bar}\/^&*()]/{a:\/foo\.bar}'
regex = r'(\/(?:(?:(?<=\\).)?[^\/]?)+\/)'
i = re.finditer(regex, text)
for m in i: print m.groups()

Problem #5: How do we avoid matching when two valid matches are split? Like "/foo/{a:b}/bar/"
Solution #5: Explicitly define the possible combinations of each regex subsection

text1='{a:\/foo\.bar}/foo\/bar/'
text2='/foo\/bar/{a:\/foo\.bar}'
regex=r'^(\{(?:(?:(?<=\\).)?[^\{\}\[\]\/\.]?)+\})(\/(?:(?:(?<=\\).)?[^\/]?)+\/)$'
i = re.finditer(regex, text1)
for m in i: print m.groups()
regex=r'^(\/(?:(?:(?<=\\).)?[^\/]?)+\/)(\{(?:(?:(?<=\\).)?[^\{\}\[\]\/\.]?)+\})$'
i = re.finditer(regex, text2)
for m in i: print m.groups()
Filed under: Python No Comments
19Mar/090

Code Highlight

Just a small test of a new code-highlighting feature.

#!/usr/bin/env python
def test():
    try:
        print "Trying something..."
        int("foo") # Should cause an exception
        return "TRY"
    except:
        print "Excepting something..."
        return "EXCEPT"
    finally:
        print "Finalizing something..."
        return "FINALLY"
    return "FUNCTION"

if __name__ == "__main__":
    print test()
Filed under: Python No Comments