#!/usr/bin/python

# Process lines from stdin with python functions
# Functions are either predefined or user supplied,
# the latter being similar to perl -n -e 'expression'.
# Update June 9 2009, I noticed a similar script called pyline:
#   http://code.activestate.com/recipes/437932/
# Update March 23 2012, Another similar tool called pyp
#   http://code.google.com/p/pyp/
# Update May 24 2012, Another similar tool called osh
#   http://geophile.com/osh/
# Update Sep 2015, Another similar tool called red (regexp)
#   https://bitbucket.org/johannestaas/red

# License: LGPLv2
# Author:
#    http://www.pixelbeat.org/
# Notes:
#    This util uses python's functional attributes to be:
#      1. scalable  (don't build lists in mem so arbitrary length input supported)
#      2. efficient (as many operations done in compiled code as possible)
# Changes:
#    V1.0, 01 Nov 2006, Initial release
#    V1.1, 30 Nov 2007, Added 'uniq' (example of many to many)
#    V1.6, 11 Jul 2011, http://github.com/pixelb/scripts/commits/master/scripts/funcpy

# Examples:
#
#  add numbers
#   echo -e "1234\n433" | ./funcpy add
#
#  The rest of the examples use user supplied expressions
#  (where the input line is represented by x)
#
#  transform numbers
#   echo -e "1234\n433" | ./funcpy "hex(int(x))[2:]"
#
#  operate on single column
#   echo "1,2,3" | ./funcpy "int(x.split(',')[1])**2"
#
#  reiterate over sub items (only reducable functions)
#   echo "1,2,3" | ./funcpy "add(int(i) for i in x.split(','))"
#
#  swartzian transform (to sort by line len) (note sort not as efficient or scalable as max)
#   ls | ./funcpy "str(len(x))+'\t'+x" | sort -k1,1n | cut -f2
#
#  filtering input (odd numbers) (note "or None" at end)
#   echo -e "1\n2\n3" | ./funcpy "int(x)%2 and x or None"
#
#  formating with printf
#   echo "12341234.432142" | ./funcpy '"%g" % float(x)'
#
#  arbitrary modules
#   ls | ./funcpy --import=base64 "base64.encodestring(x)[:-1]"
#
#  similar to perl
#   echo -e "1\n5" | perl -n -e 'print $_**2,"\n"'

import itertools
import operator
import math
import sys

used_compiled=True #Turn off for comparison


#is swartzian transform functional?


#can we add reducable decorator to functions?

#TODO: allow option=expression to filter input if true rather than and or hack?

#TODO: put human() in here?

def reduce(func, items, start=None): #foldl
    try:
        items=iter(items)
    except:
        raise TypeError("reduce() arg 2 must support iteration")
    if start==None:
        try:
            state=items.next()
        except StopIteration:
            raise TypeError("reduce() of empty sequence with no initial value")
    else:
        state=start
    for i in items:
        state = func(state,i)
    return state
if used_compiled:
    try: #builtin removed in python 3.0
        reduce=__builtins__.reduce
    except: #here in python 2.6, only here in python 3.0
        import functools
        reduce=functools.reduce

def imap(func, iter):
    for i in iter:
        yield func(i)
if used_compiled:
    if sys.version_info < (3,):
        imap=itertools.imap
    else:
        imap=map

def izip(iter1, iter2):
    for i in iter1:
        yield (i, iter2.next()) #exception raised if either exhausted as required
if used_compiled:
    if sys.version_info < (3,):
        izip=itertools.izip
    else:
        izip=zip

def irange(*args):
    start=0
    step=1
    if len(args) == 0:
        raise TypeError("irange expected at least 1 argument, got 0")
    elif len(args) == 1:
        (stop,) = args
    elif len(args) == 2:
        (start,stop) = args
    elif len(args) == 3:
        (start,stop,step) = args
    else:
        raise TypeError("irange expected at most 3 arguments, got %d" % len(args))
    while start < stop:
        yield start
        start += step
if used_compiled:
    if sys.version_info < (3,):
        irange=xrange
    else:
        irange=range

def all(iter): #every?
    return False not in imap(bool,iter)
def any(iter): #some?
    return True in imap(bool,iter)
if used_compiled:
    try: # any() and all() are available since python 2.5
        all=__builtins__.all
        any=__builtins__.any
    except:
        pass

def factorial(x):
    return reduce(operator.mul, irange(1,int(x)+1))

def add(iter):
    return reduce(operator.add, iter)
if used_compiled:
    add=__builtins__.sum #should be slightly more specific (faster)

def head(iterator, n=10):
    return (x for x,i in izip(iterator, irange(n)))
    # Note the following more general inspector
    # return itertools.islice(iterator,n)

#def tail(iterator, n=10):
#    import collections
#    return collections.deque(iterator, maxlen=n)

def count(iterator):
    return add(1 for x in iterator)

def uniq(iterator):
    state=None
    for line in iterator:
        if line != state:
            yield line
        state=line
    # Using the groupby equivalent is slightly slower:
    # return (k for k, g in itertools.groupby(iterator))

def uniq_c(iterator):
    return ("%7d %s" % (count(g), k)
            for k, g in itertools.groupby(iterator))

def uniq_d(iterator):
    return (k for k, g in itertools.groupby(iterator) if count(head(g,2)) > 1)

def min(iter):
    state=None
    for i in iter:
        if state > i: state=i
    return state
if used_compiled:
    min=__builtins__.min

def max(iter):
    state=None
    for i in iter:
        if i > state: state=i
    return state
if used_compiled:
    max=__builtins__.max

def avg(iterator):
    try:
        # Note math.fsum (available since 2.6) doesn't lose precision
        # while adding lots of small numbers
        iter1, iter2 = itertools.tee(iterator)
        return math.fsum(iter1)/count(iter2)
    except:
        total=0
        lenth=0
        for i in iterator:
            total+=i
            length+=1
            return total/length

def dec2hex(i):
    return __builtins__.hex(int(i))[2:]

def hex2dec(i):
    return int(i,16)

def bits(i):
    if i<0:  raise ValueError("negative number [%d]" % i)
    if i==0: return 1
    #shift operator slower
    return int(math.log(i,2))+1

#=============================================================

import os, errno

# The following exits cleanly on Ctrl-C or EPIPE
# while treating other exceptions as before.
def std_exceptions(etype, value, tb):
    sys.excepthook=sys.__excepthook__
    if issubclass(etype, KeyboardInterrupt):
        pass
    elif issubclass(etype, IOError) and value.errno == errno.EPIPE:
        pass
    else:
        sys.__excepthook__(etype, value, tb)
sys.excepthook=std_exceptions

functions={
    #name:       (func,       input,  type)
    'add':       (add,        float,  'mto1'),
    'max':       (max,        float,  'mto1'),
    'min':       (min,        float,  'mto1'),
    'avg':       (avg,        float,  'mto1'),
    'all':       (all,        float,  'mto1'),
    'any':       (any,        float,  'mto1'),
    'count':     (count,      str,    'mto1'),
    'head':      (head,       str,    'mtom'),
    'uniq':      (uniq,       str,    'mtom'),
    'uniq_c':    (uniq_c,     str,    'mtom'),
    'uniq_d':    (uniq_d,     str,    'mtom'),
    #Following really of minimal use
    #'dec2hex':   (dec2hex,    str,   '1to1'),
    #'hex2dec':   (hex2dec,    str,   '1to1'),
    #'factorial': (factorial,  int,   '1to1'),
    #'log10':     (math.log10, float, '1to1'),
    #'bits':      (bits,       long,  '1to1'),
}

def Usage():
    sys.stderr.write("Usage: " + os.path.split(sys.argv[0])[1] + " [OPTIONS] <" + '|'.join(sorted(functions.keys())) + ">\n")
    sys.stderr.write("       " + os.path.split(sys.argv[0])[1] + " [OPTIONS] <" + "python expression where input is x>\n")
    sys.stderr.write("Input is taken from stdin, and is processed per line\n")
    sys.stderr.write("Note python expressions must be stateless (no ifs or assigments)\n")
    sys.stderr.write("       --imports=mod1,mod2,...    modules to import for use in python expression\n")
    sys.stderr.write("       --help                     display this help\n")
    sys.stderr.close()
    sys.exit(1)


import getopt
try:
    lOpts, lArgs = getopt.getopt(sys.argv[1:], "", ["help","imports="])

    if len(lArgs) == 1:
        mode=lArgs[0]
    else:
        Usage()
        sys.exit(1)

    if ("--help","") in lOpts:
        Usage()
        sys.exit(None)

    for opt in lOpts:
        if opt[0] == "--imports":
            for mod in opt[1].split(","):
                globals()[mod]=__import__(mod)

except getopt.error:
    msg = sys.exc_info()[1]
    sys.stdout.write("%s\n" % msg)
    Usage()
    sys.exit(2)

#=============================================================

try:
    func=functions[mode][0]
    conv=functions[mode][1]
    ftype=functions[mode][2]
except:
    #shows the flexibility of interpretation over compilation
    code=compile(mode,"<expression>","eval")
    func=lambda x: eval(code)
    conv=lambda x: x[-1]=='\n' and x[:-1] or x #strip EOL
    ftype='1to1'

def convert(iterator):
    if conv==str: #shortcut for speed
        return iterator
    else:
        return (conv(line) for line in iterator)

try:
    if ftype=='mto1': #many to one
        result = func(convert(sys.stdin))
        if type(result)==bool:
            sys.exit(not result)
        else:
            sys.stdout.write("%s\n" % result)
    elif ftype=='mtom': #many to many
        #for result in func(convert(sys.stdin)):
        #    sys.stdout.write(result)
        #note the following is faster but it buffers output
        #even if stdout is a terminal. writelines really shouldn't do that.
        sys.stdout.writelines(func(convert(sys.stdin)))
    else: #one to one
        for num in convert(sys.stdin):
            result = func(num)
            if result!=None:
                sys.stdout.write("%s\n" % result)
except ValueError:
    value = sys.exc_info()[1]
    sys.stderr.write(str(value)+"\n")
    sys.exit(1)