Sleep in Windows

Many of us are aware that there is no sleep in Windows. For those of us who don't want Cygwin or something similar, I wrote this stupid little script just now.

#!/usr/bin/env python2 from time import sleep from sys import argv, stderr, exit try: duration = int(argv[1]) if duration <= 0: raise ValueError sleep(duration) except (IndexError, ValueError): print >>stderr, 'Usage: %s duration' % argv[0]

It's not significant, but I am not amused by the lack of such a command in Windows.

tmpchrome

After my dissatisfaction with incognito mode in Chromium, I ended up writing this script, which I call 'tmpchrome' (despite it using Chromium).

My main issue is how all pages share a single incognito session; I'd rather have more control over how sessions are shared.

#!/usr/bin/env python2 from subprocess import Popen from sys import exit, argv from traceback import print_exc from os import rmdir from os.path import isdir from shutil import rmtree from tempfile import mkdtemp profile = mkdtemp() returncode = 1 p = None try: print 'Profile:', profile p = Popen(['chromium', '--user-data-dir=%s' % profile] + argv[1:]) p.wait() returncode = p.returncode except: if p is not None: p.kill() p.wait() returncode = p.returncode finally: try: rmtree(profile) if isdir(profile): rmdir(profile) except Exception: print_exc() finally: exit(returncode)

It isn't perfect, but is good enough for me.

flatten3.py

I recently updated flatten.py to strip out pngcrush and use argparse:

#!/usr/bin/env python2 from sys import argv, stderr, exit from subprocess import Popen from argparse import ArgumentParser, ArgumentError from copy import copy from os import remove from os import close as os_close from re import compile as re_compile from os.path import join, isfile, exists from tempfile import mkstemp filext = re_compile('^(.+)\.(.+)$') filext_png = re_compile('\.png$') img_type = ['Bilevel', 'Grayscale', 'GrayscaleMatte', 'Palette', 'PaletteMatte', 'TrueColor', 'TrueColorMatte', 'ColorSeparate', 'ColorSeparationMatte', 'Optimize'] img_cspace = ['CMY', 'CMYK', 'Gray', 'HSB', 'HSL', 'HWB', 'Lab', 'Log', 'OHTA', 'Rec601Luma', 'Rec601YCbCr', 'Rec709Luma', 'Rec709YCbCr', 'RGB', 'sRGB', 'Transparent', 'XYZ', 'YCbCr', 'YCC', 'YIQ', 'YPbPr', 'YUV'] parser = ArgumentParser(usage = 'Usage: %(prog)s [ options ] input [ output ]') parser.add_argument('-s', '--size', dest = 'size', default = None, help = 'The output size; passed directly to ImageMagick') parser.add_argument('-n', '--no-clobber', dest = 'clobber', default = True, action = 'store_false', help = 'Don\'t overwrite the target file') parser.add_argument('-X', '--lossy-reduction', dest = 'lossy_reduction', default = True, action = 'store_false', help = 'Don\'t allow OptiPNG to perform lossy optimizations') parser.add_argument('-f', '--filter', dest = 'filter', default = 'Catrom', help = 'The filter to resize with if necessary') parser.add_argument('-b', '--background', dest = 'background', default = 'white', help = 'The background when flattening') parser.add_argument('-F', '--no-flatten', dest = 'flatten', default = True, action = 'store_false', help = 'Don\'t flatten the output') parser.add_argument('-t', '--type', dest = 'type', choices = img_type + img_cspace, default = None, help = 'The type of the output image: analogous to both -type and -colorspace in ImageMagick') parser.add_argument('-d', '--depth', dest = 'depth', choices = (1, 2, 4, 8, 16), type = int, default = None, help = 'The depth of the output image') parser.add_argument('-p', '--pre-color', dest = 'precolor', default = False, action = 'store_true', help = 'Change bit-depth/colorspace prior to scaling if applicable') parser.add_argument('input', help = 'Input file') parser.add_argument('output', nargs = '?', help = 'Destination file; will default to ${input}_small.png') args = parser.parse_args() if isfile(args.input): pngout = False if args.output: output = args.output if filext_png.search(output): pngout = True else: m = filext.search(args.input) if not m is None: output = '%s_small.%s' % (m.group(1), m.group(2)) if m.group(2) == 'png': pngout = True else: output = '%s_small.png' % args.input pngout = True if not args.clobber and exists(output): print >>stderr, 'Output file exists:', output exit(2) # ACTUAL PROCESSING pargs = ['convert', args.input] if args.precolor: if args.flatten: pargs += ['-background', args.background, '-flatten', '+matte'] if not args.size is None: pargs += ['-filter', args.filter, '-resize', args.size] if not args.type is None: if args.type in img_type: pargs += ['-type', args.type] if not args.depth is None: pargs += ['-depth', str(args.depth)] elif args.type in img_cspace: pargs += ['-colorspace', args.type] if not args.depth is None: pargs += ['-depth', str(args.depth)] if not args.precolor: if args.flatten: pargs += ['-background', args.background, '-flatten', '+matte'] if not args.size is None: pargs += ['-filter', args.filter, '-resize', args.size] if pngout: # Run convert pargs.append('png:' + output) try: Popen(pargs).wait() except KeyboardInterrupt: if isfile(output): remove(output) exit(1) # Run optipng try: optipng = ['optipng', '-o9'] if not args.lossy_reduction: optipng.append('-nx') optipng.append(output) Popen(optipng).wait() except KeyboardInterrupt: if isfile(output): remove(output) else: pargs.append(output) Popen(pargs).wait() # END else: print >>stderr, 'No such file:', args.input exit(2)

Converting Between Tabs and Spaces

If you want to use tabs in Python but everyone else uses spaces, here's something helpful you can do in git!

% echo '*.py filter=tabspace' >> .git/info/attributes
% git config --global filter.tabspace.smudge 'unexpand --tabs=4 --first-only'
% git config --global filter.tabspace.clean 'expand --tabs=4 --initial'

You can appear to comply with PEP-8 to others using your repository, but you don't need to change your habits!

flatten2.py

After reading about OptiPNG, I decided to add it into the mix of flatten.py. Since it doesn't support reducing bit depths of grayscale images, I have the script run pngcrush first and then OptiPNG.

Since the last post about flatten, I've made a few other changes as well, such as passing -bit_depth to pngcrush when a bit depth is provided. Since I use this script to compress images for my webcomic, that's an important change.

flatten.py

#!/usr/bin/python
from sys import argv, stderr, exit
from subprocess import Popen
from optparse import OptionParser, OptionValueError
from copy import copy
from os import remove
from os import close as os_close
from re import compile as re_compile
from os.path import join, isfile
from tempfile import mkstemp


size = re_compile('^(.+)x(.+)$')

def check_size(option, opt_str, value, parser):
m = size.search(value)
if m:
parser.values.size = m.group(1), m.group(2)
else:
raise OptionValueError('Invalid size: %s' % value)

filext = re_compile('^(.+)\.(.+)$')
filext_png = re_compile('\.png$')

img_type = ['Bilevel', 'Grayscale', 'GrayscaleMatte', 'Palette', 'PaletteMatte', 'TrueColor', 'TrueColorMatte', 'ColorSeparate', 'ColorSeparationMatte', 'Optimize']
img_cspace = ['CMY', 'CMYK', 'Gray', 'HSB', 'HSL', 'HWB', 'Lab', 'Log', 'OHTA', 'Rec601Luma', 'Rec601YCbCr', 'Rec709Luma', 'Rec709YCbCr', 'RGB', 'sRGB', 'Transparent', 'XYZ', 'YCbCr', 'YCC', 'YIQ', 'YPbPr', 'YUV']
pngcrush = ['pngcrush', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB']

parser = OptionParser(usage = 'Usage: %prog [ options ] input [ output ]')
parser.add_option('-s', '--size', dest = 'size', type = 'string', action = 'callback', callback = check_size, default = None, help = 'The output size: WIDTHxHEIGHT')
parser.add_option('-n', '--no-clobber', dest = 'clobber', default = True, action = 'store_false', help = 'The output size: WIDTHxHEIGHT')
parser.add_option('-f', '--filter', dest = 'filter', default = 'Catrom', help = 'The filter to resize with if necessary.')
parser.add_option('-b', '--background', dest = 'background', default = 'white', help = 'The background when flattening.')
parser.add_option('-F', '--no-flatten', dest = 'flatten', default = True, action = 'store_false', help = 'Don\'t flatten the output.')
parser.add_option('-t', '--type', dest = 'type', type = 'string', default = None, help = 'The type of the output image: analogous to both -type and -colorspace in ImageMagick')
parser.add_option('-d', '--depth', dest = 'depth', type = 'int', default = None, help = 'The depth of the output image.')

options, args = parser.parse_args()
if len(args):
input = args.pop(0)
if isfile(input):
output = None
pngout = False
if len(args):
output = args.pop(0)
if filext_png.search(output):
pngout = True
else:
m = filext.search(input)
if not m is None:
output = '%s_small.%s' % (m.group(1), m.group(2))
if m.group(2) == 'png':
pngout = True
else:
output = '%s_small.png' % input
pngout = True
if not options.clobber and isfile(output):
print >>stderr, 'Output file exists:', output
exit(2)
# ACTUAL PROCESSING
args = ['convert', input]

if not options.type is None:
if options.type in img_type:
args += ['-type', options.type]
if not options.depth is None:
args += ['-depth', str(options.depth)]
elif options.type in img_cspace:
args += ['-colorspace', options.type]
if not options.depth is None:
args += ['-depth', str(options.depth)]
if options.flatten:
args += ['-background', options.background, '-flatten', '+matte']
if not options.size is None:
args += ['-filter', options.filter, '-resize', '%sx%s' % options.size]


if pngout:
# Run convert
fd, tmpfile = mkstemp()
os_close(fd)
args.append('png:' + tmpfile)
Popen(args).communicate()
# Run pngcrush
args = copy(pngcrush)
if not options.depth is None:
args += ['-bit_depth', str(options.depth)]
args += [tmpfile, output]
Popen(args).communicate()
remove(tmpfile)
# Run optipng
Popen(['optipng', '-o9', output]).communicate()
else:
args.append(output)
Popen(args).communicate()
# END
else:
print >>stderr, 'No such file:', input
exit(2)
else:
parser.print_help()
exit(1)

The runtime options are the same as flatten.py.

Wikipedia Graph Generator

I've been working on this project for the past month or so, but since it pretty much works I've lost interest. For those interested, this sort of graph is an example of generated output, and shows pages as vertices and inter-page links as directed edges. The scripts can be found here: wikidown-20091110.zip

Since the data set is quite large (the PostgreSQL database dump compressed is over 550 MB), it'd be much easier for anyone who wants to check this out to generate the data. The process is as follows:

If the above image doesn't load, open the image location in a separate window or tab.

1. Run psql -U postgres wikidown2 < schema_create2
2. Download this file from Wikipedia: enwiki-latest-pages-articles.xml.bz2 (5.3 GB)
3. Run wikixml2csv.py enwiki-latest-pages-articles.xml.bz2 pages.lst links.lst I originally used CSV here, but switched to an ASCII group separator later. The '.csv' suffixes are vestigial.
4. Run csvlistfilter.sh links.lst links-sorted.lst.
5. Run csvpagefilter.py pages.lst pages-presorted.lst.
6. Run sort -k1nru pages-presorted.lst pages-sorted.lst.
7. Unfortunately, due to the behavior of Python's hash algorithm and Postgres's tree algorithm, there will be a duplicate title. The only real solution is to keep trying the following steps and edit pages.lst accordingly. Remember than whenever you delete a row in pages.lst, you must also delete links in links.lst that point to it.
8. Run bzip2 links.lst and bzip2 pages.lst.
9. Run csv2psql.py pages.lst.bz2 links.lst.bz2. This step is considerably faster if both files are mounted in tmpfs (RAM), but only if your machine has enough RAM. tmpfs is not available on Windows.

Once you've run those steps, you can run subgraph.py

flatten.py

Working with images a lot, thanks to my webcomic, becomes a lot easier when certain things are automated. For example, this script takes care of flattening an image (by removing alpha information), and even passes the output through pngcrush when applicable. It also includes a default output convention:

flatten.py

#!/usr/bin/python
from sys import argv, stderr, exit
from subprocess import Popen
from optparse import OptionParser, OptionValueError
from os import remove
from os import close as os_close
from re import compile as re_compile
from os.path import join, isfile
from tempfile import mkstemp


size = re_compile('^(.+)x(.+)$')

def check_size(option, opt_str, value, parser):
m = size.search(value)
if m:
parser.values.size = m.group(1), m.group(2)
else:
raise OptionValueError('Invalid size: %s' % value)

filext = re_compile('^(.+)\.(.+)$')
filext_png = re_compile('\.png$')

img_type = ['Bilevel', 'Grayscale', 'GrayscaleMatte', 'Palette', 'PaletteMatte', 'TrueColor', 'TrueColorMatte', 'ColorSeparate', 'ColorSeparationMatte', 'Optimize']
img_cspace = ['CMY', 'CMYK', 'Gray', 'HSB', 'HSL', 'HWB', 'Lab', 'Log', 'OHTA', 'Rec601Luma', 'Rec601YCbCr', 'Rec709Luma', 'Rec709YCbCr', 'RGB', 'sRGB', 'Transparent', 'XYZ', 'YCbCr', 'YCC', 'YIQ', 'YPbPr', 'YUV']

parser = OptionParser(usage = 'Usage: %prog [ options ] input [ output ]')
parser.add_option('-s', '--size', dest = 'size', type = 'string', action = 'callback', callback = check_size, default = None, help = 'The output size: WIDTHxHEIGHT')
parser.add_option('-n', '--no-clobber', dest = 'clobber', default = True, action = 'store_false', help = 'The output size: WIDTHxHEIGHT')
parser.add_option('-f', '--filter', dest = 'filter', default = 'Catrom', help = 'The filter to resize with if necessary.')
parser.add_option('-b', '--background', dest = 'background', default = 'white', help = 'The background when flattening.')
parser.add_option('-F', '--no-flatten', dest = 'flatten', default = True, action = 'store_false', help = 'Don\'t flatten the output.')
parser.add_option('-t', '--type', dest = 'type', type = 'string', default = None, help = 'The type of the output image: analogous to both -type and -colorspace in ImageMagick')
parser.add_option('-d', '--depth', dest = 'depth', type = 'int', default = None, help = 'The depth of the output image.')

options, args = parser.parse_args()
if len(args):
input = args.pop(0)
if isfile(input):
output = None
pngout = False
if len(args):
output = args.pop(0)
if filext_png.search(output):
pngout = True
else:
m = filext.search(input)
if not m is None:
output = '%s_small.%s' % (m.group(1), m.group(2))
if m.group(2) == 'png':
pngout = True
else:
output = '%s_small.png' % input
pngout = True
if not options.clobber and isfile(output):
print >>stderr, 'Output file exists:', output
exit(2)
# ACTUAL PROCESSING
args = ['convert', input]

if options.flatten:
args += ['-background', options.background, '-flatten', '+matte']
if not options.size is None:
args += ['-filter', options.filter, '-resize', '%sx%s' % options.size]

if not options.type is None:
if options.type in img_type:
args += ['-type', options.type]
if not options.depth is None:
args += ['-depth', str(options.depth)]
elif options.type in img_cspace:
args += ['-colorspace', options.type]
if not options.depth is None:
args += ['-depth', str(options.depth)]

if pngout:
fd, tmpfile = mkstemp()
os_close(fd)
args.append('png:' + tmpfile)
Popen(args).communicate()
Popen(('pngcrush', '-rem', 'sRGB', tmpfile, output)).communicate()
remove(tmpfile)
else:
args.append(output)
Popen(args).communicate()
# END
else:
print >>stderr, 'No such file:', input
exit(2)
else:
parser.print_help()
exit(1)% python flatten.py
Usage: flatten.py [ options ] input [ output ]

Options:
-h, --help show this help message and exit
-s SIZE, --size=SIZE The output size: WIDTHxHEIGHT
-n, --no-clobber The output size: WIDTHxHEIGHT
-f FILTER, --filter=FILTER
The filter to resize with if necessary.
-b BACKGROUND, --background=BACKGROUND
The background when flattening.
-F, --no-flatten Don't flatten the output.
-t TYPE, --type=TYPE The type of the output image: analogous to both -type
and -colorspace in ImageMagick
-d DEPTH, --depth=DEPTH
The depth of the output image.

hwsched

As I've mentioned earlier, I've been working on a homework scheduler. Work isn't anywhere near done, but it's functional enough (but not tested enough). Here's what I have so far:hwsched-20090615.zip

To use this script, the following files must be symlinks or hard links to hwsched.py:

• classes
• exams
• assignments
• instructors
• quarters
• sections

pdfsearch.py

For the longest time, I've wanted to search PDF files from the command line. Now, I can with pdfsearch.py! This program uses pyPdf to look for pages containing strings that match the provided regex, but due to the messy output of pyPdf, it won't print the matching lines.

pdfsearch.py

#!/usr/bin/python
from optparse import OptionParser
from sys import argv
from pyPdf import PdfFileReader
from os import walk
from re import compile as re_compile
from re import IGNORECASE
from os.path import join

parser = OptionParser(description = 'Search for text in PDF files.', usage = '%s [ options ] term [ file1..fileN ]' % argv[0])
parser.add_option('-i', '--insensitive', action = 'store_true', dest = 'insensitive', help = 'Search case-insensitively.', default = False)

def pdfgrep(expr, file):
pdf = None
with open(file, 'rb') as f:
pdf = PdfFileReader(f)
for i in xrange(pdf.getNumPages()):
content = pdf.getPage(i).extractText().strip()
if expr.search(content):
yield i



argv = [unicode(i, 'utf8') for i in argv]
options, args = parser.parse_args(argv[1:])

optionmap = {
'insensitive' : IGNORECASE
}


if len(args) >= 1:
term_flags = 0
for key, value in optionmap.iteritems():
if getattr(options, key):
term_flags |= value
term = re_compile(args[0], term_flags)
paths = ['.']
if len(args) >= 2:
paths = args[1:]
for path in paths:
for dirpath, dirnames, filenames in walk(path):
for filename in filenames:
if filename[-4:] == '.pdf':
fullfilename = join(dirpath, filename)
pages = list(pdfgrep(term, fullfilename))
if len(pages) > 0:
print u'%s:%s' % (fullfilename, u', '.join([str(i + 1) for i in pages]))
else:
parser.error('No search term provided.')% python pdfsearch.py -i undo ../Notes/CSE444
../Notes/CSE444/2009-05-07:QuizSection_Midterm.pdf:4
../Notes/CSE444/PDF/lecture14.pdf:3, 6, 10, 11
../Notes/CSE444/PDF/lecture13.pdf:17, 18, 22
../Notes/CSE444/PDF/lecture09-10.pdf:2, 16, 31, 32, 36, 37, 39, 40, 41, 42, 43, 45, 48, 49, 58, 59, 60, 62, 63, 65
../Notes/CSE444/PDF/lecture11.pdf:30