Illumination is an interactive art installation created by myself in collaboration with Yonatan Ben-Simhon. It uses light to discover and reveal poetry from within printed texts.
What happens
The basic set-up of Illumination is a table in a dark space, lit from above by a spot-light. Visitors can take any piece of paper with printed text on it (newspaper or magazine articles, pages from a book, etc) and place it on the table. Upon doing so, the light shining on the table beings to flicker, and the text on the page begins to fluctuate and shake. After a few seconds, the light goes out. Slowly, individual words of the text begin to light up. One by one, words light up and construct poetry. Take away the text and place a new text on the table, and a new poem is revealed within the text. The unique language and tone of each text reveals a unique poem.
How it was done
From a technical perspective, Illumination uses a camera, projector, and computer that work in synch to create this magical experience. The camera and projector are suspended above the table. When a visitor places a text on the table surface, the camera above the table takes a high-resolution photo of the text which is uploaded to a computer nearby. A custom Python script running on the machine saves the JPEG image received from the camera as a TIFF. The script then uses the open-source Optical Character Recognition (OCR) software Tesseract to generate an HOCR.html output file that consists of both the text of the image that was processed and the positions and locations of every words that is located.
A poetry-generation Python script trains a Markov model on hundreds of works of poetry and fiction, creating a statistical model of word sequences that appear in poetic and non-fiction texts. In addition, the texts are all tagged for parts of speech, and the Markov model is then seeded with the tagged texts as well. Then, the word and part-of-speech sequences that are generated by the Markov model are searched for in the OCR text, generating a poem of several lines long. This sequence of lines and words is sent as a multi-dimensional array of word indexes to an implementation of Processing.py that then displays the box locations of the words, animating them as they appear and fade out over time.
More Sample Illumination Poetry
As a case study of the entire process outlines above, here is a sample photo of text that was taken:
Here is the HOCR output.html file generated by Tesseact: output.html (right-click to view source to see the box data)
And here is some sample poetry generated by Illumination:
will at any sight
a will of town instincts
the same, as well
will of his actions
will and sight policy
and sight into instincts
will like a desert endgame well
and to have it cornered.
drawn profiles and instincts have
it is a will fight well
a will to turn large
to turn the instincts from
a will of some benefit,
The Code
"""
illuminate.py
by Jack Kalish and Yonatan Ben-Simhon
NYU ITP 2011
"""
import processing.video.Capture as Capture
import commands, subprocess, math, re, os, shutil, glob, random
from BeautifulSoup import BeautifulSoup
#from markov import MarkovGenerator
import imageadjuster.ImageAdjuster as ImageAdjuster;
from threading import Thread
from MarkovGenerator import MarkovGenerator
#from FakeMarkovGenerator import MarkovGenerator
#import MarkOutPoet
#markOutPoet = MarkOutPoet()
#markOutPoet.quickPoemForJack()
imgPath = 'captures/Capture_00001.JPG'
class Illuminate(object):
def setup(self):
#instantiate class vars
self.fadeAlpha = 2
self.lightColor = [255,244,208]
#self.lightColor = 0xFFEF91
self.calibrate = False
self.speed = 250
self.showImg = 1
self.wordMargin = 3
self.boxList = []
self.currentBoxNum = [0]
self.lastTime = [0]
self.mt = []
self.words = []
self.loading = 0
self.thread = None
self.generator = None
self.lightOn = True
self.generator = None
self.showTime = 1000
# calibrate
f = open('config.txt')
str = f.read()
print "str: "+str
calib = str.split(',')
print calib
try:
self.imgPos = [int(calib[0]),int(calib[1])]
except:
#set default calib value
calib = [507,324,535,297]
self.imgPos = [int(calib[0]),int(calib[1])]
self.imgSize = [int(calib[2]),int(calib[3])]
self.saveCalib();
#set screen
#frame.setLocation(1280,0);
size(1600, 900)
#myCapture = Capture(this, width, height, 30)
fill(255, 5000)
noStroke()
self.adjust = ImageAdjuster(this)
self.img = loadImage(imgPath)
self.line = []
self.lines = []
self.lineCnt = 0
self.wordCnt = 0
def draw(self):
#print 'drawing'
#clearCaptures()
#self.showLoader()
if self.lightOn:
self.light()
if self.loading == 1:
if self.thread and self.thread.isAlive():
self.showLoader()
else:
self.loading = 0
if self.onThreadComplete != None:
self.onThreadComplete()
#print "self.lines: ",
#print self.lines
if self.calibrate:
image(self.img, self.imgPos[0], self.imgPos[1], self.imgSize[0], self.imgSize[1])
if len(self.lines)>0:
#background(0)
if self.lineCnt < len(self.lines)-1: '''print "len(self.line): ", print len(self.line) print "len(self.words): ", print len(self.words) print "self.wordCnt: ", print self.wordCnt''' if millis() - self.lastTime[0] > self.showTime:
if self.wordCnt >= len(self.line):
#background(0)
#go to next line
self.wordCnt = 0
self.lineCnt += 1
self.line = self.lines[self.lineCnt]
self.showTime = 2000
self.fadeAlpha = 10
#print "new line length:"
#print len(self.line)
#print " ".join(self.line)
#print len(self.mt[self.currentBoxNum[0]])*100
else:
#background(0,10)
#lightRandomWord()
#print "self.wordCnt: ",
#print self.wordCnt
#print "line[self.wordCnt] ",
#print self.line[self.wordCnt]
self.lightWord(self.line[self.wordCnt])
#self.lightNextMt()
self.showTime = math.sqrt(self.getCurrentWordLength())*self.speed
self.wordCnt += 1
self.fadeAlpha = 2
self.lastTime[0] = millis()
fill(0,0,0,self.fadeAlpha)
rect(0,0,width,height)
else:
#we gotta end it here, no more lines!
self.end()
def light(self):
background(self.lightColor[0],self.lightColor[1],self.lightColor[2])
def end(self):
print "the poem is over"
self.runThreadOn(self.makeNewPoem)
def makeNewPoem(self):
print "thinking of a new poem...hmmm"
#make a new poem between 3 and 14 lines long
self.lineCnt = 0
self.wordCnt = 0
numLines = round(random.random()*11)+3
print "ok, it will be ",
print numLines,
print "lines long"
lines = self.generator.generateFromText(numLines)
self.lines = lines
self.line = lines[0]
self.showTime = 5000
def getCurrentWordLength(self):
return len(self.words[self.line[self.wordCnt]])
def runThreadOn(self, method, callback=None):
if self.thread and self.thread.isAlive():
print "thread already running"
return
self.onThreadComplete = callback
self.thread = Thread(target=method)
self.thread.start()
self.loading = 1
self.calibrate = False
def runThread(self):
if self.thread and self.thread.isAlive():
print "thread already running"
return
self.thread = Thread(target=self.run)
self.thread.start()
self.loading = 1
self.calibrate = False
def showLoader(self):
#print 'loading'
#text("processing...", width/2, height/2)
#jiggle the image?
randomness = 3.0
#print "random: "
#print random(1)
background(self.lightColor[0], self.lightColor[1], self.lightColor[2])
image(self.img, self.getWiggle(self.imgPos[0], randomness), self.getWiggle(self.imgPos[1], randomness), self.getWiggle(self.imgSize[0], randomness), self.getWiggle(self.imgSize[1], randomness))
#background(self.lightColor,random.random()*255)
fill(0, random.random()*50)
rect(0,0,width,height)
#image(self.img, self.imgPos[0], self.imgPos[1], self.imgSize[0], self.imgSize[1])
#image(self.img, self.imgPos[0]*random(randomness*-1, randomness), self.imgPos[1]*random(randomness*-1, randomness), self.imgSize[0]*random(randomness*-1, randomness), self.imgSize[1]*random(randomness*-1, randomness))
def getWiggle(self, val, r):
#print 'input val:',
#print val
val += (random.random()*r*2) - r
#print 'output val',
#print val
return val
def run(self):
#clear screen
background(0)
self.showImg = 0
#wait for a new image to become available
files = glob.glob('captures/*')
while len(files) < 1:
while files[0] != 'captures/capture_1.JPG':
files = glob.glob('captures/*')
print "waiting for an image"
#colorMode(HSB, 100);
#tint(0,0)
#adjust contrast
#img.adjust.contrast(g, 2)
self.capture() #convert to tiff
#delete the original jpeg
#commands.getstatusoutput('rm -f captures/capture_1.JPG')
self.performOCR()
self.words = self.parseWords()
#TRAIN THE MARKOV GENERATOR
self.generator = MarkovGenerator(n=3, max=30, min=4, words=self.words)
def onOCRComplete(self):
print "finished reading!"
self.lightOn = False
self.runThreadOn(self.makeNewPoem)
#now get generate the poetry from the text...
#doMarkov()
def performOCR(self):
print "reading text..."
subprocess.call(['tesseract',r'captures/capture.tif', 'output', '-l', 'eng' ,'+hocr.txt'])
#commands.getstatusoutput('tesseract capture.tif output -l eng +ocr/hocr.txt')
#commands.getstatusoutput('tesseract captures/capture.tif output -l eng +hocr.txt')
#commands.getstatusoutput('tesseract ocr/article.tif output -l eng +ocr/hocr.txt')
def lightRandomWord(self):
self.lightWord(random.randint(0,len(boxList)))
def lightWord(self,id):
#print "light word: ",
print self.words[id],
self.lightBox(self.boxList[id])
def lightNextBox(self):
#background(0)
#print 'light next word:'
#print boxList[currentBoxNum[0]]
self.lightBox(self.boxList[self.currentBoxNum[0]])
self.currentBoxNum[0] += 1
def lightBox(self,r):
xScale = self.imgSize[0]/float(2272)
yScale = self.imgSize[1]/float(1704)
#print r
#fill(255,10)
fill(self.lightColor[0],self.lightColor[1],self.lightColor[2])
rect((float(r[0]))*xScale+self.imgPos[0]-(self.wordMargin), (float(r[1]))*yScale+self.imgPos[1]-(self.wordMargin), float(r[2])*xScale+(self.wordMargin*2), float(r[3])*yScale+(self.wordMargin*2))
#delay(1000)
def moveImage(self,x,y):
self.imgPos[0] += x
self.imgPos[1] += y
self.saveCalib()
def scale(self,x,y):
self.imgSize[0] += x
self.imgSize[1] += y
self.saveCalib()
def saveCalib(self):
print "save calib"
f = open('config.txt', 'r+')
f.truncate()
calib = str(self.imgPos[0])+","+str(self.imgPos[1])+","+str(self.imgSize[0])+","+str(self.imgSize[1])
print "calib: "+calib
f.write(calib)
f.close()
def clearCaptures(self):
print "clear captures dir"
files = glob.glob('captures/*')
print "files:"
print files
for f in files:
print f
os.remove(f)
#if f!= 'captures/capture_1.JPG':
#os.remove(f)
#remove all images except for the first
'''for files in os.walk('/captures'):
print "file:"
print f
if f!= 'capture_1.JPG':
os.unlink(os.path.join(root, f))'''
def capture(self):
print "save tif"
self.img.save("captures/capture.tif");
def parseWords(self):
soup = BeautifulSoup(open('output.html'))
print "parsing words..."
boxes = soup.findAll('span', { "class" : "ocr_word" })
cnt = 0
words = []
for word in boxes:
#generate arrary of only words
w = word.contents[0].contents[0]
words.append(w)
title = word['title']
#print "title: " + title
r = title.split(' ')
r.pop(0)
#generate another array of box data
#convert array to processing rect object coords (x,y,W,H)
r[2] = float(r[2]) - float(r[0])
r[3] = float(r[3]) - float(r[1])
self.boxList.append([r[0], r[1], r[2], r[3]])
cnt += 1
print "words:",
print words
return words
#print "words array: "
#print words
def keyPressed(self):
print "keypressed: "
print key
if key == CODED:
if keyCode == UP:
self.moveImage(0,-1)
elif keyCode == DOWN:
self.moveImage(0,1)
elif keyCode == LEFT:
self.moveImage(-1,0)
elif keyCode == RIGHT:
self.moveImage(1,0)
elif key==61:
self.scale(1,0);
elif key==45:
self.scale(-1,0);
elif key==93:
self.scale(0,1);
elif key==91:
self.scale(0,-1);
elif key==99:
#c - turn calibration on
background(0)
self.calibrate = not self.calibrate
elif key==32:
#spacebar - run
self.runThreadOn(self.run, self.onOCRComplete)
illuminate = Illuminate()
def setup(): illuminate.setup()
def draw(): illuminate.draw()
#def mousePressed(): illuminate.mousePressed()
def keyPressed(): illuminate.keyPressed()