zengame.py

'''
Created on 2012-10-03
@author: Sebastien Ouellet sebouel@gmail.com
'''
import sys
import os
import random
import math
import time
import threading
import Queue
import pickle
import scipy

import cv2
from decaf.scripts.imagenet import DecafNet

import pygame

# Global variables #

framerate = 60

white = (255,255,255)
black = (0,0,0)

width = 512
height = 512

mapping = ["neutral", "anger", "contempt", "disgust", "fear", "happy", "sadness", "surprise"]
#inverse_mapping = {"neutral":0, "anger":1, "contempt":2, "disgust":3, "fear":4, "happy":5, "sadness":6, "surprise":7}
rectangle_margin = 50
####################

class ComputeFeatures(threading.Thread):
    def __init__(self, net, array, nextarray, results):
        threading.Thread.__init__(self)
        self.net = net
        self.array = array
        self.results = results
        self.nextarray = nextarray
        with open("classifierBestFace.sav","rb") as opened_file:
            self.classifier = pickle.load(opened_file)
    
    def run(self):
        if True: # Live gaming loop
            while True:
                #time.sleep(1)
                #scores = self.net.classify(self.array)
                #self.result = net.top_k_prediction(scores,5)[1][0:5]
                if not self.nextarray.empty():
                    self.array = self.nextarray.get(True)
                    while not self.nextarray.empty():
                        self.nextarray.get()
                    #self.result = len(self.array)
                    scores = self.net.classify(self.array, center_only=True)
                    #prediction = self.classifier.predict(self.net.feature("fc6_neuron_cudanet_out"))
                    prediction = self.classifier.predict(self.net.feature("pool5_cudanet_out").flatten())
                    self.results.put(mapping[int(prediction)])
                    #self.results.put(self.net.top_k_prediction(scores,5)[1][0:5])
                    #print self.results.qsize()
        
        else: # Personalized training
            featureArray = numpy.zeros((300,4096))
            feature_index = 0
            while True:
                if feature_index > 299:
                    numpy.save("featureEmotion", featureArray)
                #time.sleep(1)
                #scores = self.net.classify(self.array)
                #self.result = net.top_k_prediction(scores,5)[1][0:5]
                if not self.nextarray.empty():
                    self.array = self.nextarray.get(True)
                    while not self.nextarray.empty():
                        self.nextarray.get()
                    #self.result = len(self.array)
                    scores = self.net.classify(self.array, center_only=True)
                    featureArray[feature_index]=self.net.feature("fc6_neuron_cudanet_out") #original
                    feature_index += 1
                    #self.results.put(self.net.top_k_prediction(scores,5)[1][0:5])
                    #print self.results.qsize()


class Unit(pygame.sprite.Sprite):
    """ Generic class for unit sprites """
    def __init__(self,pos,image=pygame.Surface([24,24])):
        pygame.sprite.Sprite.__init__(self)
        if not isinstance(image, pygame.Surface):
            self.image = load_image(image)
        else:
            self.image = image
        self.rect = self.image.get_rect()
        self.position = pos
        self.rect.center = self.position

    def update(self):
        self.rect = self.image.get_rect()
        self.rect.center = self.position

def main():
    net = DecafNet()

    video = cv2.VideoCapture(0)
    cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
    
    arrays = Queue.LifoQueue()
    results = Queue.LifoQueue()

    detector = ComputeFeatures(net, [], arrays, results)
    detector.daemon = True
    detector.start()

    pygame.init()

    screen = pygame.display.set_mode((width, height))
    pygame.display.set_caption('This is a video game')
    
    background = pygame.Surface((width,height))
    background.fill(white)
    screen.blit(background,(0,0))
    pygame.display.flip()
    
    allsprites = pygame.sprite.RenderUpdates()

    # Some parameters #
    
    size = 10
    enemy_surface = pygame.Surface((size,size))
    speed = 200.0
    playersize = 44

    # # # # # # # # # #

    player = Unit([256.0, 256.0], pygame.Surface((playersize,playersize)))
    allsprites.add(player)
    
    enemy_counter = 1.0
    clock = pygame.time.Clock()
    elapsed = 0.0
    accumulator = 0.0

    run = True
    face = None

    emotion_window = ["neutral","neutral","neutral","neutral","neutral","neutral"]
    #emotion_accumulator = 0.0
    current_emotion = "neutral"
    emotion = "neutral"
    health = 50
    game_time = 0.0

    while run:
        seconds = elapsed/1000.0
        accumulator += seconds
        game_time += seconds
        #emotion_accumulator += seconds

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
            elif event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE:
                run = False

            #elif event.type == pygame.KEYDOWN and event.key == pygame.K_w:
            #    current = Unit((random.randint(0,512), random.randint(0,512)))
            #    allsprites.add(current)
            #elif event.type == pygame.KEYDOWN and event.key == pygame.K_s:
            #    for sprite in allsprites:
            #        sprite.position = [sprite.position[0]+random.randint(-5,5),sprite.position[1]+random.randint(-5,5)]

        if accumulator > enemy_counter: 
            allsprites.add(Unit([random.randint(0,512), 0],enemy_surface))
            accumulator = 0.0

        for sprite in allsprites.sprites():
            if sprite.image == enemy_surface:
                sprite.position[1]+=speed*seconds
            if sprite.position[1]>height-10:
                allsprites.remove(sprite)

        pressed = pygame.key.get_pressed()

        if pressed[pygame.K_RIGHT]:
            player.position[0]+=speed*seconds
        if pressed[pygame.K_LEFT]:
            player.position[0]-=speed*seconds
        if pressed[pygame.K_DOWN]:
            player.position[1]+=speed*seconds
        if pressed[pygame.K_UP]:
            player.position[1]-=speed*seconds           

        allsprites.update()

        allsprites.remove(player)
        health -= len(pygame.sprite.spritecollide(player,allsprites, True))
        allsprites.add(player)
        
        allsprites.clear(screen,background)
        changed = allsprites.draw(screen)
        pygame.display.update(changed)
        
        frame = video.read()[1]
        rects = cascade.detectMultiScale(frame, 1.3, 3, cv2.cv.CV_HAAR_SCALE_IMAGE, (150,150))

        #arrays.put(frame)

        # Idea: increase the size of the rectangle
        if len(rects) > 0:
            facerect = rects[0]
            #facerect[0] -= (rectangle_margin-30)
            #facerect[2] += rectangle_margin
            #facerect[1] -= (rectangle_margin-20)
            #facerect[3] += rectangle_margin
            face = frame[facerect[1]:facerect[1]+facerect[3], facerect[0]:facerect[0]+facerect[2]]
            face = cv2.cvtColor(face,cv2.COLOR_BGR2GRAY)
            arrays.put(face)
            if True:
                for (x,y,w,h) in rects:
                    cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
        
        if not results.empty():
            emotion = results.get()
            emotion_window.append(emotion)
            emotion_window.pop(0)
            current_emotion = max(set(emotion_window),key=emotion_window.count)
            print "Current emotion:", current_emotion, "- Last detected:", emotion#, emotion_window
            if current_emotion == "happy":
                enemy_counter += 0.03
                enemy_counter = min(0.7,enemy_counter)
            else:
                enemy_counter += -0.02
                enemy_counter = max(0.01,enemy_counter)
            print "Health:", health, "- Time:", game_time

        if health < 1:
            run = False
            print "Game over! Score:", game_time

        if face != None:
            cv2.imshow("face",face)
        cv2.imshow("frame",frame)
        c = cv2.waitKey(1)
        if c == 27:
            cv2.destroyWindow("frame")
            cv2.destroyWindow("face")
            break

        elapsed = clock.tick(framerate)

    video.release()
    cv2.destroyAllWindows()

    pygame.quit()

if __name__ == "__main__":
    main()