DinoGame_opencv_and_mediapipe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dino Game using Google Mediapipe for hand-tracking\n",
    "\n",
    "    Saad Bazaz  - i180621\n",
    "    Abraar Raza - i180544\n",
    "\n",
    "To act as a control, so we can demonstrate the effectiveness of our own hand-tracking algorithm."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install mediapipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install opencv-python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For hand-tracking\n",
    "import mediapipe as mp\n",
    "import cv2\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Game Emulation\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hand-Tracking Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_drawing = mp.solutions.drawing_utils\n",
    "mp_holistic = mp.solutions.holistic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Game Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Initialize Selenium Browser\n",
    "\n",
    "# https://sites.google.com/a/chromium.org/chromedriver/home\n",
    "# Download ChromeDriver and extract. Then enter the fullpath here.\n",
    "PATH_TO_CHROME = r\"/Users/saadbazaz/Documents/FAST Studies/Digital Image Processing Lab/Project/Dino-Game-Hand-Gestures/t-rex-runner-gh-pages/chromedriver\"\n",
    "\n",
    "# The link to your Dino Game's index.html\n",
    "DINO_GAME_LINK = r\"/Users/saadbazaz/Documents/FAST Studies/Digital Image Processing Lab/Project/Dino-Game-Hand-Gestures/t-rex-runner-gh-pages/index.html\"\n",
    "\n",
    "\n",
    "browser = webdriver.Chrome(PATH_TO_CHROME)\n",
    "res = browser.get('file://' + DINO_GAME_LINK)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Initialize Selenium Action Chains\n",
    "\n",
    "actions = ActionChains(browser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Test Actions\n",
    "\n",
    "\n",
    "# actions.key_down(Keys.DOWN)\n",
    "# actions.perform()\n",
    "\n",
    "# time.sleep(1)\n",
    "\n",
    "\n",
    "\n",
    "# actions.key_up(Keys.DOWN)\n",
    "# actions.perform()\n",
    "\n",
    "# time.sleep(1)\n",
    "\n",
    "# actions.send_keys(Keys.SPACE)\n",
    "# actions.perform()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# define a video capture object\n",
    "vid = cv2.VideoCapture(0)\n",
    "\n",
    "upper_thresh = 0.3\n",
    "lower_thresh = 0.7\n",
    "\n",
    "# 0 for Keep Running, 1 is for Jump, 2 is for Duck\n",
    "\n",
    "prev_pos = 0\n",
    "\n",
    "\n",
    "ret, frame = vid.read()\n",
    "\n",
    "scale_percent = 25# percent of original size\n",
    "width = int(frame.shape[1] * scale_percent / 100)\n",
    "height = int(frame.shape[0] * scale_percent / 100)\n",
    "dim = (width, height)\n",
    "\n",
    "\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "\n",
    "    while vid.isOpened():\n",
    "\n",
    "        # Capture the video frame\n",
    "        # by frame\n",
    "        ret, frame = vid.read()\n",
    "        \n",
    "\n",
    "        frame = cv2.resize(frame, dim, interpolation = cv2.INTER_AREA)\n",
    "        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "        \n",
    "        results = holistic.process(image)\n",
    "        \n",
    "        \n",
    "        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)        \n",
    "        mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)\n",
    "        \n",
    "\n",
    "        if results.right_hand_landmarks is not None:\n",
    "            pos_y = results.right_hand_landmarks.landmark[10].y\n",
    "\n",
    "            ## Draw threshold lines\n",
    "            y_pix = int(np.multiply(lower_thresh, image.shape[0]))\n",
    "            cv2.line(image, (0, y_pix), (image.shape[1],  y_pix), (0,255,0), thickness=2)\n",
    "            \n",
    "            \n",
    "            y_pix = int(np.multiply(upper_thresh, image.shape[0]))\n",
    "            cv2.line(image, (0, y_pix), (image.shape[1],  y_pix), (255,0,0), thickness=2)\n",
    "    \n",
    "\n",
    "            if pos_y < upper_thresh:\n",
    "                # print (\"Jump!\")\n",
    "                               \n",
    "            \n",
    "                if prev_pos == 1:\n",
    "                    pass  \n",
    "                elif prev_pos == 2:\n",
    "                    actions = ActionChains(browser)                    \n",
    "                    actions.key_up(Keys.DOWN)\n",
    "                    actions.send_keys(Keys.SPACE)\n",
    "                    actions.perform() \n",
    "                else:\n",
    "                    actions = ActionChains(browser)\n",
    "                    actions.send_keys(Keys.SPACE)\n",
    "                    actions.perform()                 \n",
    "            \n",
    "                prev_pos = 1\n",
    "    \n",
    "            elif pos_y > lower_thresh:\n",
    "                # print (\"Duck.\")\n",
    "\n",
    "                if prev_pos == 1:\n",
    "                    actions = ActionChains(browser)\n",
    "                    actions.key_down(Keys.DOWN)\n",
    "                    actions.perform()     \n",
    "                elif prev_pos == 2:\n",
    "                    pass\n",
    "                else:\n",
    "                    actions = ActionChains(browser)\n",
    "                    actions.key_down(Keys.DOWN)\n",
    "                    actions.perform()                     \n",
    "\n",
    "                prev_pos = 2\n",
    "\n",
    "\n",
    "            else:\n",
    "                # print (\"Keep running.\")\n",
    "\n",
    "                if prev_pos == 2:\n",
    "                    actions = ActionChains(browser)                    \n",
    "                    actions.key_up(Keys.DOWN)\n",
    "                    actions.perform() \n",
    "\n",
    "                prev_pos = 0            \n",
    "\n",
    "        # Display the resulting frame\n",
    "        cv2.imshow('frame', image)\n",
    "\n",
    "        # the 'q' button is set as the\n",
    "        # quitting button you may use any\n",
    "        # desired button of your choice\n",
    "        if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "# After the loop release the cap object\n",
    "vid.release()\n",
    "# Destroy all the windows\n",
    "cv2.destroyAllWindows()\n",
    "for i in range (1,5):\n",
    "    cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}