ECE5725: Security Box

Work Distribution

Team photo

Amanda

at669@cornell.edu

Hardware and software design.
Implemented facial recognition, audio feedback, and compiled master program. Collaboratively implemented NFC detection, motion detection, and voice recognition.

Michael

mr2242@cornell.edu

Hardware and software design
Implemented SMS notifications and designed the box. Collaboratively implemented NFC detection, motion detection, and voice recognition

Code Appendix

master.py

# master.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# Import modules
import nfc, pir, mic, playsound, time, servo
import pi_face_recognition as facerec
import ibmspeech as speechrec

# Misc imports
from twilio.rest import Client
import RPi.GPIO as GPIO

GPIO.setmode(GPIO.BCM)
GPIO.setup(26, GPIO.IN, pull_up_down=GPIO.PUD_UP)

# Twilio setup
account_sid = 'ACfbd404054a6297ec77dd2b94305600c1'
auth_token = '9937cade6ba6651913522d4762fb12ab'
client = Client(account_sid, auth_token)

# Calls all initialization functions
def init():
    pn532 = nfc.init()
    data, detector = facerec.init()
    service, models, model = speechrec.init()

    return pn532, data, detector, service, models, model

# Main function that loops after each open and close session
def main(pn532, data, detector, service, models, model):
    while (True):
        # Initialize values
        prev_val = GPIO.input(26) # initialize button value
        cur_val = prev_val
        pir_val = 0
        nfc_val = 0
        speech_val = 0
        face_val = 0

        # wait until motion is detected
        while(not pir_val):
            time.sleep(0.1) # sleep for poll
            pir_val = pir.PIR()

        # Motion has been detected
        playsound.main("fail")
        client.messages.create(to='+18316827737', from_='+18312564180', body="Something has been detected!")

        # wait until two-factor authentication done
        while(True):
            playsound.main("fail")
            time.sleep(0.1)
            cur_val = GPIO.input(26)
            if (not speech_val):
                # If button is pressed, begin microphone recording
                if (prev_val != cur_val):
                    mic.main()
                    speech_val = speechrec.main(service, models, model)
                    if (not speech_val): # failed speech recognition
                        prev_val = GPIO.input(26) # don't re-attempt until button press

            # Attempt one frame of facial recognition
            if (not face_val):
                face_val = facerec.main(data, detector)

            # Attempt to detect NFC device
            if (not nfc_val):
                nfc_val = nfc.read_nfc(pn532)

            # If one of two factors authenticated, play "ding"
            if (nfc_val + speech_val + face_val == 1): playsound.main("ding")

            # If two factors authenticated, play "success", break from loop
            if (nfc_val + speech_val + face_val >= 2):
                # play sfx
                playsound.main("success")
                break

        # Unlock box, wait for lock
        servo.main("unlock", 7)
        client.messages.create(to='+18316827737', from_='+18312564180', body="Unlocking...")
        prev_val = GPIO.input(26)
        cur_val = prev_val
        while(True):
            time.sleep(0.1)
            cur_val = GPIO.input(26)
            # Lock box
            if (prev_val != cur_val):
                servo.main("lock", 7.5)
                playsound.main("fail")
                client.messages.create( to='+18316827737', from_='+18312564180', body="Locking....")
                break
    p.stop()

# Call initialization function
pn532, data, detector, service, models, model = init()
# Play "ding" after initialization
playsound.main("ding")
# Run main loop forever
main(pn532, data, detector, service, models, model)

              

pir.py

# pir.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# Import modules
import RPi.GPIO as GPIO
import time

pir_sensor = 19

GPIO.setwarnings(False)

# Set to Broadcom
GPIO.setmode(GPIO.BCM)
GPIO.setup(pir_sensor, GPIO.IN)

# Check PIR for one loop
def PIR():
    # if motion detected, return 1
    if (GPIO.input(pir_sensor)):
        return 1 # detected
    else:
        return 0

# while(True):
#      time.sleep(0.10)
#      print(PIR())

              

playsound.py

# playsound.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# source: https://stackoverflow.com/questions/6951046/pyaudio-help-play-a-file

import pyaudio
import wave
import sys

def main(file):
    # length of data to read.
    counter = 50
    chunk = 1024
    filename = ""
    # Play filename
    if (file == "success"):
        filename = "/home/pi/ece5725-final/python/sounds/success.wav"
    elif (file == "fail"):
        print("is fail")
        filename = "/home/pi/ece5725-final/python/sounds/fail.wav"
    elif (file == "ding"):
        print("ding")
        filename = "/home/pi/ece5725-final/python/sounds/ding.wav"
    else:
        sys.exit(-1)

    # open the file for reading.
    wf = wave.open(filename, 'rb')

    # create an audio object
    p = pyaudio.PyAudio()

    # open stream based on the wave object which has been input.
    stream = p.open(format = p.get_format_from_width(wf.getsampwidth()), channels = wf.getnchannels(), rate = wf.getframerate(), output = True)

    # read data (based on the chunk size)
    data = wf.readframes(chunk)

    # Play audio
    while data != '' and counter > 0:
        stream.write(data)
        data = wf.readframes(chunk)
        counter -= 1

    # cleanup
    stream.close()
    p.terminate()

# main("success")
# main("fail")
              

nfc.py

# nfc.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# source: https://learn.adafruit.com/adafruit-pn532-rfid-nfc/python-circuitpython

import board
import busio
from digitalio import DigitalInOut
from adafruit_pn532.i2c import PN532_I2C

# Initialize I2C connection
def init():
    i2c = busio.I2C(board.SCL, board.SDA)

    reset_pin = DigitalInOut(board.D6)
    req_pin = DigitalInOut(board.D13)
    pn532 = PN532_I2C(i2c, debug=False, reset=reset_pin, req=req_pin)

    # Set up PN532
    ic, ver, rev, support = pn532.get_firmware_version()
    print('Found PN532 with firmware version: {0}.{1}'.format(ver, rev))

    # Configure PN532 to communicate with MiFare cards
    pn532.SAM_configuration()
    print('Waiting for RFID/NFC card...')

    return pn532

# Attempt to read NFC once
def read_nfc(pn532):
    try:
        uid = pn532.read_passive_target(timeout=0.5) # Check if a card is available to read
        if uid is None: # Try again if no card is available.
            return 0
        # Accepted cards
        if (uid.hex() == "e4bda722"):
            print("#1 detected")
            return 1
        elif (uid.hex() == "f4f7a722"):
            print("#2 detected")
            return 1
        else: # Unauthorized
            print("Unauthorized!")
            return 0
    except (KeyboardInterrupt, SystemExit):
        return 0

# pn532 = init()
# while(not read_nfc(pn532)):
#    read_nfc(pn532)

              

servo.py

# servo.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# Import modules
import RPi.GPIO as GPIO
import time

GPIO.setwarnings(False)

servo_pin = 5

# Run servo in specified direction for specified length of time
# Re-initializes and cleans up servo GPIO each call
def main(direction, length):
	GPIO.setmode(GPIO.BCM)
	GPIO.setup(servo_pin, GPIO.OUT) # pin 22 output
	GPIO.output(servo_pin, GPIO.LOW)
	p = GPIO.PWM(servo_pin,50)  # 50 Hz
	p.start(7.5)
	if (direction == "unlock"):
		val = 6.5
	elif (direction == "lock"):
		val = 8.5
	else:
		val = 7.5

	p.ChangeDutyCycle(val)
	time.sleep(length)
	p.stop()

# main("unlock", 7)
# time.sleep(2)
# main("lock", 7.5)
              

mic.py

# mic.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# source: https://makersportal.com/blog/2018/8/23/recording-audio-on-the-raspberry-pi-with-python-and-a-usb-microphone

import pyaudio
import wave
import playsound
import RPi.GPIO as GPIO
import time

GPIO.setmode(GPIO.BCM)
GPIO.setup(26, GPIO.IN, pull_up_down=GPIO.PUD_UP)

def main():
    form_1 = pyaudio.paInt16 # 16-bit resolution
    chans = 1 # 1 channel
    samp_rate = 44100 # 44.1kHz sampling rate
    chunk = 4096 # 2^12 samples for buffer
    record_secs = 5 # seconds to record
    dev_index = 2 # device index found by p.get_device_info_by_index(ii)
    wav_output_filename = '/home/pi/ece5725-final/python/audio.wav' # name of .wav file

    audio = pyaudio.PyAudio() # create pyaudio instantiation

    # create pyaudio stream
    stream = audio.open(format = form_1,rate = samp_rate,channels = chans, \
                        input_device_index = dev_index,input = True, \
                        frames_per_buffer=chunk)

    # play "success" when recording begins
    playsound.main("success")
    frames = []

    # loop through stream and append audio chunks to frame array
    for ii in range(0,int((samp_rate/chunk)*record_secs)):
        data = stream.read(chunk, exception_on_overflow = False)
        frames.append(data)

    # stop the stream, close it, and terminate the pyaudio instantiation
    stream.stop_stream()
    stream.close()
    audio.terminate()

    # save the audio frames as .wav file
    wavefile = wave.open(wav_output_filename,'wb')
    wavefile.setnchannels(chans)
    wavefile.setsampwidth(audio.get_sample_size(form_1))
    wavefile.setframerate(samp_rate)
    wavefile.writeframes(b''.join(frames))
    wavefile.close()

# main()
              

ibmspeech.py

# ibmspeech.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# source: https://github.com/watson-developer-cloud/python-sdk

# Imports
from __future__ import print_function
import json
from os.path import join, dirname
from ibm_watson import SpeechToTextV1
from ibm_watson.websocket import RecognizeCallback, AudioSource
import threading

# Example using websockets
class MyRecognizeCallback(RecognizeCallback):
    def __init__(self):
        RecognizeCallback.__init__(self)

    def on_transcription(self, transcript):
        print(transcript)

    def on_connected(self):
        print('Connection was successful')

    def on_error(self, error):
        print('Error received: {}'.format(error))

    def on_inactivity_timeout(self, error):
        print('Inactivity timeout: {}'.format(error))

    def on_listening(self):
        print('Service is listening')

    def on_hypothesis(self, hypothesis):
        print(hypothesis)

    def on_data(self, data):
        print(data)

# Initializes IBM Speech Recognition service using API key authentication
def init():
    service = SpeechToTextV1(
        url='https://gateway-wdc.watsonplatform.net/speech-to-text/api',
        iam_apikey='rEIVeCDlJxozqkYjM8TSr1u-occu7op_MG2DCO0FrCFh')
    models = service.list_models().get_result()
    model = service.get_model('en-US_BroadbandModel').get_result()
    return service, models, model

# Runs IBM Speech Recognition on specified audio file
def main(service, models, model):
    # Open recorded audio file
    with open(join(dirname(__file__), '/home/pi/ece5725-final/python/audio.wav'),
            'rb') as audio_file:
        speech_output = service.recognize(audio=audio_file, content_type='audio/wav', timestamps=True, word_confidence=True).get_result()
        print(json.dumps(speech_output, indent=2))

    # Example using threads in a non-blocking way
    mycallback = MyRecognizeCallback()
    audio_file = open(join(dirname(__file__), '/home/pi/ece5725-final/python/audio.wav'), 'rb')
    audio_source = AudioSource(audio_file)
    recognize_thread = threading.Thread(
        target=service.recognize_using_websocket,
        args=(audio_source, "audio/l16; rate=44100", mycallback))
    recognize_thread.start()
    try:
        # Parse out output string
        final_output = speech_output['results'][0]['alternatives'][0]['transcript'].rstrip()
        print("Final Output:", final_output)

        # Check output against passwrd "5725"
        if (final_output == "five seven two five"):
            return 1
        else:
            return 0
    except: # If anything goes wrong, catch the exception and return
        return 0

# service, models, model = init()
# main(service, models, model)
              

pi_face_recognition.py

# pi_face_recognition.py
# Amanda Trang (at669), Michael Rodriguez (mr2242)
# Final Project
# 5/14/2019
# Thursday Lab

# source: https://www.pyimagesearch.com/2018/06/25/raspberry-pi-face-recognition/

# Imports
from imutils.video import VideoStream
from imutils.video import FPS
import face_recognition
import argparse
import imutils
import pickle
import time
import cv2

# Load and read encodings
def init():
	# construct the argument parser and parse the arguments
	ap = argparse.ArgumentParser()
	ap.add_argument("-c", "--cascade", default="/home/pi/ece5725-final/python/haarcascade_frontalface_default.xml", required=False, help = "path to where the face cascade resides")
	ap.add_argument("-e", "--encodings", default="/home/pi/ece5725-final/python/encodings.pickle", required=False, help="path to serialized db of facial encodings")
	args = vars(ap.parse_args())

	# load the known faces and embeddings along with OpenCV's Haar cascade for face detection
	data = pickle.loads(open(args["encodings"], "rb").read())
	detector = cv2.CascadeClassifier(args["cascade"])
	return data, detector

def main(data, detector):
	try:
		# Initialize the video stream and allow the camera sensor to warm up
		print("[INFO] starting video stream...")
		vs = VideoStream(src=0).start()
		time.sleep(2.0)
		print("done sleep")

		matchFound = 0

		# grab the frame from the threaded video stream and resize
		frame = vs.read()
		frame = imutils.resize(frame, width=300)

		# convert the input frame from (1) BGR to grayscale (for face detection) and (2) from BGR to RGB (for face recognition)
		gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
		rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

		# detect faces in the grayscale frame
		rects = detector.detectMultiScale(gray, scaleFactor=1.1,
			minNeighbors=5, minSize=(30, 30),
			flags=cv2.CASCADE_SCALE_IMAGE)

		# reorder bounding obox
		boxes = [(y, x + w, y + h, x) for (x, y, w, h) in rects]

		# compute the facial embeddings for each face bounding box
		encodings = face_recognition.face_encodings(rgb, boxes)
		names = []

		# loop over the facial embeddings
		for encoding in encodings:
			# attempt to match each face
			matches = face_recognition.compare_faces(data["encodings"], encoding)
			name = "Unknown"

			# check to see if we have found a match
			if True in matches:
				# find the indexes of all matched faces
				matchedIdxs = [i for (i, b) in enumerate(matches) if b]
				counts = {}

				# loop over the matched indexes and maintain a count for each recognized face
				for i in matchedIdxs:
					name = data["names"][i]
					counts[name] = counts.get(name, 0) + 1

				# determine the recognized face with the largest number of votes
				name = max(counts, key=counts.get)

				# if Amanda or Michael's face is detected, end
				if (name == "amanda" or name == "michael"):
					matchFound = 1
					cv2.destroyAllWindows()
					vs.stop()
					return matchFound

			# update the list of names
			names.append(name)

		# cv2.imshow("Frame", frame)
		key = cv2.waitKey(1) & 0xFF

		# if the `q` key was pressed, break from the loop
		if key == ord("q"):
			return 0

		# cleanup
		cv2.destroyAllWindows()
		vs.stop()
		return matchFound

	except (KeyboardInterrupt, SystemExit):
		cv2.destroyAllWindows()
		vs.stop()
		return 0

# data, detector = init()
# while(not main(data, detector)):
# 	main(data, detector)