Google speech api will be used to control the pointer by voice. The starting code is printed in the following section. You should put LED to Pin 13 to see the results of the speech recognition.
The GUI includes a few buttons and a field for printing out the interim results, final results and status of the speech recognition. "Start" button should be pressed to begin recognition:
The connection should be secure, https. Therefore, the certificate files should be generated in Linux terminal (in the cps-iot directory):
sudo openssl genrsa -out privatekey.pem 1024
sudo openssl req -new -key privatekey.pem -out certrequest.csr
sudo openssl x509 -req -in certrequest.csr -signkey privatekey.pem -out certificate.pem
This will generate two files, privatekey.pem and certificate.pem on the disc in the cps-iot directory.
/*********************************************************************
University of Maribor ************************************************
Cybernetics & Decision Support Systems Laboratory ********************
Faculty of Organizational Sciences ***********************************
Andrej Škraba ********************************************************
*********************************************************************/
//The connection should be secure, i.e. https - don't forget to put [s] in the Chrome address i.e. https://172...
//Therefore, the certificate files should be generated in Linux terminal (in the cps-iot directory),
//you should execute the following three (3) lines in Linux terminal (bash:)
//sudo openssl genrsa -out privatekey.pem 1024
//sudo openssl req -new -key privatekey.pem -out certrequest.csr
//sudo openssl x509 -req -in certrequest.csr -signkey privatekey.pem -out certificate.pem
//This will generate two files, privatekey.pem and certificate.pem on the disc in the cps-iot directory.
var firmata = require("firmata");
var board = new firmata.Board("/dev/ttyACM0",function(){
console.log("Connection to Arduino");
console.log("Enabling pins");
board.pinMode(13, board.MODES.OUTPUT); // enable pin 13 for turning the LED on and off
});
var fs = require("fs");
var options = {
key: fs.readFileSync('privatekey.pem'),
cert: fs.readFileSync('certificate.pem')
};
var httpsServerForHTML = require("https").createServer(options, handler); // here the argument "handler" is needed, which is used latter on -> "function handler (req, res); in this line, we create the server! (https is object of our app)
var url = require("url");
// read ssl certificate
var privateKey = fs.readFileSync('privatekey.pem', 'utf8');
var certificate = fs.readFileSync('certificate.pem', 'utf8');
var credentials = { key: privateKey, cert: certificate };
var https = require('https');
//pass in your credentials to create an https server
var httpsServer = https.createServer(credentials);
httpsServer.listen(8888);
var WebSocketServer = require('ws').Server;
var wss = new WebSocketServer({
server: httpsServer
});
wss.broadcast = function broadcast(data) {
wss.clients.forEach(function each(client) {
if (client.readyState === WebSocket.OPEN) {
client.send(data);
}
});
};
var messageJSON;
function handler(req, res) {
fs.readFile(__dirname + "/example30.html",
function (err, data) {
if (err) {
res.writeHead(500, {"Content-Type": "text/plain"});
return res.end("Error loading html page.");
}
res.writeHead(200);
res.end(data);
})
}
httpsServerForHTML.listen(8080); // determine on which port we will listen | port 80 is usually used by LAMP | This could be determined on the router (http is our main object, i.e.e app)
console.log("Use (S) httpS! - System Start - Use (S) httpS!"); // we print into the console that in the Chrome browser, the httpS (S!=Secure) should be used i.e. https://...
board.on("ready", function() {
wss.on('connection', function connection(ws) { // start of wss code
ws.on("message", function (msgString) { // message comes as string -> msgString
var msg = JSON.parse(msgString); // string from ws which comes as a string is put to JSON
switch(msg.type) {
case "left":
board.digitalWrite(13, board.HIGH); // if we hear the message "left" we write HIGH value on pin 13
break;
case "center":
board.digitalWrite(13, board.LOW); // if we hear the message "center" we write LOW value on pin 13
break;
case "right":
board.digitalWrite(13, board.HIGH); // if we hear the message "right" we write HIGH value on pin 13
break;
} // end of switch(msg.type) code
}); // end of wss.on(message) code
}); // end of wss.on(connection) code
}); // end of board.on ready
<!DOCTYPE html>
<meta charset=utf-8>
<!-- ********************************************************************** -->
<!-- University of Maribor ************************************************ -->
<!-- Cybernetics & Decision Support Systems Laboratory ******************** -->
<!-- Faculty of Organizational Sciences *********************************** -->
<!-- Andrej Škraba ******************************************************** -->
<!-- ********************************************************************** -->
<html>
<head>
<title>Speech control</title>
</head>
<style>
body {font-family: courier}
</style>
<div id="interimResults" style="border-style:dotted; border-color:#c3c3c3; padding:10px; font-size:24px; font-color:#00ff00; width:477px; text-align:center;">
interimResult
</div>
<div id="finalResults" style="border-style:dotted; border-color:#c3c3c3; padding:10px; font-size:24px; font-color:#00ff00; width:477px; text-align:center;">
finalResults
</div>
<div id="speech" style="border-style:dotted; border-color:#c3c3c3; padding:10px; font-size:24px; font-color:#00ff00; width:477px; text-align:center;">
Press button START for speech input.
</div>
<div class="right">
<button id="start_button" onclick="startButton(event)" style="height: 65px; width: 162px; font-size:24px;">Start</button>
</div>
<div id="results">
<span id="final_span"></span>
<span id="interim_span"></span>
<p>
</div>
<!-- ******************************************************************************************** -->
<!-- *** Button switchboard ********************************************************************* -->
<!-- ******************************************************************************************** -->
<br></br>
<button id="buttonLeft" onClick="left()" style="height: 162px; width: 162px; font-size:35px;">LEFT</button>
<button id="buttonCenter" onClick="center()" style="height: 162px; width: 162px; font-size:35px;">CENTER</button>
<button id="buttonRight" onClick="right()" style="height: 162px; width: 162px; font-size:35px;">RIGHT</button>
<br></br>
</p>
<!-- ******************************************************************************************** -->
<!-- ******************************************************************************************** -->
<!-- ******************************************************************************************** -->
<body>
<script>
var lastResult;
var final_span = document.getElementById("final_span");
var interim_span = document.getElementById("interim_span");
function ReplaceContentInContainer(id, content) { // for replacing the div content
var container = document.getElementById(id);
container.innerHTML = content;
}
// get the socket.io library from the server
let ws = new WebSocket("wss://192.168.254.149:8888"); // create socket - connect to it
var messageJSON;
function left () {
messageJSON = {"type": "left"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
}
function center () {
messageJSON = {"type": "center"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
}
function right () {
messageJSON = {"type": "right"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
}
// *****************************************************************************
// Code for speech API START
// *****************************************************************************
var final_transcript = '';
var recognizing = false;
var ignore_onend;
var start_timestamp;
if (!('webkitSpeechRecognition' in window)) {
upgrade();
} else {
var recognition = new webkitSpeechRecognition();
recognition.continuous = true;
recognition.interimResults = true;
recognition.onstart = function() {
recognizing = true;
document.getElementById("start_button").innerText = "Speak to microphone";
};
recognition.onerror = function(event) {
if (event.error == 'no-speech') {
document.getElementById("start_button").innerText = "Start Microphone";
ReplaceContentInContainer("speech", "Speech not present.");
ignore_onend = true;
}
if (event.error == 'audio-capture') {
document.getElementById("start_button").innerText = "Start Microphone";
ReplaceContentInContainer("speech", "No microphone!");
ignore_onend = true;
}
if (event.error == 'not-allowed') {
if (event.timeStamp - start_timestamp < 100) {
ReplaceContentInContainer("speech", "Speech blocked / disabled!");
} else {
ReplaceContentInContainer("speech", "Speech input rejected.");
}
ignore_onend = true;
}
};
recognition.onend = function() { // start the recognition once again
ReplaceContentInContainer("speech", "New Restart!");
document.getElementById("start_button").innerText = "Microphone disabled";
recognition.start();
};
var final_trs;
recognition.onresult = function(event) {
var interim_transcript = '';
// interim_transcript += event.results[i][0].transcript;
for (var i = event.resultIndex; i < event.results.length; ++i) {
if (event.results[i].isFinal) {
final_transcript += event.results[i][0].transcript;
final_trs = event.results[i][0].transcript;
ReplaceContentInContainer("finalResults", final_trs);
interim_transcript += event.results[i][0].transcript;
} else {
interim_transcript += event.results[i][0].transcript;
}
}
ReplaceContentInContainer("interimResults", interim_transcript);
// *********************************************************************
// here we enter our commands
// *********************************************************************
// *********************************************************************
// LEFT
// *********************************************************************
if(interim_transcript == "left" || interim_transcript == " left" || interim_transcript == "last" || interim_transcript == " last" || interim_transcript == " let" || interim_transcript == "let" || interim_transcript == "Lyft" || interim_transcript == " Lyft" || interim_transcript == "lift" || interim_transcript == " lift") // since the pause is made during the speech, the space should also be considered, i.e. +1, t.j. " 1"
{
messageJSON = {"type": "left"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
};
// *********************************************************************
// CENTER
// *********************************************************************
if(interim_transcript == "center" || interim_transcript == " center" || interim_transcript == "Center" || interim_transcript == " Center" || interim_transcript == "centre" || interim_transcript == " centre" || interim_transcript == "Santa" || interim_transcript == " Santa" || interim_transcript == "send" || interim_transcript == " send") // since the pause is made during the speech, the space should also be considered, i.e. +1, t.j. " 1"
{
messageJSON = {"type": "center"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
};
// *********************************************************************
// RIGHT
// *********************************************************************
if(interim_transcript == "right" || interim_transcript == " right") // since the pause is made during the speech, the space should also be considered, i.e. +1, t.j. " 1"
{
messageJSON = {"type": "right"}; // JSON structure of the message
ws.send(JSON.stringify(messageJSON)); // we have to stringify JSON to send it over websocket
};
// *********************************************************************
// end of entering commands part
// *********************************************************************
}; // End of recognition.on.result
} // End of else (if we have webkitSpeechRecognition in window object, i.e. a good version of Chrome)
function upgrade() { // function to allert the user, that upgrade is needed
ReplaceContentInContainer("speech", 'You have to upgrade Chrome to version 25 of higher');
}
function startButton(event) {
if (recognizing) {
recognition.stop();
return;
}
final_transcript = '';
recognition.lang = "en-US"; // determine language
recognition.start();
ignore_onend = false;
final_span.innerHTML = '';
interim_span.innerHTML = '';
document.getElementById("start_button").innerText = "Microphone enabled";
ReplaceContentInContainer("speech", "Please talk!");
start_timestamp = event.timeStamp;
}
// *****************************************************************************
// Speech API code END
// *****************************************************************************
</script>
</body>
</html>