From 4b71fef0c1e29c89f06389b6db81cd578d028f12 Mon Sep 17 00:00:00 2001 From: Gubolin Date: Sun, 10 Aug 2014 20:04:52 +0200 Subject: add camera streaming blocks (fix #460) --- objects.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'objects.js') diff --git a/objects.js b/objects.js index fbb9e84..06bab19 100644 --- a/objects.js +++ b/objects.js @@ -569,6 +569,16 @@ SpriteMorph.prototype.initBlocks = function () { category: 'pen', spec: 'stamp' }, + doStreamCamera: { + type: 'command', + category: 'pen', + spec: 'start streaming from the camera' + }, + doStopCamera: { + type: 'command', + category: 'pen', + spec: 'stop streaming from the camera' + }, // Control receiveGo: { @@ -4911,6 +4921,9 @@ StageMorph.prototype.blockTemplates = function (category) { } else if (cat === 'pen') { blocks.push(block('clear')); + blocks.push('-'); + blocks.push(block('doStreamCamera')); + blocks.push(block('doStopCamera')); } else if (cat === 'control') { -- cgit v1.3.1 From e55c5e035a46c2a3353ce3f21bca3daf6b3fee6c Mon Sep 17 00:00:00 2001 From: Gubolin Date: Tue, 12 Aug 2014 11:32:26 +0200 Subject: add simple motion detection --- blocks.js | 1 + objects.js | 1 + threads.js | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) (limited to 'objects.js') diff --git a/blocks.js b/blocks.js index 2e297ab..5f67107 100644 --- a/blocks.js +++ b/blocks.js @@ -6571,6 +6571,7 @@ InputSlotMorph.prototype.messagesReceivedMenu = function () { InputSlotMorph.prototype.collidablesMenu = function () { var dict = { 'mouse-pointer' : ['mouse-pointer'], + 'camera motion' : ['camera motion'], edge : ['edge'], 'pen trails' : ['pen trails'] }, diff --git a/objects.js b/objects.js index 06bab19..37d66b2 100644 --- a/objects.js +++ b/objects.js @@ -4268,6 +4268,7 @@ StageMorph.prototype.init = function (globals) { this.paletteCache = {}; // not to be serialized (!) this.lastAnswer = ''; // last user input, do not persist this.activeSounds = []; // do not persist + this.lastCameraCanvas = null; this.trailsCanvas = null; this.isThreadSafe = false; diff --git a/threads.js b/threads.js index cb6dab7..4c70bd5 100644 --- a/threads.js +++ b/threads.js @@ -2355,6 +2355,24 @@ Process.prototype.objectTouchingObject = function (thisObj, name) { } else { stage = thisObj.parentThatIsA(StageMorph); if (stage) { + if (this.inputOption(name) === 'camera motion') { + var motionCanvas = this.getCameraMotion(); + + var x = thisObj.xPosition() + 210, y = 170 - thisObj.yPosition(); + // ouch! hardcoded ^ ^ + var motionData = motionCanvas.getContext('2d').getImageData( + x, y, thisObj.width(), thisObj.height()); + var i = 0, average = 0; + while (i < (motionData.data.length * 0.25)) { + average += (motionData.data[i*4] + motionData.data[i*4+1] + + motionData.data[i*4+2]) / 3; + ++i; + } + average = Math.round(average / (motionData.data.length * 0.25)); + if (average > 10) { + return true; + } + } if (this.inputOption(name) === 'edge' && !stage.bounds.containsRectangle(thisObj.bounds)) { return true; @@ -2768,12 +2786,29 @@ Process.prototype.doStreamCamera = function () { video.play(); }, this.handleError); } + stage.lastCameraCanvas = newCanvas(stage.dimensions); } else { var canvas = stage.trailsCanvas; var context = canvas.getContext('2d'); - if (video.readyState !== 0) { + if (video.readyState == 4) { try { + // https://stackoverflow.com/questions/23840880/check-whether-canvas-is-black + // check whether lastCameraCanvas is white -> copy video canvas + // otherwise you would have a 'motion' when the first camera picture is loaded + var tmp = document.createElement('canvas'), + ctx = tmp.getContext('2d'), result; + tmp.width = tmp.height = 1; + ctx.drawImage(stage.lastCameraCanvas, 0, 0, 1, 1); + result = ctx.getImageData(0, 0, 1, 1); + if (result.data[0] + result.data[1] + result.data[2] + + result.data[3] === 0) { + stage.lastCameraCanvas.getContext('2d'). + drawImage(video, 0, 0, video.width, video.height); + } else { + var dest = stage.lastCameraCanvas.getContext('2d'); + dest.drawImage(stage.trailsCanvas, 0, 0); + } context.drawImage(video, 0, 0, video.width, video.height); stage.changed(); } catch (e) { @@ -2799,6 +2834,51 @@ Process.prototype.doStopCamera = function () { } }; +Process.prototype.getCameraMotion = function () { + // see https://www.adobe.com/devnet/html5/articles/javascript-motion-detection.html + fastAbs = function (value) // faster, but less acurate + { return (value ^ (value >> 31)) - (value >> 31); }; + + threshold = function (value) + { return (value > 0x15) ? 0xFF : 0; }; + + diff = function (target, data1, data2) { + if (data1.length != data2.length) return null; + var i = 0; + while (i < (data1.length * 0.25)) { + var average1 = (data1[4*i] + data1[4*i+1] + data1[4*i+2]) / 3; + var average2 = (data2[4*i] + data2[4*i+1] + data2[4*i+2]) / 3; + var diff = threshold(fastAbs(average1 - average2)); + target[4*i] = diff; + target[4*i+1] = diff; + target[4*i+2] = diff; + target[4*i+3] = 0xFF; + ++i; + } + }; + + var stage = this.homeContext.receiver.parentThatIsA(StageMorph); + if (!stage.trailsCanvas || !stage.lastCameraCanvas) { + var canv = newCanvas(stage.dimensions); + var ctx = canv.getContext('2d'); + ctx.fill(); + return canv; + } + + var canvasSource = stage.trailsCanvas; + var contextSource = canvasSource.getContext('2d'); + var width = canvasSource.width, + height = canvasSource.height; + var sourceData = contextSource.getImageData(0, 0, width, height); + var lastImageData = + stage.lastCameraCanvas.getContext('2d').getImageData(0, 0, width, height); + var blendedData = contextSource.createImageData(width, height); + var blendedCanvas = newCanvas(stage.dimensions); + diff(blendedData.data, sourceData.data, lastImageData.data); + blendedCanvas.getContext('2d').putImageData(blendedData, 0, 0); + return blendedCanvas; +}; + // Process constant input options Process.prototype.inputOption = function (dta) { -- cgit v1.3.1 From 6560149027b29c4e2844e3bd52e8731379fb1b91 Mon Sep 17 00:00:00 2001 From: Gubolin Date: Wed, 13 Aug 2014 13:35:14 +0200 Subject: add motion reporters --- blocks.js | 1 - objects.js | 16 +++++++++++++ threads.js | 80 ++++++++++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 78 insertions(+), 19 deletions(-) (limited to 'objects.js') diff --git a/blocks.js b/blocks.js index 5f67107..2e297ab 100644 --- a/blocks.js +++ b/blocks.js @@ -6571,7 +6571,6 @@ InputSlotMorph.prototype.messagesReceivedMenu = function () { InputSlotMorph.prototype.collidablesMenu = function () { var dict = { 'mouse-pointer' : ['mouse-pointer'], - 'camera motion' : ['camera motion'], edge : ['edge'], 'pen trails' : ['pen trails'] }, diff --git a/objects.js b/objects.js index 37d66b2..9eba6a8 100644 --- a/objects.js +++ b/objects.js @@ -785,6 +785,18 @@ SpriteMorph.prototype.initBlocks = function () { category: 'sensing', spec: 'color %clr is touching %clr ?' }, + reportCameraMotion: { + only: SpriteMorph, + type: 'predicate', + category: 'sensing', + spec: 'camera motion at my position?' + }, + reportCameraDirection: { + only: SpriteMorph, + type: 'reporter', + category: 'sensing', + spec: 'camera motion direction' + }, colorFiltered: { dev: true, type: 'reporter', @@ -1868,6 +1880,9 @@ SpriteMorph.prototype.blockTemplates = function (category) { blocks.push(block('reportTouchingColor')); blocks.push(block('reportColorIsTouchingColor')); blocks.push('-'); + blocks.push(block('reportCameraMotion')); + blocks.push(block('reportCameraDirection')); + blocks.push('-'); blocks.push(block('doAsk')); blocks.push(watcherToggle('getLastAnswer')); blocks.push(block('getLastAnswer')); @@ -4269,6 +4284,7 @@ StageMorph.prototype.init = function (globals) { this.lastAnswer = ''; // last user input, do not persist this.activeSounds = []; // do not persist this.lastCameraCanvas = null; + this.lastCameraMotion = new Point(0, 0); this.trailsCanvas = null; this.isThreadSafe = false; diff --git a/threads.js b/threads.js index 4c70bd5..03625d9 100644 --- a/threads.js +++ b/threads.js @@ -2355,24 +2355,6 @@ Process.prototype.objectTouchingObject = function (thisObj, name) { } else { stage = thisObj.parentThatIsA(StageMorph); if (stage) { - if (this.inputOption(name) === 'camera motion') { - var motionCanvas = this.getCameraMotion(); - - var x = thisObj.xPosition() + 210, y = 170 - thisObj.yPosition(); - // ouch! hardcoded ^ ^ - var motionData = motionCanvas.getContext('2d').getImageData( - x, y, thisObj.width(), thisObj.height()); - var i = 0, average = 0; - while (i < (motionData.data.length * 0.25)) { - average += (motionData.data[i*4] + motionData.data[i*4+1] - + motionData.data[i*4+2]) / 3; - ++i; - } - average = Math.round(average / (motionData.data.length * 0.25)); - if (average > 10) { - return true; - } - } if (this.inputOption(name) === 'edge' && !stage.bounds.containsRectangle(thisObj.bounds)) { return true; @@ -2442,6 +2424,62 @@ Process.prototype.reportColorIsTouchingColor = function (color1, color2) { return false; }; +Process.prototype.reportCameraMotion = function () { + var thisObj = this.blockReceiver(); + var motionCanvas = this.getCameraMotion(); + + var x = thisObj.xPosition() + 210, y = 170 - thisObj.yPosition(); + // ouch! hardcoded ^ ^ + var motionData = motionCanvas.getContext('2d').getImageData( + x, y, thisObj.width(), thisObj.height()); + var i = 0, average = 0; + while (i < (motionData.data.length * 0.25)) { + average += (motionData.data[i*4] + + motionData.data[i*4+1] + motionData.data[i*4+2]) / 3; + ++i; + } + average = Math.round(average / (motionData.data.length * 0.25)); + if (average > 10) { + return true; + } else { + return false; + } +}; + +Process.prototype.reportCameraDirection = function () { + var stage = this.homeContext.receiver.parentThatIsA(StageMorph); + var motionCanvas = this.getCameraMotion(); + var canv = document.createElement('canvas'), + ctx = canv.getContext('2d'); + var data = motionCanvas.getContext('2d') + .getImageData(0, 0, motionCanvas.width, motionCanvas.height); + // scale, saves some time + canv.width = motionCanvas.width / 10; + canv.height = motionCanvas.height / 10; + ctx.drawImage(motionCanvas, 0, 0, canv.width, canv.height); + var yval = 0, xval = 0, cnt = 0; + var motion, imageData = ctx.getImageData(0, 0, canv.width, canv.height); + // find the geometric center of the moving object + for (var j = 0; j < canv.height; j++) { + for (var k = 0; k < canv.width; k++) { + motion = imageData.data[(j*canv.width+k)*4]; + yval += j * motion; + xval += k * motion; + cnt += motion; + } + } + // calculate the angle between this center and the last one + var resultX = Math.round(xval / cnt / 1) * 1; + var resultY = Math.round(yval / cnt / 1) * 1; + var diff = (resultX + resultY) - + (stage.lastCameraMotion.x - stage.lastCameraMotion.y); + var deg = 0; + deg = degrees(Math.atan2(resultX - stage.lastCameraMotion.x, + stage.lastCameraMotion.y - resultY)); + stage.lastCameraMotion = new Point(resultX, resultY); + return deg; +}; + Process.prototype.reportDistanceTo = function (name) { var thisObj = this.blockReceiver(), thatObj, @@ -2753,6 +2791,12 @@ Process.prototype.doPlayNoteForSecs = function (pitch, secs) { // Process camera streaming primitives Process.prototype.doStreamCamera = function () { + if ((Date.now() - this.context.startTime) < 3000) { + // 20 FPS is enough + this.pushContext('doYield'); + this.pushContext(); + return; + } var myself = this; var video = this.context.activeStream || null; -- cgit v1.3.1 From 48ace3a975d39bbaa2f74ec07d521dc723903371 Mon Sep 17 00:00:00 2001 From: Gubolin Date: Sun, 17 Aug 2014 18:29:23 +0200 Subject: add camera streaming predicate --- objects.js | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'objects.js') diff --git a/objects.js b/objects.js index 9eba6a8..a15aec4 100644 --- a/objects.js +++ b/objects.js @@ -797,6 +797,11 @@ SpriteMorph.prototype.initBlocks = function () { category: 'sensing', spec: 'camera motion direction' }, + reportStreamingCamera: { + type: 'predicate', + category: 'sensing', + spec: 'streaming from the camera?' + }, colorFiltered: { dev: true, type: 'reporter', @@ -1882,6 +1887,7 @@ SpriteMorph.prototype.blockTemplates = function (category) { blocks.push('-'); blocks.push(block('reportCameraMotion')); blocks.push(block('reportCameraDirection')); + blocks.push(block('reportStreamingCamera')); blocks.push('-'); blocks.push(block('doAsk')); blocks.push(watcherToggle('getLastAnswer')); @@ -4283,6 +4289,7 @@ StageMorph.prototype.init = function (globals) { this.paletteCache = {}; // not to be serialized (!) this.lastAnswer = ''; // last user input, do not persist this.activeSounds = []; // do not persist + this.streamingCamera = false; this.lastCameraCanvas = null; this.lastCameraMotion = new Point(0, 0); @@ -4997,6 +5004,8 @@ StageMorph.prototype.blockTemplates = function (category) { } else if (cat === 'sensing') { + blocks.push(block('reportStreamingCamera')); + blocks.push('-'); blocks.push(block('doAsk')); blocks.push(watcherToggle('getLastAnswer')); blocks.push(block('getLastAnswer')); -- cgit v1.3.1