zeroed-some/cue-web / 3c3843d

Browse files

maybe better gesture recog

Authored by espadonne
SHA
3c3843d270103bc590774b0d1415302e4491e771
Parents
330895a
Tree
0a7714c

3 changed files

StatusFile+-
M index.html 7 4
M js/app.js 12 8
M js/gesture-detector.js 273 112
index.htmlmodified
@@ -77,12 +77,15 @@
7777
                 <div class="info-box instructions">
7878
                     <h4>Instructions:</h4>
7979
                     <ol>
80
-                        <li>Hold a neon pink/orange post-it</li>
80
+                        <li>Hold a <span style="color: #ffff00;">YELLOW</span> object (post-it, card)</li>
8181
                         <li>Move horizontally to select effect</li>
82
-                        <li>Hold a teal post-it with other hand</li>
83
-                        <li>Move teal post-it vertically for parameter</li>
84
-                        <li>Keep post-it in same zone for cue trigger</li>
82
+                        <li>Hold a <span style="color: #00ff00;">GREEN</span> object with other hand</li>
83
+                        <li>Move GREEN vertically for parameter strength</li>
84
+                        <li>Keep object in same zone for cue trigger</li>
8585
                     </ol>
86
+                    <div style="margin-top: 10px; font-size: 12px; opacity: 0.8;">
87
+                        💡 Tip: Use bright, saturated colors for best detection
88
+                    </div>
8689
                 </div>
8790
             </aside>
8891
         </main>
js/app.jsmodified
@@ -212,13 +212,15 @@ class GestureDSPApp {
212212
     }
213213
214214
     setupGestureCallbacks() {
215
-        this.gestureDetector.onHandsDetected = (redHand, yellowHand) => {
216
-            if (redHand) {
217
-                this.handleEffectSelection(redHand);
215
+        this.gestureDetector.onHandsDetected = (yellowHand, greenHand) => {
216
+            // Yellow hand for horizontal effect selection
217
+            if (yellowHand) {
218
+                this.handleEffectSelection(yellowHand);
218219
             }
219220
220
-            if (yellowHand) {
221
-                this.handleParameterControl(yellowHand);
221
+            // Green hand for vertical parameter control
222
+            if (greenHand) {
223
+                this.handleParameterControl(greenHand);
222224
             } else {
223225
                 // Slowly return to center when no hand detected
224226
                 if (this.audioEngine.audioContext && this.audioEngine.effects) {
@@ -228,9 +230,11 @@ class GestureDSPApp {
228230
                 this.updateParameterDisplay();
229231
             }
230232
231
-            // Update hand status
232
-            document.getElementById('handStatus').textContent =
233
-                redHand || yellowHand ? 'Yes' : 'No';
233
+            // Update hand status with confidence
234
+            const status = [];
235
+            if (yellowHand) status.push(`Yellow: ${Math.round(yellowHand.confidence * 100)}%`);
236
+            if (greenHand) status.push(`Green: ${Math.round(greenHand.confidence * 100)}%`);
237
+            document.getElementById('handStatus').textContent = status.length > 0 ? status.join(', ') : 'No';
234238
         };
235239
     }
236240
js/gesture-detector.jsmodified
@@ -11,8 +11,27 @@ export class GestureDetector {
1111
         this.onHandsDetected = null;
1212
1313
         // Detection parameters
14
-        this.minHandSize = 5;   // Very small
15
-        this.maxHandY = 1.0;    // No vertical restriction
14
+        this.minHandSize = 30;  // Minimum hand size in pixels
15
+        this.maxHandSize = 150; // Maximum hand size (hand-sized)
16
+        this.idealHandRatio = 1.2; // Expected width/height ratio
17
+
18
+        // Tracking state
19
+        this.yellowHistory = [];
20
+        this.greenHistory = [];
21
+        this.maxHistory = 5;
22
+
23
+        // Adaptive thresholds
24
+        this.yellowThreshold = { h: [0.12, 0.20], s: 0.3, v: 0.4 };
25
+        this.greenThreshold = { h: [0.25, 0.40], s: 0.3, v: 0.3 };
26
+
27
+        // Smoothing
28
+        this.lastYellow = null;
29
+        this.lastGreen = null;
30
+        this.smoothingFactor = 0.3;
31
+
32
+        // Motion detection
33
+        this.previousFrame = null;
34
+        this.motionThreshold = 10;
1635
     }
1736
1837
     async start() {
@@ -34,7 +53,6 @@ export class GestureDetector {
3453
3554
         this.video.srcObject = stream;
3655
37
-        // Wait for video to be ready
3856
         await new Promise((resolve) => {
3957
             this.video.onloadedmetadata = resolve;
4058
         });
@@ -63,20 +81,20 @@ export class GestureDetector {
6381
6482
                 // Get image data and detect hands
6583
                 const imageData = this.ctx.getImageData(0, 0, this.canvas.width, this.canvas.height);
66
-                const detection = this.detectHands(imageData);
84
+                const detection = this.detectHandsSmart(imageData);
6785
6886
                 // Draw detection boxes
69
-                if (detection.redHand) {
70
-                    this.drawBox(detection.redHand, '#000000');  // Black outline for white objects
87
+                if (detection.yellowHand) {
88
+                    this.drawBox(detection.yellowHand, '#ffff00', 'Effect Select');
7189
                 }
7290
73
-                if (detection.yellowHand) {
74
-                    this.drawBox(detection.yellowHand, '#40e0d0');  // Turquoise for teal
91
+                if (detection.greenHand) {
92
+                    this.drawBox(detection.greenHand, '#00ff00', 'Parameter');
7593
                 }
7694
7795
                 // Call callback if registered
7896
                 if (this.onHandsDetected) {
79
-                    this.onHandsDetected(detection.redHand, detection.yellowHand);
97
+                    this.onHandsDetected(detection.yellowHand, detection.greenHand);
8098
                 }
8199
             }
82100
@@ -86,107 +104,277 @@ export class GestureDetector {
86104
         process();
87105
     }
88106
89
-    drawBox(box, color) {
90
-        this.ctx.strokeStyle = color;
91
-        this.ctx.lineWidth = 2;
92
-        this.ctx.strokeRect(box.x, box.y, box.width, box.height);
93
-
94
-        // Draw center point
95
-        const centerX = box.x + box.width / 2;
96
-        const centerY = box.y + box.height / 2;
97
-        this.ctx.fillStyle = color;
98
-        this.ctx.beginPath();
99
-        this.ctx.arc(centerX, centerY, 4, 0, 2 * Math.PI);
100
-        this.ctx.fill();
101
-    }
102
-
103
-    detectHands(imageData) {
107
+    detectHandsSmart(imageData) {
104108
         const data = imageData.data;
105109
         const width = imageData.width;
106110
         const height = imageData.height;
107111
108
-        // Pixel arrays for each color
109
-        const whitePixels = [];
110
-        const tealPixels = [];
112
+        // Detect motion areas first (if we have a previous frame)
113
+        const motionMask = this.previousFrame ?
114
+            this.detectMotion(data, this.previousFrame, width, height) : null;
115
+
116
+        // Store current frame for next iteration
117
+        this.previousFrame = new Uint8ClampedArray(data);
111118
112
-        // Sample center pixel for debugging
113
-        const centerX = Math.floor(width / 2);
114
-        const centerY = Math.floor(height / 2);
115
-        const centerI = (centerY * width + centerX) * 4;
116
-        const centerR = data[centerI];
117
-        const centerG = data[centerI + 1];
118
-        const centerB = data[centerI + 2];
119
+        // Color detection with motion priority
120
+        const yellowCandidates = [];
121
+        const greenCandidates = [];
119122
120
-        // Process every 2nd pixel for better performance
123
+        // Adaptive sampling - more samples in motion areas
121124
         for (let y = 0; y < height; y += 2) {
122125
             for (let x = 0; x < width; x += 2) {
123126
                 const i = (y * width + x) * 4;
124
-                const r = data[i];
125
-                const g = data[i + 1];
126
-                const b = data[i + 2];
127127
128
-                // Convert to HSV for better color discrimination
129
-                const hsv = this.rgbToHsv(r/255, g/255, b/255);
128
+                // Skip if no motion in this area (when motion mask exists)
129
+                if (motionMask && !motionMask[y * width + x]) {
130
+                    continue;
131
+                }
130132
131
-                // BRIGHT WHITE detection
132
-                // White should have very low saturation and high brightness
133
-                const isWhite = hsv.s < 0.15 && // Very low saturation (near grayscale)
134
-                               hsv.v > 0.75 && // High brightness
135
-                               r > 190 && g > 190 && b > 190 && // All channels high
136
-                               Math.abs(r - g) < 20 && // Channels close together
137
-                               Math.abs(g - b) < 20 && // Neutral color
138
-                               Math.abs(r - b) < 20;   // No color cast
133
+                const r = data[i] / 255;
134
+                const g = data[i + 1] / 255;
135
+                const b = data[i + 2] / 255;
139136
140
-                // Alternative: Very bright neutral colors
141
-                const isBrightNeutral = (r + g + b) > 650 && // Total brightness
142
-                                       Math.max(r, g, b) - Math.min(r, g, b) < 25; // Low variance
137
+                const hsv = this.rgbToHsv(r, g, b);
143138
144
-                if (isWhite || isBrightNeutral) {
145
-                    whitePixels.push({ x, y });
139
+                // YELLOW detection (bright, saturated yellow)
140
+                if (hsv.h >= this.yellowThreshold.h[0] &&
141
+                    hsv.h <= this.yellowThreshold.h[1] &&
142
+                    hsv.s > this.yellowThreshold.s &&
143
+                    hsv.v > this.yellowThreshold.v &&
144
+                    r > 0.6 && g > 0.6 && b < 0.4) {
145
+                    yellowCandidates.push({ x, y, confidence: hsv.s * hsv.v });
146146
                 }
147147
148
-                // TEAL detection (keeping exactly as is - it works!)
149
-                const isTeal = (hsv.h > 0.45 && hsv.h < 0.55) && // Cyan range
150
-                              hsv.s > 0.3 && // Some saturation
151
-                              hsv.v > 0.3 && // Not too dark
152
-                              (g > r * 1.2 || b > r * 1.2); // Green or blue dominant
148
+                // GREEN detection (bright, saturated green)
149
+                if (hsv.h >= this.greenThreshold.h[0] &&
150
+                    hsv.h <= this.greenThreshold.h[1] &&
151
+                    hsv.s > this.greenThreshold.s &&
152
+                    hsv.v > this.greenThreshold.v &&
153
+                    g > r * 1.3 && g > b * 1.2) {
154
+                    greenCandidates.push({ x, y, confidence: hsv.s * hsv.v });
155
+                }
156
+            }
157
+        }
158
+
159
+        // Cluster and filter candidates
160
+        const yellowClusters = this.clusterPoints(yellowCandidates);
161
+        const greenClusters = this.clusterPoints(greenCandidates);
162
+
163
+        // Find best hand-sized clusters
164
+        const yellowHand = this.findBestHandCluster(yellowClusters, 'yellow');
165
+        const greenHand = this.findBestHandCluster(greenClusters, 'green');
166
+
167
+        // Apply temporal smoothing
168
+        const smoothedYellow = this.smoothDetection(yellowHand, this.lastYellow);
169
+        const smoothedGreen = this.smoothDetection(greenHand, this.lastGreen);
170
+
171
+        this.lastYellow = smoothedYellow;
172
+        this.lastGreen = smoothedGreen;
173
+
174
+        return {
175
+            yellowHand: smoothedYellow,
176
+            greenHand: smoothedGreen
177
+        };
178
+    }
179
+
180
+    detectMotion(currentData, previousData, width, height) {
181
+        const motionMask = new Uint8Array(width * height);
182
+
183
+        for (let y = 0; y < height; y += 4) {
184
+            for (let x = 0; x < width; x += 4) {
185
+                const i = (y * width + x) * 4;
186
+
187
+                const diffR = Math.abs(currentData[i] - previousData[i]);
188
+                const diffG = Math.abs(currentData[i + 1] - previousData[i + 1]);
189
+                const diffB = Math.abs(currentData[i + 2] - previousData[i + 2]);
190
+
191
+                const totalDiff = diffR + diffG + diffB;
192
+
193
+                if (totalDiff > this.motionThreshold) {
194
+                    // Mark 4x4 area as motion
195
+                    for (let dy = 0; dy < 4; dy++) {
196
+                        for (let dx = 0; dx < 4; dx++) {
197
+                            if (y + dy < height && x + dx < width) {
198
+                                motionMask[(y + dy) * width + (x + dx)] = 1;
199
+                            }
200
+                        }
201
+                    }
202
+                }
203
+            }
204
+        }
205
+
206
+        return motionMask;
207
+    }
208
+
209
+    clusterPoints(points) {
210
+        if (points.length < 20) return [];
211
+
212
+        const clusters = [];
213
+        const visited = new Set();
214
+
215
+        // DBSCAN-like clustering
216
+        const epsilon = 30; // Maximum distance between points in a cluster
217
+        const minPoints = 20; // Minimum points to form a cluster
218
+
219
+        for (let i = 0; i < points.length; i++) {
220
+            if (visited.has(i)) continue;
221
+
222
+            const neighbors = [];
223
+            const cluster = [];
224
+
225
+            // Find all neighbors
226
+            for (let j = 0; j < points.length; j++) {
227
+                const dist = Math.sqrt(
228
+                    Math.pow(points[i].x - points[j].x, 2) +
229
+                    Math.pow(points[i].y - points[j].y, 2)
230
+                );
231
+
232
+                if (dist < epsilon) {
233
+                    neighbors.push(j);
234
+                }
235
+            }
236
+
237
+            if (neighbors.length >= minPoints) {
238
+                // Start a new cluster
239
+                for (const idx of neighbors) {
240
+                    if (!visited.has(idx)) {
241
+                        visited.add(idx);
242
+                        cluster.push(points[idx]);
243
+                    }
244
+                }
153245
154
-                if (isTeal) {
155
-                    tealPixels.push({ x, y });
246
+                if (cluster.length > 0) {
247
+                    clusters.push(cluster);
156248
                 }
157249
             }
158250
         }
159251
160
-        // Debug: Draw center crosshair and color info
161
-        this.ctx.strokeStyle = '#000000';
162
-        this.ctx.lineWidth = 2;
163
-        this.ctx.strokeRect(centerX - 10, centerY - 10, 20, 20);
164
-        this.ctx.strokeStyle = '#ffffff';
165
-        this.ctx.lineWidth = 1;
166
-        this.ctx.strokeRect(centerX - 9, centerY - 9, 18, 18);
167
-
168
-        // Show color at center with better contrast
169
-        this.ctx.fillStyle = '#000000';
170
-        this.ctx.fillRect(10, 10, 200, 30);
171
-        this.ctx.fillStyle = '#ffffff';
172
-        this.ctx.font = '12px monospace';
173
-        this.ctx.fillText(`Center RGB: ${centerR},${centerG},${centerB}`, 15, 30);
174
-
175
-        // Find bounding boxes
176
-        const whiteHand = this.getBoundingBox(whitePixels, height);
177
-        const tealHand = this.getBoundingBox(tealPixels, height);
178
-
179
-        // Debug logging
180
-        if (whitePixels.length > 50 || tealPixels.length > 50) {
181
-            console.log('White pixels:', whitePixels.length, 'Teal pixels:', tealPixels.length);
252
+        return clusters;
253
+    }
254
+
255
+    findBestHandCluster(clusters, color) {
256
+        if (clusters.length === 0) return null;
257
+
258
+        let bestCluster = null;
259
+        let bestScore = -1;
260
+
261
+        for (const cluster of clusters) {
262
+            // Calculate bounding box
263
+            const xs = cluster.map(p => p.x);
264
+            const ys = cluster.map(p => p.y);
265
+
266
+            const box = {
267
+                x: Math.min(...xs),
268
+                y: Math.min(...ys),
269
+                width: Math.max(...xs) - Math.min(...xs),
270
+                height: Math.max(...ys) - Math.min(...ys)
271
+            };
272
+
273
+            // Skip if too small or too large
274
+            if (box.width < this.minHandSize || box.height < this.minHandSize ||
275
+                box.width > this.maxHandSize || box.height > this.maxHandSize) {
276
+                continue;
277
+            }
278
+
279
+            // Calculate quality score
280
+            const sizeScore = 1 - Math.abs(box.width * box.height - 5000) / 10000; // Ideal area ~5000px
281
+            const ratioScore = 1 - Math.abs(box.width / box.height - this.idealHandRatio) / 2;
282
+            const densityScore = cluster.length / (box.width * box.height) * 100;
283
+            const confidenceScore = cluster.reduce((sum, p) => sum + p.confidence, 0) / cluster.length;
284
+
285
+            const totalScore = sizeScore * 0.3 + ratioScore * 0.2 + densityScore * 0.2 + confidenceScore * 0.3;
286
+
287
+            if (totalScore > bestScore) {
288
+                bestScore = totalScore;
289
+                bestCluster = box;
290
+                bestCluster.confidence = confidenceScore;
291
+                bestCluster.color = color;
292
+            }
293
+        }
294
+
295
+        // Update history for this color
296
+        if (bestCluster) {
297
+            const history = color === 'yellow' ? this.yellowHistory : this.greenHistory;
298
+            history.push(bestCluster);
299
+            if (history.length > this.maxHistory) {
300
+                history.shift();
301
+            }
182302
         }
183303
184
-        return {
185
-            redHand: whiteHand,
186
-            yellowHand: tealHand
304
+        return bestCluster;
305
+    }
306
+
307
+    smoothDetection(current, previous) {
308
+        if (!current) return null;
309
+        if (!previous) return current;
310
+
311
+        // Smooth position and size
312
+        return {
313
+            x: previous.x + (current.x - previous.x) * this.smoothingFactor,
314
+            y: previous.y + (current.y - previous.y) * this.smoothingFactor,
315
+            width: previous.width + (current.width - previous.width) * this.smoothingFactor,
316
+            height: previous.height + (current.height - previous.height) * this.smoothingFactor,
317
+            confidence: current.confidence,
318
+            color: current.color
187319
         };
188320
     }
189321
322
+    drawBox(box, color, label) {
323
+        // Draw bounding box
324
+        this.ctx.strokeStyle = color;
325
+        this.ctx.lineWidth = 3;
326
+        this.ctx.strokeRect(box.x, box.y, box.width, box.height);
327
+
328
+        // Draw corners for style
329
+        const cornerLength = 15;
330
+        this.ctx.lineWidth = 4;
331
+
332
+        // Top-left
333
+        this.ctx.beginPath();
334
+        this.ctx.moveTo(box.x, box.y + cornerLength);
335
+        this.ctx.lineTo(box.x, box.y);
336
+        this.ctx.lineTo(box.x + cornerLength, box.y);
337
+        this.ctx.stroke();
338
+
339
+        // Top-right
340
+        this.ctx.beginPath();
341
+        this.ctx.moveTo(box.x + box.width - cornerLength, box.y);
342
+        this.ctx.lineTo(box.x + box.width, box.y);
343
+        this.ctx.lineTo(box.x + box.width, box.y + cornerLength);
344
+        this.ctx.stroke();
345
+
346
+        // Bottom-left
347
+        this.ctx.beginPath();
348
+        this.ctx.moveTo(box.x, box.y + box.height - cornerLength);
349
+        this.ctx.lineTo(box.x, box.y + box.height);
350
+        this.ctx.lineTo(box.x + cornerLength, box.y + box.height);
351
+        this.ctx.stroke();
352
+
353
+        // Bottom-right
354
+        this.ctx.beginPath();
355
+        this.ctx.moveTo(box.x + box.width - cornerLength, box.y + box.height);
356
+        this.ctx.lineTo(box.x + box.width, box.y + box.height);
357
+        this.ctx.lineTo(box.x + box.width, box.y + box.height - cornerLength);
358
+        this.ctx.stroke();
359
+
360
+        // Draw label with confidence
361
+        if (label) {
362
+            this.ctx.fillStyle = 'rgba(0, 0, 0, 0.7)';
363
+            this.ctx.fillRect(box.x, box.y - 25, 120, 22);
364
+            this.ctx.fillStyle = color;
365
+            this.ctx.font = 'bold 14px monospace';
366
+            this.ctx.fillText(`${label} ${Math.round(box.confidence * 100)}%`, box.x + 5, box.y - 8);
367
+        }
368
+
369
+        // Draw center point
370
+        const centerX = box.x + box.width / 2;
371
+        const centerY = box.y + box.height / 2;
372
+        this.ctx.fillStyle = color;
373
+        this.ctx.beginPath();
374
+        this.ctx.arc(centerX, centerY, 4, 0, 2 * Math.PI);
375
+        this.ctx.fill();
376
+    }
377
+
190378
     rgbToHsv(r, g, b) {
191379
         const max = Math.max(r, g, b);
192380
         const min = Math.min(r, g, b);
@@ -208,31 +396,4 @@ export class GestureDetector {
208396
209397
         return { h, s, v };
210398
     }
211
-
212
-    getBoundingBox(pixels, frameHeight) {
213
-        // Higher threshold to avoid noise
214
-        if (pixels.length < 100) return null;
215
-
216
-        const xs = pixels.map(p => p.x);
217
-        const ys = pixels.map(p => p.y);
218
-
219
-        const box = {
220
-            x: Math.min(...xs),
221
-            y: Math.min(...ys),
222
-            width: Math.max(...xs) - Math.min(...xs),
223
-            height: Math.max(...ys) - Math.min(...ys)
224
-        };
225
-
226
-        // Reasonable size requirements
227
-        if (box.width < 20 || box.height < 20) {
228
-            return null;
229
-        }
230
-
231
-        // Reject if too large (probably background)
232
-        if (box.width > frameHeight * 0.5 || box.height > frameHeight * 0.5) {
233
-            return null;
234
-        }
235
-
236
-        return box;
237
-    }
238399
 }