[READ-ONLY] Mirror of https://github.com/flo-bit/hand-controls.
flo-bit.github.io/hand-controls/
first commit
+643
+493
hand-controls.js
+493
hand-controls.js
···
1
+
/**
2
+
* vector class for 2D, 3D and 4D vectors
3
+
*
4
+
*/
5
+
class Vector {
6
+
/**
7
+
* @param {number} x
8
+
* @param {number} y
9
+
* @param {number} z
10
+
* @param {number} w
11
+
* @returns {Vector}
12
+
* @constructor
13
+
*/
14
+
constructor(x, y, z, w) {
15
+
if (x != undefined && typeof x == "object" && x.x != undefined) {
16
+
this.copy(x);
17
+
} else {
18
+
this.set(x || 0, y || 0, z, w);
19
+
}
20
+
21
+
this.isVector = true;
22
+
23
+
this.is2D = z == undefined;
24
+
this.is3D = !this.is2D && w == undefined;
25
+
this.is4D = !this.is2D && !this.is3D;
26
+
return this;
27
+
}
28
+
copy(vec) {
29
+
this.x = vec.x;
30
+
this.y = vec.y;
31
+
this.z = vec.z;
32
+
this.w = vec.w;
33
+
34
+
return this;
35
+
}
36
+
clone() {
37
+
return new Vector(this.x, this.y, this.z, this.w);
38
+
}
39
+
set(x, y, z, w) {
40
+
if (typeof x != "number" && x.x != undefined) {
41
+
return this.copy(x);
42
+
}
43
+
if (y == undefined && z == undefined && w == undefined) {
44
+
if (this.is2D) this.set(x, x);
45
+
if (this.is3D) this.set(x, x, x);
46
+
if (this.is4D) this.set(x, x, x, x);
47
+
}
48
+
this.x = x;
49
+
this.y = y;
50
+
this.z = z;
51
+
this.w = w;
52
+
return this;
53
+
}
54
+
add(x, y, z, w) {
55
+
if (typeof x != "number" && x.x != undefined) {
56
+
return this.add(x.x, x.y, x.z, x.w);
57
+
}
58
+
if (y == undefined && z == undefined && w == undefined) {
59
+
if (this.is2D) this.add(x, x);
60
+
if (this.is3D) this.add(x, x, x);
61
+
if (this.is4D) this.add(x, x, x, x);
62
+
}
63
+
this.x += x || 0;
64
+
this.y += y || 0;
65
+
this.z += z || 0;
66
+
this.w += w || 0;
67
+
return this;
68
+
}
69
+
sub(x, y, z, w) {
70
+
if (typeof x != "number" && x.x != undefined) {
71
+
return this.sub(x.x, x.y, x.z, x.w);
72
+
}
73
+
if (y == undefined && z == undefined && w == undefined) {
74
+
if (this.is2D) this.sub(x, x);
75
+
if (this.is3D) this.sub(x, x, x);
76
+
if (this.is4D) this.sub(x, x, x, x);
77
+
}
78
+
this.x -= x || 0;
79
+
this.y -= y || 0;
80
+
this.z -= z || 0;
81
+
this.w -= w || 0;
82
+
return this;
83
+
}
84
+
mult(x, y, z, w) {
85
+
if (typeof x != "number" && x.x != undefined) {
86
+
return this.mult(x.x, x.y, x.z, x.w);
87
+
}
88
+
if (y == undefined && z == undefined && w == undefined) {
89
+
if (this.is2D) this.mult(x, x);
90
+
if (this.is3D) this.mult(x, x, x);
91
+
if (this.is4D) this.mult(x, x, x, x);
92
+
}
93
+
this.x *= x || 0;
94
+
this.y *= y || 0;
95
+
this.z *= z || 0;
96
+
this.w *= w || 0;
97
+
return this;
98
+
}
99
+
div(x, y, z, w) {
100
+
if (typeof x != "number" && x.x != undefined) {
101
+
return this.div(m.x, x.y, x.z, x.w);
102
+
}
103
+
if (y == undefined && z == undefined && w == undefined) {
104
+
if (this.is2D) this.div(x, x);
105
+
if (this.is3D) this.div(x, x, x);
106
+
if (this.is4D) this.div(x, x, x, x);
107
+
}
108
+
this.x /= x || 0;
109
+
this.y /= y || 0;
110
+
this.z /= z || 0;
111
+
this.w /= w || 0;
112
+
return this;
113
+
}
114
+
dot(vec) {
115
+
return (
116
+
this.x * vec.x +
117
+
this.y * vec.y +
118
+
(this.z != undefined ? this.z * vec.z : 0) +
119
+
(this.w != undefined ? this.w * vec.w : 0)
120
+
);
121
+
}
122
+
// 3D only
123
+
cross(vec) {
124
+
if (!this.is3D || !vec.is3D)
125
+
console.warn("cross(vec) only supports 3D vectors");
126
+
return new Vector(
127
+
this.y * vec.z - this.z * vec.y,
128
+
this.z * vec.x - this.x * vec.z,
129
+
this.x * vec.y - this.y * vec.x
130
+
);
131
+
}
132
+
133
+
dist(x, y, z, w) {
134
+
if (typeof x != "number" && x.x != undefined) {
135
+
return this.dist(x.x, x.y, x.z, x.w);
136
+
}
137
+
let sum = 0;
138
+
let dx = this.x - x;
139
+
sum += dx * dx;
140
+
let dy = (this.y || 0) - (y || 0);
141
+
sum += dy * dy;
142
+
let dz = (this.z || 0) - (z || 0);
143
+
sum += dz * dz;
144
+
let dw = (this.w || 0) - (w || 0);
145
+
sum += dw * dw;
146
+
return Math.sqrt(sum);
147
+
}
148
+
distance(x, y, z, w) {
149
+
return this.dist(x, y, z, w);
150
+
}
151
+
distanceTo(x, y, z, w) {
152
+
return this.dist(x, y, z, w);
153
+
}
154
+
length() {
155
+
return Math.sqrt(
156
+
this.x * this.x +
157
+
this.y * this.y +
158
+
(this.z || 0) * (this.z || 0) +
159
+
(this.w || 0) * (this.w || 0)
160
+
);
161
+
}
162
+
163
+
setLength(l) {
164
+
this.mult(l / this.length());
165
+
return this;
166
+
}
167
+
limit(l) {
168
+
let length = this.length();
169
+
if (length > l) this.mult(length / l);
170
+
return this;
171
+
}
172
+
173
+
// 2d only
174
+
heading() {
175
+
if (!this.is2D) console.warn("heading() only supports 2D vectors");
176
+
177
+
return Math.atan2(this.x, this.y);
178
+
}
179
+
// 2d only
180
+
rotate(a) {
181
+
if (!this.is2D) console.warn("rotate(a) only supports 2D vectors");
182
+
183
+
let ca = Math.cos(a);
184
+
let sa = Math.sin(a);
185
+
this.set(ca * this.x - sa * this.y, sa * this.x + ca * this.y);
186
+
return this;
187
+
}
188
+
189
+
angleBetween(vec) {
190
+
let d = this.dot(vec);
191
+
let l = this.length() * vec.length();
192
+
return Math.acos(d / l);
193
+
}
194
+
angleTo(vec) {
195
+
return this.angleBetween(vec);
196
+
}
197
+
equals(vec) {
198
+
return (
199
+
this.x == vec.x && this.y == vec.y && this.z == vec.z && this.w == vec.w
200
+
);
201
+
}
202
+
203
+
normalize() {
204
+
this.setLength(1);
205
+
return this;
206
+
}
207
+
mag() {
208
+
return this.length();
209
+
}
210
+
setMag(m) {
211
+
this.setLength(m);
212
+
return this;
213
+
}
214
+
manhattanLength() {
215
+
return this.x + this.y + (this.z || 0) + (this.w || 0);
216
+
}
217
+
lerp(vec, a) {
218
+
let dx = vec.x - this.x;
219
+
let dy = vec.y - this.y;
220
+
let dz = vec.z - this.z;
221
+
let dw = vec.w - this.w;
222
+
this.add(dx * a, dy * a, dz * a, dw * a);
223
+
return this;
224
+
}
225
+
226
+
stringDescription() {
227
+
let dimension = this.is2D ? "2D" : this.is3D ? "3D" : "4D";
228
+
let str = dimension + " vec (" + this.x + ", " + this.y;
229
+
if (this.z != undefined) str += ", " + this.z;
230
+
if (this.w != undefined) str += ", " + this.w;
231
+
str += ")";
232
+
return str;
233
+
}
234
+
235
+
// 2d vector from angle and optional length (default length 1)
236
+
static fromAngle2D(a, l) {
237
+
let v = new Vector(Math.cos(a), Math.sin(a));
238
+
if (l) v.setLength(l);
239
+
return v;
240
+
}
241
+
242
+
// random 2d vector with length between 0 and 1
243
+
// or set length
244
+
static random2D(l, random = Math.random) {
245
+
let v = new Vector(random() * 2 - 1, random() * 2 - 1);
246
+
while (v.length() > 1) {
247
+
v.set(random() * 2 - 1, random() * 2 - 1);
248
+
}
249
+
if (l) v.setLength(l);
250
+
return v;
251
+
}
252
+
253
+
// random 3d vector with length between 0 and 1
254
+
// or set length
255
+
static random3D(l, random = Math.random) {
256
+
let v = new Vector(random() * 2 - 1, random() * 2 - 1, random() * 2 - 1);
257
+
while (v.length() > 1) {
258
+
v.set(random() * 2 - 1, random() * 2 - 1, random() * 2 - 1);
259
+
}
260
+
if (l) v.setLength(l);
261
+
return v;
262
+
}
263
+
264
+
static breakIntoParts(a, b, parts) {
265
+
if (a == undefined || b == undefined || !a.isVector || !b.isVector) return;
266
+
267
+
parts = Math.floor(parts || 2);
268
+
let arr = [a.clone()];
269
+
for (let i = 1; i < parts; i++) {
270
+
arr.push(a.clone().lerp(b, i / parts));
271
+
}
272
+
arr.push(b.clone());
273
+
return arr;
274
+
}
275
+
}
276
+
277
+
class HandControls {
278
+
constructor(opts) {
279
+
opts = opts || {};
280
+
this.onResults = opts.onResults;
281
+
this.showResults = opts.showResults;
282
+
283
+
this.loadScripts();
284
+
}
285
+
286
+
loadScripts() {
287
+
const scripts = [
288
+
"https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js",
289
+
"https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js",
290
+
"https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js",
291
+
"https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js",
292
+
];
293
+
let scriptPromises = [];
294
+
for (var script of scripts) {
295
+
var scriptElement = document.createElement("script");
296
+
scriptPromises.push(
297
+
new Promise((resolve, reject) => {
298
+
scriptElement.onload = function () {
299
+
resolve();
300
+
};
301
+
scriptElement.onerror = function () {
302
+
reject();
303
+
};
304
+
})
305
+
);
306
+
scriptElement.src = script;
307
+
document.head.appendChild(scriptElement);
308
+
}
309
+
310
+
Promise.all(scriptPromises).then((values) => {
311
+
console.log("all scripts loaded");
312
+
this.setup();
313
+
});
314
+
}
315
+
316
+
setup() {
317
+
const hands = new Hands({
318
+
locateFile: (file) => {
319
+
return `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`;
320
+
},
321
+
});
322
+
hands.setOptions({
323
+
maxNumHands: 2,
324
+
modelComplexity: 1,
325
+
minDetectionConfidence: 0.5,
326
+
minTrackingConfidence: 0.5,
327
+
});
328
+
hands.onResults(this.gotResults.bind(this));
329
+
330
+
const video = document.createElement("video");
331
+
video.style.display = "none";
332
+
const camera = new Camera(video, {
333
+
onFrame: async () => {
334
+
await hands.send({ image: video });
335
+
},
336
+
width: 1280,
337
+
height: 720,
338
+
});
339
+
camera.start();
340
+
}
341
+
342
+
gotResults(results) {
343
+
this.processResults(results);
344
+
if (
345
+
this.onResults != undefined &&
346
+
results.multiHandLandmarks != undefined &&
347
+
results.multiHandLandmarks.length > 0
348
+
) {
349
+
this.onResults(results);
350
+
}
351
+
352
+
if (this.showResults) this.displayResults(results);
353
+
}
354
+
processResults(results) {
355
+
if (
356
+
results.multiHandLandmarks == undefined ||
357
+
results.multiHandLandmarks.length == 0
358
+
)
359
+
return;
360
+
361
+
let hands = [];
362
+
363
+
let data = [];
364
+
for (const landmarks of results.multiHandLandmarks) {
365
+
let handMin = { x: landmarks[0].x, y: landmarks[0].y };
366
+
let handMax = { x: landmarks[0].x, y: landmarks[0].y };
367
+
for (let i = 1; i < landmarks.length; i++) {
368
+
handMin.x = Math.min(handMin.x, landmarks[i].x);
369
+
handMin.y = Math.min(handMin.y, landmarks[i].y);
370
+
371
+
handMax.x = Math.max(handMax.x, landmarks[i].x);
372
+
handMax.y = Math.max(handMax.y, landmarks[i].y);
373
+
}
374
+
375
+
let normalized = [];
376
+
let maxDist = Math.max(handMax.x - handMin.x, handMax.y - handMin.y);
377
+
for (let i = 0; i < landmarks.length; i++) {
378
+
normalized.push({
379
+
x: (landmarks[i].x - handMin.x) / maxDist,
380
+
y: (landmarks[i].y - handMin.y) / maxDist,
381
+
});
382
+
}
383
+
// calculate normalized center
384
+
/*let center = {
385
+
x: (handMax.x + handMin.x) / 2,
386
+
y: (handMax.y + handMin.y) / 2,
387
+
};*/
388
+
//console.log(center);
389
+
/*
390
+
let fingers = {
391
+
thumb: [normalized[1], normalized[2], normalized[3], normalized[4]],
392
+
index: [normalized[5], normalized[6], normalized[7], normalized[8]],
393
+
middle: [normalized[9], normalized[10], normalized[11], normalized[12]],
394
+
ring: [normalized[13], normalized[14], normalized[15], normalized[16]],
395
+
pinky: [normalized[17], normalized[18], normalized[19], normalized[20]],
396
+
palm: [
397
+
normalized[0],
398
+
normalized[5],
399
+
normalized[9],
400
+
normalized[13],
401
+
normalized[17],
402
+
],
403
+
};
404
+
405
+
let hand = {
406
+
fingers: fingers,
407
+
array: normalized,
408
+
};*/
409
+
hands.push(normalized);
410
+
411
+
let palmToIndex = this.vectorBetween(landmarks[0], landmarks[5]);
412
+
let indexToTop = this.vectorBetween(landmarks[7], landmarks[8]);
413
+
414
+
let palmToIndexAngle = palmToIndex.angleTo(indexToTop);
415
+
416
+
if (this.lastPalmToIndexAngle != undefined) {
417
+
if (this.lastPalmToIndexAngle < 1 && palmToIndexAngle > 1) {
418
+
//console.log("tap");
419
+
}
420
+
if (this.lastPalmToIndexAngle > 1 && palmToIndexAngle < 1) {
421
+
//console.log("untap");
422
+
}
423
+
}
424
+
this.lastPalmToIndexAngle = palmToIndexAngle;
425
+
426
+
let indexThumbDist = this.vectorBetween(
427
+
landmarks[4],
428
+
landmarks[8]
429
+
).length();
430
+
431
+
if (this.lastIndexThumbDist != undefined) {
432
+
if (this.lastIndexThumbDist < 0.05 && indexThumbDist > 0.05) {
433
+
console.log("unpinch");
434
+
}
435
+
if (this.lastIndexThumbDist > 0.05 && indexThumbDist < 0.05) {
436
+
console.log("pinch");
437
+
}
438
+
}
439
+
this.lastIndexThumbDist = indexThumbDist;
440
+
data.push({
441
+
indexAngle: palmToIndexAngle,
442
+
indexThumbDist: indexThumbDist,
443
+
});
444
+
}
445
+
446
+
results.normalizedLandmarks = hands;
447
+
results.data = data;
448
+
}
449
+
450
+
vectorBetween(p1, p2) {
451
+
return new Vector(p2.x - p1.x, p2.y - p1.y, p2.z - p1.z);
452
+
}
453
+
454
+
displayResults(results) {
455
+
if (this.canvasCtx == undefined || this.canvasElement == undefined) {
456
+
this.canvasElement = document.createElement("canvas");
457
+
this.canvasCtx = this.canvasElement.getContext("2d");
458
+
this.canvasElement.style = "position: absolute; z-index: 100";
459
+
document.body.appendChild(this.canvasElement);
460
+
}
461
+
this.canvasCtx.save();
462
+
this.canvasCtx.clearRect(
463
+
0,
464
+
0,
465
+
this.canvasElement.width,
466
+
this.canvasElement.height
467
+
);
468
+
this.canvasCtx.drawImage(
469
+
results.image,
470
+
0,
471
+
0,
472
+
this.canvasElement.width,
473
+
this.canvasElement.height
474
+
);
475
+
// HAND_CONNECTIONS is a global variable defined in drawing_utils.js
476
+
// contains an array of connections between landmarks
477
+
// each connection is an array with two indices of the two connected landmarks
478
+
479
+
if (results.multiHandLandmarks) {
480
+
for (const landmarks of results.multiHandLandmarks) {
481
+
drawConnectors(this.canvasCtx, landmarks, HAND_CONNECTIONS, {
482
+
color: "#888888",
483
+
lineWidth: 2,
484
+
});
485
+
drawLandmarks(this.canvasCtx, landmarks, {
486
+
color: "#0",
487
+
radius: 1,
488
+
});
489
+
}
490
+
}
491
+
this.canvasCtx.restore();
492
+
}
493
+
}
+146
index.html
+146
index.html
···
1
+
<html>
2
+
<head>
3
+
<meta charset="UTF-8" />
4
+
<meta
5
+
name="viewport"
6
+
content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0"
7
+
/>
8
+
9
+
<script src="./hand-controls.js"></script>
10
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/0.148.0/three.min.js"></script>
11
+
</head>
12
+
<body style="overflow: hidden">
13
+
<script>
14
+
class THREE_SCAFFOLD {
15
+
constructor() {
16
+
let w = window.innerWidth,
17
+
h = window.innerHeight;
18
+
this.scene = new THREE.Scene();
19
+
this.camera = new THREE.PerspectiveCamera(75, w / h);
20
+
this.scene.background = new THREE.Color(0x232323);
21
+
this.scene.add(this.camera);
22
+
23
+
this.renderer = new THREE.WebGLRenderer();
24
+
this.renderer.setSize(w, h);
25
+
let d = this.renderer.domElement;
26
+
d.style.position = "absolute";
27
+
d.style.left = "0px";
28
+
d.style.top = "0px";
29
+
document.body.appendChild(d);
30
+
31
+
this.keys = {};
32
+
document.addEventListener("keydown", this.keyDown.bind(this));
33
+
document.addEventListener("keyup", this.keyUp.bind(this));
34
+
window.addEventListener("resize", this.windowResized.bind(this));
35
+
36
+
this.lights = {};
37
+
this.lights.ambi = new THREE.AmbientLight(0xffffff, 0.2);
38
+
this.scene.add(this.lights.ambi);
39
+
this.lights.dir = new THREE.DirectionalLight(0xffffff, 1.0);
40
+
this.lights.dir.position.set(-0.8, 0.5, 0.7);
41
+
this.scene.add(this.lights.dir);
42
+
43
+
this.clock = new THREE.Clock();
44
+
45
+
this.handControls = new HandControls({
46
+
onResults: this.showResults.bind(this),
47
+
showResults: true,
48
+
});
49
+
50
+
this.setupScene();
51
+
this.animate();
52
+
}
53
+
54
+
showResults(results) {
55
+
if (
56
+
results.normalizedLandmarks &&
57
+
results.normalizedLandmarks.length > 0
58
+
) {
59
+
for (let i = 0; i < results.normalizedLandmarks.length; i++) {
60
+
let hand = results.normalizedLandmarks[i];
61
+
let handedness = results.multiHandedness[i];
62
+
let handObject =
63
+
handedness.label == "Right" ? this.hand : this.hand2;
64
+
65
+
for (let j = 0; j < hand.length; j++) {
66
+
let p = hand[j];
67
+
let x = -p.x;
68
+
let y = -p.y + 0.5;
69
+
let z = results.multiHandLandmarks[i][j].z;
70
+
handObject.children[j].position.set(x, y, z);
71
+
}
72
+
if (handedness.label == "Right") {
73
+
this.data = results.data[i];
74
+
}
75
+
}
76
+
}
77
+
}
78
+
79
+
setupScene() {
80
+
/* setup your scene here */
81
+
/* START */
82
+
this.handsParent = new THREE.Object3D();
83
+
this.scene.add(this.handsParent);
84
+
this.handsParent.position.z = -1;
85
+
86
+
this.hand = new THREE.Object3D();
87
+
88
+
for (let i = 0; i < 21; i++) {
89
+
let sphere = new THREE.Mesh(
90
+
new THREE.IcosahedronGeometry(0.01, 0),
91
+
new THREE.MeshStandardMaterial({ flatShading: true })
92
+
);
93
+
sphere.position.x = Math.random() - 0.5;
94
+
sphere.position.y = Math.random() - 0.5;
95
+
this.hand.add(sphere);
96
+
}
97
+
this.hand2 = this.hand.clone();
98
+
this.hand2.position.x = 0.5;
99
+
this.handsParent.add(this.hand);
100
+
this.handsParent.add(this.hand2);
101
+
102
+
let sphere = new THREE.Mesh(
103
+
new THREE.IcosahedronGeometry(0.01, 0),
104
+
new THREE.MeshStandardMaterial({
105
+
flatShading: true,
106
+
color: 0xff0000,
107
+
})
108
+
);
109
+
this.handsParent.add(sphere);
110
+
111
+
/* END */
112
+
}
113
+
114
+
animate() {
115
+
requestAnimationFrame(this.animate.bind(this));
116
+
let delta = this.clock.getDelta();
117
+
let total = this.clock.getElapsedTime();
118
+
119
+
/* Update your scene here */
120
+
/* START */
121
+
122
+
if (this.data == undefined || this.data.indexAngle < 1)
123
+
this.hand.rotation.y += delta;
124
+
/* END */
125
+
//console.log(this.data);
126
+
127
+
this.renderer.render(this.scene, this.camera);
128
+
}
129
+
130
+
windowResized() {
131
+
this.camera.aspect = window.innerWidth / window.innerHeight;
132
+
this.camera.updateProjectionMatrix();
133
+
this.renderer.setSize(window.innerWidth, window.innerHeight);
134
+
}
135
+
keyDown(event) {
136
+
this.keys[event.key] = true;
137
+
}
138
+
keyUp(event) {
139
+
this.keys[event.key] = false;
140
+
}
141
+
}
142
+
143
+
new THREE_SCAFFOLD();
144
+
</script>
145
+
</body>
146
+
</html>