import Leap from 'leapjs';
import * as T from 'three';
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';
import { TransformControls } from 'three/examples/jsm/controls/TransformControls';
import { saveAs } from 'file-saver';
const clamp = (val, min=0, max=1) => Math.min(max, Math.max(min, val));
const lerp = (a, b, t) => a.plus(b.minus(a).times(t));
const abs = vec => vec.map(Math.abs.bind(Math));
const min = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.min(v, next[i])));
const max = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.max(v, next[i])));
const $ = x => document.getElementById(x);
const canvas = $('main');
const renderer = new T.WebGLRenderer({ canvas });
const scene = new T.Scene();
scene.background = new T.Color().setHSL(0.7, .95, .93);
const grid = new T.GridHelper(250, 10);
scene.add(grid);
const hemiLight = new T.HemisphereLight(0xffffff, 0xffffff, 0.6);
hemiLight.color.setHSL(0.6, 1, 0.6);
hemiLight.groundColor.setHSL(0.095, 1, 0.75);
hemiLight.position.set(0, 50, 0);
scene.add(hemiLight);
const dirLight = new T.DirectionalLight(0xffffff, 0.8);
dirLight.color.setHSL(0.1, 1, 0.95);
dirLight.position.set(-1, 1.75, 1);
dirLight.position.multiplyScalar(30);
scene.add(dirLight);
const camera = new T.PerspectiveCamera(75, 1, 0.1, 1000);
camera.position.set(-30, 400, 80);
let fingers;
const orbit = new OrbitControls(camera, canvas);
const transform = new TransformControls(camera, canvas);
transform.space = 'local';
transform.addEventListener('dragging-changed', e => { orbit.enabled = !e.value });
transform.addEventListener('objectChange', e => fingers.forEach(finger => finger.updateSlice()));
scene.add(transform);
window.onresize = () => {
renderer.setSize(canvas.offsetWidth, canvas.offsetHeight);
camera.aspect = canvas.offsetWidth / canvas.offsetHeight;
camera.updateProjectionMatrix();
orbit.update();
};
window.onresize();
class Finger extends T.Group {
static tipGeom = new T.SphereBufferGeometry(4, 16, 16);
static trackGeom = new T.SphereBufferGeometry(2, 4, 4);
static sliceMaterial = new T.MeshLambertMaterial({ color: 0xffffff, transparent: true, opacity: 0.5 });
static trackMaterial = new T.MeshLambertMaterial({ color: 0xff0000 });
static MAX_TRACK = 1 << 10;
constructor(i) {
super();
const color = new T.Color().setHSL(i / 5, .8, .6);
const meshMaterial = new T.MeshLambertMaterial({ color });
const trackMaterial = new T.MeshLambertMaterial({ color, transparent: true, opacity: 0.7 });
const lineMaterial = new T.LineBasicMaterial({ color, linewidth: 2 });
this.tip = new T.Mesh(Finger.tipGeom, meshMaterial);
this.add(this.tip);
this.lineGeometry = new T.Geometry();
this.lineGeometry.vertices.push(new T.Vector3(), new T.Vector3(), new T.Vector3());
this.skeleton = new T.Line(this.lineGeometry, lineMaterial);
this.add(this.skeleton);
this.trackers = new T.InstancedMesh(Finger.trackGeom.clone(), trackMaterial, Finger.MAX_TRACK);
this.add(this.trackers);
this.dataPoints = [];
this.track = false;
$('header').append(this.createControls(i));
}
createControls(i) {
const title = document.createElement('span');
title.innerText = `slice ${i}`;
const select = document.createElement('button');
select.innerText = 'select';
select.onclick = e => {
e.preventDefault();
this.selectSlice();
}
const remove = document.createElement('button');
remove.innerText = 'remove';
remove.onclick = e => {
e.preventDefault();
this.removeSlice();
}
const controls = document.createElement('div');
controls.append(title);
controls.append(select);
controls.append(remove);
return controls;
}
selectSlice() {
if (!this.slice) {
const average = this.dataPoints.reduce((sum, p) => sum.add(new T.Vector3(...p.dip)), new T.Vector3());
if (this.dataPoints.length)
average.divideScalar(this.dataPoints.length);
this.slice = new T.Mesh(new T.BoxGeometry(), Finger.sliceMaterial);
this.slice.scale.set(25, 5, 50);
this.slice.position.copy(average);
this.add(this.slice);
transform.attach(this.slice);
this.updateSlice();
} else if (transform.object === this.slice) {
transform.detach();
} else {
transform.attach(this.slice);
}
}
updateSlice() {
if (!this.slice)
return;
const sliceTransform = new T.Matrix4().getInverse(this.slice.matrixWorld);
const points = [];
this.dataPoints.map(point => {
const tip = new T.Vector3(...point.tip);
const norm = tip.clone().applyMatrix4(sliceTransform);
if (Math.abs(norm.x) <= .5 && Math.abs(norm.y) <= .5 && Math.abs(norm.z) <= .5)
points.push(tip);
});
if (this.slicedTrackers)
this.remove(this.slicedTrackers);
this.slicedTrackers = new T.InstancedMesh(Finger.trackGeom.clone(), Finger.trackMaterial.clone(), points.length);
points.forEach((point, i) => this.slicedTrackers.setMatrixAt(i, new T.Matrix4().setPosition(point)));
this.add(this.slicedTrackers);
}
removeSlice() {
if (!this.slice)
return;
if (transform.object === this.slice)
transform.detach();
this.remove(this.slice);
this.slice = null;
if (this.slicedTrackers) {
this.remove(this.slicedTrackers);
this.slicedTrackers = null;
}
}
update(finger) {
if (this.track) {
this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...finger.tipPosition));
this.trackers.instanceMatrix.needsUpdate = true;
this.dataPoints.push({
tip: finger.tipPosition,
dip: finger.dipPosition,
pip: finger.pipPosition,
});
}
this.tip.position.set(...finger.tipPosition);
this.lineGeometry.vertices[0].set(...finger.tipPosition);
this.lineGeometry.vertices[1].set(...finger.dipPosition);
this.lineGeometry.vertices[2].set(...finger.pipPosition);
this.lineGeometry.verticesNeedUpdate = true;
}
clear() {
this.removeSlice();
this.dataPoints = [];
for (let i = 0; i < Finger.MAX_TRACK; i++) {
this.trackers.setMatrixAt(i, new T.Matrix4());
}
this.trackers.instanceMatrix.needsUpdate = true;
}
store() {
return {
dataPoints: this.dataPoints,
slice: this.slice && {
position: this.slice.position.toArray(),
rotation: this.slice.rotation.toArray(),
scale: this.slice.scale.toArray(),
},
};
}
load(data) {
for (const dp of data.dataPoints) {
this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...dp.tip));
this.dataPoints.push(dp);
}
if (data.slice) {
this.selectSlice();
this.slice.position.fromArray(data.slice.position);
this.slice.rotation.fromArray(data.slice.rotation);
this.slice.scale.fromArray(data.slice.scale);
}
this.trackers.instanceMatrix.needsUpdate = true;
}
}
const material = new T.MeshLambertMaterial();
const cube = new T.BoxBufferGeometry(89, 13, 30);
scene.add(new T.Mesh(cube, material));
const palm = new T.Mesh(new T.BoxBufferGeometry(8, 2, 8), material);
scene.add(palm);
fingers = [0,1,2,3,4].map(i => {
const finger = new Finger(i);
scene.add(finger);
return finger;
});
const restore = (data) => data.forEach((data, i) => fingers[i].load(data));
let viewSliceOnly = false;
window.onkeydown = (e) => {
if (e.key === 'r') {
fingers.forEach(finger => finger.clear());
} else if (e.key === 's') {
const data = fingers.map(finger => finger.store());
saveAs(new Blob([JSON.stringify(data)]), 'finger-scan.json');
} else if (e.key === 'l') {
const input = document.createElement('input');
input.type = 'file';
input.click();
input.onchange = e => {
const reader = new FileReader();
reader.onload = e => restore(JSON.parse(e.target.result));
reader.readAsText(e.target.files[0]);
}
} else if (e.key === 'Escape') {
transform.detach();
} else if (e.key === 'Shift') {
transform.setMode('rotate');
} else if (e.key === 'Alt') {
transform.setMode('scale');
} else if (e.key === ' ') {
viewSliceOnly = !viewSliceOnly;
fingers.forEach(finger => { finger.trackers.visible = !viewSliceOnly });
}
const i = +e.key;
if (i > -1 && i < 6) {
e.preventDefault();
fingers[i % 5].track = true;
}
}
window.onkeyup = (e) => {
const i = +e.key;
if (e.key === 'Shift' || e.key === 'Alt')
transform.setMode('translate');
if (i > -1 && i < 6) {
e.preventDefault();
fingers[i % 5].track = false;
}
}
if (localStorage.getItem('data')) {
restore(JSON.parse(localStorage.getItem('data')));
} else {
fetch('examples/one.json')
.then(r => r.json())
.then(restore);
}
setInterval(() => {
const data = fingers.map(finger => finger.store());
localStorage.setItem('data', JSON.stringify(data));
}, 2000);
const loop = () => {
orbit.update();
renderer.render(scene, camera);
requestAnimationFrame(loop);
};
loop();
const controller = Leap.loop({
frameEventName: 'animationFrame',
// frameEventName: 'deviceFrame',
// optimizeHMD: true,
background: true,
}, frame => {
const hand = frame.hands[0];
if (!hand)
return;
palm.position.set(...hand.palmPosition);
palm.quaternion.setFromUnitVectors(new T.Vector3(0, -1, 0), new T.Vector3(...hand.palmNormal));
for (var finger of hand.fingers) {
const i = finger.type;
fingers[i].update(finger);
}
});