import Leap from 'leapjs'; import * as T from 'three'; import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; import { TransformControls } from 'three/examples/jsm/controls/TransformControls'; import { saveAs } from 'file-saver'; const clamp = (val, min=0, max=1) => Math.min(max, Math.max(min, val)); const lerp = (a, b, t) => a.plus(b.minus(a).times(t)); const abs = vec => vec.map(Math.abs.bind(Math)); const min = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.min(v, next[i]))); const max = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.max(v, next[i]))); const $ = x => document.getElementById(x); const canvas = $('main'); const renderer = new T.WebGLRenderer({ canvas }); const scene = new T.Scene(); scene.background = new T.Color().setHSL(0.7, .95, .93); const grid = new T.GridHelper(250, 10); scene.add(grid); const hemiLight = new T.HemisphereLight(0xffffff, 0xffffff, 0.6); hemiLight.color.setHSL(0.6, 1, 0.6); hemiLight.groundColor.setHSL(0.095, 1, 0.75); hemiLight.position.set(0, 50, 0); scene.add(hemiLight); const dirLight = new T.DirectionalLight(0xffffff, 0.8); dirLight.color.setHSL(0.1, 1, 0.95); dirLight.position.set(-1, 1.75, 1); dirLight.position.multiplyScalar(30); scene.add(dirLight); const camera = new T.PerspectiveCamera(75, 1, 0.1, 1000); camera.position.set(-30, 400, 80); let fingers; const orbit = new OrbitControls(camera, canvas); const transform = new TransformControls(camera, canvas); transform.space = 'local'; transform.addEventListener('dragging-changed', e => { orbit.enabled = !e.value }); transform.addEventListener('objectChange', e => fingers.forEach(finger => finger.updateSlice())); scene.add(transform); window.onresize = () => { renderer.setSize(canvas.offsetWidth, canvas.offsetHeight); camera.aspect = canvas.offsetWidth / canvas.offsetHeight; camera.updateProjectionMatrix(); orbit.update(); }; window.onresize(); class Finger extends T.Group { static tipGeom = new T.SphereBufferGeometry(4, 16, 16); static trackGeom = new T.SphereBufferGeometry(2, 4, 4); static sliceMaterial = new T.MeshLambertMaterial({ color: 0xffffff, transparent: true, opacity: 0.5 }); static trackMaterial = new T.MeshLambertMaterial({ color: 0xff0000 }); static MAX_TRACK = 1 << 10; constructor(i) { super(); const color = new T.Color().setHSL(i / 5, .8, .6); const meshMaterial = new T.MeshLambertMaterial({ color }); const trackMaterial = new T.MeshLambertMaterial({ color, transparent: true, opacity: 0.7 }); const lineMaterial = new T.LineBasicMaterial({ color, linewidth: 2 }); this.tip = new T.Mesh(Finger.tipGeom, meshMaterial); this.add(this.tip); this.lineGeometry = new T.Geometry(); this.lineGeometry.vertices.push(new T.Vector3(), new T.Vector3(), new T.Vector3()); this.skeleton = new T.Line(this.lineGeometry, lineMaterial); this.add(this.skeleton); this.trackers = new T.InstancedMesh(Finger.trackGeom.clone(), trackMaterial, Finger.MAX_TRACK); this.add(this.trackers); this.dataPoints = []; this.track = false; $('header').append(this.createControls(i)); } createControls(i) { const title = document.createElement('span'); title.innerText = `slice ${i}`; const select = document.createElement('button'); select.innerText = 'select'; select.onclick = e => { e.preventDefault(); this.selectSlice(); } const remove = document.createElement('button'); remove.innerText = 'remove'; remove.onclick = e => { e.preventDefault(); this.removeSlice(); } const controls = document.createElement('div'); controls.append(title); controls.append(select); controls.append(remove); return controls; } selectSlice() { if (!this.slice) { const average = this.dataPoints.reduce((sum, p) => sum.add(new T.Vector3(...p.dip)), new T.Vector3()); if (this.dataPoints.length) average.divideScalar(this.dataPoints.length); this.slice = new T.Mesh(new T.BoxGeometry(), Finger.sliceMaterial); this.slice.scale.set(25, 5, 50); this.slice.position.copy(average); this.add(this.slice); transform.attach(this.slice); this.updateSlice(); } else if (transform.object === this.slice) { transform.detach(); } else { transform.attach(this.slice); } } updateSlice() { if (!this.slice) return; const sliceTransform = new T.Matrix4().getInverse(this.slice.matrixWorld); const points = []; this.dataPoints.map(point => { const tip = new T.Vector3(...point.tip); const norm = tip.clone().applyMatrix4(sliceTransform); if (Math.abs(norm.x) <= .5 && Math.abs(norm.y) <= .5 && Math.abs(norm.z) <= .5) points.push(tip); }); if (this.slicedTrackers) this.remove(this.slicedTrackers); this.slicedTrackers = new T.InstancedMesh(Finger.trackGeom.clone(), Finger.trackMaterial.clone(), points.length); points.forEach((point, i) => this.slicedTrackers.setMatrixAt(i, new T.Matrix4().setPosition(point))); this.add(this.slicedTrackers); } removeSlice() { if (!this.slice) return; if (transform.object === this.slice) transform.detach(); this.remove(this.slice); this.slice = null; if (this.slicedTrackers) { this.remove(this.slicedTrackers); this.slicedTrackers = null; } } update(finger) { if (this.track) { this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...finger.tipPosition)); this.trackers.instanceMatrix.needsUpdate = true; this.dataPoints.push({ tip: finger.tipPosition, dip: finger.dipPosition, pip: finger.pipPosition, }); } this.tip.position.set(...finger.tipPosition); this.lineGeometry.vertices[0].set(...finger.tipPosition); this.lineGeometry.vertices[1].set(...finger.dipPosition); this.lineGeometry.vertices[2].set(...finger.pipPosition); this.lineGeometry.verticesNeedUpdate = true; } clear() { this.removeSlice(); this.dataPoints = []; for (let i = 0; i < Finger.MAX_TRACK; i++) { this.trackers.setMatrixAt(i, new T.Matrix4()); } this.trackers.instanceMatrix.needsUpdate = true; } store() { return { dataPoints: this.dataPoints, slice: this.slice && { position: this.slice.position.toArray(), rotation: this.slice.rotation.toArray(), scale: this.slice.scale.toArray(), }, }; } load(data) { for (const dp of data.dataPoints) { this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...dp.tip)); this.dataPoints.push(dp); } if (data.slice) { this.selectSlice(); this.slice.position.fromArray(data.slice.position); this.slice.rotation.fromArray(data.slice.rotation); this.slice.scale.fromArray(data.slice.scale); } this.trackers.instanceMatrix.needsUpdate = true; } } const material = new T.MeshLambertMaterial(); const cube = new T.BoxBufferGeometry(89, 13, 30); scene.add(new T.Mesh(cube, material)); const palm = new T.Mesh(new T.BoxBufferGeometry(8, 2, 8), material); scene.add(palm); fingers = [0,1,2,3,4].map(i => { const finger = new Finger(i); scene.add(finger); return finger; }); const restore = (data) => data.forEach((data, i) => fingers[i].load(data)); let viewSliceOnly = false; window.onkeydown = (e) => { if (e.key === 'r') { fingers.forEach(finger => finger.clear()); } else if (e.key === 's') { const data = fingers.map(finger => finger.store()); saveAs(new Blob([JSON.stringify(data)]), 'finger-scan.json'); } else if (e.key === 'l') { const input = document.createElement('input'); input.type = 'file'; input.click(); input.onchange = e => { const reader = new FileReader(); reader.onload = e => restore(JSON.parse(e.target.result)); reader.readAsText(e.target.files[0]); } } else if (e.key === 'Escape') { transform.detach(); } else if (e.key === 'Shift') { transform.setMode('rotate'); } else if (e.key === 'Alt') { transform.setMode('scale'); } else if (e.key === ' ') { viewSliceOnly = !viewSliceOnly; fingers.forEach(finger => { finger.trackers.visible = !viewSliceOnly }); } const i = +e.key; if (i > -1 && i < 6) { e.preventDefault(); fingers[i % 5].track = true; } } window.onkeyup = (e) => { const i = +e.key; if (e.key === 'Shift' || e.key === 'Alt') transform.setMode('translate'); if (i > -1 && i < 6) { e.preventDefault(); fingers[i % 5].track = false; } } if (localStorage.getItem('data')) { restore(JSON.parse(localStorage.getItem('data'))); } else { fetch('examples/one.json') .then(r => r.json()) .then(restore); } setInterval(() => { const data = fingers.map(finger => finger.store()); localStorage.setItem('data', JSON.stringify(data)); }, 2000); const loop = () => { orbit.update(); renderer.render(scene, camera); requestAnimationFrame(loop); }; loop(); const controller = Leap.loop({ frameEventName: 'animationFrame', // frameEventName: 'deviceFrame', // optimizeHMD: true, background: true, }, frame => { const hand = frame.hands[0]; if (!hand) return; palm.position.set(...hand.palmPosition); palm.quaternion.setFromUnitVectors(new T.Vector3(0, -1, 0), new T.Vector3(...hand.palmNormal)); for (var finger of hand.fingers) { const i = finger.type; fingers[i].update(finger); } });