git.s-ol.nu leap-finger-scan / master src / index.js
master

Tree @master (Download .tar.gz)

index.js @masterraw · history · blame

import Leap from 'leapjs';
import * as T from 'three';
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';
import { TransformControls } from 'three/examples/jsm/controls/TransformControls';
import { saveAs } from 'file-saver';

const clamp = (val, min=0, max=1) => Math.min(max, Math.max(min, val));
const lerp = (a, b, t) => a.plus(b.minus(a).times(t));
const abs = vec => vec.map(Math.abs.bind(Math));
const min = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.min(v, next[i])));
const max = (...vecs) => vecs.reduce((last, next) => last.map((v, i) => Math.max(v, next[i])));

const $ = x => document.getElementById(x);

const canvas = $('main');
const renderer = new T.WebGLRenderer({ canvas });

const scene = new T.Scene();
scene.background = new T.Color().setHSL(0.7, .95, .93);

const grid = new T.GridHelper(250, 10);
scene.add(grid);

const hemiLight = new T.HemisphereLight(0xffffff, 0xffffff, 0.6);
hemiLight.color.setHSL(0.6, 1, 0.6);
hemiLight.groundColor.setHSL(0.095, 1, 0.75);
hemiLight.position.set(0, 50, 0);
scene.add(hemiLight);

const dirLight = new T.DirectionalLight(0xffffff, 0.8);
dirLight.color.setHSL(0.1, 1, 0.95);
dirLight.position.set(-1, 1.75, 1);
dirLight.position.multiplyScalar(30);
scene.add(dirLight);

const camera = new T.PerspectiveCamera(75, 1, 0.1, 1000);
camera.position.set(-30, 400, 80);

let fingers;
const orbit = new OrbitControls(camera, canvas);
const transform = new TransformControls(camera, canvas);
transform.space = 'local';
transform.addEventListener('dragging-changed', e => { orbit.enabled = !e.value });
transform.addEventListener('objectChange', e => fingers.forEach(finger => finger.updateSlice()));

scene.add(transform);

window.onresize = () => {
  renderer.setSize(canvas.offsetWidth, canvas.offsetHeight);
  camera.aspect = canvas.offsetWidth / canvas.offsetHeight;
  camera.updateProjectionMatrix();
  orbit.update();
};
window.onresize();

class Finger extends T.Group {
  static tipGeom = new T.SphereBufferGeometry(4, 16, 16);
  static trackGeom = new T.SphereBufferGeometry(2, 4, 4);
  static sliceMaterial = new T.MeshLambertMaterial({ color: 0xffffff, transparent: true, opacity: 0.5 });
  static trackMaterial = new T.MeshLambertMaterial({ color: 0xff0000 });
  static MAX_TRACK = 1 << 10;

  constructor(i) {
    super();

    const color = new T.Color().setHSL(i / 5, .8, .6);
    const meshMaterial = new T.MeshLambertMaterial({ color });
    const trackMaterial = new T.MeshLambertMaterial({ color, transparent: true, opacity: 0.7 });
    const lineMaterial = new T.LineBasicMaterial({ color, linewidth: 2 });

    this.tip = new T.Mesh(Finger.tipGeom, meshMaterial);
    this.add(this.tip);

    this.lineGeometry = new T.Geometry();
    this.lineGeometry.vertices.push(new T.Vector3(), new T.Vector3(), new T.Vector3());
    this.skeleton = new T.Line(this.lineGeometry, lineMaterial);
    this.add(this.skeleton);

    this.trackers = new T.InstancedMesh(Finger.trackGeom.clone(), trackMaterial, Finger.MAX_TRACK);
    this.add(this.trackers);

    this.dataPoints = [];
    this.track = false;

    $('header').append(this.createControls(i));
  }

  createControls(i) {
    const title = document.createElement('span');
    title.innerText = `slice ${i}`;

    const select = document.createElement('button');
    select.innerText = 'select';
    select.onclick = e => {
      e.preventDefault();
      this.selectSlice();
    }

    const remove = document.createElement('button');
    remove.innerText = 'remove';
    remove.onclick = e => {
      e.preventDefault();
      this.removeSlice();
    }

    const controls = document.createElement('div');
    controls.append(title);
    controls.append(select);
    controls.append(remove);
    return controls;
  }

  selectSlice() {
    if (!this.slice) {
      const average = this.dataPoints.reduce((sum, p) => sum.add(new T.Vector3(...p.dip)), new T.Vector3());
      if (this.dataPoints.length)
        average.divideScalar(this.dataPoints.length);

      this.slice = new T.Mesh(new T.BoxGeometry(), Finger.sliceMaterial);
      this.slice.scale.set(25, 5, 50);
      this.slice.position.copy(average);
      this.add(this.slice);
      transform.attach(this.slice);

      this.updateSlice();
    } else if (transform.object === this.slice) {
      transform.detach();
    } else {
      transform.attach(this.slice);
    }
  }

  updateSlice() {
    if (!this.slice)
      return;

    const sliceTransform = new T.Matrix4().getInverse(this.slice.matrixWorld);
    const points = [];
    this.dataPoints.map(point => {
      const tip = new T.Vector3(...point.tip);

      const norm = tip.clone().applyMatrix4(sliceTransform);
      if (Math.abs(norm.x) <= .5 && Math.abs(norm.y) <= .5 && Math.abs(norm.z) <= .5)
        points.push(tip);
    });

    if (this.slicedTrackers)
      this.remove(this.slicedTrackers);

    this.slicedTrackers = new T.InstancedMesh(Finger.trackGeom.clone(), Finger.trackMaterial.clone(), points.length);
    points.forEach((point, i) => this.slicedTrackers.setMatrixAt(i, new T.Matrix4().setPosition(point)));
    this.add(this.slicedTrackers);
  }

  removeSlice() {
    if (!this.slice)
      return;

    if (transform.object === this.slice)
      transform.detach();

    this.remove(this.slice);
    this.slice = null;
    if (this.slicedTrackers) {
      this.remove(this.slicedTrackers);
      this.slicedTrackers = null;
    }
  }

  update(finger) {
    if (this.track) {
      this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...finger.tipPosition));
      this.trackers.instanceMatrix.needsUpdate = true;
      this.dataPoints.push({
        tip: finger.tipPosition,
        dip: finger.dipPosition,
        pip: finger.pipPosition,
      });
    }

    this.tip.position.set(...finger.tipPosition);
    this.lineGeometry.vertices[0].set(...finger.tipPosition);
    this.lineGeometry.vertices[1].set(...finger.dipPosition);
    this.lineGeometry.vertices[2].set(...finger.pipPosition);
    this.lineGeometry.verticesNeedUpdate = true;
  }

  clear() {
    this.removeSlice();
    this.dataPoints = [];
    for (let i = 0; i < Finger.MAX_TRACK; i++) {
      this.trackers.setMatrixAt(i, new T.Matrix4());
    }

    this.trackers.instanceMatrix.needsUpdate = true;
  }

  store() {
    return {
      dataPoints: this.dataPoints,
      slice: this.slice && {
        position: this.slice.position.toArray(),
        rotation: this.slice.rotation.toArray(),
        scale: this.slice.scale.toArray(),
      },
    };
  }

  load(data) {
    for (const dp of data.dataPoints) {
      this.trackers.setMatrixAt(this.dataPoints.length % Finger.MAX_TRACK, new T.Matrix4().makeTranslation(...dp.tip));
      this.dataPoints.push(dp);
    }

    if (data.slice) {
      this.selectSlice();
      this.slice.position.fromArray(data.slice.position);
      this.slice.rotation.fromArray(data.slice.rotation);
      this.slice.scale.fromArray(data.slice.scale);
    }
    this.trackers.instanceMatrix.needsUpdate = true;
  }
}

const material = new T.MeshLambertMaterial();
const cube = new T.BoxBufferGeometry(89, 13, 30);
scene.add(new T.Mesh(cube, material));

const palm = new T.Mesh(new T.BoxBufferGeometry(8, 2, 8), material);
scene.add(palm);

fingers = [0,1,2,3,4].map(i => {
  const finger = new Finger(i);
  scene.add(finger);
  return finger;
});

const restore = (data) =>  data.forEach((data, i) => fingers[i].load(data));

let viewSliceOnly = false;
window.onkeydown = (e) => {
  if (e.key === 'r') {
    fingers.forEach(finger => finger.clear());
  } else if (e.key === 's') {
    const data = fingers.map(finger => finger.store());
    saveAs(new Blob([JSON.stringify(data)]), 'finger-scan.json');
  } else if (e.key === 'l') {
    const input = document.createElement('input');
    input.type = 'file';
    input.click();

    input.onchange = e => {
      const reader = new FileReader();
      reader.onload = e => restore(JSON.parse(e.target.result));
      reader.readAsText(e.target.files[0]);
    }
  } else if (e.key === 'Escape') {
    transform.detach();
  } else if (e.key === 'Shift') {
    transform.setMode('rotate');
  } else if (e.key === 'Alt') {
    transform.setMode('scale');
  } else if (e.key === ' ') {
    viewSliceOnly = !viewSliceOnly;
    fingers.forEach(finger => { finger.trackers.visible = !viewSliceOnly });
  }

  const i = +e.key;

  if (i > -1 && i < 6) {
    e.preventDefault();
    fingers[i % 5].track = true;
  }
}

window.onkeyup = (e) => {
  const i = +e.key;

  if (e.key === 'Shift' || e.key === 'Alt')
    transform.setMode('translate');

  if (i > -1 && i < 6) {
    e.preventDefault();
    fingers[i % 5].track = false;
  }
}

if (localStorage.getItem('data')) {
  restore(JSON.parse(localStorage.getItem('data')));
} else {
  fetch('examples/one.json')
    .then(r => r.json())
    .then(restore);
}

setInterval(() => {
  const data = fingers.map(finger => finger.store());
  localStorage.setItem('data', JSON.stringify(data));
}, 2000);

const loop = () => {
  orbit.update();
  renderer.render(scene, camera);

  requestAnimationFrame(loop);
};
loop();

const controller = Leap.loop({
  frameEventName: 'animationFrame',
  // frameEventName: 'deviceFrame',
  // optimizeHMD: true,
  background: true,
}, frame => {
 const hand = frame.hands[0];
  if (!hand)
    return;

  palm.position.set(...hand.palmPosition);
  palm.quaternion.setFromUnitVectors(new T.Vector3(0, -1, 0), new T.Vector3(...hand.palmNormal));

  for (var finger of hand.fingers) {
    const i = finger.type;
    fingers[i].update(finger);
  }
});