Tracking with js-aruco on HoloJS

[lumineka]

Hello,
I'm trying to do tracking with the js-aruco library using the RGB camera.

The goal is to place a simple 3D cube on a tag in the HoloJS world.

I have some problems with the registration, there is always a shift with the 3D object.

The world of the RGB camera is the same as the HolographicCamera in ThreeJS?

Someone already had this kind of problem? Or someone ever done tracking on HoloJS? 😅

Here a small example

let frame = new Image();
let detector = new AR.Detector();
let tagSize = 0.52; // tag size, mm in aruco, m in HoloJS
let cube = new THREE.Mesh( new THREE.BoxBufferGeometry( tagSize, tagSize, tagSize ), new THREE.MeshNormalMaterial() );

frame.onload = function () {
    let canvas = document.createElement('canvas');
    canvas.width = frame.width;
    canvas.height = frame.height;
    let ctx = canvas.getContext('2d');
    ctx.drawImage(frame, 0, 0);
    let imageData = ctx.getImageData(0, 0, frame.width, frame.height);
    // detect markers
    detectTag(imageData);
}
frame.src = "camera://local/default";

function detectTag(imageData) {
    let markers = detector.detect(imageData);
    if (markers.length > 0) {
        let posit = new POS.Posit(modelSize, canvas.width);
        let corners = markers[0].corners;
        // corners must be centered on canvas
        for (var i = 0; i < corners.length; ++i) {
            corner = corners[i];
            corner.x = corner.x - (imageData.width / 2);
            corner.y = (imageData.height / 2) - corner.y;
        }
        let pose = posit.pose(corners);
        let rotation = pose.bestRotation
        let translation = pose.bestTranslation
        updateCube(rotation, translation);
    }
}

function updateCube(rotation, translation){
    cube.rotation.x = -Math.asin(-rotation[1][2]);
    cube.rotation.y = -Math.atan2(rotation[0][2], rotation[2][2]);
    cube.rotation.z = Math.atan2(rotation[1][0], rotation[1][1]);
    cube.position.x = translation[0];
    cube.position.y = translation[1];
    cube.position.z = -translation[2];
    cube.updateMatrix();
    cube.applyMatrix(camera.matrixWorld);
    cube.updateMatrix();
}

[Almost-Done]

When extracting the 3D coordinates from a photo, where does js-aruco place the origin of the coordinate system? For HoloJS, (0,0,0) in your scene is actually 2 meters in front of you:
WorldOriginRelativePosition = float3(0, 0, -2);

Try changing it to (0,0,0) to bring the scene origin roughly to the same position as the RGB camera and you might have better alignment.

Another possibility is that the 3D coordinates extracted from the photo by js-aruco do no match HoloLens's world tracking system.

[wizVR-zhangjun]

When extracting the 3D coordinates from a photo, where does js-aruco place the origin of the coordinate system? For HoloJS, (0,0,0) in your scene is actually 2 meters in front of you:
WorldOriginRelativePosition = float3(0, 0, -2);

Try changing it to (0,0,0) to bring the scene origin roughly to the same position as the RGB camera and you might have better alignment.

Another possibility is that the 3D coordinates extracted from the photo by js-aruco do no match HoloLens's world tracking system.

WorldOriginRelativePosition How did you get it