[AI] AI and MuJoCo

Tue Mar 24 01:08:49 PDT 2026

MuJoCo on the web looks like a way to simulate robots.  While taking a
broad brush to their web-based work, it looks like they are looking at the
physics of robots, which isn’t a horrible thing, it just looks like the
robots don’t have any clothes or soft-body physics involved.   I have seen
trampolines and ponchos in their non-web work.  It looks like some groups
are leveraging Three.JS for rendering, and WASM for simulation, much like I
use Ammo.js for simulation and Three.Js for rendering.  So I think if we
want to do something with MuJoCo, we should separate out the physics engine
from the rest of the project.

I have little experience interfacing with JavaScript and WASM and mainly
depends on demo work by others, so I might be able to construct a demo with
an X3D-based encoding and MuJoCo physics engine using AI, essentially
stripping out Ammo.Js, but I’m going to need to know how to include MuJoCo
in a website, that’s not clear yet.   So here goes Google AI.  I’ll be
testing this later tonight.

1. Command setup:

=========
npm init -y
npm install three mujoco
npm install -D vite

Package.json
==========
{
"name": "mujoco-poncho",
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build"
},
"dependencies": {
"mujoco": "^3.1.2",
"three": "^0.160.0"
},
"devDependencies": {
"vite": "^5.0.0"
}
}

Index.html
============

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>MuJoCo Poncho Drape</title>
<style>
body { margin: 0; overflow: hidden; background: #222; }
#loading { color: white; position: absolute; top: 20px; left: 20px;
font-family: sans-serif; }
</style>
</head>
<body>
<div id="loading">Loading MuJoCo WASM...</div>
<script type="module" src="/main.js"></script>
</body>
</html>

Scene.xml
============

<mujoco model="poncho_drape">
<compiler angle="degree"/>
<option timestep="0.005" gravity="0 0 -9.81"/>

<worldbody>
<!-- Lighting & Floor -->
<light directional="true" diffuse="0.8 0.8 0.8" pos="0 0 5" dir="0 0 -1"/>
<geom name="floor" type="plane" size="5 5 0.1" rgba="0.2 0.3 0.4 1"/>

<!-- Static Humanoid Body (Welded to the world so it stands still) -->
<body name="humanoid" pos="0 0 1.0">
<!-- Torso -->
<geom name="torso" type="capsule" size="0.15 0.25" rgba="0.8 0.6 0.4 1"/>
<!-- Head -->
<geom name="head" type="sphere" pos="0 0 0.45" size="0.12" rgba="0.8 0.6
0.4 1"/>
<!-- Arms -->
<geom name="left_arm" type="capsule" pos="-0.25 0 0.1" size="0.05 0.25"
euler="0 -45 0" rgba="0.8 0.6 0.4 1"/>
<geom name="right_arm" type="capsule" pos="0.25 0 0.1" size="0.05 0.25"
euler="0 45 0" rgba="0.8 0.6 0.4 1"/>
<!-- Legs -->
<geom name="left_leg" type="capsule" pos="-0.1 0 -0.5" size="0.06 0.35"
rgba="0.8 0.6 0.4 1"/>
<geom name="right_leg" type="capsule" pos="0.1 0 -0.5" size="0.06 0.35"
rgba="0.8 0.6 0.4 1"/>
</body>

<!-- The Poncho (Cloth Grid) -->
<!-- Placed slightly above the head, falls under gravity -->
<composite type="cloth" count="21 21 1" spacing="0.04" offset="0 0 1.8">
<geom size="0.015" rgba="0.2 0.7 0.3 1" friction="1 0.005 0.0001"/>
</composite>
</worldbody>
</mujoco>

Main.js
==========
import * as THREE from 'three';
import { OrbitControls } from
'three/examples/jsm/controls/OrbitControls.js';
import load_mujoco from 'mujoco';

async function init() {
// ==========================================
// 1. Three.js Setup
// ==========================================
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(45, window.innerWidth /
window.innerHeight, 0.1, 100);
camera.position.set(3, 3, 3);

const renderer = new THREE.WebGLRenderer({ antialias: true });
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setPixelRatio(window.devicePixelRatio);
document.body.appendChild(renderer.domElement);

const controls = new OrbitControls(camera, renderer.domElement);
controls.target.set(0, 1, 0);

const ambientLight = new THREE.AmbientLight(0xffffff, 0.4);
scene.add(ambientLight);
const dirLight = new THREE.DirectionalLight(0xffffff, 0.8);
dirLight.position.set(5, 10, 5);
scene.add(dirLight);

// Coordinate Fix: MuJoCo is Z-up, Three.js is Y-up.
// By placing all meshes in a rotated group, we avoid complex math per-mesh.
const mujocoGroup = new THREE.Group();
mujocoGroup.rotation.x = -Math.PI / 2;
scene.add(mujocoGroup);

// ==========================================
// 2. MuJoCo & VFS Setup
// ==========================================
const mujoco = await load_mujoco();

// Create a virtual file system so the C++ engine can read the XML
mujoco.FS.mkdir('/working');
mujoco.FS.mount(mujoco.MEMFS, { root: '.' }, '/working');

// Fetch our XML and write it to WASM memory
const xmlContent = await fetch('/scene.xml').then(res => res.text());
mujoco.FS.writeFile('/working/scene.xml', xmlContent);

// Initialize physics states
const model = mujoco.MjModel.loadFromXML('/working/scene.xml');
const data = new mujoco.MjData(model);

// Remove loading text
document.getElementById('loading').style.display = 'none';

// ==========================================
// 3. Mapping MuJoCo Geoms to Three.js
// ==========================================
const meshes =[];

for (let i = 0; i < model.ngeom; i++) {
const type = model.geom_type[i];

// MuJoCo flat arrays: size is index * 3, rgba is index * 4
const size = [
model.geom_size[i * 3 + 0],
model.geom_size[i * 3 + 1],
model.geom_size[i * 3 + 2]
];
const rgba =[
model.geom_rgba[i * 4 + 0],
model.geom_rgba[i * 4 + 1],
model.geom_rgba[i * 4 + 2],
model.geom_rgba[i * 4 + 3]
];

let geometry;
// 0: Plane, 2: Sphere, 3: Capsule, 6: Box
if (type === 0) {
geometry = new THREE.PlaneGeometry(size[0] * 2, size[1] * 2);
} else if (type === 2) {
geometry = new THREE.SphereGeometry(size[0], 16, 16);
} else if (type === 3) {
// MuJoCo capsule sizes: [radius, half-length]
geometry = new THREE.CapsuleGeometry(size[0], size[1] * 2, 8, 16);
// CRITICAL: Three.js aligns capsules on Y, MuJoCo aligns them on Z.
geometry.rotateX(Math.PI / 2);
} else if (type === 6) {
geometry = new THREE.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
} else {
geometry = new THREE.SphereGeometry(0.01); // Fallback
}

const material = new THREE.MeshStandardMaterial({
color: new THREE.Color(rgba[0], rgba[1], rgba[2]),
opacity: rgba[3],
transparent: rgba[3] < 1.0,
side: THREE.DoubleSide
});

const mesh = new THREE.Mesh(geometry, material);
// We will manually overwrite the transform matrix from MuJoCo each frame
mesh.matrixAutoUpdate = false;

mujocoGroup.add(mesh);
meshes.push(mesh);
}

// ==========================================
// 4. The Animation / Simulation Loop
// ==========================================
const tempMatrix = new THREE.Matrix4();

// Calculate how many physics steps to run per frame to match real time
// Default framerate is ~60fps (0.016s). MuJoCo timestep is 0.005s.
const stepsPerFrame = Math.ceil(0.016 / model.opt.timestep);

function animate() {
requestAnimationFrame(animate);

// Step physics simulation
for (let i = 0; i < stepsPerFrame; i++) {
mujoco.mj_step(model, data);
}

// Sync transforms
for (let i = 0; i < model.ngeom; i++) {
// Read 3D position
const px = data.geom_xpos[i * 3 + 0];
const py = data.geom_xpos[i * 3 + 1];
const pz = data.geom_xpos[i * 3 + 2];

// Read 3x3 rotation matrix
const m0 = data.geom_xmat[i * 9 + 0]; const m1 = data.geom_xmat[i * 9 + 1];
const m2 = data.geom_xmat[i * 9 + 2];
const m3 = data.geom_xmat[i * 9 + 3]; const m4 = data.geom_xmat[i * 9 + 4];
const m5 = data.geom_xmat[i * 9 + 5];
const m6 = data.geom_xmat[i * 9 + 6]; const m7 = data.geom_xmat[i * 9 + 7];
const m8 = data.geom_xmat[i * 9 + 8];

// Load into Three.js 4x4 Matrix (Row-major order)
tempMatrix.set(
m0, m1, m2, px,
m3, m4, m5, py,
m6, m7, m8, pz,
0, 0, 0, 1
);

meshes[i].matrix.copy(tempMatrix);
}

controls.update();
renderer.render(scene, camera);
}

animate();

// Handle Window Resize
window.addEventListener('resize', () => {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
});
}

init();

Command to run
========

npm run dev
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://web3d.org/pipermail/ai_web3d.org/attachments/20260324/d2701aaf/attachment-0001.html>