My final project will be training a model to develop an intuitive understanding of physics. When I say “intuitive,” I mean that the model will be able to predict the behavior of objects in space without explicitly being given equations describing things like momentum, force, etc. The goal is create a model that learns about its surrounding environment the way human beings do - via observation.

The model will be trained on video samples of circles bouncing around within an enclosed space. The circles are identical except in regards to color, initial position, and initial velocity. The space has no gravity and the circles move without friction. All collisions in the space are completely elastic. Obviously, this environment differs from the real world in many ways, but I wanted to keep things simple. Factors like friction and inelastic collisions add complexity that I don’t want to consider just yet.

I used the javascript physics engine matter.js to create the environment and recorded the samples using MediaRecorder. I based the code on a WebRTC sample for recording from a canvas.

var Engine = Matter.Engine,
    Render = Matter.Render,
    World = Matter.World,
    Body = Matter.Body,
    Bodies = Matter.Bodies;

var engine = Engine.create();

var render = Render.create({
    element: document.body,
    engine: engine,
    options:{
        background: '#222',
        showVelocity: true,
        wireframes: false
    }
});

// create the borders of the environment
var ground = Bodies.rectangle(400, 600, 810, 40, { isStatic: true });
var ceiling = Bodies.rectangle(400, 0, 810, 40, { isStatic: true });
var leftWall = Bodies.rectangle(0, 120, 40, 1000, { isStatic: true });
var rightWall = Bodies.rectangle(800, 120, 40, 1000, { isStatic: true });

function generateCircles(){
    // generate a random number of circles
    var numCircles = Math.floor(Math.random() * 5) + 1;
    var circles = [];

    for (i = 0; i < numCircles; i++){
        var x_coord = Math.floor(Math.random() * 700);
        var y_coord = Math.floor(Math.random() * 500);

        // randomly generate colors for circles
        var fillColor = '#' + Math.floor(Math.random()*16777215).toString(16);
        var strokeColor = '#' + Math.floor(Math.random()*16777215).toString(16);

        var ball = Bodies.circle(x_coord, y_coord, 30, {
          density: 0.04,
          friction: 0.00,
          frictionAir: 0.0000,
          restitution: 1.0,
          inertia: Infinity,
          render: {
            fillStyle: fillColor,
            strokeStyle: strokeColor,
            lineWidth: 2
          }
        });

        // choose a positive or negative velocity for the circles
        x_sign = [-1, 1][Math.floor(Math.random() * 2)];
        y_sign = [-1, 1][Math.floor(Math.random() * 2)];

        // choose the velocity of the circle
        x_velocity = Math.floor(Math.random() * 16) * x_sign;
        y_velocity = Math.floor(Math.random() * 16) * y_sign;

        Body.setVelocity(ball, { x: x_velocity, y: y_velocity })
        circles.push(ball)
    }

    // add all of the circles to the environment
    World.add(engine.world, circles);
}

generateCircles();

World.add(engine.world, [ground, ceiling, leftWall, rightWall]);
engine.world.gravity.y = 0;

Engine.run(engine);
Render.run(render);

Once the circles are in motion, we can start recording them like so:

let mediaRecorder;
let recordedBlobs;

// the part of the page to record
const canvas = document.querySelector('canvas');

const stream = canvas.captureStream();
console.log('Started stream capture from canvas element: ', stream);

// push frames from canvas onto array
function handleDataAvailable(event) {
  if (event.data && event.data.size > 0) {
    recordedBlobs.push(event.data);
  }
}

function startRecording() {
  let options = { mimeType: 'video/webm' };
  recordedBlobs = [];

  mediaRecorder = new MediaRecorder(stream, options);

  console.log('Created MediaRecorder', mediaRecorder, 'with options', options);
  mediaRecorder.ondataavailable = handleDataAvailable;
  mediaRecorder.start(100);
  console.log('MediaRecorder started', mediaRecorder);
}

function stopRecording() {
  mediaRecorder.stop();
  console.log('Recorded Blobs: ', recordedBlobs);
}

function download() {
    // creates a download link for the recording and clicks it
    const blob = new Blob(recordedBlobs, {type: 'video/webm'});
    const url = window.URL.createObjectURL(blob);
    const a = document.createElement('a');
    a.style.display = 'none';
    a.href = url;
    a.download = 'test_' + localStorage.getItem("sampleCount") + '.webm';
    document.body.appendChild(a);
    a.click();
    setTimeout(() => {
        document.body.removeChild(a);
        window.URL.revokeObjectURL(url);
    }, 100);
}

function stopAndDownload() {
    if (!localStorage.getItem("sampleCount")) {
        localStorage.setItem("sampleCount", 0);
        console.log(localStorage.getItem("sampleCount"));
    } else {
        // for adding a number to the sample file name
        previous_count = parseInt(localStorage.getItem("sampleCount"));
        localStorage.setItem("sampleCount", previous_count + 1);
        console.log(localStorage.getItem("sampleCount"));
    }

    stopRecording();
    download();
    setTimeout(refreshPage, 3000);
}

function refreshPage() {
    window.location.reload();
}

startRecording();
// records 5 sec of canvas
setTimeout(stopAndDownload, 6000);

To have the downloads proceed without interruption, you’ll need to make sure your browser is not set to ask for the location in which to place the file to be downloaded.

Pokebot

A group of researchers at Berkeley trained a model to understand how to manipulate objects by having it repeatedly move objects around a table. You can see their results here.

Photo by CMDR Shane on Unsplash