Video Input (Instead of Webcam) for Posenet P5.js

Question

I'm attempting to create a networked program that draws using a specific part of the body using P5.js, and Posenet within ML5.js. I've successfully created the networked model which uses a live camera feed using createCapture(VIDEO) in setup as seen below

function setup() {
  socket = io();
  cnv = createCanvas(windowWidth,windowHeight);
  capture = createCapture(VIDEO);
  background(255);
  
  PoseZero.init();
  
  local = new Agent();

    socket.emit('game-start', local.data)
    socket.on('heartbeat', function(data){
        world = data;
    })
  
}

var localPreviousPose = {};
var remotePreviousPose = {};

function draw() {
  
  image(capture, 0, 0, 220, 140);
  

(full code could be seen at https://glitch.com/edit/#!/makeyourmark-final)

However, for easier testing, I wanted to use a local video for the local side, I tried implementing it this way:

function setup() {
  socket = io();

  cnv = createCanvas(windowWidth, windowHeight);
  capture = createVideo(
    [
      "https://cdn.glitch.com/e36b62ca-cf57-47f9-a16d-caa525a0d130%2FScreen%20Recording%202021-05-25%20at%2013.06.25.MOV?v=1621952189772"
    ],
    onVideoLoad
  );
  background(255);
  PoseZero.init();

  local = new Agent();

  socket.emit("game-start", local.data);
  socket.on("heartbeat", function(data) {
    world = data;
  });
}

var localPreviousPose = {};
var remotePreviousPose = {};

function onVideoLoad() {
  // The media will play as soon as it is loaded.
  capture.autoplay();
  capture.loop();
}

function draw() {
  image(capture, 0, 0, 220, 140);

(full code could be seen at https://glitch.com/edit/#!/makeyourmark-video)

The error seems to be unrelated (e.g "Uncaught TypeError: Cannot read property 'autoplay' of undefined" when it is already autoplaying)

Is there any way I could fix this? Any insight/solution is appreciated.

Thank you!

Answer 1

There were are a couple of issues with your code, none of which were evident in what you included in your post:

1. Missing call to .bind(this) on callback function in pose.js

In your init function on the class declared in pose.js you pass the onVideoLoad function as a callback to createVideo. However the onVideoLoad function references this. Any time you are going to use this from a function you pass as a callback, you need to call .bind(this):

  this.init = function() {
    this.video = createVideo(
      ['https://cdn.glitch.com/e36b62ca-cf57-47f9-a16d-caa525a0d130%2FScreen%20Recording%202021-05-25%20at%2013.06.25.MOV?v=1621952189772'],
      // !!! Here, you need to use onVideoLoad.bind(this)
      onVideoLoad
    );

    // ...
  }
  
  function onVideoLoad() {
    // The media will play as soon as it is loaded.
    this.video.autoplay();
    this.video.loop();
  }

2. Broken code because i isn't declared

In your draw() function in client.js you make several references to world[i].data.pose but you haven't declared i.

3. Call to pop() without matching call to push()

In pose.js you have to following code:

    push();
    blendMode(MULTIPLY);
    // line(pose.leftWrist.x, pose.leftWrist.y, previousPose.data.leftWrist.x, previousPose.data.leftWrist.y);

    colorMode(HSB, 255);
    fill.apply(this, args.color);
    noStroke();
    beginShape();
    vertex(previousPose.data.leftWrist.x, previousPose.data.leftWrist.y);
    vertex(pose.leftWrist.x, pose.leftWrist.y);
    vertex(pose.leftElbow.x, pose.leftElbow.y);
    vertex(previousPose.data.leftElbow.x, previousPose.data.leftElbow.y);
    endShape(CLOSE);
    pop();
    
    // Presumably there should be another call to push() here
    colorMode(HSB, 255);
    fill.apply(this, args.color);
    noStroke();
    beginShape();
    vertex(previousPose.data.rightWrist.x, previousPose.data.rightWrist.y);
    vertex(pose.rightWrist.x, pose.rightWrist.y);
    vertex(pose.rightElbow.x, pose.rightElbow.y);
    vertex(previousPose.data.rightElbow.x, previousPose.data.rightElbow.y);
    endShape(CLOSE);
    pop();

4. Video playback needs to be initiated on user interaction

Calling loop() as soon as your video is loaded is not going to work in all cases. Instead you should have some prompt for the users to click on to initiate playback.

Working Remix

Here is remix that i think is "working" although I'm not 100% about your expected behavior: https://glitch.com/edit/#!/stackoverflow-67694586. I made a few changes which might not be what you want, such as making it create a single video element and passing that to PoseZero.init (instead of PoseZero having it's own video), and hiding the video element, since you're drawing it to the canvas anyway.

Conclusions

I think you should heed to following advice:

1. Add /* globals ... */ comments to your javascript files (full example below) that use p5.js so that glitch won't show errors where you are using p5.js globals without declaring them. That way where you do see errors they will be meaningful.
2. Click the Format this File button on the regular. It will make your code a lot easier to read.
3. Pay attention to the javascript console. When it starts spewing warnings don't ignore it, fix them.

Here is a sample globals comment for p5.js (may not be exhaustive):

/* globals ADD, ALT, ARROW, AUDIO, AUTO, AXES, BACKSPACE, BASELINE, BEVEL, BEZIER, BLEND, BLUR, BOLD, BOLDITALIC, BOTTOM, BURN, CENTER, CHORD, CLAMP, CLOSE, CONTROL, CORNER, CORNERS, CROSS, CURVE, DARKEST, DEGREES, DEG_TO_RAD, DELETE, DIFFERENCE, DILATE, DODGE, DOWN_ARROW, ENTER, ERODE, ESCAPE, EXCLUSION, FALLBACK, FILL, GRAY, GRID, HALF_PI, HAND, HARD_LIGHT, HSB, HSL, IMAGE, IMMEDIATE, INVERT, ITALIC, LABEL, LANDSCAPE, LEFT, LEFT_ARROW, LIGHTEST, LINEAR, LINES, LINE_LOOP, LINE_STRIP, MIRROR, MITER, MOVE, MULTIPLY, NEAREST, NORMAL, OPAQUE, OPEN, OPTION, OVERLAY, P2D, PI, PIE, POINTS, PORTRAIT, POSTERIZE, PROJECT, QUADRATIC, QUADS, QUAD_STRIP, QUARTER_PI, RADIANS, RADIUS, RAD_TO_DEG, REMOVE, REPEAT, REPLACE, RETURN, RGB, RIGHT, RIGHT_ARROW, ROUND, SCREEN, SHIFT, SOFT_LIGHT, SQUARE, STROKE, SUBTRACT, TAB, TAU, TESS, TEXT, TEXTURE, THRESHOLD, TOP, TRIANGLES, TRIANGLE_FAN, TRIANGLE_STRIP, TWO_PI, UP_ARROW, VIDEO, WAIT, WEBGL, accelerationX, accelerationY, accelerationZ, deltaTime, deviceOrientation, displayHeight, displayWidth, focused, frameCount, height, isKeyPressed, key, keyCode, keyIsPressed, mouseButton, mouseIsPressed, mouseX, mouseY, movedX, movedY, pAccelerationX, pAccelerationY, pAccelerationZ, pRotateDirectionX, pRotateDirectionY, pRotateDirectionZ, pRotationX, pRotationY, pRotationZ, pixels, pmouseX, pmouseY, pwinMouseX, pwinMouseY, rotationX, rotationY, rotationZ, touches, turnAxis, width, winMouseX, winMouseY, windowHeight, windowWidth, abs, acos, alpha, ambientLight, ambientMaterial, angleMode, append, applyMatrix, arc, arrayCopy, asin, atan, atan2, background, beginContour, beginShape, bezier, bezierDetail, bezierPoint, bezierTangent, bezierVertex, blend, blendMode, blue, boolean, box, brightness, byte, camera, ceil, char, circle, clear, clearStorage, color, colorMode, concat, cone, constrain, copy, cos, createA, createAudio, createButton, createCamera, createCanvas, createCapture, createCheckbox, createColorPicker, createDiv, createElement, createFileInput, createGraphics, createImage, createImg, createInput, createNumberDict, createP, createRadio, createSelect, createShader, createSlider, createSpan, createStringDict, createVector, createVideo, createWriter, cursor, curve, curveDetail, curvePoint, curveTangent, curveTightness, curveVertex, cylinder, day, debugMode, degrees, describe, describeElement, directionalLight, displayDensity, dist, downloadFile, ellipse, ellipseMode, ellipsoid, emissiveMaterial, endContour, endShape, erase, exitPointerLock, exp, fill, filter, float, floor, fract, frameRate, frustum, fullscreen, get, getFrameRate, getItem, getURL, getURLParams, getURLPath, green, gridOutput, hex, hour, httpDo, httpGet, httpPost, hue, image, imageMode, int, isLooping, join, keyIsDown, lerp, lerpColor, lightFalloff, lightness, lights, line, loadBytes, loadFont, loadImage, loadJSON, loadModel, loadPixels, loadShader, loadStrings, loadTable, loadXML, log, loop, mag, map, match, matchAll, max, millis, min, minute, model, month, nf, nfc, nfp, nfs, noCanvas, noCursor, noDebugMode, noErase, noFill, noLights, noLoop, noSmooth, noStroke, noTint, noise, noiseDetail, noiseSeed, norm, normalMaterial, orbitControl, ortho, perspective, pixelDensity, plane, point, pointLight, pop, popMatrix, popStyle, pow, print, push, pushMatrix, pushStyle, quad, quadraticVertex, radians, random, randomGaussian, randomSeed, rect, rectMode, red, redraw, registerPromisePreload, removeElements, removeItem, requestPointerLock, resetMatrix, resetShader, resizeCanvas, reverse, rotate, rotateX, rotateY, rotateZ, round, saturation, save, saveCanvas, saveFrames, saveGif, saveJSON, saveJSONArray, saveJSONObject, saveStrings, saveTable, scale, second, select, selectAll, set, setAttributes, setCamera, setFrameRate, setMoveThreshold, setShakeThreshold, shader, shearX, shearY, shininess, shorten, shuffle, sin, smooth, sort, specularColor, specularMaterial, sphere, splice, split, splitTokens, spotLight, sq, sqrt, square, storeItem, str, stroke, strokeCap, strokeJoin, strokeWeight, subset, tan, text, textAlign, textAscent, textDescent, textFont, textLeading, textOutput, textSize, textStyle, textWidth, texture, textureMode, textureWrap, tint, torus, translate, triangle, trim, unchar, unhex, updatePixels, vertex, writeFile, year */