CART 351 – MEDIA API’s

The Media Capture API

The next examples will demonstrate how to use the the Media Capture API in order to be able to capture live streaming video and audio media.

Capture Live Video I

The getUserMedia API provides access to multimedia streams (video, audio, or both) from local devices. Previously, we needed a 3rd party plugin – now HTML 5 has support. You need to use this API to access the camera, microphone etc…
The getUserMedia API exposes just one method called getUserMedia().
The first example, seen below demonstrates how to setup the basic code template for capturing video from the users web cam, storing it, and displaying it using the html5 video element:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

<!DOCTYPE html>
<html>
  <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <title>getUserMedia Demo</title>
    <style>
 
    main{
      width:50%;
      margin:0 auto;
      background:grey;
    }
      #video
      {
        display: block;
      }
 </style>
 <script type="text/javascript" src="getUserMedia.js"></script>
 
  </head>
  <body>
 
    <h1>Access Camera Example</h1>
    <main>
    <video id="video" autoplay="autoplay" controls="true"></video>
  </main>
</body>
</html>

The HTML page sets up the necessary video element – without a source. Why? Because we will set up the source using javascript:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

window.onload = function() {
  console.log("loaded");
  let  video = document.getElementById("video");
 
navigator.mediaDevices.getUserMedia({video: true})
      .then(
        //stream is what is returned
        (stream) => {
          video.srcObject = stream;
    }) 
    .catch(function(err) {
    /* handle the error */
    console.log("had an error getting the camera");
    });
 
};

The navigator object in a nutshell contains information about the browser- and exposes properties and methods that one can program against.
The MediaDevices.getUserMedia() method prompts the user for permission to use a media input which produces a MediaStream with tracks containing the requested types of media. That stream can include a video track (produced by video source such as a camera), an audio track (produced by a physical or virtual audio source like a microphone).
It returns a Promise that resolves to a MediaStream object.
If the user denies permission, or matching media is not available, then the promise is rejected with NotAllowedError or NotFoundError respectively.
Upon success we thus assign the stream object to the srcObject of the video element.
Instead of feeding the video the URL of a media file, we’re giving it a MediaStream from the webcam.
Note: we also set the video to autoplay, otherwise it would be frozen on the first frame and we would need to specify it to play within the code.

Capture Live Video II

The getUserMedia() allows for one to specify if one also requires audio as well as properties like size. Lets alter the function call to include parameters for width and height.

1
2
3
4

navigator.mediaDevices.getUserMedia(
  {video: {
    width:320,
    height:240}})

Capture Live Video III and the Canvas

We can now show live video displayed using the video element. But we can do even better – we can also use the canvas to display every frame of video and then we can also manipulate/change/adapt the video pixels! … We will still use the video element as the container to hold the streaming content – but then we will display the video content using the canvas context.
The new HTML markup:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

<!DOCTYPE html>
<html>
  <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <title>getUserMedia Demo</title>
    <style>
 
    main{
      width:50%;
      margin:0 auto;
      background:grey;
    }
 
 </style>
 <script type="text/javascript" src="getUserMediaAndCanvas.js"></script>
 
  </head>
  <body>
 
    <h1>Access Camera Example</h1>
    <main>
      <!-- set video to hidden - we only  want it as a temp container for streaming  -->
    <video id="video"  autoplay="autoplay" controls="true" hidden></video>
    <canvas id="testCanvas" width =640 height=240></canvas>
  </main>
</body>
</html>

Note how we have set the video element to be hidden ….
Lets write the JavaScript: first we set up the canvas and context, set up an animation loop and within that use the context’s drawImage() method to display the next video frame:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

window.onload = function() {
  console.log("loaded");
  let  video = document.getElementById("video");
  let canvas = document.getElementById("testCanvas");
  let context = canvas.getContext('2d');
 
 
      navigator.mediaDevices.getUserMedia({video: {
        width:320,
        height:240}})
      .then(
        //stream is what is returned
        (stream) => {
          video.srcObject = stream;
 
    })
    .catch(function(err) {
    /* handle the error */
    console.log("had an error getting the camera");
    });
 
/*** instead of using the video object we can use the canvas **/
requestAnimationFrame(run);
function run(){
context.clearRect(0,0,canvas.width, canvas.height);
context.drawImage(video, 0, 0, canvas.width/2, canvas.height);
context.fillStyle = "#FFFFFF";
context.fillRect(canvas.width/2+50, canvas.height/2,50,50);
requestAnimationFrame(run);
}
};

If you run the page – you should now see the video displaying in the canvas ..
Let’s now go one step further and use some inbuilt methods to access the pixels in the canvas and modify them … the new run() should now be:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

function run(){
context.clearRect(0,0,canvas.width, canvas.height);
context.drawImage(video, 0, 0, canvas.width/2, canvas.height);
context.fillStyle = "#FFFFFF";
context.fillRect(canvas.width/2+50, canvas.height/2,50,50);
 let frame = context.getImageData(0, 0, canvas.width/2, canvas.height);
  // every pixel has an r,g,b,a value ... and so in the array - every 4 values== 1 pixel
    for (let i = 0; i < frame.data.length;i+=4) {
    let r = frame.data[i];
    let g = frame.data[i+1];
    let b = frame.data[i+ 2];
    let a = frame.data[i+ 3];
       frame.data[i] =r;
       frame.data[i+1] =0;
       frame.data[i+2]=0;
       // make every 32nd frame have an alpha of 0/
       if(i%32==0){
          frame.data[i+3]=0;
       }
    }
    context.putImageData(frame, 0, 0);
    //can draw on top ...
    context.fillRect(0, canvas.height/2,50,50);
 
 requestAnimationFrame(run);
}

Two new methods have been introduced here: the getImageData() returns an ImageData object.The ImageData object represents the underlying pixel data of an area of a canvas object.It contains the following read-only attributes:

  • width:: The width of the image in pixels.
  • height:: The height of the image in pixels.
  • data: An integer array representing a one-dimensional array containing the data in the RGBA order,
    with integer values between 0 and 255 (included).

And we use this method to access the specific pixel data that we want to manipulate. Second – the putImageData(img, dx,dy) method is used to paint pixel data into a context.The dx and dy parameters indicate the device coordinates within the context that you want to draw into.

One amazing usage for video capture is to render live input as a WebGL texture. If you are interested please give Jerome Etienne’s tutorial a look. It talks about how to use getUserMedia() and this Three.js tutorial to render live video into WebGL.

The Microphone and the Web Audio API

You can also get live microphone input from the getUserMedia() method – and then if you want to – you can pipe the stream to the Web Audio API in order to create and output some real-time effects.
We use the same navigator.mediaDevices.getUserMedia() method – which once again will return the media stream. Now, though, instead of just assiging the stram to the source object associated with an audio element, we can pipe the stream to the Web Audio API:

The Web Audio API is a high-level JavaScript library for processing and synthesizing audio in web applications. The goal of this library is to include capabilities found in modern game audio engines and some of the mixing, processing, and filtering tasks that are found in modern desktop audio production applications.

The basic setup for the Web Audio API is as follows:
1: An AudioContext is needed for managing and playing all sounds.
2: To produce a sound using the Web Audio API, create one or more sound sources and
connect them to the sound destination provided by the AudioContext instance.
3: A single instance of AudioContext can support multiple sound inputs -> sources could be
files, microphone, self-creation …

In this example we will use the microphone as an input source, then connect that the inbuilt analyser module. This module will allow us to anaylze the frequencies coming from the microphone (amongst other stuff) – and then we can use this data somehow -> here we will just get a rough average frequency of the input stream and use this value to change the width of a rectangle of the canvas in real time.

The HTML markup:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

<html>
<head>
  <title> Microphone Tutorial </title>
  <style>
  body{
    margin:0;
    padding:0;
  }
  canvas{
    background:#000000;
  }
  </style>
  <!-- REFERENCE OUR SCRIPTS -->
 
  <script type="text/javascript" src="MicrophoneExRev.js"></script>
 
</head>
<body>
  <canvas id="testCanvas" width =500 height=500></canvas>
</body>
</html>

The JavaScript:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

window.onload = function(){
 
  window.AudioContext = window.AudioContext ||
                      window.webkitAudioContext;
 let audioContext = new AudioContext(); //suing the web audio library
 // get the canvas
 let canvas = document.getElementById("testCanvas");
 
 //get the context
 let context = canvas.getContext("2d");
 
navigator.mediaDevices.getUserMedia({audio: true})
  .then(
    (stream) => {
    //returns a MediaStreamAudioSourceNode.
        const microphoneIn = audioContext.createMediaStreamSource(stream);
        const filter = audioContext.createBiquadFilter();
        const analyser = audioContext.createAnalyser();
        // microphone -> filter ->  analyzer->destination
          microphoneIn.connect(filter);
        //use the analyzer object to get some properties ....
        filter.connect(analyser);
 
        //we do not need a destination (out)
        //analyser.connect(audioContext.destination);
        analyser.fftSize = 32;
        let frequencyData = new Uint8Array(analyser.frequencyBinCount);
 
        //call loop ...
       requestAnimationFrame (callBackLoop);
 
       /****our looping callback function */
       function callBackLoop(){
        context.clearRect(0, 0, canvas.width, canvas.height);
         analyser.getByteFrequencyData(frequencyData);
         let average =0;
         let sum=0;
 
         for(let i = 0; i<frequencyData.length; i++){
           sum+=frequencyData[i];
         }
         average = sum/frequencyData.length;
         console.log(average);
         context.fillStyle = "#FF0000";
         //use the average frequency
        context.fillRect(canvas.width/2,canvas.height/2,average,30);
        requestAnimationFrame(callBackLoop);
 
       }
 
})
.catch(function(err) {
/* handle the error */
console.log("had an error getting the microphone");
});
}