play twilio call media stream directly in the browser

281 views Asked by At

I am trying to stream call audio from a twilio call direclty in the browser so it can be played once the call is initiated

I have a already established a python app that once the call is started it tell twilio to stream the call to a websocket (w/ node js)

websocket:

const WebSocket = require('ws');
const express = require('express')
const wav = require('wav');
const fs = require('fs');
const WaveFile = require('wavefile').WaveFile;
const app = express()
const server = require('http').createServer(app);
const wss = new WebSocket.Server({ server });
const port = 4000
const TwilioMediaStreamSaveAudioFile = require("twilio-media-stream-save-audio-file");

const Speaker = require('speaker');
const alawmulaw = require("alawmulaw").alaw;

const mediaStreamSaver = new TwilioMediaStreamSaveAudioFile({
    saveLocation: __dirname,
    saveFilename: "my-twilio-media-stream-output",
    onSaved: () => console.log("File was saved!"),
});


app.set('view engine', 'ejs');


connectionState = "Not Established"
payloadData = ""

const speaker = new Speaker({
    channels: 1,          // 2 channels
    bitDepth: 16,         // 16-bit samples
    sampleRate: 8000     // 44,100 Hz sample rate
  });

const wavFile = new WaveFile();



wss.on('connection', function connection(ws) {
    console.log('New connection')
    mediaStreamSaver.setWebsocket(ws);

    ws.on('message', function incoming(message) {
        const msg = JSON.parse(message)

        switch (msg.event) {
            case 'connected':
                console.log('A new call has been connected')
                connectionState = " Connected";
                break
            case 'start':
                console.log('The call has started')
                mediaStreamSaver.twilioStreamStart();
                connectionState = " Started";
                break
            case 'media':
                connectionState = " Call in Progress"
                
                payloadData = msg.media.payload

                // ! play the audio in the background
                // let buff = Buffer.from(payloadData, 'base64');
                // let PCM = Buffer.from(alawmulaw.decode(buff));
                // speaker.write(PCM);

                // ! save the audio to a file
                // mediaStreamSaver.twilioStreamMedia(msg.media.payload);


                break
            case 'stop':
                connectionState = " Call Ended"
                mediaStreamSaver.twilioStreamStop();
                console.log(' The call has ended')
                break
            default:
                connectionState = " Unknown"
                console.log('Something went wrong')
                break
        }
    }
    );
});

app.post('/', (req, res) => {
    res.setHeader('Content-Type', 'text/plain')
    res.send('Hello World!')
    console.log(req.body)
})
app.get('/', (req, res) => {
    res.render('page', { connectionState, payloadData })
});


// SSE endpoint to send data updates
app.get('/events', (req, res) => {
    res.setHeader('Content-Type', 'text/event-stream');
    res.setHeader('Cache-Control', 'no-cache');
    res.setHeader('Connection', 'keep-alive');

    // Send initial data to the client
    res.write(`data: ${JSON.stringify({ connectionState,payloadData })}\n\n`);
// Continuously send updated data to the client
    setInterval(() => {
        res.write(`data: ${JSON.stringify({ connectionState,payloadData })}\n\n`);

    }, 1000);
});

console.log('Server started on port ' + port)
server.listen(port)

i am also able to recive the state of the call in the browser along with the media payload

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Real-Time Updates</title>
</head>
<body>
  <h1>Data from the server:</h1>
  <div id="data"  class="center-div"></div>
  <div id="payload"  class="center-div"></div>
  
  <!-- <figure>
    
    <audio controls src="/media/cc0-audio/t-rex-roar.mp3">
      <a href="/media/cc0-audio/t-rex-roar.mp3"> Download audio </a>
    </audio>
  </figure> -->

  <script>
    const eventSource = new EventSource('/events');

    eventSource.onmessage = (event) => {
      const data = JSON.parse(event.data);

      document.getElementById('data').innerText = `Connection State: ${data.connectionState}`;
        document.getElementById('payload').innerText = `PayloadData: ${data.payloadData}`;
    };
  </script>
</body>
</html>

I want to be able to listen to call in browser along with the ability to controle (pause play , adjust volume)

Any help would be much appreacited

I have tried to:

  • play the call via the speaker but the audio is terrible it only play a pulse of static every second
  • save the audio being gathered from the stream with the help of this package twilio-media-stream-save-audio-file but the result wav file is playing in slow-mo and he audio is terrible aswell
1

There are 1 answers

1
Yasir Ijaz On

Not an expert on Python but here is an overview.

Add Twilio client library on front side

<script type="text/javascript" src="https://sdk.twilio.com/js/client/v1.14/twilio.js"></script>

Then Send a request on the server side to fetch the access token. While making an access token you will provide a unique identity like this

  $accessToken = new AccessToken($accountSid, $apiKeySid, $apiKeySecret,43200,$uniqueIdentity);
    // $uniqueIdentity is unique for every user. It can be an email or phone number. Preferably encoded email.

    // Create a Voice grant and add it to the token
    $voiceGrant = new VoiceGrant();
    $voiceGrant->setIncomingAllow(TRUE); // Set your TwiML application SID
    $voiceGrant->setOutgoingApplicationSid($applicationSid); // Set your TwiML application SID
    $accessToken->addGrant($voiceGrant);

    // Serialize the token as a JWT
    $token = $accessToken->toJWT();
    $data = array();
    $data['data']['success'] = true;
    $data['data']['token'] = $token;
    $data['data']['identity'] = $twilioName;

Return this to front side and on front side use this token to initialize the Twilio device

  const Device =  Twilio.Device.setup(status.data.token);
  Device.on('ready',function(){
            console.log('ready')
  })
  Device.on('incoming', (connection) => {
            $('#accept').show();
            console.log(connection);
            $('#accept').on('click',function(){
                connection.accept();
                $('#reject').show();
            })
            // Handle incoming call
        });
        Device.on('connect', (connection) => {
            console.log(connection);
            $('#reject').on('click',function(){
                connection.disconnect();
                $('#reject').hide();
                $('#accept').hide();
            })
        });

I assume you already gave the callback URL to the twiml app associated with the twilio number.

Now When I call comes on Twilio number. It will hit the callback URL where you will forward it to browser agent like this

$response = new VoiceResponse();
    $dial = $response->dial('',['callerId'=>$_POST['From'],'record'=>'record-from-answer','timeout'=>'30','action' => url('ivr/voiceMailHandler')]);
    $dial->client($uniqueIdentity); 
return $response;

This will place the call to the user who have this $uniqueIdentity.

Hope this helps.