I am trying to stream call audio from a twilio call direclty in the browser so it can be played once the call is initiated
I have a already established a python app that once the call is started it tell twilio to stream the call to a websocket (w/ node js)
websocket:
const WebSocket = require('ws');
const express = require('express')
const wav = require('wav');
const fs = require('fs');
const WaveFile = require('wavefile').WaveFile;
const app = express()
const server = require('http').createServer(app);
const wss = new WebSocket.Server({ server });
const port = 4000
const TwilioMediaStreamSaveAudioFile = require("twilio-media-stream-save-audio-file");
const Speaker = require('speaker');
const alawmulaw = require("alawmulaw").alaw;
const mediaStreamSaver = new TwilioMediaStreamSaveAudioFile({
saveLocation: __dirname,
saveFilename: "my-twilio-media-stream-output",
onSaved: () => console.log("File was saved!"),
});
app.set('view engine', 'ejs');
connectionState = "Not Established"
payloadData = ""
const speaker = new Speaker({
channels: 1, // 2 channels
bitDepth: 16, // 16-bit samples
sampleRate: 8000 // 44,100 Hz sample rate
});
const wavFile = new WaveFile();
wss.on('connection', function connection(ws) {
console.log('New connection')
mediaStreamSaver.setWebsocket(ws);
ws.on('message', function incoming(message) {
const msg = JSON.parse(message)
switch (msg.event) {
case 'connected':
console.log('A new call has been connected')
connectionState = " Connected";
break
case 'start':
console.log('The call has started')
mediaStreamSaver.twilioStreamStart();
connectionState = " Started";
break
case 'media':
connectionState = " Call in Progress"
payloadData = msg.media.payload
// ! play the audio in the background
// let buff = Buffer.from(payloadData, 'base64');
// let PCM = Buffer.from(alawmulaw.decode(buff));
// speaker.write(PCM);
// ! save the audio to a file
// mediaStreamSaver.twilioStreamMedia(msg.media.payload);
break
case 'stop':
connectionState = " Call Ended"
mediaStreamSaver.twilioStreamStop();
console.log(' The call has ended')
break
default:
connectionState = " Unknown"
console.log('Something went wrong')
break
}
}
);
});
app.post('/', (req, res) => {
res.setHeader('Content-Type', 'text/plain')
res.send('Hello World!')
console.log(req.body)
})
app.get('/', (req, res) => {
res.render('page', { connectionState, payloadData })
});
// SSE endpoint to send data updates
app.get('/events', (req, res) => {
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
// Send initial data to the client
res.write(`data: ${JSON.stringify({ connectionState,payloadData })}\n\n`);
// Continuously send updated data to the client
setInterval(() => {
res.write(`data: ${JSON.stringify({ connectionState,payloadData })}\n\n`);
}, 1000);
});
console.log('Server started on port ' + port)
server.listen(port)
i am also able to recive the state of the call in the browser along with the media payload
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Real-Time Updates</title>
</head>
<body>
<h1>Data from the server:</h1>
<div id="data" class="center-div"></div>
<div id="payload" class="center-div"></div>
<!-- <figure>
<audio controls src="/media/cc0-audio/t-rex-roar.mp3">
<a href="/media/cc0-audio/t-rex-roar.mp3"> Download audio </a>
</audio>
</figure> -->
<script>
const eventSource = new EventSource('/events');
eventSource.onmessage = (event) => {
const data = JSON.parse(event.data);
document.getElementById('data').innerText = `Connection State: ${data.connectionState}`;
document.getElementById('payload').innerText = `PayloadData: ${data.payloadData}`;
};
</script>
</body>
</html>
I want to be able to listen to call in browser along with the ability to controle (pause play , adjust volume)
Any help would be much appreacited
I have tried to:
- play the call via the speaker but the audio is terrible it only play a pulse of static every second
- save the audio being gathered from the stream with the help of this package twilio-media-stream-save-audio-file but the result wav file is playing in slow-mo and he audio is terrible aswell
Not an expert on Python but here is an overview.
Add Twilio client library on front side
Then Send a request on the server side to fetch the access token. While making an access token you will provide a unique identity like this
Return this to front side and on front side use this token to initialize the Twilio device
I assume you already gave the callback URL to the twiml app associated with the twilio number.
Now When I call comes on Twilio number. It will hit the callback URL where you will forward it to browser agent like this
This will place the call to the user who have this $uniqueIdentity.
Hope this helps.