WebRTC, getDisplayMedia() does not capture sound from the remote stream

Question

I have a web application of my own, which is based on the peerjs library (It is a video conference).

I'm trying to make a recording with 'MediaRecorder', but I'm facing a very unpleasant case.

The code for capturing my desktop stream is the following:

let chooseScreen = document.querySelector('.chooseScreenBtn')
chooseScreen.onclick = async () => {
    let desktopStream = await navigator.mediaDevices.getDisplayMedia({ video:true, audio: true });
}

I then successfully apply my received desktopStream to videoElement in DOM:

const videoElement = doc.querySelector('.videoElement')
  videoElement.srcObject = desktopStream 
  videoElement.muted = false;
  videoElement.onloadedmetadata = ()=>{videoElement.play();}

For example, I get desktopStream on the page with an active conference where everyone hears and sees each other.

To check the video and audio in desktopStream I play some video on the video player on the desktop. I can hear any audio from my desktop but audio from any participant cannot be heard. Of course, when I put the desktopStream in MediaRecorder I get a video file with no sound from anyone except my desktop. Any ideas on how to solve it?

Answer 1

Chrome's MediaRecorder API can only output one track. The createMediaStreamSource can take streams from desktop audio and microphone, by connecting both together into one object created by createMediaStreamDestination it gives you the ability to pipe this one stream into the MediaRecorder API.

const mergeAudioStreams = (desktopStream, voiceStream) => {
    const context = new AudioContext();

    // Create a couple of sources
    const source1 = context.createMediaStreamSource(desktopStream);
    const source2 = context.createMediaStreamSource(voiceStream);
    const destination = context.createMediaStreamDestination();

    const desktopGain = context.createGain();
    const voiceGain = context.createGain();

    desktopGain.gain.value = 0.7;
    voiceGain.gain.value = 0.7;

    source1.connect(desktopGain).connect(destination);
    // Connect source2
    source2.connect(voiceGain).connect(destination);

    return destination.stream.getAudioTracks();
}; 

It is also possible to use two or more audio inputs + video input.

window.onload = () => {
    const warningEl = document.getElementById('warning');
    const videoElement = document.getElementById('videoElement');
    const captureBtn = document.getElementById('captureBtn');
    const startBtn = document.getElementById('startBtn');
    const stopBtn = document.getElementById('stopBtn');
    const download = document.getElementById('download');
    const audioToggle = document.getElementById('audioToggle');
    const micAudioToggle = document.getElementById('micAudioToggle');
    
    if('getDisplayMedia' in navigator.mediaDevices) warningEl.style.display = 'none';

    let blobs;
    let blob;
    let rec;
    let stream;
    let voiceStream;
    let desktopStream;
    
    const mergeAudioStreams = (desktopStream, voiceStream) => {
        const context = new AudioContext();
        const destination = context.createMediaStreamDestination();
        let hasDesktop = false;
        let hasVoice = false;
        if (desktopStream && desktopStream.getAudioTracks().length > 0) {
        // If you don't want to share Audio from the desktop it should still work with just the voice.
        const source1 = context.createMediaStreamSource(desktopStream);
        const desktopGain = context.createGain();
        desktopGain.gain.value = 0.7;
        source1.connect(desktopGain).connect(destination);
        hasDesktop = true;
        }
        
        if (voiceStream && voiceStream.getAudioTracks().length > 0) {
        const source2 = context.createMediaStreamSource(voiceStream);
        const voiceGain = context.createGain();
        voiceGain.gain.value = 0.7;
        source2.connect(voiceGain).connect(destination);
        hasVoice = true;
        }
        
        return (hasDesktop || hasVoice) ? destination.stream.getAudioTracks() : [];
    };

    captureBtn.onclick = async () => {
        download.style.display = 'none';
        const audio = audioToggle.checked || false;
        const mic = micAudioToggle.checked || false;
        
        desktopStream = await navigator.mediaDevices.getDisplayMedia({ video:true, audio: audio });
        
        if (mic === true) {
        voiceStream = await navigator.mediaDevices.getUserMedia({ video: false, audio: mic });
        }
    
        const tracks = [
        ...desktopStream.getVideoTracks(), 
        ...mergeAudioStreams(desktopStream, voiceStream)
        ];
        
        console.log('Tracks to add to stream', tracks);
        stream = new MediaStream(tracks);
        console.log('Stream', stream)
        videoElement.srcObject = stream;
        videoElement.muted = true;
        
        blobs = [];
    
        rec = new MediaRecorder(stream, {mimeType: 'video/webm; codecs=vp8,opus'});
        rec.ondataavailable = (e) => blobs.push(e.data);
        rec.onstop = async () => {
        
        blob = new Blob(blobs, {type: 'video/webm'});
        let url = window.URL.createObjectURL(blob);
        download.href = url;
        download.download = 'test.webm';
        download.style.display = 'block';
        };
        startBtn.disabled = false;
        captureBtn.disabled = true;
        audioToggle.disabled = true;
        micAudioToggle.disabled = true;
    };

    startBtn.onclick = () => {
        startBtn.disabled = true;
        stopBtn.disabled = false;
        rec.start();
    };

    stopBtn.onclick = () => {
        captureBtn.disabled = false;
        audioToggle.disabled = false;
        micAudioToggle.disabled = false;
        startBtn.disabled = true;
        stopBtn.disabled = true;
        
        rec.stop();
        
        stream.getTracks().forEach(s=>s.stop())
        videoElement.srcObject = null
        stream = null;
    };
};