Gstreamer recording video with audio

Question

I'm trying to record on a file a video from my webcam along with audio using Gstreamer on my Ubuntu 16 machine through glib library.
I'm able to watch the video streaming from the webcam through these code lines

#include <gst/gst.h>

int main(int argc, char *argv[]) {
    GstElement *pipeline, *source, *sink, *convert;
    GstBus *bus;
    GstMessage *msg;
    GstStateChangeReturn ret;

    /* Initialize GStreamer */
    gst_init (&argc, &argv);

    /* Create the elements */
    source = gst_element_factory_make ("v4l2src", "source");
    sink = gst_element_factory_make ("autovideosink", "sink");
    convert =gst_element_factory_make("videoconvert","convert");
    //convert = gst_element_factory_make ("audioconvert", "convert");
    //sink = gst_element_factory_make ("autoaudiosink", "sink");

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    if (!pipeline || !source || !sink || !convert) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /*set der source*/
    g_object_set (source, "device", "/dev/video0", NULL);

    /* Build the pipeline */
    gst_bin_add_many (GST_BIN (pipeline), source, sink, convert, NULL);
    if (gst_element_link (convert, sink) != TRUE) {
        g_printerr ("Elements could not be linked confert sink.\n");
        gst_object_unref (pipeline);
        return -1;
    }


    if (gst_element_link (source, convert) != TRUE) {
        g_printerr ("Elements could not be linked source -convert.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Start playing */
    ret = gst_element_set_state (pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline to the playing state.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Wait until error or EOS */
    bus = gst_element_get_bus (pipeline);
    msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,(GstMessageType) (GST_MESSAGE_ERROR | GST_MESSAGE_EOS));

    /* Parse message */
    if (msg != NULL) {
        GError *err;
        gchar *debug_info;

        switch (GST_MESSAGE_TYPE (msg)) {
            case GST_MESSAGE_ERROR:
                gst_message_parse_error (msg, &err, &debug_info);
                g_printerr ("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
                g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
                g_clear_error (&err);
                g_free (debug_info);
                break;
            case GST_MESSAGE_EOS:
                g_print ("End-Of-Stream reached.\n");
                break;
            default:
                /* We should not reach here because we only asked for ERRORs and EOS */
                g_printerr ("Unexpected message received.\n");
                break;
        }
        gst_message_unref (msg);
    }

    /* Free resources */
    gst_object_unref (bus);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    gst_object_unref (pipeline);
    return 0;
}

and to capture audio from microphone and listen it through the speakers using these code lines

#include <gst/gst.h>
#include <glib.h>

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data){
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      g_print ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      g_printerr ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

/* Main function for audio pipeline initialization and looping streaming process  */
gint
main (gint argc, gchar **argv) {
    GMainLoop *loop;
    GstElement *pipeline, *audio_source, *sink; 
    GstBus *bus;
    guint bus_watch_id;
    GstCaps *caps;
    gboolean ret;

    /* Initialization of gstreamer */
    gst_init (&argc, &argv);
    loop = g_main_loop_new (NULL, FALSE);

    /* Elements creation */
    pipeline     = gst_pipeline_new ("audio_stream");
    audio_source = gst_element_factory_make ("alsasrc", "audio_source");
    sink   = gst_element_factory_make ("alsasink", "audio_sink");

    // video_source = gst_element_factory_make ("v4l2src", "source");
    // video_sink   = gst_element_factory_make ("autovideosink", "sink");
    // video_convert= gst_element_factory_make("videoconvert","convert");

    if (!pipeline) {
        g_printerr ("Audio: Pipeline couldn't be created\n");
        return -1;
    }
    if (!audio_source) {
        g_printerr ("Audio: alsasrc couldn't be created\n");
        return -1;
    }
    if (!sink) {
        g_printerr ("Audio: Output file couldn't be created\n");
        return -1;
    }

    g_object_set (G_OBJECT (audio_source), "device", "hw:1,0", NULL);
    g_object_set (G_OBJECT (sink), "device", "hw:1,0", NULL);

    bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
    bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
    gst_object_unref (bus);

    gst_bin_add_many (GST_BIN(pipeline), audio_source, sink, NULL);

    caps = gst_caps_new_simple ("audio/x-raw", "format", G_TYPE_STRING, "S16LE",  "layout", G_TYPE_STRING, "interleaved", "rate", G_TYPE_INT, (int)44100, "channels", G_TYPE_INT, (int)2, NULL);
    ret = gst_element_link_filtered (audio_source, sink, caps);
    if (!ret) {
        g_print ("audio_source and sink couldn't be linked\n");
        gst_caps_unref (caps);
        return FALSE;
    }

    gst_element_set_state (pipeline, GST_STATE_PLAYING);

    g_print ("streaming...\n");
    g_main_loop_run (loop);

    g_print ("Returned, stopping stream\n");
    gst_element_set_state (pipeline, GST_STATE_NULL);

    g_print ("Deleting pipeline\n");
    gst_object_unref (GST_OBJECT (pipeline));
    g_source_remove (bus_watch_id);
    g_main_loop_unref (loop);

    return 0;
}

What i really don't understand is how to get video from the webcam and audio from my alsa hw at the same time and save them into a file (such as .mp4 for ex). Can anyone help me? I tried to find something useful, but there's nothing on the board. In addition, it would be really appreciate also how to save just the video stream or just the audio stream in separated files.

UPDATE
I looked again to the tutorials and to the git link gave by @nayana, so i tried myself to code something. I have two results:

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio;
static GstBus *bus;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);
  queue_video = gst_element_factory_make("queue", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !encoder_video || !src_audio || !encoder_audio
        || !queue_video || !queue_audio || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);


  gst_bin_add_many(GST_BIN(pipeline), src_video, encoder_video, queue_video, 
    src_audio, encoder_audio, queue_audio, muxer, sink, NULL);

  gst_element_link_many (src_video,encoder_video,queue_video, muxer,NULL);

  gst_element_link_many (src_audio,encoder_audio,queue_audio, muxer,NULL);

  if (!gst_element_link_many(muxer, sink, NULL)){
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

With this upon i am able to record the video from the cam, but the audio is recorded for just one second somewhere randomly during the recording and it gives me this error

ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Can't record audio fast enough
Additional debug info:
gstaudiobasesrc.c(869): gst_audio_base_src_create (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
Dropped 206388 samples. This is most likely because downstream can't keep up and is consuming samples too slowly.<br>

So i tried to add some setting and queues

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video, *rate_video, *scale_video, *capsfilter_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio, *queue_audio2, *capsfilter_audio, *rate_audio;
static GstBus *bus;
static GstCaps *caps;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  rate_video = gst_element_factory_make ("videorate", NULL);
  scale_video = gst_element_factory_make ("videoscale", NULL);
  capsfilter_video = gst_element_factory_make ("capsfilter", NULL);
  queue_video = gst_element_factory_make("queue", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  capsfilter_audio = gst_element_factory_make ("capsfilter", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);
  rate_audio = gst_element_factory_make ("audiorate", NULL);
  queue_audio2 = gst_element_factory_make("queue", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !rate_video || !scale_video || !capsfilter_video 
     || !queue_video || !encoder_video || !src_audio || !capsfilter_audio 
     || !queue_audio || !rate_audio || !queue_audio2 || !encoder_audio 
     || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  // Set up the pipeline
  g_object_set(src_video, "device", "/dev/video0", NULL); 
  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);

  // video settings
  caps = gst_caps_from_string("video/x-raw,format=(string)I420,width=480,height=384,framerate=(fraction)25/1");
  g_object_set (G_OBJECT (capsfilter_video), "caps", caps, NULL);
  gst_caps_unref (caps); 

  // audio settings
  caps = gst_caps_from_string("audio/x-raw,rate=44100,channels=1");
  g_object_set (G_OBJECT (capsfilter_audio), "caps", caps, NULL);
  gst_caps_unref (caps);

  // add all elements into the pipeline 
  gst_bin_add_many(GST_BIN(pipeline), src_video, rate_video, scale_video, capsfilter_video, 
    queue_video, encoder_video, src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer, sink, NULL);

  if (!gst_element_link_many (src_video,rate_video,scale_video, capsfilter_video,
    queue_video, encoder_video, muxer,NULL))
  {
    g_error("Failed to link video elements");
    return -2;
  }

  if (!gst_element_link_many (src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer,NULL))
  {
    g_error("Failed to link audio elements");
    return -2;
  }

  if (!gst_element_link_many(muxer, sink, NULL))
  {
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

This time the code doesnt record anything and give me the following error

   ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Internal data flow error.
Additional debug info:
gstbasesrc.c(2948): gst_base_src_loop (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
streaming task paused, reason not-negotiated (-4)

Can you address me to fix the error?
Thanks in advance

Answer 1

What you need is the multiplexer - such GStreamer element that can merge two streams into one.

mp4, mkv, avi.. are just a container formats which contains multiple "data streams", which can be audio, video, subtitles (not all formats support this).

I don't know about your use case, but you don't need C code for what you do. You can just use gst-launch-1.0 tool which has its own GStreamer kind-of-scripting language.

For simplicity I will use debugging elements videotestsrc and audiotestsrc for simulating input (instead of actual camera etc).

gst-launch-1.0 -e videotestsrc ! x264enc ! mp4mux name=mux ! filesink location="bla.mp4"  audiotestsrc ! lamemp3enc ! mux.

videotestsrc --> x264enc -----\
                               >---> mp4mux ---> filesink
audiotestsrc --> lamemp3enc --/ 

Explanation:

Videotestsrc generates raw video which is in GStreamer terms called "video/x-raw".

However mp4 cannot hold raw video, so we need to encode it with for example x264enc which makes our data "video/x-h264".

Then we can finally mux this into our mp4 with mp4mux element.

When we take a look into GStreamer docs using gst-inspect-1.0 mp4mux we see that this element supports various formats amongst which there is also video/x-h264.

The same thing we do with audio with either faac for AAC format or lamemp3enc for mp3.

With gst-launch-1.0 I did two tricks and one bonus trick:

1. ability to have separate branches in one line. This is achieved by just separating those branches with space instead of !
2. ability to make alias with name=mux and later on using it with adding dot right at the end of name like mux. . You can make up any name for that element you like.
3. Write EOS after hitting ctrl+c to stop the recording. This is achieved with parameter -e

Finally the output goes to filesink which just writes anything you give it to file.

Now for a homework you:

1. Use your elements for what you need - v4l2, alsasrc

2. Add queue elements to add buffering and thread separation