Comment développer un analogue de Zoom pour les boîtiers TV sur RDK et Linux. Comprendre le framework GStreamer

La pandémie COVID-19 est devenue un catalyseur pour de nouveaux services utiles. Par exemple, Zoom a connu un tel succès qu'il a dépassé IBM en valeur ce mois-ci. Nous nous sommes inspirés de cet exemple, et nous avons décidé d'aller encore plus loin: et si les conférences en ligne étaient implémentées sur des décodeurs et des Smart TV afin de communiquer non seulement au travail, mais d'organiser des rassemblements à distance sur le canapé avec des amis? Mais ensuite, vous pouvez crier ensemble au football, regarder un film ou faire du sport sous la supervision d'un entraîneur. 

- , - Linux/Android RDK. « Zoom» Smart TV. GStreamer. , .

- . , desktop-, , , embedded- .

, -:

1. . STB-   ARM-, , / . , — .

2. . Android, — RDK, — Linux . . desktop-. .

3. . Ethernet wifi. / — .

4. . .

5. .

. Zoom - :

• /

• /

:

GStreamer, .. .

/  

1) GStreamer

, . , 30 640x480. , RGB24 :

640 480 3 30 = 27 648 000 , .. 26 , .

— - . , , GStreamer. ? :

1. Linux Android.

2. RDK Gstreamer / -.

3. , . FFmpeg, , - GStreamer’.

4. (pipeline). / , , .

5. API /C++ .

6. /, OpenMAX API — -.

2) GStreamer  

, , .  GStreamer , :

gst-inspect-1.0 , , , .

gst-launch-1.0 (pipeline).

GStreamer , , source, sink-. source — , ,  (sink) — , , ( RTP).

. mp4-:

gst-launch-1.0 filesrc location=file.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink

mp4-, mp4 — qtdemux, h264, , , , .

autovideosink filesink .

3) GStreamer C/C++ API.

, gst-launch-1.0, , . : (pipeline), GStreamer glib-.

filesrc filesink — «GStreamer: ». H264-.

GStreamer-

gstinit (NULL, NULL);

,

gst_debug_set_active(TRUE);
gst_debug_set_default_threshold(GST_LEVEL_LOG);

: , gstinit .

event-loop, :

GMainLoop *loop;
loop = g_main_loop_new (NULL, FALSE);

:

, GstElement:

GstElement *pipeline, *source, *demuxer, *parser, *decoder, *conv, *sink;

pipeline = gst_pipeline_new ("video-decoder");
source   = gst_element_factory_make ("filesrc",       "file-source");
demuxer  = gst_element_factory_make ("qtdemux",      "h264-demuxer");
parser   = gst_element_factory_make ("h264parse",      "h264-parser");
decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");
conv     = gst_element_factory_make ("videoconvert",  "converter");
sink     = gst_element_factory_make ("appsink", "video-output");

gst_element_factory_make, , — GStreamer, , , .

, , gst_element_factory_make NULL.

if (!pipeline || !source || !demuxer || !parser || !decoder || !conv || !sink) {
		//     - 
		return;
}

location gob_ject_set:

gob_ject_set (G_OBJECT (source), "location", argv[1], NULL);

.

GStreamer, bus_call:

GstBus *bus;
	guint bus_watch_id;
	bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
	bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
	gst_object_unref (bus);

gst_object_unref .

:

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data)
{
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      LOGI ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      LOGE ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

: , gst-launch. , , :

gst_bin_add_many (GST_BIN (pipeline), source, demuxer, parser, decoder, conv, sink, NULL);
gst_element_link_many (source, demuxer, parser, decoder, conv, sink, NULL);

, , (autovideosink) :

  gst_element_link (source, demuxer);
  gst_element_link_many (parser, decoder, conv, sink, NULL);
  g_signal_connect (demuxer, "pad-added", G_CALLBACK (on_pad_added), parser);

static void
on_pad_added (GstElement *element,
              GstPad     *pad,
              gpointer    data)
{
  GstPad *sinkpad;
  GstElement *decoder = (GstElement *) data;

  /* We can now link this pad with the sink pad */
  g_print ("Dynamic pad created, linking demuxer/decoder\n");

  sinkpad = gst_element_get_static_pad (decoder, "sink");

  gst_pad_link (pad, sinkpad);

  gst_object_unref (sinkpad);
}

, .

, , :

gst_element_set_state (pipeline, GST_STATE_PLAYING);

event-loop:

g_main_loop_run (loop);

:

gst_element_set_state (pipeline, GST_STATE_NULL);
  gst_object_unref (GST_OBJECT (pipeline));
  g_source_remove (bus_watch_id);
  g_main_loop_unref (loop);

4) .

, — , .

gst_element_factory_find, , factory :

if(gst_element_factory_find("omxh264dec"))
		decoder  = gst_element_factory_make ("omxh264dec",     "h264-decoder");
	else
		decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");

OMX RDK, .

, , GstElement ( ):

gst_plugin_feature_get_name(gst_element_get_factory(encoder))

.

5)

, . YUV, RGB.

YUYV. , GStreamer, I420. , gl-, I420-. . , .

GStreamer’ , , - . 

1)

  . , , filesrc filesink .

appsrc / appsink. - . 

, ? , . , I420. , ? ?

need-data, :

g_signal_connect (source, "need-data", G_CALLBACK (encoder_cb_need_data), NULL);

:

encoder_cb_need_data (GstElement *appsrc,
          guint       unused_size,
          gpointer    user_data)
{
  GstBuffer *buffer;
  GstFlowReturn ret;
  GstMapInfo map;

   int size;
   uint8_t* image;
  // get image

  buffer = gst_buffer_new_allocate (NULL, size, NULL);
  gst_buffer_map (buffer, &map, GST_MAP_WRITE);
  
  memcpy((guchar *)map.data, image,  gst_buffer_get_size( buffer ) );
  gst_buffer_unmap(buffer, &map);

  g_signal_emit_by_name (appsrc, "push-buffer", buffer, &ret);
  gst_buffer_unref(buffer);
}

image — , , I420.

gst_buffer_new_allocate , .

gst_buffer_map , memcpy, . 

, , GStream’ , .

: gst_buffer_unmap, gst_buffer_unref. . , , , .

, , : caps .

need-data:

	g_object_set (G_OBJECT (source),
        "stream-type", 0,
        "format", GST_FORMAT_TIME, NULL);

	g_object_set (G_OBJECT (source), "caps",
		gst_caps_new_simple ("video/x-raw",
					"format", G_TYPE_STRING, "I420",
					"width", G_TYPE_INT, 640,
					"height", G_TYPE_INT, 480,
					"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL), 
	NULL);

GstElement, g_object_set.

, caps — . , appsrc I420 c 640x480 30 .

, , . , GStreamer - need-data .

, . 

2)

, .

sink pad:

	GstPad *pad = gst_element_get_static_pad (sink, "sink");
  	gst_pad_add_probe  (pad, GST_PAD_PROBE_TYPE_BUFFER, encoder_cb_have_data, NULL, NULL);
  	gst_object_unref (pad);

sink pad — GST_PAD_PROBE_TYPE_BUFFER, — sink pad.

static GstPadProbeReturn
encoder_cb_have_data (GstPad * pad,
                        GstPadProbeInfo * info,
                        gpointer user_data) {

  GstBuffer *buf = gst_pad_probe_info_get_buffer (info);
  GstMemory *bufMem = gst_buffer_get_memory(buf, 0);
  GstMapInfo bufInfo;
  gst_memory_map(bufMem, &bufInfo, GST_MAP_READ);

  // bufInfo.data, bufInfo.size

  gst_memory_unmap(bufMem, &bufInfo);

  return GST_PAD_PROBE_OK;
}

. . GstBuffer, , gst_buffer_get_memory 0 ( ). , , gst_memory_map, bufInfo.data bufInfo.size.

— .

, Smart TV — Zoom -: , / GStreamer, / .

. — — embedded- RDK, Linux Android. — , .

Cette idée avec un service de visioconférence via Smart TV peut être développée davantage, à la fois en termes de solutions d'ingénierie et de scénarios d'utilisation. Alors partagez vos réflexions dans les commentaires.