screen sharing for desktop. (#135)

* screen sharing for desktop (WIP).

* use git repo for testing.

* update.

* Add sourceId/frameRate to ScreenShareCaptureOptions.

* Compatible with flutter web.

* Rendering of camera and screenshare video tracks at same time.

* Remote screen sharing is shown first, and fixed sub/unsub button for screenshare widget.

* resolved conflict.

* modify flutter-webrtc to pub version.

* remove unused import.

* chore: Common params for Camera and ScreenShare.

Author: cloudwebrtc

example/lib/pages/room.dart

@@ -7,6 +7,7 @@ import 'package:livekit_client/livekit_client.dart';
 import '../exts.dart';
 import '../widgets/controls.dart';
 import '../widgets/participant.dart';
+import '../widgets/participant_info.dart';
 
 class RoomPage extends StatefulWidget {
   //
@@ -25,7 +26,7 @@ class RoomPage extends StatefulWidget {
 
 class _RoomPageState extends State<RoomPage> {
   //
-  List<Participant> participants = [];
+  List<ParticipantTrack> participantTracks = [];
   EventsListener<RoomEvent> get _listener => widget.listener;
   bool get fastConnection => widget.room.engine.fastConnectOptions != null;
   @override
@@ -57,6 +58,8 @@ class _RoomPageState extends State<RoomPage> {
       WidgetsBindingCompatible.instance
           ?.addPostFrameCallback((timeStamp) => Navigator.pop(context));
     })
+    ..on<LocalTrackPublishedEvent>((_) => _sortParticipants())
+    ..on<LocalTrackUnpublishedEvent>((_) => _sortParticipants())
     ..on<DataReceivedEvent>((event) {
       String decoded = 'Failed to decode';
       try {
@@ -90,47 +93,74 @@ class _RoomPageState extends State<RoomPage> {
   }
 
   void _sortParticipants() {
-    List<Participant> participants = [];
-    participants.addAll(widget.room.participants.values);
+    List<ParticipantTrack> userMediaTracks = [];
+    List<ParticipantTrack> screenTracks = [];
+    for (var participant in widget.room.participants.values) {
+      for (var t in participant.videoTracks) {
+        if (t.isScreenShare) {
+          screenTracks.add(ParticipantTrack(
+            participant: participant,
+            videoTrack: t.track,
+            isScreenShare: true,
+          ));
+        } else {
+          userMediaTracks.add(ParticipantTrack(
+            participant: participant,
+            videoTrack: t.track,
+            isScreenShare: false,
+          ));
+        }
+      }
+    }
     // sort speakers for the grid
-    participants.sort((a, b) {
+    userMediaTracks.sort((a, b) {
       // loudest speaker first
-      if (a.isSpeaking && b.isSpeaking) {
-        if (a.audioLevel > b.audioLevel) {
+      if (a.participant.isSpeaking && b.participant.isSpeaking) {
+        if (a.participant.audioLevel > b.participant.audioLevel) {
           return -1;
         } else {
           return 1;
         }
       }
 
       // last spoken at
-      final aSpokeAt = a.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
-      final bSpokeAt = b.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
+      final aSpokeAt = a.participant.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
+      final bSpokeAt = b.participant.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
 
       if (aSpokeAt != bSpokeAt) {
         return aSpokeAt > bSpokeAt ? -1 : 1;
       }
 
       // video on
-      if (a.hasVideo != b.hasVideo) {
-        return a.hasVideo ? -1 : 1;
+      if (a.participant.hasVideo != b.participant.hasVideo) {
+        return a.participant.hasVideo ? -1 : 1;
       }
 
       // joinedAt
-      return a.joinedAt.millisecondsSinceEpoch -
-          b.joinedAt.millisecondsSinceEpoch;
+      return a.participant.joinedAt.millisecondsSinceEpoch -
+          b.participant.joinedAt.millisecondsSinceEpoch;
     });
 
-    final localParticipant = widget.room.localParticipant;
-    if (localParticipant != null) {
-      if (participants.length > 1) {
-        participants.insert(1, localParticipant);
-      } else {
-        participants.add(localParticipant);
+    final localParticipantTracks = widget.room.localParticipant?.videoTracks;
+    if (localParticipantTracks != null) {
+      for (var t in localParticipantTracks) {
+        if (t.isScreenShare) {
+          screenTracks.add(ParticipantTrack(
+            participant: widget.room.localParticipant!,
+            videoTrack: t.track,
+            isScreenShare: true,
+          ));
+        } else {
+          userMediaTracks.add(ParticipantTrack(
+            participant: widget.room.localParticipant!,
+            videoTrack: t.track,
+            isScreenShare: false,
+          ));
+        }
       }
     }
     setState(() {
-      this.participants = participants;
+      participantTracks = [...screenTracks, ...userMediaTracks];
     });
   }
 
@@ -139,18 +169,19 @@ class _RoomPageState extends State<RoomPage> {
         body: Column(
           children: [
             Expanded(
-                child: participants.isNotEmpty
-                    ? ParticipantWidget.widgetFor(participants.first)
+                child: participantTracks.isNotEmpty
+                    ? ParticipantWidget.widgetFor(participantTracks.first)
                     : Container()),
             SizedBox(
               height: 100,
               child: ListView.builder(
                 scrollDirection: Axis.horizontal,
-                itemCount: math.max(0, participants.length - 1),
+                itemCount: math.max(0, participantTracks.length - 1),
                 itemBuilder: (BuildContext context, int index) => SizedBox(
                   width: 100,
                   height: 100,
-                  child: ParticipantWidget.widgetFor(participants[index + 1]),
+                  child:
+                      ParticipantWidget.widgetFor(participantTracks[index + 1]),
                 ),
               ),
             ),

example/lib/widgets/controls.dart

@@ -6,6 +6,7 @@ import 'package:eva_icons_flutter/eva_icons_flutter.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_background/flutter_background.dart';
 import 'package:livekit_client/livekit_client.dart';
+import 'package:flutter_webrtc/flutter_webrtc.dart';
 
 import '../exts.dart';
 
@@ -86,9 +87,30 @@ class _ControlsWidgetState extends State<ControlsWidget> {
   }
 
   void _enableScreenShare() async {
-    await participant.setScreenShareEnabled(true);
-
-    if (Platform.isAndroid) {
+    if (WebRTC.platformIsDesktop) {
+      try {
+        final source = await showDialog<DesktopCapturerSource>(
+          context: context,
+          builder: (context) => ScreenSelectDialog(),
+        );
+        if (source == null) {
+          print('cancelled screenshare');
+          return;
+        }
+        print('DesktopCapturerSource: ${source.id}');
+        var track = await LocalVideoTrack.createScreenShareTrack(
+          ScreenShareCaptureOptions(
+            sourceId: source.id,
+            maxFrameRate: 15.0,
+          ),
+        );
+        await participant.publishVideoTrack(track);
+      } catch (e) {
+        print('could not publish video: $e');
+      }
+      return;
+    }
+    if (WebRTC.platformIsAndroid) {
       // Android specific
       try {
         // Required for android screenshare.
@@ -105,6 +127,7 @@ class _ControlsWidgetState extends State<ControlsWidget> {
         print('could not publish video: $e');
       }
     }
+    await participant.setScreenShareEnabled(true);
   }
 
   void _disableScreenShare() async {

example/lib/widgets/participant.dart

@@ -10,17 +10,25 @@ import 'participant_info.dart';
 
 abstract class ParticipantWidget extends StatefulWidget {
   // Convenience method to return relevant widget for participant
-  static ParticipantWidget widgetFor(Participant participant) {
-    if (participant is LocalParticipant) {
-      return LocalParticipantWidget(participant);
-    } else if (participant is RemoteParticipant) {
-      return RemoteParticipantWidget(participant);
+  static ParticipantWidget widgetFor(ParticipantTrack participantTrack) {
+    if (participantTrack.participant is LocalParticipant) {
+      return LocalParticipantWidget(
+          participantTrack.participant as LocalParticipant,
+          participantTrack.videoTrack,
+          participantTrack.isScreenShare);
+    } else if (participantTrack.participant is RemoteParticipant) {
+      return RemoteParticipantWidget(
+          participantTrack.participant as RemoteParticipant,
+          participantTrack.videoTrack,
+          participantTrack.isScreenShare);
     }
     throw UnimplementedError('Unknown participant type');
   }
 
   // Must be implemented by child class
   abstract final Participant participant;
+  abstract final VideoTrack? videoTrack;
+  abstract final bool isScreenShare;
   final VideoQuality quality;
 
   const ParticipantWidget({
@@ -32,9 +40,15 @@ abstract class ParticipantWidget extends StatefulWidget {
 class LocalParticipantWidget extends ParticipantWidget {
   @override
   final LocalParticipant participant;
+  @override
+  final VideoTrack? videoTrack;
+  @override
+  final bool isScreenShare;
 
   const LocalParticipantWidget(
-    this.participant, {
+    this.participant,
+    this.videoTrack,
+    this.isScreenShare, {
     Key? key,
   }) : super(key: key);
 
@@ -45,9 +59,15 @@ class LocalParticipantWidget extends ParticipantWidget {
 class RemoteParticipantWidget extends ParticipantWidget {
   @override
   final RemoteParticipant participant;
+  @override
+  final VideoTrack? videoTrack;
+  @override
+  final bool isScreenShare;
 
   const RemoteParticipantWidget(
-    this.participant, {
+    this.participant,
+    this.videoTrack,
+    this.isScreenShare, {
     Key? key,
   }) : super(key: key);
 
@@ -60,7 +80,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
   //
   bool _visible = true;
   VideoTrack? get activeVideoTrack;
-  TrackPublication? get firstVideoPublication;
+  TrackPublication? get videoPublication;
   TrackPublication? get firstAudioPublication;
 
   @override
@@ -89,12 +109,12 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
   void _onParticipantChanged() => setState(() {});
 
   // Widgets to show above the info bar
-  List<Widget> extraWidgets() => [];
+  List<Widget> extraWidgets(bool isScreenShare) => [];
 
   @override
   Widget build(BuildContext ctx) => Container(
         foregroundDecoration: BoxDecoration(
-          border: widget.participant.isSpeaking
+          border: widget.participant.isSpeaking && !widget.isScreenShare
               ? Border.all(
                   width: 5,
                   color: LKColors.lkBlue,
@@ -112,7 +132,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
               child: activeVideoTrack != null
                   ? VideoTrackRenderer(
                       activeVideoTrack!,
-                      fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover,
+                      fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
                     )
                   : const NoVideoWidget(),
             ),
@@ -124,14 +144,15 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
                 crossAxisAlignment: CrossAxisAlignment.stretch,
                 mainAxisSize: MainAxisSize.min,
                 children: [
-                  ...extraWidgets(),
+                  ...extraWidgets(widget.isScreenShare),
                   ParticipantInfoWidget(
                     title: widget.participant.name.isNotEmpty
                         ? '${widget.participant.name} (${widget.participant.identity})'
                         : widget.participant.identity,
                     audioAvailable: firstAudioPublication?.muted == false &&
                         firstAudioPublication?.subscribed == true,
                     connectionQuality: widget.participant.connectionQuality,
+                    isScreenShare: widget.isScreenShare,
                   ),
                 ],
               ),
@@ -144,60 +165,48 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
 class _LocalParticipantWidgetState
     extends _ParticipantWidgetState<LocalParticipantWidget> {
   @override
-  LocalTrackPublication<LocalVideoTrack>? get firstVideoPublication =>
-      widget.participant.videoTracks.firstOrNull;
+  LocalTrackPublication<LocalVideoTrack>? get videoPublication =>
+      widget.participant.videoTracks
+          .where((element) => element.sid == widget.videoTrack?.sid)
+          .firstOrNull;
 
   @override
   LocalTrackPublication<LocalAudioTrack>? get firstAudioPublication =>
       widget.participant.audioTracks.firstOrNull;
 
   @override
-  VideoTrack? get activeVideoTrack {
-    if (firstVideoPublication?.subscribed == true &&
-        firstVideoPublication?.muted == false &&
-        _visible) {
-      return firstVideoPublication?.track;
-    }
-    return null;
-  }
+  VideoTrack? get activeVideoTrack => widget.videoTrack;
 }
 
 class _RemoteParticipantWidgetState
     extends _ParticipantWidgetState<RemoteParticipantWidget> {
   @override
-  RemoteTrackPublication<RemoteVideoTrack>? get firstVideoPublication =>
-      widget.participant.videoTracks.firstOrNull;
+  RemoteTrackPublication<RemoteVideoTrack>? get videoPublication =>
+      widget.participant.videoTracks
+          .where((element) => element.sid == widget.videoTrack?.sid)
+          .firstOrNull;
 
   @override
   RemoteTrackPublication<RemoteAudioTrack>? get firstAudioPublication =>
       widget.participant.audioTracks.firstOrNull;
 
   @override
-  VideoTrack? get activeVideoTrack {
-    for (final trackPublication in widget.participant.videoTracks) {
-      print(
-          'video track ${trackPublication.sid} subscribed ${trackPublication.subscribed} muted ${trackPublication.muted}');
-      if (trackPublication.subscribed && !trackPublication.muted && _visible) {
-        return trackPublication.track;
-      }
-    }
-    return null;
-  }
+  VideoTrack? get activeVideoTrack => widget.videoTrack;
 
   @override
-  List<Widget> extraWidgets() => [
+  List<Widget> extraWidgets(bool isScreenShare) => [
         Row(
           mainAxisSize: MainAxisSize.max,
           mainAxisAlignment: MainAxisAlignment.end,
           children: [
             // Menu for RemoteTrackPublication<RemoteVideoTrack>
-            if (firstVideoPublication != null)
+            if (videoPublication != null)
               RemoteTrackPublicationMenuWidget(
-                pub: firstVideoPublication!,
-                icon: EvaIcons.video,
+                pub: videoPublication!,
+                icon: isScreenShare ? EvaIcons.monitor : EvaIcons.video,
               ),
             // Menu for RemoteTrackPublication<RemoteAudioTrack>
-            if (firstAudioPublication != null)
+            if (firstAudioPublication != null && !isScreenShare)
               RemoteTrackPublicationMenuWidget(
                 pub: firstAudioPublication!,
                 icon: EvaIcons.volumeUp,