=========================
Part VI - Streaming
=========================

In this part, we will learn how to visualize the drone's video stream.

Create a Janus Media Server Client Service
-----------------------

Let's start by installing the janus-gateway library, by running the following command:

``npm install janus-gateway-js --save``

Now, create a new service to develop your logic and connections to the beXStream Janus Media Server.

Run the following command to generate the necessary files:

``ng generate service lib/services/janus``

On the drone janus Service, add the client variables, methods, and instantiate a janusClient:

.. highlight:: ts

.. code-block::
    :linenos:
    :caption: janus.service.ts

    import {ElementRef, Injectable} from '@angular/core';

    @Injectable({
      providedIn: 'root'
    })
    export class JanusService {
      // @ts-ignore
      Janus = require('janus-gateway-js');
      janusPath = 'wss://bexstream-preprod.beyond-vision.com/janus';
      janusClient: any; // Holds the Janus Client object, used to generate connections
      streaming: any; // Streaming object
      videoElement: ElementRef<HTMLVideoElement> | undefined; // DOM element to retain the stream
      connection: any; // Holds the connection to Janus
      videoStopped: boolean; // Used to maintain status of a video stream. You may want to change to an array if you want to have multiple streams
      currentWatch: number | null; // Current streaming being watched

      constructor() {
        this.janusClient = new this.Janus.Client(this.janusPath, {
          debug: 'all',
          keepalive: 'true',
          pc: {config: {
              iceServers: [{
                username: 'coturn',
                credential: 'coturn',
                urls: [ 'turn:213.63.138.90:3478?transport=udp',
                ],
              }]
            },
            iceTransportPolicy: 'relay', },
        });
      }

      // Helper methods to attach/reattach a stream to a video element (previously part of adapter.js)
      attachMediaStream(element: HTMLVideoElement, stream: any) {
        try {
          element.srcObject = stream;

        } catch (e) {
          try {
            element.src = URL.createObjectURL(stream);
          } catch (e) {
            console.log('Janus:error: Error attaching stream to element', e);
          }
        }
      }

      // creates a Janus session
      createConnection(videoViewer: ElementRef<HTMLVideoElement>) {
        this.videoElement = videoViewer;
        if (!this.connection) {
          this.janusClient.createConnection().then((connection: any) => {
            this.connection = connection;
            console.log('New connection', this.connection);
            this.connection.createSession().then((session: any) => {
              session.attachPlugin('janus.plugin.streaming').then((streaming: any) => {
                this.streaming = streaming;
                console.log('Streaming plugin', this.streaming);
                this.streaming.on('pc:track:remote', (event: any) => {
                  console.log('Remote event', event);
                  const pc = event.target;
                  if (pc) {
                    pc.iceconnectionstatechange = (e: any) => {
                      console.log('iceconnectionstatechange', e);
                      if (pc.iceConnectionState === 'failed') {
                        pc.restartIce();
                      }
                    };
                  }
                  if (this.videoElement && this.videoElement.nativeElement) {
                    const vid = this.videoElement.nativeElement;
                    console.log('Attach Video');
                    this.attachMediaStream(vid, event.streams[0]);
                  }
                });
                // ACK from Janus after stream is stopped
                this.streaming.on('message', (event: any) => {
                  if (!this.videoStopped && event._plainMessage.janus === 'hangup') {
                    console.log('Hangup Event', event._plainMessage.janus);
                    this.videoStopped = true;
                  }
                });
                // ACK from Janus when Asset disconnects
                this.streaming.on('detach', (event: any) => {
                  if (!this.videoStopped && event._plainMessage.janus === 'hangup') {
                    console.log('Detach Event', event._plainMessage.janus);
                    this.videoStopped = true;
                  }
                });
              });
            });
          }, ((error: any) => {
              console.log('Error connecting janus', error);
            }
          ));
        } else {
          if (this.streaming) {
            console.log('Streaming plugin already created');
          }
        }
      }

      // obtain the stream link, requested by the watch method
      watchJanusStream(streamID: number) {
          const callback = () => {
              this.streaming.watch(streamID).then( () => {
                this.currentWatch = streamID;
              }).catch((error: any) => {
                console.log('Janus:error: Attempt to watch', error);
              });
          };
          setTimeout(callback, 50);
      }

    }

In general, when using the Janus features, you would normally do the following (and that's exactly what the provided code does):

#. include the Janus JavaScript library in your web page;
#. initialize the Janus JavaScript library and (optionally) passing its dependencies;
#. connect to the server and create a session;
#. create one or more handles to attach to a plugin (e.g., echo test and/or streaming);
#. interact with the plugin (sending/receiving messages, negotiating a PeerConnection);
#. eventually, close all the handles and shutdown the related PeerConnections;
#. destroy the session.

If you look closely, this logic is divided in the presented methods

#. ``constructor()``: initialize the Janus JavaScript library
#. ``createConnection()``: connect to the server and create a session and create one handler to attach to the streaming plugin
#. ``watchJanusStream()``: requests a stream (sending/receiving messages, negotiating a PeerConnection)
#. ``attachMediaStream()``: attachs the janus object to the DOM video object

Once the library has been initialized, you can start creating sessions. Normally, each browser tab will need a single session with the server: in fact, each Janus session can contain several different plugin handles at the same time, meaning you can start several different WebRTC sessions with the same or different plugins for the same user using the same Janus session. That said, you're free to set up different Janus sessions in the same page, should you prefer so.

Creating a session is quite easy. You just need to use the new constructor to create a new Janus object that will handle your interaction with the server. Considering the dynamic and asynchronous nature of Janus sessions (events may occur at any time), there are several properties and callbacks you can configure when creating a session:

* ``server``: the address of the server as a specific address (e.g., wss://bexstream-preprod.beyond-vision.com/janus for WebSockets; **on the tutorial we've choosen to use WebSockets communications**) or as an array of addresses to try sequentially to allow automatic for fallback/failover during setup;
* ``iceServers``: a list of STUN/TURN servers to use (a default STUN server will be used if you skip this property);
* ``ipv6``: whether IPv6 candidates should be gathered or not;
* ``withCredentials``: whether the withCredentials property of XHR requests should be enabled or not (false by default, and only valid when using HTTP as a transport, ignored for WebSockets);
* ``max_poll_events``: the number of events that should be returned when polling; the default is 1 (polling returns an object), passing a higher number will have the backend return an array of objects instead (again, only valid for HTTP usage as this is strictly related to long polling, ignored for WebSockets);
* ``destroyOnUnload``: whether we should destroy automatically try and destroy this session via Janus API when onbeforeunload is called (true by default);
* ``token`` , ``apisecret``: optional parameters only needed in case you're Authenticating the Janus API ;
* a set of callbacks to be notified about events, namely:
    * ``success``: the session was successfully created and is ready to be used;
    * ``error``: the session was NOT successfully created;
    * ``destroyed``: the session was destroyed and can't be used any more.

These properties and callbacks are passed to the method as properties of a single parameter object: that is, the Janus constructor takes a single parameter, which although acts as a container for all the available options. The success callback is where you typically start your application logic, e.g., attaching the peer to a plugin and start a media session.


Drone Video Player
--------------------------------

Now it's time to create the required DOM object and logic to display the stream. Start by creating a dedicated component for the video-stream.

Video-stream component
==========================

The video stream component will be only responsible to attach a Janus Stream to a DOM video object. All the remaining logic will be left for other fragments.

To initialize the video-stream component, run the command:

``ng generate component drone-video/component/video/video-stream``

Afterwards, update the typescript code, with the following block:

.. highlight:: ts

.. code-block::
    :linenos:
    :caption: video-stream.component.ts

    import {AfterViewInit, Component, ElementRef, OnDestroy, ViewChild} from '@angular/core';
    import {JanusService} from "../../../../lib/services/janus.service";

    @Component({
      selector: 'app-video-stream',
      templateUrl: './video-stream.component.html',
      styleUrls: ['./video-stream.component.css']
    })
    export class VideoStreamComponent implements AfterViewInit, OnDestroy {

      constructor(private janusService: JanusService) {}

      @ViewChild('videostream', { static: false, read: ElementRef }) videoElement: ElementRef<HTMLVideoElement> | undefined;
      ngAfterViewInit(): void {
        if (this.videoElement) {
          this.janusService.createConnection(this.videoElement);
        }
      }

      ngOnDestroy(): void {
        console.log('Closing Video Stream');
      }
    }


Update the video-stream.component.html

.. code-block:: html+ng2
    :linenos:
    :caption: video-stream.component.html

    <video class="rounded centered" style="width: 80%" #videostream id="videostream"  [muted]="true" controls autoplay playsinline></video>


Viewer component
==========================

To test if all the previous developments work, let's create a simple viewer component. You can init it by running the following command:

``ng generate component drone-video/component/viewer``

On the viewer, add Janus Service and Asset Service to the new component. Note that to simplify the tutorial, we have choosen to visualize the first drone with a stream.
*As an exercise, we suggest to you modify the provided code, and try to connect to another aircraft within your organization!*
If you want to dive any deeper, you can explore the possibility to visualize multiple drone streams in parallel or render 360º streams using canvas!
All of this is possible with our media server, and the possibilities are endless.

Start by adding a new variable ``Stream`` to our Asset class. Generate it using:

``ng generate class --skip-tests=true drone/models/stream``

And add the variable ``mountPoint``, which will represent the stream id for a given drone.

.. code-block:: ts
    :caption: stream.ts

    export class Stream {
      mountPoint: number;
    }

You will also need to add import and add this variable to the asset class (drone itself):



.. code-block::
    :linenos:
    :caption: asset.ts
    :emphasize-lines: 2, 12

    import { Drone } from "./drone";
    import { Stream } from "./stream";

    export class Asset {

        id: string;
        name: string;
        mountPoint: number | null;
        isActive: boolean;
        drone?: Drone;
        lastConnected: Date | null;
        stream?: Stream;

        constructor(type: string) {
            if (type === 'Drone') {
              this.drone = new Drone();
            }
            this.id = '';
            this.name = '';
            this.mountPoint = null;
            this.isActive = false;
            this.lastConnected = null;

        }
    }



Modify the viewer component typescript with the following code:




.. code-block:: ts
    :linenos:
    :caption: viewer.component.ts

    import { Component, OnInit } from '@angular/core';
    import {JanusService} from "../../../lib/services/janus.service";
    import {AssetService} from "../../../drone/services/asset.service";
    import {PaginatorDto} from "../../../lib/models/paginator.dto";
    import {Asset} from "../../../drone/models/asset";

    @Component({
      selector: 'app-viewer',
      templateUrl: './viewer.component.html',
      styleUrls: ['./viewer.component.css']
    })
    export class ViewerComponent implements OnInit {

      paginator: PaginatorDto = new PaginatorDto();
      hasStream = false;
      selectedDrone: Asset;

      constructor(
        private assetService: AssetService,
        private janusService: JanusService,
      ) { }

      ngOnInit(): void {
        this.assetService
          .getAllDrones(this.paginator)
          .subscribe((assets) => {
            // You can change this logic for your own
            // For the demonstration purposes, we will just find the first available stream

            for (let i = 0; i < assets.length; i++) {
              const asset = assets[i]
              if(asset.stream) {
                this.selectedDrone = asset;
                this.hasStream = true;
                setTimeout( () => {this.janusService.watchJanusStream(asset.stream.mountPoint)}, 1000);
                // Nothing else to init, so we can just break here...
                break;
              }
            }
          });
      }
    }

Also, update the viewer.component DOM:

.. code-block:: html+ng2
    :linenos:
    :caption: viewer.component.html

    <h2>Welcome to the Drone Video Player</h2>
    <div *ngIf="hasStream">
      <p>Drone {{selectedDrone.name}}</p>
      <!-- video stream -->
      <app-video-stream></app-video-stream>
    </div>
    <p *ngIf="!hasStream">Looking for a drone with an active stream. If this take too long, check your simulation/video-stream and trying refreshing this page.</p>


If you manage to watch the stream of your drone, congratulations! You have already have everything to visualize drone streams. If there's a drone with a stream at the Tutorial Organization, you should have something like the following:

.. image:: ../../_static/images/bexstream/tutorial/stream.avif
    :align: center

If there are not any drone connected with a stream online, either you can run a simulated HEIFU with stream capabilities, or you can request Beyond Vision to launch an aircraft with streaming capabilities. Email us at info@beyond-vision.com

Now, feel free to adapt the tutorial to any combination that better suits your needs, such as multiple streams in parallel, video stream relay, and much more.


Adding stream viewer to drone list
--------------------------------

Wouldn't it be cool, if you could pilot and visualize the stream at the same time? Let's get it done!

Start by modifying the drone-list component. The modifications are quire simple. Add the Janus Service to your typescript, and call the watch method to whenever a new drone is selected:

First modification - add the Janus Service:

.. code-block::
    :linenos:
    :caption: drone-list.component.ts
    :emphasize-lines: 10, 28

    import { Component, OnDestroy, OnInit } from '@angular/core';
    import { Subscription } from 'rxjs';
    import { PaginatorDto } from 'src/app/lib/models/paginator.dto';
    import { Asset } from '../../models/asset';
    import { MavrosState } from '../../models/mavrosState';
    import { MissionDrone } from '../../models/missionDrone';
    import { Position } from '../../models/position';
    import { Velocity } from '../../models/velocity';
    import { AssetService } from '../../services/asset.service';
    import { JanusService } from "../../../lib/services/janus.service";

    @Component({
      selector: 'app-drone-list',
      templateUrl: './drone-list.component.html',
      styleUrls: ['./drone-list.component.less']
    })
    export class DroneListComponent implements OnInit, OnDestroy {

      assets: Asset[] = [];
      paginator: PaginatorDto = new PaginatorDto();
      subscriptions: Subscription = new Subscription();

      selectedAsset: Asset = null;

      DRONE_FE_MAX_SPEED = 3.0; // Drone Frontend Constant Default Value

      constructor(private assetService: AssetService,
                  private janusService: JanusService,) { }



Secondly, call the watch method whenever a new drone is selected:

.. code-block::
    :linenos:
    :caption: drone-list.component.ts
    :emphasize-lines: 6

      public selectAsset(selectedAsset: Asset) {
        this.selectedAsset = selectedAsset;
        this.selectedAsset.drone.md = new MissionDrone();
        if(selectedAsset.stream) {
          setTimeout( () => {
            this.janusService.watchJanusStream(selectedAsset.stream.mountPoint)}, 1000);

        }
        this.initSubscriptions();
      }


Now, all that is required, is to add our ``app-video-stream`` to the drone-list dom. Let's do it:


.. code-block:: html+ng2
    :linenos:
    :caption: drone-list.component.html
    :emphasize-lines: 25, 26, 47-49, 52-53

    <h3>Drones List</h3>
    <hr><hr>
    <div *ngFor="let asset of assets">
        <h5>{{ asset.name }}</h5>
        <p>{{ asset.drone.id }}</p>
        <p *ngIf="asset.drone.latitude">Latitude: {{ asset.drone.latitude }}, Longitude: {{ asset.drone.longitude }}</p>
        <p *ngIf="asset.drone.onMission">On Mission</p>
        <p *ngIf="!asset.drone.onMission">Not on Mission</p>
        <p *ngIf="asset.isActive">Active</p>
        <p>Last Connected: {{asset.lastConnected}}</p>
        <p>
            <button (click)="downloadConfig(asset.id)">
                Download Config File
            </button>
        </p>

        <p *ngIf="asset.isActive && (!selectedAsset || asset.id !== selectedAsset.id)"><button (click)="selectAsset(asset)">Select Drone</button></p>

        <p *ngIf="asset.isActive && selectedAsset && asset.id === selectedAsset.id">
            <span><b>Latitude:</b> {{ asset.drone.md.position.latitude }}</span> <br>
            <span><b>Longitude:</b> {{ asset.drone.md.position.longitude }}</span> <br>
            <span><b>Altitude:</b> {{ asset.drone.md.position.altitude }}</span> <br>
        </p>

        <div *ngIf="asset.isActive && selectedAsset && asset.id === selectedAsset.id">
          <table>
              <tbody>
                  <tr>
                      <td *ngIf="selectedAsset.drone.flightStatus.isLanded"><button (click)="takeOffDrone()">TakeOff</button></td>
                      <td *ngIf="!selectedAsset.drone.flightStatus.isLanded"><button (click)="landDrone()">Land</button></td>
                  </tr>

                  <tr>
                      <td>&nbsp;</td>
                  </tr>

                  <tr *ngIf="!selectedAsset.drone.flightStatus.isLanded">
                      <td colspan="2" align="center"><button (click)="moveDroneForward()">Move Forward</button></td>
                  </tr>
                  <tr>
                      <td><button (click)="moveDroneLeft()">Move Left</button></td>
                      <td><button (click)="moveDroneRight()">Move Right</button></td>
                  </tr>
                  <tr>
                      <td colspan="2" align="center"><button (click)="moveDroneBack()">Move Back</button></td>
                  </tr>
                  <tr>
                    <td>&nbsp;</td>
                  </tr>
              </tbody>
          </table>
          <app-video-stream></app-video-stream>
        </div>

      <hr>
    </div>




Easy enough, right? You are all set now!

Hammer Time!!!
--------------------------------

If you reached this point, there's nothing stopping you! Not even the sky is the limit!

Start developing your applications on top of what you just learned, and pilot more and more aircraft! If you want to do it in the real-world, rather than in the simulated environment, purchase an HEIFU aircraft.

.. image:: ../../_static/images/bexstream/tutorial/pilot_and_see.avif
    :align: center