Modify IBasicVideoEffect values on the fly

Question

I'm trying to create a UWP App which allows you to modify the colors of what the camera is showing on the fly. My current code is based on the Effective example by Nikola Metulev, and it modifies the camera view by using a IBasicVideoEffect. But I need to modify the colors while the camera is active (showing more red, for example).

I've searched for days, and I think my problem is that, as an IBasicVideoEffect must be in a separated class, I don't know how to pass a value (a Matrix5x4 in this case) from the MainPage to the Effect class.

Here is my code:

MainPage Class:

using ColorEffects;
using System;
using Windows.Media.Capture;
using Windows.Media.Effects;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Navigation;

// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409

namespace Color
{
    /// <summary>
    /// An empty page that can be used on its own or navigated to within a Frame.
    /// </summary>
    public sealed partial class MainPage : Page
    {
        private MediaCapture _mediaCapture;

        public MainPage()
        {
            this.InitializeComponent();
        }

        protected async override void OnNavigatedTo(NavigationEventArgs e)
        {
            base.OnNavigatedTo(e);

            _mediaCapture = new MediaCapture();
            var settings = new MediaCaptureInitializationSettings()
            {
                StreamingCaptureMode = StreamingCaptureMode.Video,
            };
            await _mediaCapture.InitializeAsync(settings);
            captureElement.Source = _mediaCapture;
            await _mediaCapture.StartPreviewAsync();

            var effect = await _mediaCapture.AddVideoEffectAsync(
                new VideoEffectDefinition(typeof(TheEffect).FullName),
                                            MediaStreamType.VideoPreview);
        }
    }
}

ColorEffect Class:

using Microsoft.Graphics.Canvas;
using Microsoft.Graphics.Canvas.Brushes;
using Microsoft.Graphics.Canvas.Effects;
using Microsoft.Graphics.Canvas.Text;
using System;
using System.Collections.Generic;
using System.Numerics;
using Windows.Foundation.Collections;
using Windows.Graphics.DirectX.Direct3D11;
using Windows.Media.Effects;
using Windows.Media.MediaProperties;
using Windows.UI.Text;

namespace ColorEffects
{
    public sealed class TheEffect : IBasicVideoEffect
    {
        private IPropertySet _configuration;
        private CanvasDevice _canvasDevice;

        public void ProcessFrame(ProcessVideoFrameContext context)
        {
            using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
            using (var drawingSurface = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface).CreateDrawingSession())
            {
                var matrix = new Matrix5x4() { M11 = 0.5f, M22 = 0.5f, M33 = 1.0f, M44 = 1.0f, M54 = 0.0f };
                var colorEffect = new ColorMatrixEffect()
                {
                    ColorMatrix = matrix,
                    Source = inputBitmap
                };

                drawingSurface.DrawImage(colorEffect);
            }
        }

        public void SetEncodingProperties(VideoEncodingProperties encodingProperties, IDirect3DDevice device)
        {
            _canvasDevice = CanvasDevice.CreateFromDirect3D11Device(device);
        }

        public bool IsReadOnly { get { return false; } }

        public IReadOnlyList<VideoEncodingProperties> SupportedEncodingProperties
        {
            get
            {
                var properties = new List<VideoEncodingProperties>();
                properties.Add(VideoEncodingProperties.CreateUncompressed("ARGB32", 640, 480));
                return properties;
            }
        }

        public MediaMemoryTypes SupportedMemoryTypes { get { return MediaMemoryTypes.Gpu; } }

        public bool TimeIndependent { get { return false; } }

        public void DiscardQueuedFrames() { }

        public void Close(MediaEffectClosedReason reason)
        {
            if (_canvasDevice != null)
                _canvasDevice.Dispose();
        }

        public void SetProperties(IPropertySet configuration)
        {
            _configuration = configuration;
        }
    }
}

Can anyone help my out? Thank you very much!

Answer 1

With a little adjustment to your code, you can do the color change during video running, till you share the same object as a parameter in effect as well as a source object to change color i.e.

public class TheEffectParameter
{
    public Matrix5x4 ColorMatrix { get; set; }
}

public sealed class TheEffect: IBasicVideoEffect
{
    private IPropertySet _configuration;
    private CanvasDevice _canvasDevice;

    public void ProcessFrame(ProcessVideoFrameContext context)
    {
        using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
        using (var drawingSurface = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface).CreateDrawingSession())
        {
            Matrix5x4 matrix;
            if (ColorParameter == null)
            {
                //Default if parameter not passed
                matrix = new Matrix5x4() { M11 = 0.5f, M22 = 0.5f, M33 = 1.0f, M44 = 1.0f, M54 = 0.0f };
            }
            else
                matrix = ColorParameter.ColorMatrix;

            var colorEffect = new ColorMatrixEffect()
            {
                ColorMatrix = matrix,
                Source = inputBitmap
            };

            drawingSurface.DrawImage(colorEffect);
        }
    }

    public void SetEncodingProperties(VideoEncodingProperties encodingProperties, IDirect3DDevice device)
    {
        _canvasDevice = CanvasDevice.CreateFromDirect3D11Device(device);
    }

    public bool IsReadOnly { get { return false; } }

    public IReadOnlyList<VideoEncodingProperties> SupportedEncodingProperties
    {
        get
        {
            var properties = new List<VideoEncodingProperties>();
            properties.Add(VideoEncodingProperties.CreateUncompressed("ARGB32", 640, 480));
            return properties;
        }
    }

    public MediaMemoryTypes SupportedMemoryTypes { get { return MediaMemoryTypes.Gpu; } }

    public bool TimeIndependent { get { return false; } }

    public void DiscardQueuedFrames()
    {
    }

    public void Close(MediaEffectClosedReason reason)
    {
        if (_canvasDevice != null)
            _canvasDevice.Dispose();
    }

    public void SetProperties(IPropertySet configuration)
    {
        _configuration = configuration;
    }

    public TheEffectParameter ColorParameter
    {
        get
        {
            if (_configuration != null && _configuration.TryGetValue(nameof(ColorParameter), out object val))
                return (TheEffectParameter)val;
            return null;
        }
    }
}

and little Change on the main page :

TheEffectParameter parameter;
protected async override void OnNavigatedTo(NavigationEventArgs e)
{
    base.OnNavigatedTo(e);

    _mediaCapture = new MediaCapture();
    var settings = new MediaCaptureInitializationSettings()
    {
        StreamingCaptureMode = StreamingCaptureMode.Video,
    };
    await _mediaCapture.InitializeAsync(settings);
    captureElement.Source = _mediaCapture;
    await _mediaCapture.StartPreviewAsync();
    parameter = new TheEffectParameter {
        ColorMatrix = new Matrix5x4() { M11 = 0.5f, M22 = 0.5f, M33 = 1.0f, M44 = 1.0f, M54 = 0.0f }
    };
    var propertySet = new PropertySet();
    propertySet.Add(nameof(TheEffect.ColorParameter), parameter);

    var effect = await _mediaCapture.AddVideoEffectAsync(
        new VideoEffectDefinition(typeof(TheEffect).FullName,propertySet),
                                    MediaStreamType.VideoPreview);
}

public void ChangeColor(Matrix5x4 matrix)
{
    if (parameter == null)
        return;
    parameter.ColorMatrix = matrix;
}

this might have a bug forgive me for that because I write this code using your's on the fly and haven't tested but it should work