Backtesting custom data sources

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Reflection;
using cAlgo.API;
using cAlgo.API.Collections;
using cAlgo.API.Indicators;
using cAlgo.API.Internals;
using System.Linq;

namespace cAlgo.Plugins
{
    [Plugin(AccessRights = AccessRights.FullAccess)]
    public class CustomDataSourceCSVTest : Plugin
    {
        private BacktestingDataSource _dataSource;
        private TextBox _symbolNameTextBox;
        private ComboBox _timeFrameCombobox;
        private string _selectedFilePath;

        private readonly Dictionary<DataId, Data> _addedData = new();
        private readonly Dictionary<DataId, Dictionary<DateOnly, List<BacktestingTick>>> _cachedTickData = new();
        private readonly Dictionary<DataId, List<BacktestingBar>> _cachedBarData = new();

        private StackPanel _panel;
        private Button _selectFileButton;

        private CustomTimeFrame _customTimeFrame;

        protected override void OnStart()
        {
            // System.Diagnostics.Debugger.Launch();

            LoadFromLocalStorage();

            var options = new BacktestingDataSourceOptions(BacktestingDataSourceDataType.Tick | BacktestingDataSourceDataType.M1OpenPrices | BacktestingDataSourceDataType.OpenPrices, OnMinMaxTimeRequested, OnDataRequested);

            _dataSource = Backtesting.DataSources.Add("CSV", options);

            var aspBlock = Asp.SymbolTab.AddBlock("Custom Data Source CSV Test");

            aspBlock.Child = GetAddDataPanel();
            aspBlock.Height = 400;
        }

        private Panel GetAddDataPanel()
        {
            _panel = new StackPanel() {Orientation = Orientation.Vertical, Margin = 3};

            _symbolNameTextBox = new TextBox() {Text = "Enter Symbol Name", Margin = 3};

            _panel.AddChild(_symbolNameTextBox);

            _timeFrameCombobox = new ComboBox() {Margin = 3, Padding = 2};

            foreach (var timeFrame in GetTimeFrames())
            {
                _timeFrameCombobox.AddItem(timeFrame.Name);
            }

            _timeFrameCombobox.SelectedItem = TimeFrame.Minute.Name;

            _panel.AddChild(_timeFrameCombobox);

            _selectFileButton = new Button() {Text = "Select File", Margin = 3};

            _selectFileButton.Click += OnSelectFileButtonClicked;

            _panel.AddChild(_selectFileButton);

            var addButton = new Button() {Text = "Add Data", Margin = 3};

            addButton.Click += OnAddClick;

            _panel.AddChild(addButton);

            var showDataButton = new Button() {Text = "Show Data", Margin = 3};

            showDataButton.Click += OnShowDataClick;

            _panel.AddChild(showDataButton);

            var removeAllDataButton = new Button() {Text = "Remove All Data", Margin = 3};

            removeAllDataButton.Click += OnRemoveAllDataClick;

            _panel.AddChild(removeAllDataButton);

            var clearCacheButton = new Button() {Text = "Clear Cache", Margin = 3};

            clearCacheButton.Click += OnClearCacheClick;

            _panel.AddChild(clearCacheButton);

            var addCustomTimeFrameButton = new Button() {Text = "Add Custom Time Frame", Margin = 3};

            addCustomTimeFrameButton.Click += OnAddCustomTimeFrameClick;

            _panel.AddChild(addCustomTimeFrameButton);

            var removeCustomTimeFrameButton = new Button() {Text = "Remove Custom Time Frame", Margin = 3};

            removeCustomTimeFrameButton.Click += OnRemoveCustomTimeFrameClick;

            _panel.AddChild(removeCustomTimeFrameButton);

            return _panel;
        }

        private void OnRemoveCustomTimeFrameClick(ButtonClickEventArgs obj)
        {
            if (TimeFrameManager.Custom.Remove(_customTimeFrame))
            {
                Print($"Custom time frame removed.");
                return;
            }

            Print($"Removing custom time frame failed.");
        }

        private void OnAddCustomTimeFrameClick(ButtonClickEventArgs obj)
        {
            _customTimeFrame = TimeFrameManager.Custom.Add("Custom Data Source Test (H1)");

            _customTimeFrame.BarsNeeded = args => 
            {
                Print($"Custom time frame bars needed for symbol '{args.CustomBars.Symbol.Name}'.");

                var bars = MarketData.GetBars(TimeFrame.Hour, args.CustomBars.Symbol.Name);

                Print($"Appending custom time frame bars '{bars.Count}' for symbol '{args.CustomBars.Symbol.Name}'.");

                args.CustomBars.AppendBars(bars.Select(b => new CustomBar(b.OpenTime, b.Open, b.High, b.Low, b.Close, b.TickVolume)));
            };

            Print("Custom time frame added.");
        }

        private void OnClearCacheClick(ButtonClickEventArgs obj)
        {
            if (_dataSource.ClearCache())
            {
                Print($"Data source '{_dataSource.Name}' cached data cleared.");
                return;
            }

            Print($"Data source '{_dataSource.Name}' cached data clear failed.");
        }

        private void OnRemoveAllDataClick(ButtonClickEventArgs obj)
        {
            foreach (var (_, data) in _addedData)
            {
                File.Delete(data.FilePath);
            }

            LocalStorage.Remove("Data", LocalStorageScope.Type);
            LocalStorage.Flush(LocalStorageScope.Type);

            _addedData.Clear();

            Print("All added data has been removed.");
        }

        private void OnAddClick(ButtonClickEventArgs obj)
        {
            if (_symbolNameTextBox.Text.Length == 0 || Symbols.GetSymbol(_symbolNameTextBox.Text) is not { } symbol)
            {
                Print($"Invalid or empty symbol name: {_symbolNameTextBox.Text}");
                return;
            }

            if (string.IsNullOrWhiteSpace(_timeFrameCombobox.SelectedItem) || !TimeFrame.TryParse(_timeFrameCombobox.SelectedItem, out var selectedTimeFrame))
            {
                Print($"Invalid time frame: {_timeFrameCombobox.SelectedItem}");
                return;
            }

            if (string.IsNullOrWhiteSpace(_selectedFilePath))
            {
                Print($"Invalid file path: {_selectedFilePath}");
                return;
            }

            var dataType = selectedTimeFrame == TimeFrame.Tick 
                ? BacktestingDataSourceDataType.Tick 
                : selectedTimeFrame == TimeFrame.Minute 
                    ? BacktestingDataSourceDataType.M1OpenPrices 
                    : BacktestingDataSourceDataType.OpenPrices;

            var dataId = new DataId(symbol.Name, selectedTimeFrame.Name, dataType);

            _panel.IsEnabled = false;

            BeginInvokeOnMainThread(() =>
            {
                try
                {
                    var minMaxTime = GetMinMaxTime(_selectedFilePath);
                    var data = new Data(_selectedFilePath, minMaxTime);

                    _addedData[dataId] = data;

                    SaveToLocalStorage(dataId, data);

                    Print($"Data for symbol {symbol} and time frame {selectedTimeFrame} is added: {_selectedFilePath}, Min/Max Time: {minMaxTime}");
                }
                finally
                {
                    _panel.IsEnabled = true;
                }
            });
        }

        private void SaveToLocalStorage(DataId dataId, Data data)
        {
            var existingData = LocalStorage.GetObject<List<(DataId DataId, Data Data)>>("Data", LocalStorageScope.Type) 
                               ?? new List<(DataId DataId, Data Data)>();

            existingData.Add((dataId, data));

            LocalStorage.SetObject("Data", existingData, LocalStorageScope.Type);
            LocalStorage.Flush(LocalStorageScope.Type);
        }


        private void OnShowDataClick(ButtonClickEventArgs obj)
        {
            var existingData =
                LocalStorage.GetObject<List<(DataId DataId, Data Data)>>("Data", LocalStorageScope.Type);

            if (existingData is null)
                return;

            Print("Showing data");

            foreach (var (dataId, data) in existingData)
            {
                Print($"Data '{dataId}' with '{data}'");
            }
        }

        private void LoadFromLocalStorage()
        {
            var existingData =
                LocalStorage.GetObject<List<(DataId DataId, Data Data)>>("Data", LocalStorageScope.Type);

            if (existingData is null)
                return;

            foreach (var (dataId, data) in existingData)
            {
                _addedData[dataId] = data with {MinMaxTime = data.MinMaxTime with
                {
                    MinTime = DateTime.SpecifyKind(data.MinMaxTime.MinTime, DateTimeKind.Utc),
                    MaxTime = DateTime.SpecifyKind(data.MinMaxTime.MaxTime, DateTimeKind.Utc)
                }};

                Print($"Added data '{dataId}' from local storage");
            }
        }

        private void OnSelectFileButtonClicked(ButtonClickEventArgs obj)
        {
            var openFileDialog = new OpenFileDialog();

            _panel.IsEnabled = false;
            _selectFileButton.Text = "Please wait...";

            BeginInvokeOnMainThread(() =>
            {
                try
                {
                    var result = openFileDialog.ShowDialog();

                    if (result != FileDialogResult.OK)
                        return;

                    _selectedFilePath = openFileDialog.FileName;
                }
                finally
                {
                    _selectFileButton.Text = "Select File";
                    _panel.IsEnabled = true;
                }
            });
        }

        private IEnumerable<TimeFrame> GetTimeFrames()
        {
            var timeFrameFields = typeof(TimeFrame).GetFields(BindingFlags.Static | BindingFlags.Public);

            foreach (var timeFrameField in timeFrameFields)
            {
                if (timeFrameField.GetValue(null) is not TimeFrame timeFrame)
                    continue;

                yield return timeFrame;
            }
        }

        private void OnDataRequested(BacktestingDataRequest request)
        {
            request.StatusChanged += request_StatusChanged;

            Print($"Data Request from {request.StartTime} to {request.EndTime}");

            var dataId = GetDataId(request.PriceDataSourceId, request.DataType);

            if (!_addedData.TryGetValue(dataId, out var data))
            {
                request.Fail($"Data not found for '{dataId}'.");

                return;
            }

            switch (request.DataType)
            {
                case BacktestingDataSourceDataType.Tick:
                    var ticks = GetRequestTicks(dataId, data, request);
                    Print($"Sending ticks.");

                    request.Complete(ticks);
                    break;
                default:
                    var bars = GetRequestBars(dataId, data, request);
                    Print($"Sending bars.");

                    request.Complete(bars);
                    break;
            }
        }

        private BacktestingData GetRequestTicks(DataId dataId, Data data, BacktestingDataRequest request)
        {
            if (!_cachedTickData.TryGetValue(dataId, out var tickData))
            {
                tickData = CacheTickData(dataId, data);

                _cachedTickData[dataId] = tickData;
            }

            var requestData = new List<BacktestingTick>();

            for (var i = DateOnly.FromDateTime(request.StartTime); i <= DateOnly.FromDateTime(request.EndTime); i = i.AddDays(1))
            {
                if (!tickData.TryGetValue(i, out var ticks))
                    continue;

                requestData.AddRange(ticks);
            }

            return new BacktestingTickData(requestData);
        }

        private BacktestingData GetRequestBars(DataId dataId, Data data, BacktestingDataRequest request)
        {
            if (!_cachedBarData.TryGetValue(dataId, out var barsData))
            {
                barsData = CacheBarData(dataId, data);

                _cachedBarData[dataId] = barsData;
            }

            return new BacktestingBarsData(barsData.Where(b => b.Time >= request.StartTime && b.Time <= request.EndTime).ToArray());
        }

        private List<BacktestingBar> CacheBarData(DataId dataId, Data data)
        {
            var result = new List<BacktestingBar>();

            using var fileStream = File.OpenRead(data.FilePath);
            using var streamReader = new StreamReader(fileStream);

            while (streamReader.ReadLine() is {} line)
            {
                var lineSplit = line.Split(',');

                if (lineSplit.Length != 6)
                {
                    throw new InvalidOperationException($"Invalid data format for '{dataId}'.");
                }

                result.Add(new BacktestingBar(DateTime.Parse(lineSplit[0]).ToUniversalTime(), double.Parse(lineSplit[1]), double.Parse(lineSplit[2]), double.Parse(lineSplit[3]), double.Parse(lineSplit[4]), long.Parse(lineSplit[5])));
            }

            return result;
        }

        private Dictionary<DateOnly, List<BacktestingTick>> CacheTickData(DataId dataId, Data data)
        {
            var result = new Dictionary<DateOnly, List<BacktestingTick>>();

            using var fileStream = File.OpenRead(data.FilePath);
            using var streamReader = new StreamReader(fileStream);

            while (streamReader.ReadLine() is {} line)
            {
                var lineSplit = line.Split(',');

                if (lineSplit.Length != 3)
                {
                    throw new InvalidOperationException($"Invalid data format for '{dataId}'.");
                }

                var time = DateTime.Parse(lineSplit[0]).ToUniversalTime();
                var date = DateOnly.FromDateTime(time);

                if (!result.TryGetValue(date, out var dateTicks))
                {
                    dateTicks = new List<BacktestingTick>();

                    result.Add(date, dateTicks);
                }

                dateTicks.Add(new BacktestingTick(time, double.Parse(lineSplit[1]), double.Parse(lineSplit[2])));
            }

            return result;
        }

        private void request_StatusChanged(BacktestingDataRequestStatusChangedEventArgs args)
        {
            Print($"Request '{args.DataRequest.PriceDataSourceId}' status changed: {args.DataRequest.Status}");
        }

        private BacktestingMinMaxTime OnMinMaxTimeRequested(BacktestingDataSourceMinMaxTimeArgs args)
        {
            Print($"Min/Max time requested, {args.PriceDataSourceId}, {args.DataType}");

            var dataId = GetDataId(args.PriceDataSourceId, args.DataType);

            Print($"Min/Max time request DataId: {dataId}");

            if (!_addedData.TryGetValue(dataId, out var data))
            {
                throw new InvalidOperationException("No data");
                //return default;    
            }

            Print($"Min/Max time sent, {data.MinMaxTime}");

            return data.MinMaxTime;
        }

        private DataId GetDataId(PriceDataSourceId priceDataSourceId, BacktestingDataSourceDataType dateType)
        {
            return new DataId(priceDataSourceId.Symbol.Name, dateType switch
            {
                BacktestingDataSourceDataType.Tick => TimeFrame.Tick.Name,
                BacktestingDataSourceDataType.M1OpenPrices => TimeFrame.Minute.Name,
                BacktestingDataSourceDataType.OpenPrices when _customTimeFrame != null && priceDataSourceId.TimeFrame == _customTimeFrame.TimeFrame => TimeFrame.Hour.Name,
                _ => priceDataSourceId.TimeFrame.Name
            }, dateType);
        }

        private BacktestingMinMaxTime GetMinMaxTime(string filePath)
        {
            using var fileStream = File.OpenRead(filePath);
            using var streamReader = new StreamReader(fileStream);

            string firstLine = null;
            string lastLine = null;

            while (streamReader.ReadLine() is {} line)
            {
                if (firstLine == null)
                    firstLine = line;

                lastLine = line;
            }

            Print($"First Line: {firstLine}");
            Print($"Last Line: {lastLine}");

            return new(
                !string.IsNullOrWhiteSpace(firstLine) ? DateTime.Parse(firstLine.Split(',')[0]).ToUniversalTime() : default,
                !string.IsNullOrWhiteSpace(lastLine) ? DateTime.Parse(lastLine.Split(',')[0]).ToUniversalTime() : default);
        }

        private readonly record struct DataId(string SymbolName, string TimeFrameName, BacktestingDataSourceDataType DataType);
        private readonly record struct Data(string FilePath, BacktestingMinMaxTime MinMaxTime);
    }
}