Differences between optimization and backtesting results

cysecsbin.01
02 Feb 2019, 18:54

by the way, i tried the same procedure both on fxpro's ctrader and on the open beta frome spotware's site, same strange outcome

PanagiotisCharalampous
04 Feb 2019, 10:26

Hi cysecsbin.01,

Can you share the cBot with us so that we can reproduce? Also please let us know the cBot parameters you are optimizing.

Best Regards,

Panagiotis

cysecsbin.01
04 Feb 2019, 11:07 ( Updated at: 21 Dec 2023, 09:21 )

Thanks for your answer, i'll send the indicators' and bot's sources

using System;
using cAlgo.API;
using cAlgo.API.Internals;
using cAlgo.API.Indicators;
using cAlgo.Indicators;

namespace cAlgo
{
    [Indicator(IsOverlay = false, TimeZone = TimeZones.UTC, AccessRights = AccessRights.None)]
    public class ShutForce : Indicator
    {
        [Parameter("RSI Periods", DefaultValue = 14)]
        public int Rsi_per { get; set; }
        [Parameter("ATR Periods", DefaultValue = 14)]
        public int ATR_per { get; set; }
        [Parameter("TimeFrame")]
        public TimeFrame tf { get; set; }

        [Output("Main", Color = Colors.Black)]
        public IndicatorDataSeries Result { get; set; }

        private AverageTrueRange atr;
        private RelativeStrengthIndex rsi;
        private MarketSeries series;


        protected override void Initialize()
        {
            series = MarketData.GetSeries(tf);
            rsi = Indicators.RelativeStrengthIndex(series.Close, Rsi_per);
            atr = Indicators.AverageTrueRange(series, ATR_per, MovingAverageType.Simple);
        }

        public override void Calculate(int index)
        {
            int i = series.OpenTime.GetIndexByTime(MarketSeries.OpenTime[index]);
            if (rsi.Result[i] > rsi.Result[i - 1] && atr.Result[i] > atr.Result[i - 1])
            {
                Result[index] = 1;
                ChartObjects.DrawLine("line" + index, index, -1, index, 1, Colors.Green, 5, LineStyle.Solid);
            }
            else if (rsi.Result[i] < rsi.Result[i - 1] && atr.Result[i] > atr.Result[i - 1])
            {
                Result[index] = -1;
                ChartObjects.DrawLine("line" + index, index, -1, index, 1, Colors.Red, 5, LineStyle.Solid);
            }
            else
            {
                Result[index] = 0;
                ChartObjects.DrawLine("line" + index, index, -1, index, 1, Colors.Blue, 5, LineStyle.Solid);
            }


        }
    }
}

indicator 1

using System;
using cAlgo.API;
using cAlgo.API.Internals;
using cAlgo.API.Indicators;
using cAlgo.Indicators;

namespace cAlgo
{
    [Indicator(IsOverlay = false, TimeZone = TimeZones.UTC, AccessRights = AccessRights.None)]
    public class TickVolume : Indicator
    {
        [Parameter(DefaultValue = false)]
        public bool Segno { get; set; }
        [Parameter(DefaultValue = 10)]
        public int Periodi { get; set; }

        [Output("Main", PlotType = PlotType.Histogram)]
        public IndicatorDataSeries Result { get; set; }
        [Output("EMA", Color = Colors.Cyan)]
        public IndicatorDataSeries EMA { get; set; }

        private ExponentialMovingAverage ema;


        protected override void Initialize()
        {
            ema = Indicators.ExponentialMovingAverage(MarketSeries.TickVolume, Periodi);
        }

        public override void Calculate(int index)
        {
            if (!Segno)
                Result[index] = MarketSeries.TickVolume[index];
            else if (Segno)
                if (Math.Max(MarketSeries.Close[index], MarketSeries.Open[index]) == MarketSeries.Close[index])
                {
                    Result[index] = MarketSeries.TickVolume[index];
                }
                else
                {
                    Result[index] = -MarketSeries.TickVolume[index];
                }
            EMA[index] = ema.Result[index];
        }
    }
}

indicator 2

using System;
using System.Linq;
using cAlgo.API;
using cAlgo.API.Indicators;
using cAlgo.API.Internals;
using cAlgo.Indicators;

namespace cAlgo.Robots
{
    [Robot(TimeZone = TimeZones.UTC, AccessRights = AccessRights.FullAccess)]
    public class ShutForce_Bot : Robot
    {
        [Parameter("RSI Periods", DefaultValue = 14)]
        public int Rsi_per { get; set; }
        [Parameter("ATR Periods", DefaultValue = 14)]
        public int ATR_per { get; set; }
        [Parameter("MTF RSI Periods", DefaultValue = 14)]
        public int mtf_Rsi_per { get; set; }
        [Parameter("MTF ATR Periods", DefaultValue = 14)]
        public int mtf_ATR_per { get; set; }
        [Parameter("Volume Average Periods", DefaultValue = 14)]
        public int tv_per { get; set; }
        [Parameter("SL Multiplier", DefaultValue = 1)]
        public double sl_mul { get; set; }
        [Parameter("TP to SL Ratio", DefaultValue = 1)]
        public double tp_to_sl { get; set; }
        [Parameter("Rischio %", DefaultValue = 1)]
        public double risk { get; set; }
        /*[Parameter("Trailing Start", DefaultValue = 10)]
        public double trl_start { get; set; }
        [Parameter("Trailing Step", DefaultValue = 1)]
        public double trl_step { get; set; }*/
        [Parameter("TimeFrame Secondario")]
        public TimeFrame tf { get; set; }

        private ShutForce sf, sf_mtf;
        private MarketSeries mtf_series;
        private ExponentialMovingAverage ema;
        private TickVolume tv;

        protected override void OnStart()
        {
            mtf_series = MarketData.GetSeries(tf);
            sf = Indicators.GetIndicator<ShutForce>(Rsi_per, ATR_per, TimeFrame);
            sf_mtf = Indicators.GetIndicator<ShutForce>(mtf_Rsi_per, mtf_ATR_per, tf);
            ema = Indicators.ExponentialMovingAverage(MarketSeries.Close, 200);
            tv = Indicators.GetIndicator<TickVolume>(false, tv_per);
        }
//&& MarketSeries.Close.LastValue > ema.Result.LastValue&& MarketSeries.Close.LastValue < ema.Result.LastValue
        protected override void OnBar()
        {
            CancelOrders();
            double sl = sl_mul * (MarketSeries.High.Last(1) - MarketSeries.Low.Last(1)) / Symbol.PipSize;
            double tp = sl * tp_to_sl;
            string label = MarketSeries.OpenTime.GetIndexByTime(MarketSeries.OpenTime.Last(1)).ToString();
            int index_mtf = mtf_series.OpenTime.GetIndexByTime(MarketSeries.OpenTime.Last(0));
            if (sf.Result.Last(1) == 1 && sf_mtf.Result[index_mtf - 1] == 1 && tv.Result.Last(1) > tv.EMA.Last(1))
            {
                PlaceStopOrder(TradeType.Buy, Symbol, Volume(sl), MarketSeries.High.Last(1), label, sl, tp);
            }
            else if (sf.Result.Last(1) == -1 && sf_mtf.Result[index_mtf - 1] == -1 && tv.Result.Last(1) > tv.EMA.Last(1))
            {
                PlaceStopOrder(TradeType.Sell, Symbol, Volume(sl), MarketSeries.Low.Last(1), label, sl, tp);
            }
        }

        protected override void OnTick()
        {
            //Trailing();
        }

        private void CancelOrders()
        {
            foreach (var order in PendingOrders)
            {
                CancelPendingOrder(order);
            }
        }

        private int Volume(double sl)
        {
            double _2_percent = risk * Account.Balance / 100;
            double vol = (_2_percent * Symbol.Ask) / (Symbol.PipSize * sl);
            int vol_int = 1000 * (int)Math.Round(vol / 1000, 0);
            if (vol_int < 1000)
                return 1000;
            return vol_int;
        }

        /* private void Trailing()
        {
            foreach (var pos in Positions)
            {
                if (pos.Pips > trl_start)
                    ModifyPosition(pos, pos.EntryPrice);
            }
        }*/

        protected override void OnStop()
        {
            foreach (var pos in Positions)
            {
                ClosePosition(pos);
            }
        }
    }
}

bot

these are the optimization parameters, i used grid search, tick data and a period from 21/06/2018 to 30/01/2019.

ctraders used are both spotware's open beta and fxpro's

the optimization takes a while but the issue is displayed on any positive result when transported in the backtest section.

i kindly thank you for your help, it is much appreciated.

PanagiotisCharalampous
04 Feb 2019, 11:30

Hi cysecsbin.01,

Thanks I will have a look. However in the previous images timeframe is set to h1 while in the last one it is set to h4. Can you tell me which one is the correct?

Best Regards,

Panagiotis

cysecsbin.01
04 Feb 2019, 15:41

the correct setting for the timeframe to apply the bot to is h1, the 'timeftrame secondario' = auxiliary timeframe from italian, should be set to h4.

however the last pic with the optimization parameters settings is correct.

PanagiotisCharalampous
05 Feb 2019, 17:20

Hi cysecsbin.01,

Thanks, we managed to reproduce this. The problem is with the optimization results. The backtesting results are correct. We will fix in in an upcoming update.

Best Regards,

Panagiotis

leohermoso
13 Aug 2019, 17:06

RE:

Panagiotis Charalampous said:

Hi cysecsbin.01,

Thanks, we managed to reproduce this. The problem is with the optimization results. The backtesting results are correct. We will fix in in an upcoming update.

Best Regards,

Panagiotis

Hi, Is this problem solved? I am getting different results between opt and backtests. I think its caused by MarketData.GetSeries that MAYBE is not pre allocating the correct amount of data to initialize the indicators. 

leohermoso
13 Aug 2019, 17:23 ( Updated at: 21 Dec 2023, 09:21 )

Both are backtesting results

These both are optimization results.

I use daily data inside this 1-minute bot, and the parameter attached to daily data is 35 periods and its roughly the period that optimization and backtest begin to behave accordingly, so it makes me think to backtest are not correct pre-allocating the necessary data to correctly load the indicator

PanagiotisCharalampous
13 Aug 2019, 17:28

Hi leohermoso,

No it has not been fixed yet.

Best Regards,

Panagiotis

leohermoso
13 Aug 2019, 19:06

So backtest is correct? 

leohermoso
13 Aug 2019, 19:25

Is there any place with the know bugs? If now it will be nice to have. I just spent 220 USD in an aws instance at amazon with optimization purposes and looks like the results are entirely wrong. Can you please enumerate know bugs? Tks 

PanagiotisCharalampous
14 Aug 2019, 08:54

Hi leoheermoso,

There isn't such a place. Usually known bugs are fixed in the next couple of versions. This is an exception since we are currently refactoring cTrader Automate and we will fix this as soon as we finish.

Best Regards,

Panagiotis

leohermoso
14 Aug 2019, 21:37

RE:

Panagiotis Charalampous said:

Hi leoheermoso,

There isn't such a place. Usually known bugs are fixed in the next couple of versions. This is an exception since we are currently refactoring cTrader Automate and we will fix this as soon as we finish.

Best Regards,

Panagiotis

Hello Panagiotis,

Tks for your answer, I am looking forward to this fix! 

sylvain.rochette@gmail.com
29 Feb 2020, 16:01

RE: RE:

leohermoso said:

Panagiotis Charalampous said:

Hi leoheermoso,

There isn't such a place. Usually known bugs are fixed in the next couple of versions. This is an exception since we are currently refactoring cTrader Automate and we will fix this as soon as we finish.

Best Regards,

Panagiotis

Hello Panagiotis,

 

Tks for your answer, I am looking forward to this fix! 

elmpie
04 May 2020, 22:09

RE:

PanagiotisCharalampous said:

Hi leoheermoso,

There isn't such a place. Usually known bugs are fixed in the next couple of versions. This is an exception since we are currently refactoring cTrader Automate and we will fix this as soon as we finish.

Best Regards,

Panagiotis

Is this fixed yet? When will it be fixed? I what cases do we get wrong results in the optimization?

Thanks.

PanagiotisCharalampous
05 May 2020, 08:40

Hi elpmie,

No there is no fix for this issue yet.

Best Regards,

Panagiotis

bury1992
25 May 2020, 20:11

RE:

PanagiotisCharalampous said:

Hi elpmie,

No there is no fix for this issue yet.

Best Regards,

Panagiotis

Hi,

Is there something new about the fix? Because it seems like my bot returns different values each time I run the backtesting, even though I've fixed everything i could (dataset, spread, time).

Best regards,

Wojtek

PanagiotisCharalampous
26 May 2020, 08:07

Hi Wojtek,

No there is no fix yet.

Best Regards,

Panagiotis 

Join us on Telegram

bury1992
13 Jun 2020, 17:51

RE:

PanagiotisCharalampous said:

Hi Wojtek,

No there is no fix yet.

Best Regards,

Panagiotis 

Join us on Telegram

 

I did some small research and i've noticed that, although some values in my bot are used only in thread-safe mode (using Interlocked.* methods), they dont behave like they should in optimization mode, but they do in backtesting. Going even further, I can't reproduce the error if bot will not update properties that are shared between threads. Is there a way to fix the thread pool size for one optimization run? It would help me in my debugging.

Regards,

Wojtek

PanagiotisCharalampous
15 Jun 2020, 09:51

Hi Wojtek,

No there is no such option.

Best Regards,

Panagiotis 

Join us on Telegram

dave.anderson.consulting
25 Aug 2023, 13:52

This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23.
I needed to optimize a client's cBot and ran into this bug today.
To make sure it's not my fault I grabbed the above passed cBot and tested it and saw that there is still a difference between Optimization and Backtest results. Exactly as it was reported here in 2019.

PanagiotisChar
26 Aug 2023, 14:53 ( Updated at: 26 Aug 2023, 15:00 )

RE: Differences between optimization and backtesting results

dave.anderson.consulting said: 

This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23.
I needed to optimize a client's cBot and ran into this bug today.
To make sure it's not my fault I grabbed the above passed cBot and tested it and saw that there is still a difference between Optimization and Backtest results. Exactly as it was reported here in 2019.

Hi there,

The same symptom does not mean that it has the same cause. Better share cBot code and exact steps to reproduce.

Aieden Technologies

Need help? Join us on Telegram

ctid6328327
06 Nov 2023, 17:29 ( Updated at: 07 Nov 2023, 06:28 )

RE: Differences between optimization and backtesting results

dave.anderson.consulting said: 

This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23.
I needed to optimize a client's cBot and ran into this bug today.
To make sure it's not my fault I grabbed the above passed cBot and tested it and saw that there is still a difference between Optimization and Backtest results. Exactly as it was reported here in 2019.

This is not the only issue. With some bots there is a difference between Visual and Non-Visual backtest. 

PanagiotisCharalampous
07 Nov 2023, 06:52

RE: RE: Differences between optimization and backtesting results

ctid6328327 said: 

dave.anderson.consulting said: 

This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23.
I needed to optimize a client's cBot and ran into this bug today.
To make sure it's not my fault I grabbed the above passed cBot and tested it and saw that there is still a difference between Optimization and Backtest results. Exactly as it was reported here in 2019.

This is not the only issue. With some bots there is a difference between Visual and Non-Visual backtest. 

Hi there,

There is no known issue at the moment. If you can reproduce such issue, please share the information requested above.

Best regards,

Panagiotis

ctid6328327
07 Nov 2023, 19:38 ( Updated at: 21 Dec 2023, 09:23 )

RE: RE: RE: Differences between optimization and backtesting results

PanagiotisCharalampous said: 

ctid6328327 said: 

dave.anderson.consulting said: 

This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23.
I needed to optimize a client's cBot and ran into this bug today.
To make sure it's not my fault I grabbed the above passed cBot and tested it and saw that there is still a difference between Optimization and Backtest results. Exactly as it was reported here in 2019.

This is not the only issue. With some bots there is a difference between Visual and Non-Visual backtest. 

Hi there,

There is no known issue at the moment. If you can reproduce such issue, please share the information requested above.

Best regards,

Panagiotis

Sure that won't be difficult to do.  See below the best parameters set selected after the completion of the optimisation process. As you can see the Non-Visual backtest produces the same outcome and the same results as per the optimisation, however, the Visual test tells a different story. I hired a programmer who created a simple bot to count the ticks and the number of ticks is somehow quite low, which makes me wonder how accurate the backtests are. So the statement made by the above user: “This issue has never been resolved since it was reported. We are now 4 years later and still no fix on cTrader version 4.8.23” is still valid, the issue is there and that makes the optimisation process pretty much useless because if the platform can have such a basic issue then it rises questions of the accuracy for the results produced. 

Thanks