Prevent QuantBook composer reset on notebook initialization

Engine/Initializer.cs

@@ -26,6 +26,9 @@ namespace QuantConnect.Lean.Engine
     /// </summary>
     public static class Initializer
     {
+        private static LeanEngineSystemHandlers _systemHandlers;
+        private static LeanEngineAlgorithmHandlers _algorithmHandlers;
+
         /// <summary>
         /// Basic common Lean initialization
         /// </summary>
@@ -62,20 +65,37 @@ public static void Start()
         /// </summary>
         public static LeanEngineSystemHandlers GetSystemHandlers()
         {
-            var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
+            if (_systemHandlers == null)
+            {
+                _systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
 
-            //Setup packeting, queue and controls system: These don't do much locally.
-            systemHandlers.Initialize();
+                //Setup packeting, queue and controls system: These don't do much locally.
+                _systemHandlers.Initialize();
+            }
 
-            return systemHandlers;
+            return _systemHandlers;
         }
 
         /// <summary>
         /// Get and initializes Algorithm Handler
         /// </summary>
         public static LeanEngineAlgorithmHandlers GetAlgorithmHandlers(bool researchMode = false)
         {
-            return LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance, researchMode);
+            if (_algorithmHandlers == null)
+            {
+                _algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance, researchMode);
+            }
+
+            return _algorithmHandlers;
+        }
+
+        /// <summary>
+        /// Reset the handlers to null, so they are recreated on the next request
+        /// </summary>
+        public static void ResetHandlers()
+        {
+            _systemHandlers = null;
+            _algorithmHandlers = null;
         }
     }
 }

Research/QuantBook.cs

@@ -38,7 +38,6 @@
 using QuantConnect.Lean.Engine.Setup;
 using QuantConnect.Indicators;
 using QuantConnect.Scheduling;
-using System.Collections;
 
 namespace QuantConnect.Research
 {
@@ -52,6 +51,9 @@ public class QuantBook : QCAlgorithm
         private IDataProvider _dataProvider;
         private static bool _isPythonNotebook;
 
+        private static LeanEngineSystemHandlers _systemHandlers;
+        private static LeanEngineAlgorithmHandlers _algorithmHandlers;
+
         static QuantBook()
         {
             //Determine if we are in a Python Notebook
@@ -115,17 +117,8 @@ public QuantBook() : base()
                 // Sets PandasConverter
                 SetPandasConverter();
 
-                // Reset our composer; needed for re-creation of QuantBook
-                Composer.Instance.Reset();
-                var composer = Composer.Instance;
-                Config.Reset();
-
-                // Create our handlers with our composer instance
-                var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
-                // init the API
-                systemHandlers.Initialize();
-                var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer, researchMode: true);
-;
+                // Reset composer and initialize handlers
+                PrepareForInstantiation();
 
                 var algorithmPacket = new BacktestNodePacket
                 {
@@ -137,16 +130,16 @@ public QuantBook() : base()
                 };
 
                 ProjectId = algorithmPacket.ProjectId;
-                systemHandlers.LeanManager.Initialize(systemHandlers,
-                    algorithmHandlers,
+                _systemHandlers.LeanManager.Initialize(_systemHandlers,
+                    _algorithmHandlers,
                     algorithmPacket,
                     new AlgorithmManager(false));
-                systemHandlers.LeanManager.SetAlgorithm(this);
+                _systemHandlers.LeanManager.SetAlgorithm(this);
 
 
-                algorithmHandlers.DataPermissionsManager.Initialize(algorithmPacket);
+                _algorithmHandlers.DataPermissionsManager.Initialize(algorithmPacket);
 
-                algorithmHandlers.ObjectStore.Initialize(algorithmPacket.UserId,
+                _algorithmHandlers.ObjectStore.Initialize(algorithmPacket.UserId,
                     algorithmPacket.ProjectId,
                     algorithmPacket.UserToken,
                     new Controls
@@ -157,10 +150,10 @@ public QuantBook() : base()
                         StorageFileCount = Config.GetInt("storage-file-count", 10000),
                         StoragePermissions = (FileAccess) Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
                     });
-                SetObjectStore(algorithmHandlers.ObjectStore);
+                SetObjectStore(_algorithmHandlers.ObjectStore);
 
-                _dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
-                _dataProvider = algorithmHandlers.DataProvider;
+                _dataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider);
+                _dataProvider = _algorithmHandlers.DataProvider;
 
                 var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
                 var registeredTypes = new RegisteredSecurityDataTypesProvider();
@@ -174,27 +167,27 @@ public QuantBook() : base()
                 Securities.SetSecurityService(securityService);
                 SubscriptionManager.SetDataManager(
                     new DataManager(new NullDataFeed(),
-                        new UniverseSelection(this, securityService, algorithmHandlers.DataPermissionsManager, algorithmHandlers.DataProvider),
+                        new UniverseSelection(this, securityService, _algorithmHandlers.DataPermissionsManager, _algorithmHandlers.DataProvider),
                         this,
                         TimeKeeper,
                         MarketHoursDatabase,
                         false,
                         registeredTypes,
-                        algorithmHandlers.DataPermissionsManager));
+                        _algorithmHandlers.DataPermissionsManager));
 
-                var mapFileProvider = algorithmHandlers.MapFileProvider;
+                var mapFileProvider = _algorithmHandlers.MapFileProvider;
                 HistoryProvider = new HistoryProviderManager();
                 HistoryProvider.Initialize(
                     new HistoryProviderInitializeParameters(
                         null,
                         null,
-                        algorithmHandlers.DataProvider,
+                        _algorithmHandlers.DataProvider,
                         _dataCacheProvider,
                         mapFileProvider,
-                        algorithmHandlers.FactorFileProvider,
+                        _algorithmHandlers.FactorFileProvider,
                         null,
                         true,
-                        algorithmHandlers.DataPermissionsManager,
+                        _algorithmHandlers.DataPermissionsManager,
                         ObjectStore,
                         Settings
                     )
@@ -210,6 +203,12 @@ public QuantBook() : base()
             {
                 throw new Exception("QuantBook.Main(): " + exception);
             }
+            finally
+            {
+                // Reset the handlers so they (and the composer) can be re-initialized for the next instance
+                _systemHandlers = null;
+                _algorithmHandlers = null;
+            }
         }
 
         /// <summary>
@@ -863,6 +862,24 @@ public PyDict GetPortfolioStatistics(PyObject dataFrame)
             }
         }
 
+        /// <summary>
+        /// Prepares the environment for a new <see cref="QuantBook"/> instantiation, resetting the composer and handlers
+        /// </summary>
+        public static void PrepareForInstantiation()
+        {
+            if (_systemHandlers != null && _algorithmHandlers != null)
+            {
+                return;
+            }
+
+            Composer.Instance.Reset();
+            Config.Reset();
+
+            Initializer.ResetHandlers();
+            _systemHandlers = Initializer.GetSystemHandlers();
+            _algorithmHandlers = Initializer.GetAlgorithmHandlers();
+        }
+
         /// <summary>
         /// Helper method to perform selection on the given data and filter it
         /// </summary>

Research/QuantConnect.csx

@@ -84,5 +84,6 @@ using QuantConnect.Lean.Engine;
 
 Config.Reset();
 Initializer.Start();
+QuantBook.PrepareForInstantiation();
 Api api = (Api)Initializer.GetSystemHandlers().Api;
 var algorithmHandlers = Initializer.GetAlgorithmHandlers(researchMode: true);

Research/start.py

@@ -34,10 +34,11 @@
 
 Config.Reset()
 Initializer.Start()
+QuantBook.PrepareForInstantiation()
 api = Initializer.GetSystemHandlers().Api
 algorithmHandlers = Initializer.GetAlgorithmHandlers(researchMode=True)
 
-# Required to configure pythonpath with additional paths the user may have 
+# Required to configure pythonpath with additional paths the user may have
 # set in the config, like a project library.
 PythonInitializer.Initialize(False)
 

Tests/AlgorithmRunner.cs

@@ -64,6 +64,7 @@ public static AlgorithmRunnerResults RunLocalBacktest(
             SymbolCache.Clear();
             TextSubscriptionDataSourceReader.ClearCache();
             MarketOnCloseOrder.SubmissionTimeBuffer = MarketOnCloseOrder.DefaultSubmissionTimeBuffer;
+            Initializer.ResetHandlers();
 
             // clean up object storage
             var objectStorePath = LocalObjectStore.DefaultObjectStore;