Bump pythonNet version 2.0.12 (QuantConnect#6310)

* Updates after pythonNet rebase * Bump pythonNet version 2.0.12 * Fix exception types being thrown
zjokerxxx · May 2, 2022 · b9d3d99 · b9d3d99
1 parent 6412566
commit b9d3d99
Show file tree

Hide file tree

Showing 64 changed files with 384 additions and 347 deletions.
diff --git a/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj b/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj
@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

diff --git a/Algorithm.Framework/Portfolio/BlackLittermanOptimizationPortfolioConstructionModel.py b/Algorithm.Framework/Portfolio/BlackLittermanOptimizationPortfolioConstructionModel.py
@@ -51,6 +51,7 @@ def __init__(self,
             risk_free_rate(float): The risk free rate
             delta(float): The risk aversion coeffficient of the market portfolio
             tau(float): The model parameter indicating the uncertainty of the CAPM prior"""
+        super().__init__()
         self.lookback = lookback
         self.period = period
         self.resolution = resolution

diff --git a/Algorithm.Framework/Portfolio/EqualWeightingPortfolioConstructionModel.py b/Algorithm.Framework/Portfolio/EqualWeightingPortfolioConstructionModel.py
@@ -28,6 +28,7 @@ def __init__(self, rebalance = Resolution.Daily, portfolioBias = PortfolioBias.L
                               The function returns null if unknown, in which case the function will be called again in the
                               next loop. Returning current time will trigger rebalance.
             portfolioBias: Specifies the bias of the portfolio (Short, Long/Short, Long)'''
+        super().__init__()
         self.portfolioBias = portfolioBias
 
         # If the argument is an instance of Resolution or Timedelta

diff --git a/Algorithm.Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModel.py b/Algorithm.Framework/Portfolio/MeanVarianceOptimizationPortfolioConstructionModel.py
@@ -39,6 +39,7 @@ def __init__(self,
             period(int): The time interval of history price to calculate the weight
             resolution: The resolution of the history price
             optimizer(class): Method used to compute the portfolio weights"""
+        super().__init__()
         self.lookback = lookback
         self.period = period
         self.resolution = resolution

diff --git a/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj b/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj
@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

diff --git a/Algorithm.Python/CustomModelsAlgorithm.py b/Algorithm.Python/CustomModelsAlgorithm.py
@@ -58,6 +58,7 @@ def OnData(self, data):
 # If we want to use methods from other models, you need to inherit from one of them
 class CustomFillModel(ImmediateFillModel):
     def __init__(self, algorithm):
+        super().__init__()
         self.algorithm = algorithm
         self.absoluteRemainingByOrderId = {}
         self.random = Random(387510346)
@@ -85,6 +86,7 @@ def MarketFill(self, asset, order):
 
 class CustomFeeModel(FeeModel):
     def __init__(self, algorithm):
+        super().__init__()
         self.algorithm = algorithm
 
     def GetOrderFee(self, parameters):
@@ -107,6 +109,7 @@ def GetSlippageApproximation(self, asset, order):
 
 class CustomBuyingPowerModel(BuyingPowerModel):
     def __init__(self, algorithm):
+        super().__init__()
         self.algorithm = algorithm
 
     def HasSufficientBuyingPowerForOrder(self, parameters):

diff --git a/Algorithm.Python/CustomWarmUpPeriodIndicatorAlgorithm.py b/Algorithm.Python/CustomWarmUpPeriodIndicatorAlgorithm.py
@@ -97,6 +97,7 @@ def OnData(self, data):
 # Represents the traditional simple moving average indicator (SMA) without Warm Up Period parameter defined
 class CSMANotWarmUp(PythonIndicator):
     def __init__(self, name, period):
+        super().__init__()
         self.Name = name
         self.Value = 0
         self.queue = deque(maxlen=period)

diff --git a/Algorithm.Python/QuantConnect.Algorithm.Python.csproj b/Algorithm.Python/QuantConnect.Algorithm.Python.csproj
@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

diff --git a/Algorithm.Python/RegisterIndicatorRegressionAlgorithm.py b/Algorithm.Python/RegisterIndicatorRegressionAlgorithm.py
@@ -107,6 +107,7 @@ def OnEndOfAlgorithm(self):
 
 class CustomIndicator(PythonIndicator):
     def __init__(self):
+        super().__init__()
         self.Name = "Jose"
         self.Value = 0
 

diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs
@@ -1121,7 +1121,8 @@ public string Download(string address, PyObject headers, string userName, string
                     // In order to convert it into a C# Dictionary
                     if (PyDict.IsDictType(headers))
                     {
-                        foreach (PyObject pyKey in headers)
+                        using var iterator = headers.GetIterator();
+                        foreach (PyObject pyKey in iterator)
                         {
                             var key = (string)pyKey.AsManagedObject(typeof(string));
                             var value = (string)headers.GetItem(pyKey).AsManagedObject(typeof(string));

diff --git a/Algorithm/QuantConnect.Algorithm.csproj b/Algorithm/QuantConnect.Algorithm.csproj
@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs
@@ -634,7 +634,7 @@ public void OnEndOfDay()
             // Only throws if there is an error in its implementation body
             catch (PythonException exception)
             {
-                if (!exception.Message.StartsWith("TypeError : OnEndOfDay()"))
+                if (!exception.Message.StartsWith("OnEndOfDay()"))
                 {
                     _baseAlgorithm.SetRunTimeError(exception);
                 }
@@ -662,7 +662,7 @@ public void OnEndOfDay(Symbol symbol)
             // Only throws if there is an error in its implementation body
             catch (PythonException exception)
             {
-                if (!exception.Message.StartsWith("TypeError : OnEndOfDay()"))
+                if (!exception.Message.StartsWith("OnEndOfDay()"))
                 {
                     _baseAlgorithm.SetRunTimeError(exception);
                 }
@@ -690,7 +690,8 @@ public void OnMarginCall(List<SubmitOrderRequest> requests)
 
                     requests.Clear();
 
-                    foreach (PyObject pyRequest in pyRequests)
+                    using var iterator = pyRequests.GetIterator();
+                    foreach (PyObject pyRequest in iterator)
                     {
                         SubmitOrderRequest request;
                         if (TryConvert(pyRequest, out request))

diff --git a/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj b/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj
@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

diff --git a/Common/Exceptions/InvalidTokenPythonExceptionInterpreter.cs b/Common/Exceptions/InvalidTokenPythonExceptionInterpreter.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -36,7 +36,6 @@ public class InvalidTokenPythonExceptionInterpreter : PythonExceptionInterpreter
         public override bool CanInterpret(Exception exception)
         {
             return base.CanInterpret(exception) &&
-                exception.Message.Contains("SyntaxError") &&
                 exception.Message.Contains("invalid token");
         }
 

diff --git a/Common/Exceptions/KeyErrorPythonExceptionInterpreter.cs b/Common/Exceptions/KeyErrorPythonExceptionInterpreter.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -37,8 +37,17 @@ public class KeyErrorPythonExceptionInterpreter : PythonExceptionInterpreter
         /// <returns>True if the exception can be interpreted, false otherwise</returns>
         public override bool CanInterpret(Exception exception)
         {
-            return base.CanInterpret(exception) &&
-                exception.Message.Contains("KeyError");
+            var pythonException = exception as PythonException;
+            if (pythonException == null)
+            {
+                return false;
+            }
+
+            using (Py.GIL())
+            {
+                return base.CanInterpret(exception) &&
+                    pythonException.Type.Name.Contains("KeyError", StringComparison.InvariantCultureIgnoreCase);
+            }
         }
         /// <summary>
         /// Interprets the specified exception into a new exception

diff --git a/Common/Exceptions/NoMethodMatchPythonExceptionInterpreter.cs b/Common/Exceptions/NoMethodMatchPythonExceptionInterpreter.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -37,7 +37,6 @@ public class NoMethodMatchPythonExceptionInterpreter : PythonExceptionInterprete
         public override bool CanInterpret(Exception exception)
         {
             return base.CanInterpret(exception) &&
-                exception.Message.Contains("TypeError") &&
                 exception.Message.Contains("No method match");
         }
 

diff --git a/Common/Exceptions/UnsupportedOperandPythonExceptionInterpreter.cs b/Common/Exceptions/UnsupportedOperandPythonExceptionInterpreter.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -37,7 +37,6 @@ public class UnsupportedOperandPythonExceptionInterpreter : PythonExceptionInter
         public override bool CanInterpret(Exception exception)
         {
             return base.CanInterpret(exception) &&
-                exception.Message.Contains("TypeError") &&
                 exception.Message.Contains("unsupported operand type");
         }
 
@@ -51,7 +50,7 @@ public override Exception Interpret(Exception exception, IExceptionInterpreter i
         {
             var pe = (PythonException)exception;
 
-            var types = pe.Message.Split(':')[2].Trim();
+            var types = pe.Message.Split(':')[1].Trim();
             var message = $"Trying to perform a summation, subtraction, multiplication or division between {types} objects throws a TypeError exception. To prevent the exception, ensure that both values share the same type.";
             message += PythonUtil.PythonExceptionStackParser(pe.StackTrace);
 

diff --git a/Common/Extensions.cs b/Common/Extensions.cs
@@ -588,7 +588,7 @@ public static int GetPythonArgCount(this PyObject method)
             using (Py.GIL())
             {
                 int argCount;
-                var pyArgCount = PythonEngine.ModuleFromString(Guid.NewGuid().ToString(),
+                var pyArgCount = PyModule.FromString(Guid.NewGuid().ToString(),
                     "from inspect import signature\n" +
                     "def GetArgCount(method):\n" +
                     "   return len(signature(method).parameters)\n"
@@ -2651,7 +2651,7 @@ public static bool TryConvertToDelegate<T>(this PyObject pyObject, out T result)
                     var name = type.FullName.Substring(0, type.FullName.IndexOf('`'));
                     code = $"import System; delegate = {name}[{code.Substring(1)}](pyObject)";
 
-                    PythonEngine.Exec(code, null, locals.Handle);
+                    PythonEngine.Exec(code, null, locals);
                     result = (T)locals.GetItem("delegate").AsManagedObject(typeof(T));
                     locals.Dispose();
                     return true;
@@ -2816,7 +2816,8 @@ public static IEnumerable<Symbol> ConvertToSymbolEnumerable(this PyObject pyObje
                     pyObject = new PyList(new[] {pyObject});
                 }
 
-                foreach (PyObject item in pyObject)
+                using var iterator = pyObject.GetIterator();
+                foreach (PyObject item in iterator)
                 {
                     if (PyString.IsStringType(item))
                     {

diff --git a/Common/Python/MarginCallModelPythonWrapper.cs b/Common/Python/MarginCallModelPythonWrapper.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -62,7 +62,7 @@ public List<OrderTicket> ExecuteMarginCall(IEnumerable<SubmitOrderRequest> gener
                 // Since ExecuteMarginCall may return a python list
                 // Need to convert to C# list
                 var tickets = new List<OrderTicket>();
-                var iterator = marginCalls.GetIterator();
+                using var iterator = marginCalls.GetIterator();
                 foreach (PyObject pyObject in iterator)
                 {
                     OrderTicket ticket;
@@ -105,7 +105,8 @@ public List<SubmitOrderRequest> GetMarginCallOrders(out bool issueMarginCallWarn
                 // Since GetMarginCallOrders may return a python list
                 // Need to convert to C# list
                 var requests = new List<SubmitOrderRequest>();
-                foreach (PyObject pyObject in marginCallOrders)
+                using var iterator = marginCallOrders.GetIterator();
+                foreach (PyObject pyObject in iterator)
                 {
                     SubmitOrderRequest request;
                     if (pyObject.TryConvert(out request))
@@ -120,4 +121,4 @@ public List<SubmitOrderRequest> GetMarginCallOrders(out bool issueMarginCallWarn
             }
         }
     }
-}
+}
diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs
@@ -60,7 +60,7 @@ public PandasData(object data)
                 using (Py.GIL())
                 {
                     // Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs
-                    _pandas = PythonEngine.ImportModule("PandasMapper");
+                    _pandas = Py.Import("PandasMapper");
                 }
             }
 

diff --git a/Common/Python/PythonSlice.cs b/Common/Python/PythonSlice.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -32,7 +32,7 @@ public class PythonSlice : Slice
         static PythonSlice()
         {
             // Python Data class: Converts custom data (PythonData) into a python object'''
-            _converter = PythonEngine.ModuleFromString("converter",
+            _converter = PyModule.FromString("converter",
                 "class Data(object):\n" +
                 "    def __init__(self, data):\n" +
                 "        self.data = data\n" +
@@ -161,4 +161,4 @@ public override bool TryGetValue(Symbol symbol, out dynamic data)
             return _slice.TryGetValue(symbol, out data);
         }
     }
-}
+}
diff --git a/Common/QuantConnect.csproj b/Common/QuantConnect.csproj
@@ -35,7 +35,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="CloneExtensions" Version="1.3.0" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

diff --git a/Common/Util/PythonUtil.cs b/Common/Util/PythonUtil.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -256,7 +256,7 @@ private static bool TryGetArgLength(PyObject pyObject, out long length)
         /// <returns>PyObject with a python module</returns>
         private static PyObject GetModule()
         {
-            return PythonEngine.ModuleFromString("x",
+            return PyModule.FromString("x",
                 "from clr import AddReference\n" +
                 "AddReference(\"System\")\n" +
                 "from System import Action, Func\n" +

diff --git a/Engine/QuantConnect.Lean.Engine.csproj b/Engine/QuantConnect.Lean.Engine.csproj
@@ -43,7 +43,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="FSharp.Core" Version="4.5.2" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

diff --git a/Indicators/QuantConnect.Indicators.csproj b/Indicators/QuantConnect.Indicators.csproj
@@ -32,7 +32,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

diff --git a/Report/QuantConnect.Report.csproj b/Report/QuantConnect.Report.csproj
@@ -42,7 +42,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.12" />
     <PackageReference Include="Deedle" Version="2.1.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">

diff --git a/Report/ReportElements/ChartReportElement.cs b/Report/ReportElements/ChartReportElement.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -32,11 +32,11 @@ protected ChartReportElement()
 
             using (Py.GIL())
             {
-                dynamic module = PythonEngine.ImportModule("ReportCharts");
+                dynamic module = Py.Import("ReportCharts");
                 var classObj = module.ReportCharts;
 
                 Charting = classObj.Invoke();
             }
         }
     }
-}
+}
diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs
@@ -56,9 +56,10 @@ static QuantBook()
             //Determine if we are in a Python Notebook
             try
             {
+                PythonEngine.Initialize();
                 using (Py.GIL())
                 {
-                    var isPython = PythonEngine.ModuleFromString(Guid.NewGuid().ToString(),
+                    var isPython = PyModule.FromString(Guid.NewGuid().ToString(),
                         "try:\n" +
                         "   import IPython\n" +
                         "   def IsPythonNotebook():\n" +