Support list operations on universe selection

Common/Data/UniverseSelection/CoarseFundamentalUniverse.cs

@@ -17,6 +17,7 @@
 using System.Linq;
 using Python.Runtime;
 using System.Collections.Generic;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -87,7 +88,7 @@ public CoarseFundamentalUniverse(Symbol symbol, UniverseSettings universeSetting
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            return _selector(data.Data.OfType<CoarseFundamental>());
+            return _selector(new CastingEnumerable<BaseData, CoarseFundamental>(data.Data));
         }
 
         /// <summary>

Common/Data/UniverseSelection/FineFundamentalUniverse.cs

@@ -17,6 +17,7 @@
 using System.Linq;
 using System.Collections.Generic;
 using QuantConnect.Data.Fundamental;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -60,7 +61,7 @@ public FineFundamentalUniverse(Symbol symbol, UniverseSettings universeSettings,
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            return _selector(data.Data.OfType<FineFundamental>());
+            return _selector(new CastingEnumerable<BaseData, FineFundamental>(data.Data));
         }
 
         /// <summary>

Common/Data/UniverseSelection/FuncUniverse.cs

@@ -17,6 +17,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Python.Runtime;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -62,7 +63,7 @@ public FuncUniverse(SubscriptionDataConfig configuration, UniverseSettings unive
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            return _universeSelector(data.Data.Cast<T>());
+            return _universeSelector(new CastingEnumerable<BaseData, T>(data.Data));
         }
     }
 

Common/Data/UniverseSelection/FundamentalUniverseFactory.cs

@@ -18,6 +18,7 @@
 using Python.Runtime;
 using System.Collections.Generic;
 using QuantConnect.Data.Fundamental;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -84,7 +85,7 @@ public FundamentalUniverseFactory(Symbol symbol, UniverseSettings universeSettin
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            return _selector(data.Data.OfType<Fundamental.Fundamental>());
+            return _selector(new CastingEnumerable<BaseData, Fundamental.Fundamental>(data.Data));
         }
 
         /// <summary>

Common/Data/UniverseSelection/FuturesChainUniverse.cs

@@ -19,6 +19,7 @@
 using System.Linq;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -84,7 +85,7 @@ public override UniverseSettings UniverseSettings
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
             var localEndTime = utcTime.ConvertFromUtc(Future.Exchange.TimeZone);
-            var availableContracts = data.Data.Cast<FutureUniverse>().ToList();
+            var availableContracts = new CastingEnumerable<BaseData, FutureUniverse>(data.Data);
             var results = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, localEndTime));
 
             return results.Select(x => x.Symbol);

Common/Data/UniverseSelection/OptionChainUniverse.cs

@@ -20,6 +20,7 @@
 using QuantConnect.Securities;
 using System.Collections.Generic;
 using QuantConnect.Securities.Option;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data.UniverseSelection
 {
@@ -92,7 +93,7 @@ public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataColl
         {
             var localEndTime = utcTime.ConvertFromUtc(Option.Exchange.TimeZone);
             // we will only update unique strikes when there is an exchange date change
-            _optionFilterUniverse.Refresh(data.Data.Cast<OptionUniverse>().ToList(), data.Underlying, localEndTime);
+            _optionFilterUniverse.Refresh(new CastingEnumerable<BaseData, OptionUniverse>(data.Data), data.Underlying, localEndTime);
 
             var results = Option.ContractFilter.Filter(_optionFilterUniverse);
 

Common/Securities/ContractSecurityFilterUniverse.cs

@@ -32,7 +32,7 @@ public abstract class ContractSecurityFilterUniverse<T, TData> : IDerivativeSecu
     {
         private bool _alreadyAppliedTypeFilters;
 
-        private IEnumerable<TData> _data;
+        private IReadOnlyList<TData> _data;
 
         /// <summary>
         /// Defines listed contract types with Flags attribute
@@ -74,7 +74,7 @@ protected enum ContractExpirationType : int
         /// <remarks>
         /// Setting it will also set AllSymbols
         /// </remarks>
-        internal IEnumerable<TData> Data
+        internal IReadOnlyList<TData> Data
         {
             get
             {
@@ -118,7 +118,7 @@ protected ContractSecurityFilterUniverse()
         /// <summary>
         /// Constructs ContractSecurityFilterUniverse
         /// </summary>
-        protected ContractSecurityFilterUniverse(IEnumerable<TData> allData, DateTime localTime)
+        protected ContractSecurityFilterUniverse(IReadOnlyList<TData> allData, DateTime localTime)
         {
             Data = allData;
             LocalTime = localTime;
@@ -188,7 +188,7 @@ internal T ApplyTypesFilter()
         /// </summary>
         /// <param name="allData">All data for contracts in the Universe</param>
         /// <param name="localTime">The local exchange current time</param>
-        public virtual void Refresh(IEnumerable<TData> allData, DateTime localTime)
+        public virtual void Refresh(IReadOnlyList<TData> allData, DateTime localTime)
         {
             Data = allData;
             LocalTime = localTime;

Common/Securities/Future/FutureFilterUniverse.cs

@@ -31,7 +31,7 @@ public class FutureFilterUniverse : ContractSecurityFilterUniverse<FutureFilterU
         /// <summary>
         /// Constructs FutureFilterUniverse
         /// </summary>
-        public FutureFilterUniverse(IEnumerable<FutureUniverse> allData, DateTime localTime)
+        public FutureFilterUniverse(IReadOnlyList<FutureUniverse> allData, DateTime localTime)
             : base(allData, localTime)
         {
         }

Common/Securities/Option/OptionFilterUniverse.cs

@@ -71,7 +71,7 @@ public OptionFilterUniverse(Option.Option option)
         /// Constructs OptionFilterUniverse
         /// </summary>
         /// <remarks>Used for testing only</remarks>
-        public OptionFilterUniverse(Option.Option option, IEnumerable<OptionUniverse> allData, BaseData underlying, decimal underlyingScaleFactor = 1)
+        public OptionFilterUniverse(Option.Option option, IReadOnlyList<OptionUniverse> allData, BaseData underlying, decimal underlyingScaleFactor = 1)
             : base(allData, underlying.EndTime)
         {
             _option = option;
@@ -86,7 +86,7 @@ public OptionFilterUniverse(Option.Option option, IEnumerable<OptionUniverse> al
         /// <param name="allContractsData">All data for the option contracts</param>
         /// <param name="underlying">The current underlying last data point</param>
         /// <param name="localTime">The current local time</param>
-        public void Refresh(IEnumerable<OptionUniverse> allContractsData, BaseData underlying, DateTime localTime)
+        public void Refresh(IReadOnlyList<OptionUniverse> allContractsData, BaseData underlying, DateTime localTime)
         {
             base.Refresh(allContractsData, localTime);
 
@@ -1037,7 +1037,7 @@ private IEnumerable<Symbol> GetContractsForExpiry(IEnumerable<Symbol> symbols, i
         /// </summary>
         private OptionFilterUniverse Empty()
         {
-            Data = Enumerable.Empty<OptionUniverse>();
+            Data = Enumerable.Empty<OptionUniverse>().ToList();
             return this;
         }
 

Common/Util/CastingEnumerable.cs

@@ -0,0 +1,81 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); 
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Util
+{
+    /// <summary>
+    /// Defines a list that casts the elements of a source list to a derived type.
+    /// This is useful to avoid materializing another list after using, for example, the <see cref="Enumerable.Cast{TResult}(IEnumerable)"/> LINQ method.
+    /// </summary>
+    /// <typeparam name="TBase">The base type of the elements in the source enumerable.</typeparam>
+    /// <typeparam name="TDerived">The type to cast the elements to.</typeparam>
+    public class CastingEnumerable<TBase, TDerived> : IReadOnlyList<TDerived>
+        where TDerived : class, TBase
+    {
+        private IReadOnlyList<TBase> _data;
+
+        /// <summary>
+        /// Gets the count of items in the enumerable.
+        /// </summary>
+        public int Count => _data.Count;
+
+        /// <summary>
+        /// Gets the element at the specified index.
+        /// </summary>
+        /// <param name="index">The zero-based index of the element to get.</param>
+        /// <returns>The element at the specified index.</returns>
+        public TDerived this[int index] => _data[index] as TDerived;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CastingEnumerable{TBase, TDerived}"/> class
+        /// </summary>
+        public CastingEnumerable(IReadOnlyList<TBase> data)
+        {
+            _data = data;
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through the collection.
+        /// </summary>
+        /// <returns>
+        /// An enumerator that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<TDerived> GetEnumerator()
+        {
+            foreach (var item in _data)
+            {
+                yield return (TDerived)item;
+            }
+        }
+
+        /// <summary>
+        /// Returns an enumerator that iterates through a collection.
+        /// </summary>
+        /// <returns>
+        /// An enumerator object that can be used to iterate through the collection.
+        /// </returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

Research/QuantBook.cs

@@ -864,7 +864,7 @@ public PyDict GetPortfolioStatistics(PyObject dataFrame)
         /// <summary>
         /// Get's the universe data for the specified date
         /// </summary>
-        private IEnumerable<T> GetChainHistory<T>(Symbol canonicalSymbol, DateTime date, out BaseData underlyingData)
+        private IReadOnlyList<T> GetChainHistory<T>(Symbol canonicalSymbol, DateTime date, out BaseData underlyingData)
             where T : BaseChainUniverseData
         {
             // Use this GetEntry extension method since it's data type dependent, so we get the correct entry for the option universe
@@ -878,17 +878,17 @@ private IEnumerable<T> GetChainHistory<T>(Symbol canonicalSymbol, DateTime date,
             if (universeData is not null)
             {
                 underlyingData = universeData.Underlying;
-                return universeData.Data.Cast<T>();
+                return new CastingEnumerable<BaseData, T>(universeData.Data);
             }
 
             underlyingData = null;
-            return Enumerable.Empty<T>();
+            return Enumerable.Empty<T>().ToList();
         }
 
         /// <summary>
         /// Helper method to get option/future chain historical data for a given date range
         /// </summary>
-        private IEnumerable<(DateTime Date, IEnumerable<T> ChainData, BaseData UnderlyingData)> GetChainHistory<T>(
+        private IEnumerable<(DateTime Date, IReadOnlyList<T> ChainData, BaseData UnderlyingData)> GetChainHistory<T>(
             Security security, DateTime start, DateTime end, bool extendedMarketHours)
             where T : BaseChainUniverseData
         {

Tests/Common/Securities/FutureFilterTests.cs

@@ -53,7 +53,7 @@ public void FiltersExpiryRange()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, time)).Select(x => x.Symbol).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), time)).Select(x => x.Symbol).ToList();
             Assert.AreEqual(5, filtered.Count);
             Assert.AreEqual(symbols[3], filtered[0]);
             Assert.AreEqual(symbols[4], filtered[1]);
@@ -86,7 +86,7 @@ public void FutureContractFiltering(bool standardsOnly, int expectedCount)
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, time)).Select(x => x.Symbol).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), time)).Select(x => x.Symbol).ToList();
 
             Assert.AreEqual(expectedCount, filtered.Count);
 
@@ -126,7 +126,7 @@ public void WeeklysFilterDoesNotFilterStandardContractWithExpiryMonthPriorOrAfte
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var standardContracts = filter.Filter(new FutureFilterUniverse(data, new DateTime(2020, 1, 1))).Select(x => x.Symbol).ToList();
+            var standardContracts = filter.Filter(new FutureFilterUniverse(data.ToList(), new DateTime(2020, 1, 1))).Select(x => x.Symbol).ToList();
             Assert.AreEqual(6, standardContracts.Count);
             Assert.AreEqual(symbols[0], standardContracts[0]);
             Assert.AreEqual(symbols[1], standardContracts[1]);
@@ -159,7 +159,7 @@ public void FilterAllowBothTypes()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, time)).Select(x => x.Symbol).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), time)).Select(x => x.Symbol).ToList();
             Assert.AreEqual(6, filtered.Count);
             Assert.AreEqual(symbols, filtered);
         }
@@ -187,7 +187,7 @@ public void FilterOutStandards()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, time)).Select(x => x.Symbol).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), time)).Select(x => x.Symbol).ToList();
             Assert.AreEqual(4, filtered.Count);
             Assert.AreEqual(symbols[1], filtered[0]);
             Assert.AreEqual(symbols[2], filtered[1]);
@@ -224,7 +224,7 @@ public void FiltersFrontMonth()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, new DateTime(2016, 02, 26))).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), new DateTime(2016, 02, 26))).ToList();
             Assert.AreEqual(4, filtered.Count);
         }
 
@@ -257,7 +257,7 @@ public void FiltersBackMonth()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, new DateTime(2016, 02, 26))).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), new DateTime(2016, 02, 26))).ToList();
             Assert.AreEqual(3, filtered.Count);
         }
 
@@ -290,7 +290,7 @@ public void FiltersExpirationCycles()
             };
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
-            var filtered = filter.Filter(new FutureFilterUniverse(data, new DateTime(2016, 02, 26))).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), new DateTime(2016, 02, 26))).ToList();
             Assert.AreEqual(5, filtered.Count);
         }
 
@@ -314,7 +314,7 @@ public void FilterTypeDoesNotBreakOnMissingExpiryFunction()
             var data = symbols.Select(x => new FutureUniverse() { Symbol = x });
 
             // Since this is a unidentifiable symbol for our expiry functions it will return true and be passed through
-            var filtered = filter.Filter(new FutureFilterUniverse(data, time)).Select(x => x.Symbol).ToList();
+            var filtered = filter.Filter(new FutureFilterUniverse(data.ToList(), time)).Select(x => x.Symbol).ToList();
             Assert.AreEqual(1, filtered.Count);
             Assert.AreEqual(symbols[0], filtered[0]);
         }