# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from AlgorithmImports import *

### <summary>
### In this algorithm, we fetch a list of tickers with corresponding dates from a file on Dropbox.
### We then create a fine fundamental universe which contains those symbols on their respective dates.###
### </summary>
### <meta name="tag" content="download" />
### <meta name="tag" content="universes" />
### <meta name="tag" content="custom data" />
class DropboxCoarseFineAlgorithm(QCAlgorithm):

 def initialize(self):
 self.set_start_date(2019, 9, 23) # Set Start Date
 self.set_end_date(2019, 9, 30) # Set End Date
 self.set_cash(100000) # Set Strategy Cash
 self.add_universe(self.select_coarse, self.select_fine)

 self.universe_data = None
 self.next_update = datetime(1, 1, 1) # Minimum datetime
 self.url = "https://www.dropbox.com/s/x2sb9gaiicc6hm3/tickers_with_dates.csv?dl=1"

 def on_end_of_day(self, symbol: Symbol) -> None:
 self.debug(f"{self.time.date()}{symbol.value} with Market Cap: ${self.securities[symbol].fundamentals.market_cap}")

 def select_coarse(self, coarse):
 return self.get_symbols()

 def select_fine(self, fine):
 symbols = self.get_symbols()

 # Return symbols from our list which have a market capitalization of at least 10B
 return [f.symbol for f in fine if f.market_cap > 1e10 and f.symbol in symbols]

 def get_symbols(self):

 # In live trading update every 12 hours
 if self.live_mode:
 if self.time < self.next_update:
 # Return today's row
 return self.universe_data[self.time.date()]
 # When updating set the new reset time.
 self.next_update = self.time + timedelta(hours=12)
 self.universe_data = self.parse(self.url)

 # In backtest load once if not set, then just use the dates.
 if not self.universe_data:
 self.universe_data = self.parse(self.url)

 # Check if contains the row we need
 if self.time.date() not in self.universe_data:
 return Universe.UNCHANGED

 return self.universe_data[self.time.date()]


 def parse(self, url):
 # Download file from url as string
 file = self.download(url).split("\n")

 # # Remove formatting characters
 data = [x.replace("\r", "").replace(" ", "") for x in file]

 # # Split data by date and symbol
 split_data = [x.split(",") for x in data]

 # Dictionary to hold list of active symbols for each date, keyed by date
 symbols_by_date = {}

 # Parse data into dictionary
 for arr in split_data:
 date = datetime.strptime(arr[0], "%Y%m%d").date()
 symbols = [Symbol.create(ticker, SecurityType.EQUITY, Market.USA) for ticker in arr[1:]]
 symbols_by_date[date] = symbols

 return symbols_by_date

 def on_securities_changed(self, changes):
 self.log(f"Added Securities: {[security.symbol.value for security in changes.added_securities]}")