added demo_project and generate

.gitignore

@@ -0,0 +1 @@
+config.py

README.md

@@ -1,13 +1,24 @@
 # lean-lite
-A hack to enable options data generation and backtesting in QuantConnect Lean (https://github.com/QuantConnect/Lean).
-As of this release, there is no need to change/modify/hack the lean source code.
+This is a simple hack to complement QuantConnect Lean (https://github.com/QuantConnect/Lean) and QuantConnect lean-cli (https://github.com/QuantConnect/lean-cli) to:
+- address some of the shortcomings with lean-cli
+    - simulated data generation is broken - especially for options
+    - processing simulated data does not currently work in the docker
+- create the ability to backtest with the lean source code - no docker
+    - great for running lean in debug which is useful if you want to understand the framework
+    - running with lean source is easy - perhaps even faster than docker (lean-cli)
+
+As of this release, there is no need to change/modify/hack the lean source code - except for some 
+configuration files (config.json). [Lean is very inconsistent in how it uses configuration files.  Furthermore, the requirement
+that the configuration files reside in the execution directory is just wrong.]
+
+Note: this distribution use Python version 3.9 - not the official QuantConnect version.  It works.
 
 # Installation
 Requirements
 - Since much of lean is in C#, I found it convienent to work in windows.  There is no reason why this
 will not work in linux.
 ## Prepare your environment
-1. Install anaconda
+1. Install anaconda https://www.anaconda.com/products/distribution
 2. Create a virtual environment with python 3.9:
     ~~~
     conda create --name lean python=3.9
@@ -28,11 +39,12 @@ is merely a convenient way to install the necessary python libraries for
 execution in the lean environment. These scripts (lean-lite) is a replacement for 
 lean-cli.*
 
-4. Create an environment variable for python:
+4. Create an environment variable for python (in windows):
     ~~~
     set PYTHONNET_PYDLL=ANACONDA\envs\lean\python39.dll
     ~~~
-where *ANACONDA* is the directory where anaconda is installed.
+where *ANACONDA* is the directory where anaconda is installed.  For more information about this
+see https://github.com/QuantConnect/Lean/tree/master/Algorithm.Python
 
 4. Clone lean into a different directory, i.e. projects:
     ~~~
@@ -62,8 +74,161 @@ the *data* subdirectory in a lean-cli project folder (after running *lean init*)
 ## Generate data (options)
 Option generation with lean-cli is *broken* [on purpose?]- especially, if you want options data. However,
 lean works.
+
+Problems with *lean data generate*:
+- unable to generate multiple contracts because the *--chain-symbol-count* flag is hidden.
+- generated data does not work in the current docker (why?)
+
+The generate.py script encapsulates the QuantConnect.Toolbox for RandomDataGenerator. The options are different 
+from *lean data generate*.
+
+To see the options, use:
+~~~
+python generate.py --help
+~~~
+This will return the Toolbox help:
+~~~
+Lean Engine ToolBox
+
+Usage: QuantConnect.ToolBox.exe [options]
+
+Options:
+  -?|-h|--help                         Show help information
+  -V|--version                         Show version information
+  --app                                [REQUIRED] Target tool, CASE INSENSITIVE: GDAXDownloader or
+                                       GDAXDL/CryptoiqDownloader or CDL/DukascopyDownloader or DDL/IEXDownloader or
+                                       IEXDL/FxcmDownloader or FDL/FxcmVolumeDownload or FVDL/GoogleDownloader or
+                                       GDL/IBDownloader or IBDL/KrakenDownloader or KDL/OandaDownloader or
+                                       ODL/QuandlBitfinexDownloader or QBDL/YahooDownloader or
+                                       YDL/AlgoSeekFuturesConverter or ASFC/AlgoSeekOptionsConverter or
+                                       ASOC/IVolatilityEquityConverter or IVEC/KaikoDataConverter or
+                                       KDC/NseMarketDataConverter or NMDC/QuantQuoteConverter or
+                                       QQC/CoarseUniverseGenerator or CUG/
+                                       RandomDataGenerator or RDG
+                                       Example 1: --app=DDL
+                                       Example 2: --app=NseMarketDataConverter
+                                       Example 3: --app=RDG
+  --tickers                            [REQUIRED ALL downloaders (except QBDL)] --tickers=SPY,AAPL,etc
+  --resolution                         [REQUIRED ALL downloaders (except QBDL, CDL) and IVolatilityEquityConverter,
+                                       QuantQuoteConverter] *Not all downloaders support all resolutions. Send empty for
+                                       more information.* CASE SENSITIVE: --resolution=Tick/Second/Minute/Hour/Daily/All
+                                       [OPTIONAL for RandomDataGenerator - same format as downloaders, Options only
+                                       support Minute
+  --from-date                          [REQUIRED ALL downloaders] --from-date=yyyyMMdd-HH:mm:ss
+  --to-date                            [OPTIONAL for downloaders] If not provided 'DateTime.UtcNow' will be used.
+                                       --to-date=yyyyMMdd-HH:mm:ss
+  --exchange                           [REQUIRED for CryptoiqDownloader] [Optional for KaikoDataConverter] The exchange
+                                       to process, if not defined, all exchanges will be processed.
+  --api-key                            [REQUIRED for QuandlBitfinexDownloader, IEXDownloader]
+  --date                               [REQUIRED for AlgoSeekFuturesConverter, AlgoSeekOptionsConverter,
+                                       KaikoDataConverter]Date for the option bz files: --date=yyyyMMdd
+  --source-dir                         [REQUIRED for IVolatilityEquityConverter, KaikoDataConverter,
+                                       CoinApiDataConverter, NseMarketDataConverter, QuantQuoteConverter]
+  --destination-dir                    [REQUIRED for IVolatilityEquityConverter, NseMarketDataConverter,
+                                       QuantQuoteConverter]
+  --source-meta-dir                    [REQUIRED for IVolatilityEquityConverter]
+  --start                              [REQUIRED for RandomDataGenerator. Format yyyyMMdd Example: --start=20010101]
+  --end                                [REQUIRED for RandomDataGenerator. Format yyyyMMdd Example: --end=20020101]
+  --market                             [OPTIONAL for RandomDataGenerator. Market of generated symbols. Defaults to
+                                       default market for security type: Example: --market=usa]
+  --symbol-count                       [REQUIRED for RandomDataGenerator. Number of symbols to generate data for:
+                                       Example: --symbol-count=10]
+  --security-type                      [OPTIONAL for RandomDataGenerator. Security type of generated symbols, defaults
+                                       to Equity: Example: --security-type=Equity/Option/Forex/Future/Cfd/Crypto]
+  --data-density                       [OPTIONAL for RandomDataGenerator. Defaults to Dense. Valid values:
+                                       --data-density=Dense/Sparse/VerySparse ]
+  --include-coarse                     [OPTIONAL for RandomDataGenerator. Only used for Equity, defaults to true:
+                                       Example: --include-coarse=true]
+  --quote-trade-ratio                  [OPTIONAL for RandomDataGenerator. Sets the ratio of generated quotes to
+                                       generated trades. Values larger than 1 mean more quotes than trades. Only used
+                                       for Option, Future and Crypto, defaults to 1: Example: --quote-trade-ratio=1.75 ]
+  --random-seed                        [OPTIONAL for RandomDataGenerator. Sets the random number generator seed.
+                                       Defaults to null (random seed). Example: --random-seed=11399 ]
+  --ipo-percentage                     [OPTIONAL for RandomDataGenerator. Sets the probability each equity generated
+                                       will have an IPO event. Note that this is not the total probability for all
+                                       symbols generated. Only used for Equity. Defaults to 5.0: Example:
+                                       --ipo-percentage=43.25 ]
+  --rename-percentage                  [OPTIONAL for RandomDataGenerator. Sets the probability each equity generated
+                                       will have a rename event. Note that this is not the total probability for all
+                                       symbols generated. Only used for Equity. Defaults to 30.0: Example:
+                                       --rename-percentage=20.0 ]
+  --splits-percentage                  [OPTIONAL for RandomDataGenerator. Sets the probability each equity generated
+                                       will have a stock split event. Note that this is not the total probability for
+                                       all symbols generated. Only used for Equity. Defaults to 15.0: Example:
+                                       --splits-percentage=10.0 ]
+  --dividends-percentage               [OPTIONAL for RandomDataGenerator. Sets the probability each equity generated
+                                       will have dividends. Note that this is not the probability for all symbols
+                                       genearted. Only used for Equity. Defaults to 60.0: Example:
+                                       --dividends-percentage=25.5 ]
+  --dividend-every-quarter-percentage  [OPTIONAL for RandomDataGenerator. Sets the probability each equity generated
+                                       will have a dividend event every quarter. Note that this is not the total
+                                       probability for all symbols generated. Only used for Equity. Defaults to 30.0:
+                                       Example: --dividend-every-quarter-percentage=15.0 ]
+  --option-price-engine                [OPTIONAL for RandomDataGenerator. Sets the stochastic process, and returns new
+                                       pricing engine to run calculations for that option. Defaults to
+                                       BaroneAdesiWhaleyApproximationEngine: Example:
+                                       --option-price-engine=BaroneAdesiWhaleyApproximationEngine ]
+  --volatility-model-resolution        [OPTIONAL for RandomDataGenerator. Sets the volatility model period span.
+                                       Defaults to Daily: Example: --volatility-model-resolution=Daily ]
+  --chain-symbol-count                 [OPTIONAL for RandomDataGenerator. Sets the size of the option chain. Defaults to
+                                       1 put and 1 call: Example: --chain-symbol-count=2 ]
+
+The ToolBox is a wrapper of >15 tools. Each require a different set of parameters. Example: --app=YahooDownloader --tickers=SPY,AAPL --resolution=Daily --from-date=yyyyMMdd-HH:mm:ss --to-date=yyyyMMdd-HH:mm:ss
+~~~
+*Note: do not specify the --app flag!*
+
+To generate options for one ticker with 10 strikes:
+~~~
+python generate.py --start 20220101 --end 20220501 --symbol-count 1 --security-type Option --resolution Minute --chain-symbol-count 1
+~~~
+
 ## Run a backtest
 At this point, we need a project.  You can use *lean cloud pull* (after you *lean login*) to retrieve
-project you have on QuantConnect.com.  Otherwise, use the sample project included with this distribution.
+project you have on QuantConnect.com.  Otherwise, use the *demo_project* included with this distribution.
+
+With this demo, we will use the options generated in the Generate section above.  Look in the DATADIR/options/usa/minute to get the ticker symbol.
+
+The demo project uses parameters for the ticker.  Edit demo_project/config.json and change the ticker:
+~~~
+{
+    "algorithm-language": "Python",
+    "parameters": {
+        "TAKE_PROFIT": "0.5",
+        "MAX_DELTA_PUT": "0.4",
+        "MAX_DELTA_CALL": "0.4",
+        "ticker": "NXJ"
+    },
+    "description": "exploring",
+    "cloud-id": 11592212,
+    "local-id": 840173305
+}
+~~~
+**IF YOU DO NOT CHANGE THE TICKER, YOU WILL NOT GET ANY RESULTS**
+
+To process a backtest and generate a report:
+~~~
+python backtest.py --main demo_project/main.py
+~~~
+
+The python class name for this demo_project is OptionWhelAlgorithm. This will generate files in the demo_project/backtests/[timestamp] directory:
+- log.txt
+- OptionWhelAlgorithm - financial analysis 
+- OptionWhelAlgorithm_config.json - the Lean configuration file
+- OptionWhelAlgorithm-log.txt - more logs
+- OptionWhelAlgorithm-order-events.json - the trade transactions
+- report.html - the tearsheet report
+- report-backtesting-portfolio.json - portfolio stats
+- report-live-portfolio.json - portfolio stats (repeat?)
+- main.py - the code used in this backtest
+
+
+## Summary
+Please:
+- fork and improve
+- report issues
+- send comments / suggestions to [email protected]
+
+Fini.
+
 
 

backtest.py

@@ -49,10 +49,6 @@
 # FIXME: default values should be in a configuration
 parser = argparse.ArgumentParser("lean runner")
 parser.add_argument('--main',type=str,default='main.py')
-# parser.add_argument('--datadir',type=str,default="s:/PROJECTS/Lean.Github/Data")
-parser.add_argument('--datadir',type=str,default=DATADIR)
-parser.add_argument('--leandir',type=str,default=LEANDIR+"/Launcher/bin/Debug")
-parser.add_argument('--leanreportdir',type=str,default=LEANDIR+"/Report/bin/Debug")
 args = parser.parse_args()
 
 # useful values
@@ -67,77 +63,79 @@ def getclassname(fn):
     """
     with open(fn,"r") as f:
         for line in f:
-
-            # checking condition for string found or not
             if "class" in line: 
-                # print(line)
                 cn = line.split()[1].split("(")[0]
-                # print(cn)
                 return cn
 
 def getparams(fn):
     with open(fn,"r") as f:
-        # content = f.readlines()
         d = json.load(f)
     return d
 
+print("ENVIRONMENT:")
+print(f"\tDATDIR={DATADIR}")
+print(f"\tLEANDIR={LEANDIR}\n")
+
 # prepare a backtest
 wdir = str(Path(args.main).resolve())
 basedir = str(Path(os.path.dirname(wdir)))
+# the project parameters
 configfile = str(Path(basedir+"/config.json"))
+
+# create results folder
+btfolder = str(Path(basedir+"/backtests"))
+exedir = f"{LEANDIR}/Launcher/bin/debug"
+if not os.path.exists(btfolder):
+  os.mkdir(btfolder)
+# prepare a report folder
+fdrname = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+btfldr = str(Path(btfolder+"/"+fdrname))
+os.mkdir(btfldr)
+# get the class name
 cn = getclassname(args.main)
 configd = getparams(configfile)
+#define the lean backtest config
+config = f"{btfldr}/{cn}_config.json"
+config = str(Path(config).resolve())
 # add the config to the config
-core = {**core,**configd}
-core["algorithm-type-name"]=cn
-core["algorithm-location"]=wdir
-core["data-dir"]=str(Path(args.datadir))
-core["data-directory"]=str(Path(args.datadir))
-core["cache-location"]=str(Path(args.datadir))
-core[ "data-folder"]=str(Path(args.datadir))
-core["close-automatically"]=True
-# core["results-destination-folder"]=wdir
+launcher_template["algorithm-type-name"]=cn
+launcher_template["algorithm-location"]=wdir
+launcher_template["data-directory"]=str(Path(DATADIR))
+launcher_template["cache-location"]=str(Path(DATADIR))
+launcher_template[ "data-folder"]=str(Path(DATADIR))
+launcher_template["close-automatically"]=True
+launcher_template["results-destination-folder"]=btfldr
+launcher_template = {**launcher_template,**configd}
 # write the config
-config = f"{cn}.json"
 with open(config,"w") as outfile:
-    json.dump(core,outfile,indent=4)
+    json.dump(launcher_template,outfile,indent=4)
 
-cdir = str(Path(config).resolve())
 # run the program
-os.chdir(args.leandir)
-cmd = f"{lean} --config {cdir}"
+os.chdir(str(Path(exedir)))
+cmd = f"{lean} --config {config}"
 os.system(cmd)
 # the results are the launcher file
 orderfn = f"{cn}-order-events.json"
 resultfn= f"{cn}.json"
-# create results folder
-btfolder = str(Path(basedir+"/backtests"))
-if not os.path.exists(btfolder):
-  os.mkdir(btfolder)
-
-# prepare a report
-fdrname = datetime.now().strftime("%Y-%m-%d-%H-%M")
-btfldr = str(Path(btfolder+"/"+fdrname))
-os.mkdir(btfldr)
-# copy
-print("copy ",f"{args.leandir}/{orderfn}",f"{btfldr}/orders.json")
-shutil.copy(str(Path(f"{args.leandir}/{resultfn}")),str(Path(f"{btfldr}/results.json")))
-shutil.copy(str(Path(f"{args.leandir}/{orderfn}")),str(Path(f"{btfldr}/orders.json")))
+#copy the code
 shutil.copy(str(Path(f"{wdir}")),str(Path(f"{btfldr}/{os.path.basename(wdir)}")))
+
 # generate a report
-report_config["strategy-name"]=cn
-report_config["strategy-version"]=fdrname
-report_config["strategy-description"]=f"Created {fdrname}"
-report_config["backtest-data-source-file"]=str(Path(f"{btfldr}/results.json"))
-report_config["report-destination"]=str(Path(f"{btfldr}/report.html"))
-report_config["live-data-source-file"]=str(Path(f"{btfldr}/results.json"))
-report_config[ "data-folder"]=str(Path(args.datadir))
+report_template["strategy-name"]=cn
+report_template["strategy-version"]=fdrname
+report_template["strategy-description"]=f"Created {fdrname}"
+report_template["backtest-data-source-file"]=str(Path(f"{btfldr}/{resultfn}"))
+report_template["report-destination"]=str(Path(f"{btfldr}/report.html"))
+report_template["live-data-source-file"]=str(Path(f"{btfldr}/{resultfn}"))
+report_template["data-folder"]=str(Path(DATADIR))
 
-config = f"{args.leanreportdir}/config.json"
+exedir = f"{LEANDIR}/Report/bin/debug"
+# lean requires the configuration file in the exe dir. yep.
+config = f"{exedir}/config.json"
 with open(config,"w") as f:
-  json.dump(report_config,f)
-
-os.chdir(args.leanreportdir)
+  json.dump(report_template,f)
+os.chdir(exedir)
 cmd = f"{reportexe}"
 os.system(cmd)
+
 print("Fini")

config.py

@@ -36,4 +36,4 @@
 DATADIR="s:/data"
 
 # the directory where lean is cloned into:
-LEANDIR="s:/Lean"
+LEANDIR="s:/PROJECTS/Lean.github"

demo_project/config.json

@@ -0,0 +1,12 @@
+{
+    "algorithm-language": "Python",
+    "parameters": {
+        "TAKE_PROFIT": "0.5",
+        "MAX_DELTA_PUT": "0.4",
+        "MAX_DELTA_CALL": "0.4",
+        "ticker": "RVW"
+    },
+    "description": "exploring",
+    "cloud-id": 11592212,
+    "local-id": 840173305
+}