feat!: Type definition specification with env vars and JSON files (#399)

* feat!: Type definition specification with env vars and JSON files

BREAKING CHANGE remove /config/types.json; Replaced with SAS_SUBSTRATE_TYPES.

* Add links to readme

* chore(deps): Bump deps

README.md

@@ -39,7 +39,6 @@ This service requires Node version 12 or higher.
 - [Chain integration guide](/CHAIN_INTEGRATION.md)
 - [Docker](#docker)
 - [Note for maintainers](#note-for-maintainers)
-- [Roadmap](#roadmap)
 
 ## NPM package installation and usage
 
@@ -89,7 +88,7 @@ Simply run `yarn`.
 
 ### Rust development installation
 
-If you are looking to hack on the `calc` Rust crate make sure your machine has an [up-to-date version of `rustup`](https://www.rust-lang.org/tools/install) 
+If you are looking to hack on the `calc` Rust crate make sure your machine has an [up-to-date version of `rustup`](https://www.rust-lang.org/tools/install)
 installed to manage Rust dependencies.
 
 Install `wasm-pack` if your machine does not already have it:
@@ -142,20 +141,45 @@ For more information on our configuration manager visit its readme [here](https:
 #### Custom substrate types
 
 Some chains require custom type definitions in order for Sidecar to know how to decode the data
-retrieved from the node. You can define chain specific types in `config/types.json`. Read more about [defining
-types for polkadot-js here.](https://polkadot.js.org/api/start/types.extend.html)
+retrieved from the node. Sidecar pulls types for chains from [@polkadot/apps-config](https://github.com/polkadot-js/apps/tree/master/packages/apps-config), but in some cases
+the types for the chain you are trying to connect to may be out of date or may simply not exist in
+@polkadot/apps-config.
 
-If you are connecting to [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), please add the following custom types in `config/types.json`.
+Sidecar affords environment variables which allow the user to specify an absolute path to a JSON file
+that contains type definitions in the corresponding formats. Consult polkadot-js/api for more info on
+the type formats (see `RegisteredTypes`).
+
+**N.B** Types set from environment variables will override the corresponding types pulled from
+@polkadot/apps-config.
+
+-   `SAS_SUBSTRATE_TYPES_BUNDLE`: a bundle of types with versioning info, type aliases, derives, and
+    rpc definitions. Format: `OverrideBundleType` (see [`typesBundle`](https://github.com/polkadot-js/api/blob/21039dec1fcad36061a96bf5526248c5fab38780/packages/types/src/types/registry.ts#L72)).
+-   `SAS_SUBSTRATE_TYPES_CHAIN`: type definitions keyed by `chainName`. Format: `Record<string, RegistryTypes>` (see [`typesChain`](https://github.com/polkadot-js/api/blob/21039dec1fcad36061a96bf5526248c5fab38780/packages/types/src/types/registry.ts#L76)).
+-   `SAS_SUBSTRATE_TYPES_SPEC`: type definitions keyed by `specName`. Format: `Record<string, RegistryTypes>` (see [`typesSpec`](https://github.com/polkadot-js/api/blob/21039dec1fcad36061a96bf5526248c5fab38780/packages/types/src/types/registry.ts#L80)).
+-   `SAS_SUBSTRATE_TYPES`: type definitions and overrides, not keyed. Format: `RegistryTypes` (see [`types`](https://github.com/polkadot-js/api/blob/21039dec1fcad36061a96bf5526248c5fab38780/packages/types/src/types/registry.ts#L64)).
+
+You can read more about [defining types for polkadot-js here.](https://polkadot.js.org/api/start/types.extend.html)
+
+##### Connecting a modified node template
+
+Polkadot-js can recognize the standard node template and inject the correct types, but if you have
+modified the name of your chain in the node template you will need to add the types manually in a
+JSON `types` file like so:
 
 ```json
+// my-chains-types.json
 {
-	"CUSTOM_TYPES": {
-		"Address": "AccountId",
-		"LookupSource": "AccountId"
-	}
+  "Address": "AccountId",
+  "LookupSource": "AccountId"
 }
 ```
 
+and then set the enviroment variable to point to your definitions:
+
+```bash
+export SAS_SUBSTRATE_TYPES=/path/to/my-chains-types.json
+```
+
 ### Logging
 
 -   `SAS_LOG_LEVEL`: the lowest priority log level to surface, defaults to `info`. Tip: set to `http`
@@ -271,10 +295,3 @@ $ rimraf lib/ && tsc
 
 To publish the new package, just follow the instructions: `git push --follow-tags origin master && npm publish.`
 You must have access to the @substrate organization on npm to publish.
-
-## Roadmap
-
-- Investigate and implement support for parachains in Sidecar. At this moment there is no concrete
-plan, but options that allow configuration and plugins specified by parachain development teams is
-one possible path forward. Initial support will be focused on enabling the workflow for core balance
-transfer and monitoring features.

package.json

@@ -53,10 +53,10 @@
     "@types/jest": "^26.0.20",
     "@types/morgan": "^1.9.2",
     "@types/triple-beam": "^1.3.2",
-    "@typescript-eslint/eslint-plugin": "4.13.0",
-    "@typescript-eslint/parser": "4.13.0",
+    "@typescript-eslint/eslint-plugin": "4.14.0",
+    "@typescript-eslint/parser": "4.14.0",
     "eslint": "^7.17.0",
-    "eslint-config-prettier": "^7.1.0",
+    "eslint-config-prettier": "^7.2.0",
     "eslint-plugin-prettier": "^3.3.1",
     "eslint-plugin-simple-import-sort": "^7.0.0",
     "jest": "^26.6.3",

src/SidecarConfig.ts

@@ -1,6 +1,5 @@
 import { ConfigManager } from 'confmgr';
 
-import * as configTypes from '../config/types.json';
 import { Specs } from './Specs';
 import { CONFIG, ISidecarConfig, MODULES } from './types/sidecar-config';
 
@@ -17,7 +16,7 @@ export class SidecarConfig {
 	 * Gather env vars for config and make sure they are valid.
 	 */
 	private static create(): ISidecarConfig {
-		// Instantiate ConfigManager which is used to read in the specs.yml
+		// Instantiate ConfigManager which is used to read in the specs
 		const config = ConfigManager.getInstance(Specs.specs).getConfig();
 
 		if (!config.Validate()) {
@@ -43,7 +42,19 @@ export class SidecarConfig {
 			},
 			SUBSTRATE: {
 				WS_URL: config.Get(MODULES.SUBSTRATE, CONFIG.WS_URL) as string,
-				CUSTOM_TYPES: configTypes[CONFIG.CUSTOM_TYPES],
+				TYPES_BUNDLE: config.Get(
+					MODULES.SUBSTRATE,
+					CONFIG.TYPES_BUNDLE
+				) as string,
+				TYPES_CHAIN: config.Get(
+					MODULES.SUBSTRATE,
+					CONFIG.TYPES_CHAIN
+				) as string,
+				TYPES_SPEC: config.Get(
+					MODULES.SUBSTRATE,
+					CONFIG.TYPES_SPEC
+				) as string,
+				TYPES: config.Get(MODULES.SUBSTRATE, CONFIG.TYPES) as string,
 			},
 			LOG: {
 				LEVEL: config.Get(MODULES.LOG, CONFIG.LEVEL) as string,

src/Specs.ts

@@ -2,6 +2,8 @@ import { ConfigSpecs, SpecsFactory } from 'confmgr';
 
 import { CONFIG, MODULES } from './types/sidecar-config';
 
+const APPEND_SPEC_ERROR = 'Must create SpecFactory first.';
+
 /**
  * Access a singleton specification for config enviroment variables that will
  * be initialized on first use.
@@ -30,7 +32,7 @@ export class Specs {
 	 */
 	private static appendExpressSpecs() {
 		if (!this._specs) {
-			throw 'Must create SpecFactory first.';
+			throw APPEND_SPEC_ERROR;
 		}
 
 		// HOST
@@ -66,9 +68,10 @@ export class Specs {
 	 */
 	private static appendSubstrateSpecs() {
 		if (!this._specs) {
-			throw 'Must create SpecFactory first.';
+			throw APPEND_SPEC_ERROR;
 		}
 
+		// WS
 		this._specs.appendSpec(
 			MODULES.SUBSTRATE,
 			this._specs.getSpec(CONFIG.WS_URL, 'Websocket URL', {
@@ -77,14 +80,64 @@ export class Specs {
 				regexp: /^wss?:\/\/.*(:\d{4,5})?$/,
 			})
 		);
+
+		// TYPES_BUNDLE
+		this._specs.appendSpec(
+			MODULES.SUBSTRATE,
+			this._specs.getSpec(
+				CONFIG.TYPES_BUNDLE,
+				'absolute path to file with `typesBundle` type definitions for @polkadot/api',
+				{
+					default: '',
+					mandatory: false,
+				}
+			)
+		);
+
+		// TYPES_CHAIN
+		this._specs.appendSpec(
+			MODULES.SUBSTRATE,
+			this._specs.getSpec(
+				CONFIG.TYPES_CHAIN,
+				'absolute path to file with `typesChain` type definitions for @polkadot/api',
+				{
+					default: '',
+					mandatory: false,
+				}
+			)
+		);
+
+		// TYPES_SPEC
+		this._specs.appendSpec(
+			MODULES.SUBSTRATE,
+			this._specs.getSpec(
+				CONFIG.TYPES_SPEC,
+				'absolute path to file with `typesSpec` type definitions for @polkadot/api',
+				{
+					default: '',
+					mandatory: false,
+				}
+			)
+		);
+		this._specs.appendSpec(
+			MODULES.SUBSTRATE,
+			this._specs.getSpec(
+				CONFIG.TYPES,
+				'absolute path to file with `typesSpec` type definitions for @polkadot/api',
+				{
+					default: '',
+					mandatory: false,
+				}
+			)
+		);
 	}
 
 	/**
 	 * LOG module of the enviroment variable configuration specification.
 	 */
 	private static appendLogSpecs() {
 		if (!this._specs) {
-			throw 'Must create SpecFactory first.';
+			throw APPEND_SPEC_ERROR;
 		}
 
 		// LEVEL

src/main.ts

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+
 // Copyright 2017-2020 Parity Technologies (UK) Ltd.
 // This file is part of Substrate API Sidecar.
 //
@@ -16,8 +17,9 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { ApiPromise } from '@polkadot/api';
-import { typesBundle, typesChain, typesSpec } from '@polkadot/apps-config/api';
+import * as apps from '@polkadot/apps-config/api';
 import { WsProvider } from '@polkadot/rpc-provider';
+import { OverrideBundleType, RegistryTypes } from '@polkadot/types/types';
 import { json } from 'express';
 
 import App from './App';
@@ -34,15 +36,22 @@ async function main() {
 	// Overide console.{log, error, warn, etc}
 	consoleOverride(logger);
 
-	// Instantiate a web socket connection to the node for basic polkadot-js use
+	const { TYPES_BUNDLE, TYPES_SPEC, TYPES_CHAIN, TYPES } = config.SUBSTRATE;
+	// Instantiate a web socket connection to the node and load types
 	const api = await ApiPromise.create({
 		provider: new WsProvider(config.SUBSTRATE.WS_URL),
-		typesBundle,
-		typesChain,
-		typesSpec,
-		types: {
-			...config.SUBSTRATE.CUSTOM_TYPES,
-		},
+		/* eslint-disable @typescript-eslint/no-var-requires */
+		typesBundle: TYPES_BUNDLE
+			? (require(TYPES_BUNDLE) as OverrideBundleType)
+			: apps.typesBundle,
+		typesChain: TYPES_CHAIN
+			? (require(TYPES_CHAIN) as Record<string, RegistryTypes>)
+			: apps.typesChain,
+		typesSpec: TYPES_SPEC
+			? (require(TYPES_SPEC) as Record<string, RegistryTypes>)
+			: apps.typesSpec,
+		types: TYPES ? (require(TYPES) as RegistryTypes) : undefined,
+		/* eslint-enable @typescript-eslint/no-var-requires */
 	});
 
 	// Gather some basic details about the node so we can display a nice message

src/types/sidecar-config/CONFIG.ts

@@ -5,9 +5,12 @@ export enum CONFIG {
 	BIND_HOST = 'BIND_HOST',
 	PORT = 'PORT',
 	WS_URL = 'WS_URL',
-	CUSTOM_TYPES = 'CUSTOM_TYPES',
 	LEVEL = 'LEVEL',
 	JSON = 'JSON',
 	FILTER_RPC = 'FILTER_RPC',
 	STRIP_ANSI = 'STRIP_ANSI',
+	TYPES_BUNDLE = 'TYPES_BUNDLE',
+	TYPES_CHAIN = 'TYPES_CHAIN',
+	TYPES_SPEC = 'TYPES_SPEC',
+	TYPES = 'TYPES',
 }

src/types/sidecar-config/SidecarConfig.ts

@@ -1,5 +1,3 @@
-import { RegistryTypes } from '@polkadot/types/types';
-
 /**
  * Object to house the values of all the configurable components for Sidecar.
  */
@@ -11,7 +9,10 @@ export interface ISidecarConfig {
 
 interface ISidecarConfigSubstrate {
 	WS_URL: string;
-	CUSTOM_TYPES: RegistryTypes | undefined;
+	TYPES_BUNDLE: string;
+	TYPES_CHAIN: string;
+	TYPES_SPEC: string;
+	TYPES: string;
 }
 
 interface ISidecarConfigExpress {