LIVE-2174 Dynamic ERC20 CAL (#1288)

* Dynamic cal

* dynamic call script edit

* readd spl token

* mobile dyn cal

* remove spl tokens

* cal importer with s3 staging

* move to ubuntu latest for aws

* cal importer in folder

* staging dyn cal

* fix caching on staging env dyn cal

* dyncal erc20-signatures

* Feedback from github

* fix lint

* staging change role

* try-selfhosted

* selfrunner for aws

* change yalm runner

* change yalm runner

* reomve runner

* update doc

* Test + bugfix on integration

Co-authored-by: Kévin Lambert <[email protected]>

* update lock + add schedule to importer

* cal importer with prod env

* change find to getCryptoCurrencyById

* prod cal importer

* prd cal importer

* redoing cache

* fix: different ref

* typing

* lint fix

* add tojson method

* feedback fix

* add cache for findERC20Signature

* better with save file

* cache fix lint

* remove cache from dyn cal et add it to static

* add return

* revert pnpm-lock

Co-authored-by: Kévin Lambert <[email protected]>

.changeset/friendly-rocks-yawn.md

@@ -0,0 +1,5 @@
+---
+"@ledgerhq/hw-app-eth": minor
+---
+
+Dynamic cal for erc20

.changeset/tame-birds-sparkle.md

@@ -0,0 +1,5 @@
+---
+"live-mobile": minor
+---
+
+Dynamic cal for erc20

.changeset/twenty-rice-prove.md

@@ -0,0 +1,5 @@
+---
+"ledger-live-desktop": minor
+---
+
+Dynamic cal for erc20

.github/workflows/cal-importer.yml

@@ -0,0 +1,58 @@
+name: "Dynamic cal importer ERC20"
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+  push:
+    branches:
+      - feat/LIVE-2174-dynamic-cal
+
+concurrency:
+  group: bot-dyn-cal
+
+jobs:
+  run-bot:
+    runs-on: [self-hosted, ledger-live-assets]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Retrieving crypto-assets
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          repository: LedgerHQ/crypto-assets
+          token: ${{ secrets.CI_BOT_TOKEN }}
+          path: crypto-assets
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+      - uses: pnpm/[email protected]
+        id: pnpm-install
+        with:
+          version: latest
+          run_install: false
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+      - name: install and build
+        continue-on-error: true
+        run: |
+          pnpm i --filter="live-cli..." --filter="ledger-live" --filter="ledger-libs" --frozen-lockfile
+          pnpm build:ljs
+        shell: bash
+      - name: run script
+        run: |
+          mkdir cal
+          node libs/ledgerjs/script/crypto-assets-importer/index.js $PWD/crypto-assets true
+          mv libs/ledgerjs/packages/cryptoassets/data/**.json cal
+      - name: Deploy json to s3
+        run: aws s3 sync ./cal/ s3://ledger-live-cryptoassets-prd/cryptoassets --delete
+      - name: Cache invalidate
+        run: aws cloudfront create-invalidation --distribution-id E53IORS7MW883 --paths "/cryptoassets/*" --debug

apps/ledger-live-desktop/src/renderer/init.jsx

@@ -20,6 +20,8 @@ import "~/renderer/styles/global";
 import "~/renderer/live-common-setup";
 import { getLocalStorageEnvs } from "~/renderer/experimental";
 import "~/renderer/i18n/init";
+import { prepareCurrency } from "~/renderer/bridge/cache";
+import { getCryptoCurrencyById } from "@ledgerhq/live-common/currencies/index";
 
 import logger, { enableDebugLogger } from "~/logger";
 import LoggerTransport from "~/logger/logger-transport-renderer";
@@ -130,6 +132,11 @@ async function init() {
   if (accounts) {
     accounts = implicitMigration(accounts);
     await store.dispatch(setAccounts(accounts));
+
+    // preload currency that's not in accounts list
+    if (accounts.some(a => a.currency.id !== "ethereum")) {
+      prepareCurrency(getCryptoCurrencyById("ethereum"));
+    }
   } else {
     store.dispatch(lock());
   }

apps/ledger-live-mobile/src/live-common-setup.ts

@@ -6,7 +6,10 @@ import HIDTransport from "@ledgerhq/react-native-hid";
 import withStaticURLs from "@ledgerhq/hw-transport-http";
 import { retry } from "@ledgerhq/live-common/promise";
 import { setEnv } from "@ledgerhq/live-common/env";
-import { setSupportedCurrencies } from "@ledgerhq/live-common/currencies/index";
+import {
+  getCryptoCurrencyById,
+  setSupportedCurrencies,
+} from "@ledgerhq/live-common/currencies/index";
 import { setPlatformVersion } from "@ledgerhq/live-common/platform/version";
 import { registerTransportModule } from "@ledgerhq/live-common/hw/index";
 import type { TransportModule } from "@ledgerhq/live-common/hw/index";
@@ -18,6 +21,7 @@ import { Platform } from "react-native";
 import axios from "axios";
 import { setSecp256k1Instance } from "@ledgerhq/live-common/families/bitcoin/wallet-btc/crypto/secp256k1";
 import { setGlobalOnBridgeError } from "@ledgerhq/live-common/bridge/useBridgeTransaction";
+import { prepareCurrency } from "./bridge/cache";
 import BluetoothTransport from "./react-native-hw-transport-ble";
 import "./experimental";
 import logger from "./logger";
@@ -162,3 +166,5 @@ if (process.env.NODE_ENV === "production") {
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 setSecp256k1Instance(require("./logic/secp256k1"));
+
+prepareCurrency(getCryptoCurrencyById("ethereum"));

libs/ledger-live-common/src/currencies/index.ts

@@ -27,9 +27,10 @@ import {
   findTokenById,
   findTokenByAddress,
   hasTokenId,
-  getTokenById,
   findCompoundToken,
   getAbandonSeedAddress,
+  getTokenById,
+  addTokens,
 } from "@ledgerhq/cryptoassets";
 export * from "./support";
 export * from "./helpers";
@@ -66,7 +67,6 @@ export {
   findCryptoCurrencyByKeyword,
   findFiatCurrencyByTicker,
   hasFiatCurrencyTicker,
-  listTokens,
   listTokensForCryptoCurrency,
   listTokenTypesForCryptoCurrency,
   findTokenByAddress,
@@ -88,4 +88,6 @@ export {
   sanitizeValueString,
   getCurrencyColor,
   findCompoundToken,
+  listTokens,
+  addTokens,
 };

libs/ledger-live-common/src/env.ts

@@ -666,6 +666,11 @@ const envDefinitions = {
     parser: boolParser,
     desc: "use the staging URL for the learn page",
   },
+  DYNAMIC_CAL_BASE_URL: {
+    def: "https://cdn.live.ledger.com/cryptoassets",
+    parser: stringParser,
+    desc: "bucket S3 of the dynamic cryptoassets list",
+  },
   FEATURE_FLAGS: {
     def: "",
     parser: jsonParser,

libs/ledger-live-common/src/families/ethereum/modules/erc20.ts

@@ -5,17 +5,32 @@ import eip55 from "eip55";
 import { BigNumber } from "bignumber.js";
 import type { ModeModule } from "../types";
 import { AmountRequired } from "@ledgerhq/errors";
-import { findTokenByAddressInCurrency } from "@ledgerhq/cryptoassets";
+import {
+  findTokenByAddressInCurrency,
+  convertERC20,
+  ERC20Token,
+} from "@ledgerhq/cryptoassets";
 import { inferTokenAccount, validateRecipient } from "../transaction";
 import {
   getAccountCurrency,
   getAccountUnit,
   getMainAccount,
 } from "../../../account";
-import { findTokenById, formatCurrencyUnit } from "../../../currencies";
+import type { CryptoCurrency } from "@ledgerhq/types-cryptoassets";
+import {
+  findTokenById,
+  formatCurrencyUnit,
+  addTokens,
+} from "../../../currencies";
 import { getAccountCapabilities } from "../../../compound/logic";
 import { CompoundLowerAllowanceOfActiveAccountError } from "../../../errors";
 import { DeviceTransactionField } from "../../../transaction";
+import { getEnv } from "../../../env";
+import { log } from "@ledgerhq/logs";
+import network from "../../../network";
+import { makeLRUCache } from "../../../cache";
+import { findERC20SignaturesInfo } from "@ledgerhq/hw-app-eth/erc20";
+import { LoadConfig } from "@ledgerhq/hw-app-eth/lib/services/types";
 
 const infinite = new BigNumber(2).pow(256).minus(1);
 
@@ -144,3 +159,48 @@ const erc20approve: ModeModule = {
 export const modes: Record<Modes, ModeModule> = {
   "erc20.approve": erc20approve,
 };
+
+export const fetchERC20Tokens: () => Promise<ERC20Token[]> = async () => {
+  let tokens: ERC20Token[];
+
+  try {
+    const { data } = await network({
+      url: `${getEnv("DYNAMIC_CAL_BASE_URL")}/erc20.json`,
+    });
+
+    tokens = data;
+  } catch (e: any) {
+    log("preload-erc20", `failed to preload erc20 ${e.toString()}`);
+    tokens = [];
+  }
+
+  return tokens;
+};
+
+export async function preload(
+  currency: CryptoCurrency
+): Promise<ERC20Token[] | null | undefined> {
+  if (currency.id !== "ethereum") {
+    return Promise.resolve(null);
+  }
+
+  const tokens = await fetchERC20Tokens();
+  addTokens(tokens.map(convertERC20));
+  return tokens;
+}
+
+export function hydrate(
+  value: ERC20Token[] | null | undefined,
+  currency: CryptoCurrency
+): void {
+  if (currency.id !== "ethereum" || !value) return;
+  addTokens(value.map(convertERC20));
+  log("ethereum/preload", "hydrate " + value.length + " tokens");
+}
+
+export const erc20SignatureInfo: (
+  loadConfig: LoadConfig
+) => Promise<string | undefined | null> = makeLRUCache(
+  async (loadConfig: LoadConfig) => findERC20SignaturesInfo(loadConfig),
+  () => "erc20-signatures"
+);

libs/ledger-live-common/src/families/ethereum/modules/erc20.unit.test.ts

@@ -0,0 +1,44 @@
+import { fetchERC20Tokens } from "./erc20";
+
+beforeEach(() => {
+  jest.resetModules();
+});
+
+jest.mock("../../../network", () => {
+  return jest.fn().mockImplementationOnce(() => ({
+    data: [
+      [
+        "ethereum",
+        "$aapl",
+        "$AAPL",
+        18,
+        "$AAPL",
+        "3045022100a1e0859e2ad886121b0c5bb374622dcee83b6b0b26a5552559b56a328e4d50ad02202efc09d46a0770a40c6a650a9eec00ba9d8a6727a369398a5f8e3f1d698ccc71",
+        "0x41eFc0253ee7Ea44400abB5F907FDbfdEbc82bec",
+        true,
+        false,
+      ],
+      [
+        "ethereum",
+        "$based",
+        "$BASED",
+        18,
+        "$BASED",
+        "304402207e5be628591465b9ee45b62d5d067beb99ead27485874371b213037137b60a9702205339b3d713d990b5c614a98c20b113978ae36ff72324183d35673a40b71493fc",
+        "0x68A118Ef45063051Eac49c7e647CE5Ace48a68a5",
+        false,
+        false,
+      ],
+    ],
+  }));
+});
+
+describe("fetch ERC20 Tokens", () => {
+  it("normal behaviour", async () => {
+    expect((await fetchERC20Tokens()).length).toBe(2);
+  });
+
+  it("not able to fetch", async () => {
+    expect((await fetchERC20Tokens()).length).toBe(0);
+  });
+});

libs/ledger-live-common/src/families/ethereum/signOperation.ts

@@ -23,6 +23,7 @@ import { isNFTActive } from "../../nft";
 import { getEnv } from "../../env";
 import type { LoadConfig } from "@ledgerhq/hw-app-eth/lib/services/types";
 import { Transaction as EthereumTx } from "@ethereumjs/tx";
+import { erc20SignatureInfo } from "./modules/erc20";
 export const signOperation = ({
   account,
   deviceId,
@@ -70,7 +71,9 @@ export const signOperation = ({
               const rawData = tx.raw();
               rawData[6] = Buffer.from([common.chainIdBN().toNumber()]);
               const txHex = Buffer.from(encode(rawData)).toString("hex");
-              const loadConfig: LoadConfig = {};
+              const loadConfig: LoadConfig = {
+                cryptoassetsBaseURL: getEnv("DYNAMIC_CAL_BASE_URL"),
+              };
               if (isNFTActive(account.currency)) {
                 loadConfig.nftExplorerBaseURL =
                   getEnv("NFT_ETH_METADATA_SERVICE") + "/v1/ethereum";
@@ -98,10 +101,13 @@ export const signOperation = ({
                   fillTransactionDataResult.erc20contracts) ||
                 [];
 
+              const erc20SignatureBlob = await erc20SignatureInfo(loadConfig);
+
               for (const addr of addrs) {
                 const tokenInfo = byContractAddressAndChainId(
                   addr,
-                  account.currency.ethereumLikeInfo?.chainId || 0
+                  account.currency.ethereumLikeInfo?.chainId || 0,
+                  erc20SignatureBlob
                 );
 
                 if (tokenInfo) {

libs/ledgerjs/packages/cryptoassets/src/tokens.ts

@@ -12,6 +12,7 @@ import asatokens from "../data/asa";
 import esdttokens from "../data/esdt";
 import cardanoNativeTokens from "../data/cardanoNative";
 import stellarTokens from "../data/stellar";
+import { ERC20Token } from "./types";
 //import spltokens from "../data/spl";
 const emptyArray = [];
 const tokensArray: TokenCurrency[] = [];
@@ -153,8 +154,9 @@ function comparePriority(a: TokenCurrency, b: TokenCurrency) {
   return Number(!!b.disableCountervalue) - Number(!!a.disableCountervalue);
 }
 
-function addTokens(list: TokenCurrency[]) {
+export function addTokens(list: TokenCurrency[]): void {
   list.forEach((token) => {
+    if (tokensById[token.id]) return;
     if (!token.delisted) tokensArray.push(token);
     tokensArrayWithDelisted.push(token);
     tokensById[token.id] = token;
@@ -184,7 +186,7 @@ function addTokens(list: TokenCurrency[]) {
   });
 }
 
-function convertERC20([
+export function convertERC20([
   parentCurrencyId,
   token,
   ticker,
@@ -196,7 +198,7 @@ function convertERC20([
   delisted,
   countervalueTicker,
   compoundFor,
-]): TokenCurrency {
+]: ERC20Token): TokenCurrency {
   const parentCurrency = getCryptoCurrencyById(parentCurrencyId);
   return {
     type: "TokenCurrency",

libs/ledgerjs/packages/cryptoassets/src/types.ts

@@ -3,3 +3,17 @@ import { cryptocurrenciesById } from "./currencies";
 // FIXME This should be the other way around, CryptoCurrencyIds should be defined on its own and cryptocurrenciesById should be derived from it.
 // That way CryptoCurrencyIds could be moved to the types-cryptoassets lib where it belongs.
 export type CryptoCurrencyIds = keyof typeof cryptocurrenciesById;
+
+export type ERC20Token = [
+  string,
+  string,
+  string,
+  number,
+  string,
+  string,
+  string,
+  boolean,
+  boolean,
+  string,
+  string
+];