Ultimate_Accompaniment_Transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Ultimate Accompaniment Transformer (ver. 3.0)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2024\n",
        "\n",
        "***"
      ],
      "metadata": {
        "id": "gpy3qsulqHa5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GPU CHECK)"
      ],
      "metadata": {
        "id": "W_So4w8fqPGL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X3rABEpKCO02",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title NVIDIA GPU check\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (SETUP ENVIRONMENT)"
      ],
      "metadata": {
        "id": "C0XxnXGFqVyh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vK40g6V_BTNj",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install dependencies\n",
        "!git clone --depth 1 https://github.com/asigalov61/Ultimate-Accompaniment-Transformer\n",
        "!pip install huggingface_hub\n",
        "!pip install einops\n",
        "!pip install torch-summary\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DzCOZU_gBiQV",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Import modules\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading core Ultimate Accompaniment Transformer modules...')\n",
        "\n",
        "import os\n",
        "import copy\n",
        "import pickle\n",
        "import secrets\n",
        "import statistics\n",
        "from time import time\n",
        "import tqdm\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading main Ultimate Accompaniment Transformer modules...')\n",
        "import torch\n",
        "\n",
        "%cd /content/Ultimate-Accompaniment-Transformer\n",
        "\n",
        "import TMIDIX\n",
        "\n",
        "from midi_to_colab_audio import midi_to_colab_audio\n",
        "\n",
        "from x_transformer_1_23_2 import *\n",
        "\n",
        "import random\n",
        "\n",
        "%cd /content/\n",
        "print('=' * 70)\n",
        "print('Loading aux Ultimate Accompaniment Transformer modules...')\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from torchsummary import summary\n",
        "from sklearn import metrics\n",
        "\n",
        "from IPython.display import Audio, display\n",
        "\n",
        "from huggingface_hub import hf_hub_download\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('Enjoy! :)')\n",
        "print('=' * 70)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eI3aQtHzqSnp"
      },
      "source": [
        "# (LOAD MODEL)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load Chords Progressions Transformer Pre-Trained Model\n",
        "\n",
        "#@markdown Choose model\n",
        "\n",
        "select_model_to_load = \"187M-Improved-Very-Fast-Small\" # @param [\"187M-Original-Very-Fast-Small\", \"187M-Improved-Very-Fast-Small\"]\n",
        "\n",
        "#@markdown Model precision option\n",
        "\n",
        "model_precision = \"bfloat16\" # @param [\"bfloat16\", \"float16\"]\n",
        "\n",
        "#@markdown bfloat16 == Half precision/faster speed (if supported, otherwise the model will default to float16)\n",
        "\n",
        "#@markdown float16 == Full precision/fast speed\n",
        "\n",
        "plot_tokens_embeddings = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading Chords Progressions Transformer Pre-Trained Model...')\n",
        "print('Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "full_path_to_models_dir = \"/content/Ultimate-Accompaniment-Transformer/Models\"\n",
        "\n",
        "if select_model_to_load == '187M-Original-Very-Fast-Small':\n",
        "\n",
        "  dim = 2048\n",
        "  depth = 4\n",
        "  heads = 16\n",
        "\n",
        "  model_checkpoint_file_name = 'Ultimate_Accompaniment_Transformer_Small_Trained_Model_13414_steps_0.5291_loss_0.8338_acc.pth'\n",
        "  model_path = full_path_to_models_dir+'/Small/'+model_checkpoint_file_name\n",
        "  if os.path.isfile(model_path):\n",
        "    print('Model already exists...')\n",
        "\n",
        "  else:\n",
        "    hf_hub_download(repo_id='asigalov61/Ultimate-Accompaniment-Transformer',\n",
        "                    filename=model_checkpoint_file_name,\n",
        "                    local_dir='/content/Ultimate-Accompaniment-Transformer/Models/Small',\n",
        "                    local_dir_use_symlinks=False)\n",
        "else:\n",
        "\n",
        "  dim = 2048\n",
        "  depth = 4\n",
        "  heads = 16\n",
        "\n",
        "  model_checkpoint_file_name = 'Ultimate_Accompaniment_Transformer_Small_Improved_Trained_Model_13649_steps_0.3229_loss_0.898_acc.pth'\n",
        "  model_path = full_path_to_models_dir+'/Small_Improved/'+model_checkpoint_file_name\n",
        "  if os.path.isfile(model_path):\n",
        "    print('Model already exists...')\n",
        "\n",
        "  else:\n",
        "    hf_hub_download(repo_id='asigalov61/Ultimate-Accompaniment-Transformer',\n",
        "                    filename=model_checkpoint_file_name,\n",
        "                    local_dir='/content/Ultimate-Accompaniment-Transformer/Models/Small_Improved',\n",
        "                    local_dir_use_symlinks=False)\n",
        "\n",
        "print('=' * 70)\n",
        "print('Instantiating model...')\n",
        "\n",
        "device_type = 'cuda'\n",
        "\n",
        "if model_precision == 'bfloat16' and torch.cuda.is_bf16_supported():\n",
        "  dtype = 'bfloat16'\n",
        "else:\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float16':\n",
        "  dtype = 'float16'\n",
        "\n",
        "ptdtype = {'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]\n",
        "ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)\n",
        "\n",
        "SEQ_LEN = 8192 # Models seq len\n",
        "PAD_IDX = 767 # Models pad index\n",
        "\n",
        "# instantiate the model\n",
        "\n",
        "model = TransformerWrapper(\n",
        "    num_tokens = PAD_IDX+1,\n",
        "    max_seq_len = SEQ_LEN,\n",
        "    attn_layers = Decoder(dim = dim, depth = depth, heads = heads, attn_flash = True)\n",
        "    )\n",
        "\n",
        "model = AutoregressiveWrapper(model, ignore_index = PAD_IDX)\n",
        "\n",
        "model.cuda()\n",
        "print('=' * 70)\n",
        "\n",
        "print('Loading model checkpoint...')\n",
        "\n",
        "model.load_state_dict(torch.load(model_path))\n",
        "print('=' * 70)\n",
        "\n",
        "model.eval()\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Model will use', dtype, 'precision...')\n",
        "print('=' * 70)\n",
        "\n",
        "# Model stats\n",
        "print('Model summary...')\n",
        "summary(model)\n",
        "\n",
        "# Plot Token Embeddings\n",
        "if plot_tokens_embeddings:\n",
        "  tok_emb = model.net.token_emb.emb.weight.detach().cpu().tolist()\n",
        "\n",
        "  cos_sim = metrics.pairwise_distances(\n",
        "    tok_emb, metric='cosine'\n",
        "  )\n",
        "  plt.figure(figsize=(7, 7))\n",
        "  plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"nearest\")\n",
        "  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]\n",
        "  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "  plt.xlabel(\"Position\")\n",
        "  plt.ylabel(\"Position\")\n",
        "  plt.tight_layout()\n",
        "  plt.plot()\n",
        "  plt.savefig(\"/content/Ultimate-Accompaniment-Transformer-Tokens-Embeddings-Plot.png\", bbox_inches=\"tight\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "V4s_G8yUL0cH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GENERATE)"
      ],
      "metadata": {
        "id": "7xNyANjZsCOi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (IMPROV)"
      ],
      "metadata": {
        "id": "BxepTeHVmmKO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Standard Improv Generator\n",
        "\n",
        "#@markdown Generation settings\n",
        "\n",
        "melody_MIDI_patch_number = 40 # @param {type:\"slider\", min:0, max:127, step:1}\n",
        "accompaniment_MIDI_patch_number = 0 # @param {type:\"slider\", min:0, max:127, step:1}\n",
        "\n",
        "number_of_tokens_to_generate = 512 # @param {type:\"slider\", min:30, max:8190, step:3}\n",
        "number_of_batches_to_generate = 4 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "temperature = 0.9 # @param {type:\"slider\", min:0.1, max:1, step:0.05}\n",
        "\n",
        "#@markdown Other settings\n",
        "\n",
        "render_MIDI_to_audio = True # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Ultimate Accompaniment Transformer Standard Improv Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "outy = [0]\n",
        "\n",
        "print('Selected Improv sequence:')\n",
        "print(outy)\n",
        "print('=' * 70)\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "inp = [outy] * number_of_batches_to_generate\n",
        "\n",
        "inp = torch.LongTensor(inp).cuda()\n",
        "\n",
        "with ctx:\n",
        "  out = model.generate(inp,\n",
        "                        number_of_tokens_to_generate,\n",
        "                        temperature=temperature,\n",
        "                        return_prime=True,\n",
        "                        verbose=True)\n",
        "\n",
        "out0 = out.tolist()\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "#======================================================================\n",
        "\n",
        "print('Rendering results...')\n",
        "\n",
        "for i in range(number_of_batches_to_generate):\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Batch #', i)\n",
        "  print('=' * 70)\n",
        "\n",
        "  out1 = out0[i]\n",
        "\n",
        "  print('Sample INTs', out1[:12])\n",
        "  print('=' * 70)\n",
        "\n",
        "  if len(out1) != 0:\n",
        "\n",
        "      song = out1\n",
        "      song_f = []\n",
        "\n",
        "      time = 0\n",
        "      dur = 128\n",
        "      vel = 90\n",
        "      pitch = 0\n",
        "      channel = 0\n",
        "\n",
        "      patches = [0] * 16\n",
        "      patches[3] = melody_MIDI_patch_number\n",
        "      patches[0] = accompaniment_MIDI_patch_number\n",
        "\n",
        "      for s in song:\n",
        "\n",
        "        if 0 <= s < 128:\n",
        "\n",
        "            time += s * 32\n",
        "\n",
        "        if 128 <= s < 256:\n",
        "\n",
        "            dur = (s-128) * 32\n",
        "\n",
        "        if 256 <= s < 512:\n",
        "\n",
        "            chan = (s-256) // 128\n",
        "\n",
        "            if chan == 0:\n",
        "              channel = 0\n",
        "              pat = accompaniment_MIDI_patch_number\n",
        "            else:\n",
        "              channel = 3\n",
        "              pat = melody_MIDI_patch_number\n",
        "\n",
        "            pitch = (s-256) % 128\n",
        "\n",
        "            vel = max(40, pitch)\n",
        "\n",
        "            song_f.append(['note', time, dur, channel, pitch, vel, pat])\n",
        "\n",
        "      data = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                      output_signature = 'Ultimate Accompaniment Transformer',\n",
        "                                                      output_file_name = '/content/Ultimate-Accompaniment-Transformer-Composition_'+str(i),\n",
        "                                                      track_name='Project Los Angeles',\n",
        "                                                      list_of_MIDI_patches=patches\n",
        "                                                      )\n",
        "\n",
        "\n",
        "      print('=' * 70)\n",
        "      print('Displaying resulting composition...')\n",
        "      print('=' * 70)\n",
        "\n",
        "      fname = '/content/Ultimate-Accompaniment-Transformer-Composition_'+str(i)\n",
        "\n",
        "      if render_MIDI_to_audio:\n",
        "        midi_audio = midi_to_colab_audio(fname + '.mid')\n",
        "        display(Audio(midi_audio, rate=16000, normalize=False))\n",
        "\n",
        "      TMIDIX.plot_ms_SONG(song_f, plot_title=fname)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Jwxz-eaF0K1y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (ACCOMPANIMENT GENERATION)"
      ],
      "metadata": {
        "id": "fmm3KjOtoVp9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4QXbFLsKqSnt",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Load Seed MIDI\n",
        "\n",
        "#@markdown Press play button to to upload your own seed MIDI or to load one of the provided sample seed MIDIs from the dropdown list below\n",
        "\n",
        "select_seed_MIDI = \"Upload your own custom MIDI\" # @param [\"Upload your own custom MIDI\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-1\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-2\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-3\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-4\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-5\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-6\", \"Ultimate-Accompaniment-Transformer-Melody-Seed-7\"]\n",
        "render_MIDI_to_audio = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Ultimate Accompaniment Transformer Seed MIDI Loader')\n",
        "print('=' * 70)\n",
        "\n",
        "f = ''\n",
        "\n",
        "if select_seed_MIDI != \"Upload your own custom MIDI\":\n",
        "  print('Loading seed MIDI...')\n",
        "  f = '/content/Ultimate-Accompaniment-Transformer/Seeds/'+select_seed_MIDI+'.mid'\n",
        "\n",
        "else:\n",
        "  print('Upload your own custom MIDI...')\n",
        "  print('=' * 70)\n",
        "  uploaded_MIDI = files.upload()\n",
        "  if list(uploaded_MIDI.keys()):\n",
        "    f = list(uploaded_MIDI.keys())[0]\n",
        "\n",
        "if f != '':\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('File:', f)\n",
        "  print('=' * 70)\n",
        "\n",
        "  #=======================================================\n",
        "  # START PROCESSING\n",
        "\n",
        "  #===============================================================================\n",
        "  # Raw single-track ms score\n",
        "\n",
        "  raw_score = TMIDIX.midi2single_track_ms_score(f)\n",
        "\n",
        "  #===============================================================================\n",
        "  # Enhanced score notes\n",
        "\n",
        "  escore_notes = TMIDIX.advanced_score_processor(raw_score, return_enhanced_score_notes=True)[0]\n",
        "\n",
        "  escore_notes = [e for e in escore_notes if e[3] != 9]\n",
        "\n",
        "  if len(escore_notes) > 0:\n",
        "\n",
        "      #=======================================================\n",
        "      # PRE-PROCESSING\n",
        "\n",
        "      #===============================================================================\n",
        "      # Augmented enhanced score notes\n",
        "\n",
        "      escore_notes = TMIDIX.augment_enhanced_score_notes(escore_notes, timings_divider=32)\n",
        "\n",
        "      cscore = TMIDIX.chordify_score([1000, escore_notes])\n",
        "\n",
        "      melody = TMIDIX.fix_monophonic_score_durations([sorted(e, key=lambda x: x[4], reverse=True)[0] for e in cscore])\n",
        "\n",
        "      #=======================================================\n",
        "      # FINAL PROCESSING\n",
        "\n",
        "      melody_chords = []\n",
        "\n",
        "      #=======================================================\n",
        "      # MAIN PROCESSING CYCLE\n",
        "      #=======================================================\n",
        "\n",
        "      pe = cscore[0][0]\n",
        "\n",
        "      mpe = melody[0]\n",
        "\n",
        "      midx = 1\n",
        "\n",
        "      for i, c in enumerate(cscore):\n",
        "\n",
        "          c.sort(key=lambda x: (x[3], x[4]), reverse=True)\n",
        "\n",
        "          # Next melody note\n",
        "\n",
        "          if midx < len(melody):\n",
        "\n",
        "            # Time\n",
        "            mtime = melody[midx][1]-mpe[1]\n",
        "\n",
        "            mdur = melody[midx][2]\n",
        "\n",
        "            mdelta_time = max(0, min(127, mtime))\n",
        "\n",
        "            # Durations\n",
        "            mdur = max(0, min(127, mdur))\n",
        "\n",
        "            # Pitch\n",
        "            mptc = melody[midx][4]\n",
        "\n",
        "          else:\n",
        "            mtime = 127-mpe[1]\n",
        "\n",
        "            mdur = mpe[2]\n",
        "\n",
        "            mdelta_time = max(0, min(127, mtime))\n",
        "\n",
        "            # Durations\n",
        "            mdur = max(0, min(127, mdur))\n",
        "\n",
        "            # Pitch\n",
        "            mptc = mpe[4]\n",
        "\n",
        "\n",
        "          e = melody[i]\n",
        "\n",
        "          #=======================================================\n",
        "          # Timings...\n",
        "\n",
        "          time = e[1]-pe[1]\n",
        "\n",
        "          dur = e[2]\n",
        "\n",
        "          delta_time = max(0, min(127, time))\n",
        "\n",
        "          # Durations\n",
        "\n",
        "          dur = max(0, min(127, dur))\n",
        "\n",
        "          # Pitches\n",
        "\n",
        "          ptc = max(1, min(127, e[4]))\n",
        "\n",
        "          if ptc < 60:\n",
        "            ptc = 60 + (ptc % 12)\n",
        "\n",
        "          cha = e[3]\n",
        "\n",
        "          #=======================================================\n",
        "          # FINAL NOTE SEQ\n",
        "\n",
        "          if midx < len(melody):\n",
        "            melody_chords.append([delta_time, dur+128, ptc+384, mdelta_time+512, mptc+640])\n",
        "            mpe = melody[midx]\n",
        "            midx += 1\n",
        "          else:\n",
        "            melody_chords.append([delta_time, dur+128, ptc+384, mdelta_time+512, mptc+640])\n",
        "\n",
        "          pe = e\n",
        "\n",
        "  #=======================================================\n",
        "\n",
        "  song = melody_chords\n",
        "  song_f = []\n",
        "\n",
        "\n",
        "\n",
        "  time = 0\n",
        "  dur = 128\n",
        "  vel = 90\n",
        "  pitch = 0\n",
        "  pat = 40\n",
        "  channel = 3\n",
        "\n",
        "  patches = [0] * 16\n",
        "  patches[3] = 40\n",
        "  patches[0] = 0\n",
        "\n",
        "  for ss in song:\n",
        "\n",
        "\n",
        "\n",
        "    time += ss[0] * 32\n",
        "\n",
        "    dur = (ss[1]-128) * 32\n",
        "\n",
        "    pitch = (ss[2]-256) % 128\n",
        "\n",
        "    vel = max(40, pitch)\n",
        "\n",
        "    song_f.append(['note', time, dur, channel, pitch, vel, pat])\n",
        "\n",
        "\n",
        "\n",
        "  detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                            output_signature = 'Ultimate Accompaniment Transformer',\n",
        "                                                            output_file_name = '/content/Ultimate-Accompaniment-Transformer-Seed-Composition',\n",
        "                                                            track_name='Project Los Angeles',\n",
        "                                                            list_of_MIDI_patches=patches\n",
        "                                                            )\n",
        "  #=======================================================\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Composition stats:')\n",
        "  print('Composition has', len(cscore), 'chords')\n",
        "  print('Composition has', len(melody_chords), 'tokens')\n",
        "  print('Composition MIDI patches:', sorted(set(patches)))\n",
        "  print('=' * 70)\n",
        "\n",
        "  print('Displaying resulting composition...')\n",
        "  print('=' * 70)\n",
        "\n",
        "  fname = '/content/Ultimate-Accompaniment-Transformer-Seed-Composition'\n",
        "\n",
        "  if render_MIDI_to_audio:\n",
        "    midi_audio = midi_to_colab_audio(fname + '.mid')\n",
        "    display(Audio(midi_audio, rate=16000, normalize=False))\n",
        "\n",
        "  TMIDIX.plot_ms_SONG(song_f, plot_title=fname)\n",
        "\n",
        "else:\n",
        "  print('=' * 70)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dkvXYwR_qSnx",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Accompaniment generation\n",
        "\n",
        "#@markdown NOTE: You can stop the generation at any time to render partial results\n",
        "\n",
        "#@markdown Output settings\n",
        "\n",
        "melody_MIDI_patch_number = 40 # @param {type:\"slider\", min:0, max:127, step:1}\n",
        "accompaniment_MIDI_patch_number = 0 # @param {type:\"slider\", min:0, max:127, step:1}\n",
        "\n",
        "#@markdown Generation settings\n",
        "\n",
        "number_of_prime_melody_notes = 1 # @param {type:\"slider\", min:0, max:64, step:1}\n",
        "use_harmonized_melody = False # @param {type:\"boolean\"}\n",
        "harmonized_melody_octave = 4 # @param {type:\"slider\", min:3, max:5, step:1}\n",
        "force_accompaniment_generation_for_each_melody_note = False # @param {type:\"boolean\"}\n",
        "tokens_sampling_rate = 1 # @param {type:\"slider\", min:1, max:16, step:1}\n",
        "number_of_memory_tokens = 4096 # @param {type:\"slider\", min:32, max:8188, step:16}\n",
        "temperature = 0.9 # @param {type:\"slider\", min:0.1, max:1, step:0.05}\n",
        "\n",
        "#@markdown Other settings\n",
        "render_MIDI_to_audio = True # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Ultimate Accompaniment Transformer Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "def generate_acc(input_seq,\n",
        "                  next_note_time,\n",
        "                  force_acc = False,\n",
        "                  num_samples = 2,\n",
        "                  num_batches = 8,\n",
        "                  num_memory_tokens = 4096,\n",
        "                  temperature=0.9\n",
        "                  ):\n",
        "\n",
        "    input_seq = input_seq[-num_memory_tokens:]\n",
        "\n",
        "    if force_acc:\n",
        "      x = torch.LongTensor([input_seq+[0]] * num_batches).cuda()\n",
        "    else:\n",
        "      x = torch.LongTensor([input_seq] * num_batches).cuda()\n",
        "\n",
        "    cur_time = 0\n",
        "    ctime = 0\n",
        "\n",
        "    o = 0\n",
        "\n",
        "    while cur_time < next_note_time and o < 384:\n",
        "\n",
        "      samples = []\n",
        "\n",
        "      for _ in range(num_samples):\n",
        "\n",
        "        with ctx:\n",
        "          out = model.generate(x,\n",
        "                              1,\n",
        "                              temperature=temperature,\n",
        "                              return_prime=True,\n",
        "                              verbose=False)\n",
        "\n",
        "          with torch.no_grad():\n",
        "            test_loss, test_acc = model(out)\n",
        "\n",
        "        samples.append([out[:,-1].tolist(), test_acc.tolist()])\n",
        "\n",
        "      accs = [y[1] for y in samples]\n",
        "      max_acc = max(accs)\n",
        "      o = statistics.mode(samples[accs.index(max_acc)][0])\n",
        "\n",
        "      if 0 <= o < 128:\n",
        "        cur_time += o\n",
        "\n",
        "      if cur_time < next_note_time and o < 384:\n",
        "\n",
        "        ctime = cur_time\n",
        "\n",
        "        out = torch.LongTensor([[o]] * num_batches).cuda()\n",
        "        x = torch.cat((x, out), dim=1)\n",
        "\n",
        "    return list(statistics.mode([tuple(t) for t in x[:, len(input_seq):].tolist()])), ctime\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "melody_tones_chords = TMIDIX.harmonize_enhanced_melody_score_notes(melody)\n",
        "\n",
        "harm_melody_chords =  []\n",
        "\n",
        "for i, m in enumerate(melody_tones_chords):\n",
        "  cho = []\n",
        "  mm = sorted(m, reverse=True)\n",
        "  for mmm in mm:\n",
        "    cho.extend([0, max(0, min(127,melody[i][2]))+128, (harmonized_melody_octave*12)+mmm+256])\n",
        "\n",
        "  harm_melody_chords.append(cho)\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "output1 = []\n",
        "output2 = []\n",
        "\n",
        "for i in range(number_of_prime_melody_notes):\n",
        "\n",
        "  output1.extend(melody_chords[i])\n",
        "\n",
        "  if use_harmonized_melody:\n",
        "    output1.extend(harm_melody_chords[i])\n",
        "    output2.append(harm_melody_chords[i])\n",
        "  else:\n",
        "    output2.append([])\n",
        "\n",
        "hidx = number_of_prime_melody_notes\n",
        "cur_time = 0\n",
        "\n",
        "for m in tqdm.tqdm(melody_chords[number_of_prime_melody_notes:]):\n",
        "\n",
        "  try:\n",
        "    mel = copy.deepcopy(m)\n",
        "    mel[0] = mel[0] - cur_time\n",
        "    next_note_time = mel[3]-512\n",
        "\n",
        "    if use_harmonized_melody:\n",
        "      input = mel + harm_melody_chords[hidx]\n",
        "    else:\n",
        "      input = mel\n",
        "\n",
        "    output1.extend(input)\n",
        "\n",
        "    out, cur_time = generate_acc(output1,\n",
        "                                next_note_time,\n",
        "                                force_acc=force_accompaniment_generation_for_each_melody_note,\n",
        "                                num_samples=tokens_sampling_rate,\n",
        "                                num_batches=tokens_sampling_rate,\n",
        "                                num_memory_tokens=number_of_memory_tokens,\n",
        "                                temperature=temperature\n",
        "                                )\n",
        "    output1.extend(out)\n",
        "\n",
        "    if use_harmonized_melody:\n",
        "      output2.append(harm_melody_chords[hidx] + out)\n",
        "    else:\n",
        "      output2.append(out)\n",
        "\n",
        "    hidx += 1\n",
        "\n",
        "  except KeyboardInterrupt:\n",
        "    print('Stopping...')\n",
        "    print('=' * 70)\n",
        "    print('Stopping generation...')\n",
        "    break\n",
        "\n",
        "  except Exception as e:\n",
        "    print('=' * 70)\n",
        "    print('Error:', e)\n",
        "    break\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "#===============================================================================\n",
        "print('Rendering results...')\n",
        "\n",
        "print('=' * 70)\n",
        "print('Sample INTs', output1[:12])\n",
        "print('=' * 70)\n",
        "\n",
        "out1 = output2\n",
        "\n",
        "if len(out1) != 0:\n",
        "\n",
        "    song = out1\n",
        "    song_f = []\n",
        "\n",
        "    time = 0\n",
        "    ntime = 0\n",
        "    ndur = 0\n",
        "    vel = 90\n",
        "    npitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    patches = [0] * 16\n",
        "    patches[0] = accompaniment_MIDI_patch_number\n",
        "    patches[3] = melody_MIDI_patch_number\n",
        "\n",
        "    for i, ss in enumerate(song):\n",
        "\n",
        "            ntime += melody_chords[i][0] * 32\n",
        "            ndur = (melody_chords[i][1]-128) * 32\n",
        "            nchannel = 1\n",
        "            npitch = (melody_chords[i][2]-256) % 128\n",
        "            vel = max(40, npitch)+20\n",
        "\n",
        "            song_f.append(['note', ntime, ndur, 3, npitch, vel, melody_MIDI_patch_number ])\n",
        "\n",
        "            time = ntime\n",
        "\n",
        "            for s in ss:\n",
        "\n",
        "              if 0 <= s < 128:\n",
        "\n",
        "                  time += s * 32\n",
        "\n",
        "              if 128 <= s < 256:\n",
        "\n",
        "                  dur = (s-128) * 32\n",
        "\n",
        "              if 256 <= s < 384:\n",
        "\n",
        "                  pitch = (s-256)\n",
        "\n",
        "                  vel = max(40, pitch)\n",
        "\n",
        "                  song_f.append(['note', time, dur, 0, pitch, vel, accompaniment_MIDI_patch_number])\n",
        "\n",
        "detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                          output_signature = 'Ultimate Accompaniment Transformer',\n",
        "                                                          output_file_name = '/content/Ultimate-Accompaniment-Transformer-Composition',\n",
        "                                                          track_name='Project Los Angeles',\n",
        "                                                          list_of_MIDI_patches=patches\n",
        "                                                          )\n",
        "\n",
        "#=========================================================================\n",
        "\n",
        "print('=' * 70)\n",
        "print('Displaying resulting composition...')\n",
        "print('=' * 70)\n",
        "\n",
        "fname = '/content/Ultimate-Accompaniment-Transformer-Composition'\n",
        "\n",
        "if render_MIDI_to_audio:\n",
        "  midi_audio = midi_to_colab_audio(fname + '.mid')\n",
        "  display(Audio(midi_audio, rate=16000, normalize=False))\n",
        "\n",
        "TMIDIX.plot_ms_SONG(song_f, plot_title=fname)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Congrats! You did it! :)"
      ],
      "metadata": {
        "id": "eoWDEy6CwDr6"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4",
      "gpuClass": "premium"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}