simple upserts

docker-compose.yml

@@ -1,9 +1,8 @@
 ---
-version: '3'
 services:
   musicbot_db_prod:
     container_name: musicbot-db-prod
-    image: "edgedb/edgedb:5.0-beta.2"
+    image: "edgedb/edgedb:5"
     restart: always
     ports:
       - 5656:5656
@@ -22,7 +21,7 @@ services:
       - musicbot-prod-data:/var/lib/edgedb/data
   musicbot_db_test:
     container_name: musicbot-db-test
-    image: "edgedb/edgedb:5.0-beta.2"
+    image: "edgedb/edgedb:5"
     restart: always
     ports:
       - 5657:5656

musicbot/commands/local.py

@@ -128,16 +128,11 @@ async def scan(
     save: bool,
     output: str,
 ) -> None:
-    music_inputs = await local.scan(
-        musicdb=musicdb,
-        scan_folders=scan_folders,
-    )
     if clean:
         _ = await musicdb.clean_musics()
-    music_outputs = await local.upsert_musics(
-        musicdb=musicdb,
-        music_inputs=music_inputs,
-    )
+
+    music_outputs = await scan_folders.scan(musicdb=musicdb)
+
     _ = await musicdb.soft_clean()
 
     if output == "json":
@@ -358,15 +353,8 @@ async def custom_playlists(
 ) -> None:
     scan_folders = ScanFolders(directories=[scan_folder])
     if not fast:
-        music_inputs = await local.scan(
-            musicdb=musicdb,
-            scan_folders=scan_folders,
-        )
         _ = await musicdb.clean_musics()
-        _ = await local.upsert_musics(
-            musicdb=musicdb,
-            music_inputs=music_inputs,
-        )
+        _ = await scan_folders.scan(musicdb=musicdb)
 
     musicdb.set_readonly()
 

musicbot/local.py

@@ -8,7 +8,6 @@
 
 from musicbot import (
     File,
-    Issue,
     Music,
     MusicbotObject,
     MusicDb,
@@ -74,40 +73,6 @@ async def bests(
     MusicbotObject.success(f"Playlists: {len(bests)}")
 
 
-async def scan(
-    musicdb: MusicDb,
-    scan_folders: ScanFolders,
-) -> list[MusicInput]:
-    max_value = len(scan_folders.folders_and_paths)
-    if not max_value:
-        MusicbotObject.warn(f"No music folder or paths discovered from directories {scan_folders.directories}")
-        return []
-
-    MusicbotObject.echo(f"{musicdb} : loading {max_value} files")
-    music_inputs = []
-    with MusicbotObject.progressbar(desc="Loading files", max_value=max_value) as pbar:
-        for folder_and_path in scan_folders.folders_and_paths:
-            try:
-                folder, path = folder_and_path
-                if not (file := File.from_path(folder=folder, path=path)):
-                    continue
-
-                issues = file.issues
-                if Issue.NO_TITLE in issues or Issue.NO_ARTIST in issues or Issue.NO_ALBUM in issues:
-                    MusicbotObject.warn(f"{file} : missing mandatory fields title/album/artist : {issues}")
-                    continue
-                if not (music_input := file.music_input):
-                    MusicbotObject.err(f"{file} : cannot upsert music without physical folder !")
-                    continue
-
-                music_inputs.append(music_input)
-            finally:
-                pbar.value += 1
-                _ = pbar.update()
-
-    return music_inputs
-
-
 async def upsert_musics(
     musicdb: MusicDb,
     music_inputs: list[MusicInput],

musicbot/musicdb.py

@@ -1,8 +1,9 @@
 import logging
 import os
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from urllib.parse import urlparse
+from uuid import UUID
 
 import edgedb
 import httpx
@@ -21,23 +22,43 @@
 from musicbot.queries.gen_playlist_async_edgeql import GenPlaylistResult, gen_playlist
 from musicbot.queries.remove_async_edgeql import remove
 from musicbot.queries.soft_clean_async_edgeql import soft_clean
-from musicbot.queries.upsert_async_edgeql import upsert
+from musicbot.queries.upsert_album_async_edgeql import upsert_album
+from musicbot.queries.upsert_artist_async_edgeql import upsert_artist
+from musicbot.queries.upsert_folder_async_edgeql import upsert_folder
+from musicbot.queries.upsert_genre_async_edgeql import upsert_genre
+from musicbot.queries.upsert_keyword_async_edgeql import upsert_keyword
+from musicbot.queries.upsert_music_async_edgeql import upsert_music
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class ArtistAlbums:
+    id: UUID
+    albums: dict[str, UUID] = field(default_factory=dict)
+
+
+@dataclass
+class UpsertCache:
+    folders: dict[str, UUID] = field(default_factory=dict)
+    artists_and_albums: dict[str, ArtistAlbums] = field(default_factory=dict)
+    genres: dict[str, UUID] = field(default_factory=dict)
+    keywords: dict[str, UUID] = field(default_factory=dict)
+
+
 @beartype
 @dataclass(unsafe_hash=True)
 class MusicDb(MusicbotObject):
     client: AsyncIOClient
     graphql: str
+    upsert_cache: UpsertCache = field(default_factory=UpsertCache, hash=False)
 
     def __repr__(self) -> str:
         return self.dsn
 
     def set_readonly(self, readonly: bool = True) -> None:
         """set client to read only mode"""
-        options = TransactionOptions(readonly=readonly)
+        options = TransactionOptions(readonly=readonly, deferrable=True)
         self.client = self.client.with_transaction_options(options)
 
     @property
@@ -165,9 +186,60 @@ async def upsert_music(self, music_input: MusicInput) -> Music | None:
 
         retries = 3
         last_error = None
+
         while retries > 0:
             try:
-                result = await upsert(self.client, **asdict(music_input))
+                if music_input.artist not in self.upsert_cache.artists_and_albums:
+                    artist_result = await upsert_artist(self.client, artist=music_input.artist)
+                    artist_id = artist_result.id
+                    self.upsert_cache.artists_and_albums[music_input.artist] = ArtistAlbums(id=artist_id)
+                else:
+                    artist_id = self.upsert_cache.artists_and_albums[music_input.artist].id
+
+                if music_input.folder not in self.upsert_cache.folders:
+                    folder_result = await upsert_folder(self.client, username=music_input.username, ipv4=music_input.ipv4, folder=music_input.folder)
+                    folder_id = folder_result.id
+                    self.upsert_cache.folders[music_input.folder] = folder_id
+                else:
+                    folder_id = self.upsert_cache.folders[music_input.folder]
+
+                if music_input.genre not in self.upsert_cache.genres:
+                    genre_result = await upsert_genre(self.client, genre=music_input.genre)
+                    genre_id = genre_result.id
+                    self.upsert_cache.genres[music_input.genre] = genre_id
+                else:
+                    genre_id = self.upsert_cache.genres[music_input.genre]
+
+                keyword_ids = []
+                for keyword in music_input.keywords:
+                    if keyword not in self.upsert_cache.keywords:
+                        keyword_result = await upsert_keyword(self.client, keyword=keyword)
+                        keyword_id = keyword_result.id
+                        self.upsert_cache.keywords[keyword] = keyword_id
+                    else:
+                        keyword_id = self.upsert_cache.keywords[keyword]
+                    keyword_ids.append(keyword_id)
+
+                if music_input.album not in self.upsert_cache.artists_and_albums[music_input.artist].albums:
+                    album_result = await upsert_album(self.client, album=music_input.album, artist=artist_id)
+                    album_id = album_result.id
+                    self.upsert_cache.artists_and_albums[music_input.artist].albums[music_input.album] = album_id
+                else:
+                    album_id = self.upsert_cache.artists_and_albums[music_input.artist].albums[music_input.album]
+
+                result = await upsert_music(
+                    self.client,
+                    title=music_input.title,
+                    folder=folder_id,
+                    path=music_input.path,
+                    size=music_input.size,
+                    length=music_input.length,
+                    rating=music_input.rating,
+                    keywords=keyword_ids,
+                    album=album_id,
+                    genre=genre_id,
+                    track=music_input.track,
+                )
                 keywords = frozenset(keyword.name for keyword in result.keywords)
                 folders = [Folder(path=Path(folder.path), name=folder.name, ipv4=folder.ipv4, username=folder.username) for folder in result.folders if folder.path is not None]
                 music = Music(
@@ -182,7 +254,6 @@ async def upsert_music(self, music_input: MusicInput) -> Music | None:
                     rating=result.rating,
                     folders=frozenset(folders),
                 )
-                # output_music = file.music
                 self.success(f"{self} : updated {music_input}")
                 return music
             except edgedb.errors.TransactionSerializationError as error:
@@ -195,7 +266,7 @@ async def upsert_music(self, music_input: MusicInput) -> Music | None:
             except OSError as error:
                 self.warn(f"{music_input} : unknown error", error=error)
             return None
-        self.err(f"{self} : too many transaction failures", errpr=last_error)
+        self.err(f"{self} : too many transaction failures", error=last_error)
         return None
 
     async def make_bests(

musicbot/queries/gen_bests_async_edgeql.py

@@ -1,5 +1,5 @@
 # AUTOGENERATED FROM 'musicbot/queries/gen_bests.edgeql' WITH:
-#     $ edgedb-py -I musicbot-prod --tls-security insecure --dir musicbot/queries
+#     $ edgedb-py -I musicbot-prod --dir musicbot/queries
 
 
 from __future__ import annotations
@@ -20,8 +20,16 @@
 
 
 class NoPydanticValidation:
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type, _handler):
+        # Pydantic 2.x
+        from pydantic_core.core_schema import any_schema
+
+        return any_schema()
+
     @classmethod
     def __get_validators__(cls):
+        # Pydantic 1.x
         from pydantic.dataclasses import dataclass as pydantic_dataclass
 
         pydantic_dataclass(cls)

musicbot/queries/gen_playlist_async_edgeql.py

@@ -1,5 +1,5 @@
 # AUTOGENERATED FROM 'musicbot/queries/gen_playlist.edgeql' WITH:
-#     $ edgedb-py -I musicbot-prod --tls-security insecure --dir musicbot/queries
+#     $ edgedb-py -I musicbot-prod --dir musicbot/queries
 
 
 from __future__ import annotations
@@ -20,8 +20,16 @@
 
 
 class NoPydanticValidation:
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type, _handler):
+        # Pydantic 2.x
+        from pydantic_core.core_schema import any_schema
+
+        return any_schema()
+
     @classmethod
     def __get_validators__(cls):
+        # Pydantic 1.x
         from pydantic.dataclasses import dataclass as pydantic_dataclass
 
         pydantic_dataclass(cls)

musicbot/queries/remove_async_edgeql.py

@@ -1,5 +1,5 @@
 # AUTOGENERATED FROM 'musicbot/queries/remove.edgeql' WITH:
-#     $ edgedb-py -I musicbot-prod --tls-security insecure --dir musicbot/queries
+#     $ edgedb-py -I musicbot-prod --dir musicbot/queries
 
 
 from __future__ import annotations
@@ -11,8 +11,16 @@
 
 
 class NoPydanticValidation:
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type, _handler):
+        # Pydantic 2.x
+        from pydantic_core.core_schema import any_schema
+
+        return any_schema()
+
     @classmethod
     def __get_validators__(cls):
+        # Pydantic 1.x
         from pydantic.dataclasses import dataclass as pydantic_dataclass
 
         pydantic_dataclass(cls)

musicbot/queries/soft_clean_async_edgeql.py

@@ -1,5 +1,5 @@
 # AUTOGENERATED FROM 'musicbot/queries/soft_clean.edgeql' WITH:
-#     $ edgedb-py -I musicbot-prod --tls-security insecure --dir musicbot/queries
+#     $ edgedb-py -I musicbot-prod --dir musicbot/queries
 
 
 from __future__ import annotations
@@ -11,8 +11,16 @@
 
 
 class NoPydanticValidation:
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type, _handler):
+        # Pydantic 2.x
+        from pydantic_core.core_schema import any_schema
+
+        return any_schema()
+
     @classmethod
     def __get_validators__(cls):
+        # Pydantic 1.x
         from pydantic.dataclasses import dataclass as pydantic_dataclass
 
         pydantic_dataclass(cls)