splitted scan and insert

musicbot/__init__.py

@@ -1,10 +1,10 @@
 from warnings import filterwarnings
 
 from musicbot.config import Config
-from musicbot.file import File
+from musicbot.file import File, Issue
 from musicbot.folder import Folder
 from musicbot.helpers import syncify
-from musicbot.music import Music
+from musicbot.music import Music, MusicInput
 from musicbot.music_filter import MusicFilter
 from musicbot.musicdb import MusicDb
 from musicbot.object import MusicbotObject
@@ -17,9 +17,11 @@
 
 __all__ = [
     "MusicbotObject",
+    "Issue",
     "Config",
     "File",
     "Music",
+    "MusicInput",
     "MusicFilter",
     "Playlist",
     "PlaylistOptions",

musicbot/cli/options.py

@@ -7,6 +7,7 @@
 
 from musicbot.defaults import (
     DEFAULT_CLEAN,
+    DEFAULT_COROUTINES,
     DEFAULT_DRY,
     DEFAULT_OUTPUT,
     DEFAULT_SAVE,
@@ -40,6 +41,25 @@ def sane_dry(ctx: click.Context, param: click.Parameter, value: bool) -> None:
     expose_value=False,
 )
 
+
+@beartype
+def sane_coroutines(ctx: click.Context, param: click.Parameter, value: int) -> None:  # pylint: disable=unused-argument
+    """Overwrite global concurrency"""
+    if not param.name:
+        logger.error("no param name set")
+        raise click.Abort()
+    MusicbotObject.coroutines = value
+
+
+coroutines_option = click.option(
+    "--coroutines",
+    help="Limit number of coroutines",
+    default=DEFAULT_COROUTINES,
+    callback=sane_coroutines,
+    show_default=True,
+    expose_value=False,
+)
+
 true_values = ("enabled", "y", "yes", "t", "true", "on", "1")
 false_values = ("", "none", "disabled", "n", "no", "f", "false", "off", "0")
 
@@ -209,13 +229,6 @@ def confirm(ctx: click.Context, param: Any, value: bool) -> None:  # pylint: dis
     type=click.Choice(["json", "table", "m3u"]),
 )
 
-coroutines_option = click.option(
-    "--coroutines",
-    help="Limit number of coroutines",
-    default=64,
-    show_default=True,
-)
-
 
 @beartype
 def config_string(ctx: click.Context, param: click.Parameter, value: str | None) -> Any:

musicbot/commands/local.py

@@ -11,6 +11,7 @@
 from watchfiles import Change, DefaultFilter, awatch
 
 from musicbot import (
+    File,
     MusicbotObject,
     MusicDb,
     MusicFilter,
@@ -126,16 +127,25 @@ async def scan(
     clean: bool,
     save: bool,
     output: str,
-    coroutines: int,
 ) -> None:
-    await local.scan(
+    music_inputs = await local.scan(
         musicdb=musicdb,
         scan_folders=scan_folders,
-        clean=clean,
-        save=save,
-        output=output,
-        coroutines=coroutines,
     )
+    if clean:
+        _ = await musicdb.clean_musics()
+    music_outputs = await local.upsert_musics(
+        musicdb=musicdb,
+        music_inputs=music_inputs,
+    )
+    _ = await musicdb.soft_clean()
+
+    if output == "json":
+        MusicbotObject.print_json([asdict(mo) for mo in music_outputs])
+
+    if save:
+        MusicbotObject.config.configfile["musicbot"]["folders"] = scan_folders.unique_directories
+        MusicbotObject.config.write()
 
 
 @cli.command(help="Watch files changes in folders", aliases=["watcher"])
@@ -164,10 +174,16 @@ async def soft_clean_periodically() -> None:
         except (asyncio.CancelledError, KeyboardInterrupt):
             pass
 
-    async def update_music(path: str) -> None:
+    async def update_music(path: Path) -> None:
         for directory in scan_folders.directories:
-            if path.startswith(str(directory)):
-                _ = await musicdb.upsert_path((directory, Path(path)))
+            if str(path).startswith(str(directory)):
+                if (file := File.from_path(folder=directory, path=path)) is None:
+                    continue
+
+                if (music_input := file.music_input) is None:
+                    continue
+
+                _ = await musicdb.upsert_music(music_input)
 
     async def watcher() -> None:
         class MusicWatchFilter(DefaultFilter):
@@ -184,7 +200,7 @@ def __call__(self, change: Change, path: str) -> bool:
                     for change_path in changes:
                         change, path = change_path
                         if change in (Change.added, Change.modified):
-                            await update_music(path)
+                            await update_music(Path(path))
                         elif change == Change.deleted:
                             _ = await musicdb.remove_music_path(path)
                 except edgedb.ClientConnectionFailedTemporarilyError as error:
@@ -336,20 +352,20 @@ async def sync(
 async def custom_playlists(
     musicdb: MusicDb,
     scan_folder: Path,
-    coroutines: int,
     min_playlist_size: int,
     playlist_options: PlaylistOptions,
     fast: bool,
 ) -> None:
     scan_folders = ScanFolders(directories=[scan_folder])
     if not fast:
-        await local.scan(
+        music_inputs = await local.scan(
             musicdb=musicdb,
             scan_folders=scan_folders,
-            clean=True,
-            save=False,
-            output="table",
-            coroutines=coroutines,
+        )
+        _ = await musicdb.clean_musics()
+        _ = await local.upsert_musics(
+            musicdb=musicdb,
+            music_inputs=music_inputs,
         )
 
     musicdb.set_readonly()

musicbot/file.py

@@ -23,7 +23,7 @@
 )
 from musicbot.folder import Folder
 from musicbot.helpers import current_user
-from musicbot.music import Music
+from musicbot.music import Music, MusicInput
 from musicbot.object import MusicbotObject
 
 logger = logging.getLogger(__name__)
@@ -114,22 +114,23 @@ def music(self) -> Music | None:
         )
 
     @property
-    def music_input(self) -> dict[str, Any] | None:
-        if self.music is None:
+    def music_input(self) -> MusicInput | None:
+        if (music := self.music) is None:
             return None
-        data = asdict(self.music)
-        if not data["folders"]:
+        if not music.folders:
             self.err(f"{self} : no folder set")
             return None
 
+        data = asdict(music)
+
         folders = data.pop("folders")
         folder = next(iter(folders))
         data["keywords"] = list(data["keywords"])
         data["ipv4"] = folder.ipv4
         data["username"] = folder.username
         data["folder"] = folder.name
         data["path"] = str(folder.path)
-        return data
+        return MusicInput(**data)
 
     def set_tags(
         self,

musicbot/local.py

@@ -1,16 +1,19 @@
+import asyncio
 import codecs
 import logging
 import shutil
-from dataclasses import asdict
 from pathlib import Path
 
 import click
 
 from musicbot import (
     File,
+    Issue,
+    Music,
     MusicbotObject,
     MusicDb,
     MusicFilter,
+    MusicInput,
     PlaylistOptions,
     ScanFolders,
 )
@@ -74,26 +77,64 @@ async def bests(
 async def scan(
     musicdb: MusicDb,
     scan_folders: ScanFolders,
-    clean: bool,
-    save: bool,
-    output: str,
-    coroutines: int,
-) -> None:
-    if clean:
-        _ = await musicdb.clean_musics()
+) -> list[MusicInput]:
+    max_value = len(scan_folders.folders_and_paths)
+    if not max_value:
+        MusicbotObject.warn(f"No music folder or paths discovered from directories {scan_folders.directories}")
+        return []
+
+    MusicbotObject.echo(f"{musicdb} : loading {max_value} files")
+    music_inputs = []
+    with MusicbotObject.progressbar(desc="Loading files", max_value=max_value) as pbar:
+        for folder_and_path in scan_folders.folders_and_paths:
+            try:
+                folder, path = folder_and_path
+                if not (file := File.from_path(folder=folder, path=path)):
+                    continue
 
-    files = await musicdb.upsert_folders(
-        scan_folders=scan_folders,
-        coroutines=coroutines,
-    )
-    _ = await musicdb.soft_clean()
+                issues = file.issues
+                if Issue.NO_TITLE in issues or Issue.NO_ARTIST in issues or Issue.NO_ALBUM in issues:
+                    MusicbotObject.warn(f"{file} : missing mandatory fields title/album/artist : {issues}")
+                    continue
+                if not (music_input := file.music_input):
+                    MusicbotObject.err(f"{file} : cannot upsert music without physical folder !")
+                    continue
+
+                music_inputs.append(music_input)
+            finally:
+                pbar.value += 1
+                _ = pbar.update()
+
+    return music_inputs
+
+
+async def upsert_musics(
+    musicdb: MusicDb,
+    music_inputs: list[MusicInput],
+) -> list[Music]:
+    failed_inputs = []
+    music_outputs = []
+
+    with MusicbotObject.progressbar(desc="Inserting musics", max_value=len(music_inputs)) as pbar:
+
+        async def upsert_worker(music_input: MusicInput) -> None:
+            try:
+                if (music_output := await musicdb.upsert_music(music_input)) is None:
+                    MusicbotObject.err(f"{music_input} : unable to insert")
+                    failed_inputs.append(music_input)
+                else:
+                    music_outputs.append(music_output)
+            finally:
+                pbar.value += 1
+                _ = pbar.update()
 
-    if output == "json":
-        MusicbotObject.print_json([asdict(file.music) for file in files if file.music is not None])
+        async with asyncio.TaskGroup() as tg:
+            for music_input in music_inputs:
+                _ = tg.create_task(upsert_worker(music_input))
 
-    if save:
-        MusicbotObject.config.configfile["musicbot"]["folders"] = scan_folders.unique_directories
-        MusicbotObject.config.write()
+    if failed_inputs:
+        MusicbotObject.warn(f"Unable to insert {len(failed_inputs)} files")
+    return music_outputs
 
 
 async def sync(

musicbot/music.py

@@ -53,3 +53,21 @@ def slug(self) -> str:
             stopwords=STOPWORDS,
             replacements=REPLACEMENTS,
         )
+
+
+@beartype
+@dataclass
+class MusicInput(MusicbotObject):
+    title: str
+    album: str
+    artist: str
+    genre: str
+    size: int
+    rating: float
+    length: int
+    keywords: list[str]
+    ipv4: str
+    username: str
+    folder: str
+    path: str
+    track: int | None = None