Merge pull request #322 from pyinat/i18n

Add support for displaying and searching common names for configured locale

HISTORY.md

@@ -7,6 +7,9 @@
 * Display observation count and leaf taxon count on taxon info cards
 * In refresh mode, check for taxonomy changes and update tags with the new taxon (1:1 changes only)
 * Add support for alternate XMP sidecar path format, if it already exists (`basename.ext.xmp` instead of `basename.xmp`)
+* Add support for displaying and searching common names in any language supported by iNaturalist.org
+    * Uses the `locale` setting from settings menu
+    * The packaged taxon DB only contains english names, but a full version can be downloaded [here](https://github.com/pyinat/naturtag/releases/download/untagged-e757223556c30fa118ba/taxonomy_full.tar.gz)
 
 **CLI:**
 * Split CLI into subcommands:

assets/locales.json

@@ -0,0 +1 @@
+{"af": "Afrikaans", "ar": "\u0627\u0644\u0639\u0631\u0628\u064a\u0629", "be": "\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f", "bg": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438", "br": "brezhoneg", "ca": "catal\u00e0", "cs": "\u010de\u0161tina", "da": "dansk", "de": "Deutsch", "el": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac", "en": "English", "eo": "esperanto", "es": "espa\u00f1ol", "et": "eesti", "eu": "euskara", "fa": "\u0641\u0627\u0631\u0633\u06cc", "fi": "suomi", "fil": "Filipino", "fr": "fran\u00e7ais", "gl": "galego", "haw": "\u02bb\u014clelo Hawai\u02bbi", "he": "\u05e2\u05d1\u05e8\u05d9\u05ea", "hr": "hrvatski", "hu": "magyar", "id": "Indonesia", "it": "italiano", "ja": "\u65e5\u672c\u8a9e", "ka": "\u10e5\u10d0\u10e0\u10d7\u10e3\u10da\u10d8", "kk": "\u049b\u0430\u0437\u0430\u049b \u0442\u0456\u043b\u0456", "kn": "\u0c95\u0ca8\u0ccd\u0ca8\u0ca1", "ko": "\ud55c\uad6d\uc5b4", "lb": "L\u00ebtzebuergesch", "lt": "lietuvi\u0173", "lv": "latvie\u0161u", "mi": "M\u0101ori", "mk": "\u043c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438", "mr": "\u092e\u0930\u093e\u0920\u0940", "nb": "norsk bokm\u00e5l", "nl": "Nederlands", "oc": "occitan", "pl": "polski", "pt": "portugu\u00eas", "ro": "rom\u00e2n\u0103", "ru": "\u0440\u0443\u0441\u0441\u043a\u0438\u0439", "sat": "\u1c65\u1c5f\u1c71\u1c5b\u1c5f\u1c72\u1c64", "si": "\u0dc3\u0dd2\u0d82\u0dc4\u0dbd", "sk": "sloven\u010dina", "sl": "sloven\u0161\u010dina", "sq": "shqip", "sr": "\u0441\u0440\u043f\u0441\u043a\u0438", "sv": "svenska", "sw": "Kiswahili", "th": "\u0e44\u0e17\u0e22", "tr": "T\u00fcrk\u00e7e", "uk": "\u0443\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430", "vi": "Ti\u1ebfng Vi\u1ec7t", "zh": "\u4e2d\u6587", "zh-CN": "\u4e2d\u6587 (\u7b80\u4f53, \u4e2d\u56fd)"}

naturtag/app/settings_menu.py

@@ -16,6 +16,7 @@
 
 from naturtag.controllers import BaseController
 from naturtag.settings import Settings
+from naturtag.utils import read_locales
 from naturtag.widgets import FAIcon, HorizontalLayout, ToggleSwitch, VerticalLayout
 
 logger = getLogger(__name__)
@@ -28,11 +29,23 @@ def __init__(self, settings: Settings):
         super().__init__(settings)
         self.settings = settings
         self.settings_layout = VerticalLayout(self)
+        # Dictionary of locale codes and display names
+        self.locales = {k: f'{v} ({k})' for k, v in read_locales().items()}
 
         # iNaturalist settings
         inat = self.add_group('iNaturalist', self.settings_layout)
         inat.addLayout(TextSetting(settings, icon_str='fa.user', setting_attr='username'))
-        inat.addLayout(TextSetting(settings, icon_str='fa.globe', setting_attr='locale'))
+        inat.addLayout(
+            ChoiceAltDisplaySetting(
+                settings,
+                icon_str='fa.globe',
+                setting_attr='locale',
+                choices=self.locales,
+            )
+        )
+        inat.addLayout(
+            ToggleSetting(settings, icon_str='fa.language', setting_attr='search_locale')
+        )
         inat.addLayout(
             IntSetting(
                 settings, icon_str='mdi.home-city-outline', setting_attr='preferred_place_id'
@@ -180,6 +193,35 @@ def set_text(text):
         self.addWidget(widget)
 
 
+class ChoiceAltDisplaySetting(SettingContainer):
+    def __init__(
+        self,
+        settings: Settings,
+        icon_str: str,
+        setting_attr: str,
+        setting_title: Optional[str] = None,
+        choices: Optional[dict] = None,
+    ):
+        """Modified ChoiceSetting that allows the display value to differ from the setting value"""
+        super().__init__(icon_str, setting_attr, setting_title)
+        choices = choices or {}
+        self.lookup = {v: k for k, v in choices.items()}
+        reverse_lookup = choices
+
+        current_value = str(getattr(settings, setting_attr))
+        current_display_value = reverse_lookup.get(current_value) or current_value
+
+        # On setting value, first look up correct value based on display text
+        def set_text(text):
+            setattr(settings, setting_attr, self.lookup[text])
+
+        widget = QComboBox()
+        widget.addItems(choices.values() or [])
+        widget.setCurrentText(current_display_value)
+        widget.currentTextChanged.connect(set_text)
+        self.addWidget(widget)
+
+
 class TextSetting(SettingContainer):
     """Text input setting"""
 

naturtag/constants.py

@@ -11,6 +11,7 @@
 ICONS_DIR = ASSETS_DIR / 'icons'
 CLI_COMPLETE_DIR = ASSETS_DIR / 'autocomplete'
 PACKAGED_TAXON_DB = ASSETS_DIR / 'taxonomy.tar.gz'
+LOCALES_PATH = ASSETS_DIR / 'locales.json'
 APP_ICON = ICONS_DIR / 'logo.ico'
 APP_LOGO = ICONS_DIR / 'logo.png'
 SPINNER = ICONS_DIR / 'spinner_250px.svg'

naturtag/controllers/taxon_controller.py

@@ -74,7 +74,8 @@ def select_taxon(self, taxon_id: int):
         if self.tabs._init_complete:
             self.threadpool.cancel()
         future = self.threadpool.schedule(
-            lambda: INAT_CLIENT.taxa(taxon_id), priority=QThread.HighPriority
+            lambda: INAT_CLIENT.taxa(taxon_id, locale=self.settings.locale),
+            priority=QThread.HighPriority,
         )
         future.on_result.connect(lambda taxon: self.display_taxon(taxon))
 
@@ -155,7 +156,9 @@ def load_user_taxa(self):
 
         def get_recent_taxa():
             logger.info(f'Loading {len(display_ids)} user taxa')
-            return INAT_CLIENT.taxa.from_ids(*display_ids, accept_partial=True).all()
+            return INAT_CLIENT.taxa.from_ids(
+                *display_ids, locale=self.settings.locale, accept_partial=True
+            ).all()
 
         future = self.threadpool.schedule(get_recent_taxa, priority=QThread.LowPriority)
         future.on_result.connect(self.display_recent)

naturtag/settings.py

@@ -62,12 +62,12 @@ class YamlMixin:
     path: Optional[Path] = field(default=None)
 
     @classmethod
-    def read(cls, path: Path) -> 'YamlMixin':
+    def read(cls, path: Optional[Path]) -> 'YamlMixin':
         """Read settings from config file"""
         path = path or cls.path
 
         # New file; no contents to read
-        if not path.is_file():
+        if not path or not path.is_file():
             return cls(path=path)
 
         logger.debug(f'Reading {cls.__name__} from {path}')
@@ -130,6 +130,9 @@ class Settings(YamlMixin):
     preferred_place_id: int = doc_field(
         default=1, converter=int, doc='Place preference for regional species common names'
     )
+    search_locale: bool = doc_field(
+        default=True, doc='Search common names for only your selected locale'
+    )
     username: str = doc_field(default='', doc='Your iNaturalist username')
 
     # Metadata
@@ -155,7 +158,7 @@ class Settings(YamlMixin):
     last_obs_check: Optional[datetime] = field(default=None)
 
     @classmethod
-    def read(cls, path: Path = CONFIG_PATH) -> 'Settings':
+    def read(cls, path: Path = CONFIG_PATH) -> 'Settings':  # type: ignore
         return super(Settings, cls).read(path)  # type: ignore
 
     # Shortcuts for application files within the user data dir
@@ -228,7 +231,7 @@ def __attrs_post_init__(self):
         self.frequent = Counter(self.history)
 
     @classmethod
-    def read(cls, path: Path = USER_TAXA_PATH) -> 'UserTaxa':
+    def read(cls, path: Path = USER_TAXA_PATH) -> 'UserTaxa':  # type: ignore
         return super(UserTaxa, cls).read(path)  # type: ignore
 
     @property

naturtag/utils/__init__.py

@@ -1,3 +1,4 @@
 # flake8: noqa: F401
+from naturtag.utils.i18n import read_locales
 from naturtag.utils.image_glob import get_valid_image_paths
 from naturtag.utils.thumbnails import generate_thumbnail

naturtag/utils/i18n.py

@@ -0,0 +1,42 @@
+"""Internationalization utilities"""
+import json
+import sqlite3
+from logging import getLogger
+
+from naturtag.constants import DB_PATH, LOCALES_PATH, PathOrStr
+
+logger = getLogger(__name__)
+
+
+def get_locales(db_path: PathOrStr = DB_PATH) -> dict[str, str]:
+    """Get all locale codes represented in the FTS table and their localised names"""
+    from babel import Locale, UnknownLocaleError
+
+    with sqlite3.connect(db_path) as conn:
+        results = conn.execute('SELECT DISTINCT(language_code) from taxon_fts').fetchall()
+        locales = sorted([r[0] for r in results if r[0]])
+
+    locale_dict = {}
+    for locale in locales:
+        try:
+            locale_dict[locale] = Locale.parse(locale.replace('-', '_')).display_name
+        except UnknownLocaleError as e:
+            logger.warning(e)
+
+    locale_dict.pop('und')  # "Undefined"; seems to be a mix of languages
+    return locale_dict
+
+
+def write_locales(db_path: PathOrStr = DB_PATH):
+    locale_dict = get_locales(db_path)
+    with open(LOCALES_PATH, 'w') as f:
+        f.write(json.dumps(locale_dict))
+
+
+def read_locales() -> dict[str, str]:
+    try:
+        with open(LOCALES_PATH) as f:
+            return json.loads(f.read())
+    except IOError as e:
+        logger.warning(e)
+        return {'en': 'English'}

naturtag/widgets/autocomplete.py

@@ -22,6 +22,7 @@ def __init__(self, settings: Settings):
         super().__init__()
         self.setClearButtonEnabled(True)
         self.findChild(QToolButton).setIcon(fa_icon('mdi.backspace'))
+        self.settings = settings
         self.taxa: dict[str, int] = {}
 
         completer = QCompleter()
@@ -52,7 +53,9 @@ def next_result(self):
     # TODO: Input delay
     def search(self, q: str):
         if len(q) > 1 and q not in self.taxa:
-            self.taxa = {t.name: t.id for t in self.taxon_completer.search(q)}
+            language = self.settings.locale if self.settings.search_locale else None
+            results = self.taxon_completer.search(q, language=language)
+            self.taxa = {t.name: t.id for t in results}
             self.model.setStringList(self.taxa.keys())
 
     @Slot(str)

pyproject.toml

@@ -56,6 +56,7 @@ sphinx-design               = {optional=true, version='>=0.2'}
 sphinxcontrib-apidoc        = {optional=true, version='^0.3'}
 
 [tool.poetry.dev-dependencies]
+babel = '>=2.0'
 coverage = '^7.0'
 nox = '^2023.4'
 nox-poetry = '^1.0'

test/test_image_glob.py

@@ -54,7 +54,7 @@ def test_get_valid_image_paths__glob():
 
 
 def test_get_valid_image_paths__recursive():
-    assert len(get_valid_image_paths([ASSETS_DIR], recursive=True)) == 28
+    assert len(get_valid_image_paths([ASSETS_DIR], recursive=True)) == 29
 
 
 def test_get_valid_image_paths__uri():