HCK21_2024_Janelia_NDRH/tutorials/advanced_asset_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "62326652-fb78-4838-9bbd-6f29e09793fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "from dandi.dandiapi import DandiAPIClient\n",
    "from tqdm.notebook import tqdm\n",
    "from isodate import parse_duration, Duration\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3c949acf-7661-41ec-801b-7f7aec844ee0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "client = DandiAPIClient()\n",
    "dandisets = list(client.get_dandisets())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a226e74-0f13-40c6-9e8d-9a3eb3fbd3ad",
   "metadata": {
    "tags": []
   },
   "source": [
    "# More specific identification of NWB dandisets\n",
    "\n",
    "The simpler tutorial only tested if the phrase \"NWB\" was in the name of any of the data standards for a dandiset.\n",
    "\n",
    "The more official and precise method is to use the specific [RRID of NWB](https://scicrunch.org/resolver/RRID:SCR_015242), which is `\"RRID:SCR_015242\"`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "01b611f1-6d7f-40d7-8922-26eb1c5b2264",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2988d690aff4294aaf1a13e597bd6df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/465 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are currently 277 NWB datasets on DANDI!\n"
     ]
    }
   ],
   "source": [
    "nwb_dandisets = []\n",
    "\n",
    "for dandiset in tqdm(dandisets):\n",
    "    raw_metadata = dandiset.get_raw_metadata()\n",
    "\n",
    "    if any(\n",
    "        data_standard['identifier'] == \"RRID:SCR_015242\"  # this is the RRID for NWB\n",
    "        for data_standard in raw_metadata['assetsSummary'].get('dataStandard', [])\n",
    "    ):\n",
    "        nwb_dandisets.append(dandiset)\n",
    "print(f\"There are currently {len(nwb_dandisets)} NWB datasets on DANDI!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7962c8a-f93c-4de2-aa9a-bee6516da634",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Average age of subjects used in a dandiset\n",
    "\n",
    "Let's consider a more advanced calculation - finding the average age of all the subjects used in a particular dandiset.\n",
    "\n",
    "For this we will be directly accessing the asset level fields `wasAttributedTo` as a key of the `asset_metadata`, instead of as an attribute.\n",
    "\n",
    "We will also have to do some manual data manipulation to parse the form of the [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f5dbbb89-0b66-4557-8cea-829e92a4f4fd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d262c3538174436832a3ce7ce46d6a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/42 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The average age of the subjects in dandiset #398 is: 170.74276620370375 days\n"
     ]
    }
   ],
   "source": [
    "def iso_to_fractional_days(age_iso: str, experiment_date: str) -> float:\n",
    "    \"\"\"\n",
    "    Defining a helper function which parses the ISO 8601 age and returns it in float-valued seconds.\n",
    "    \n",
    "    This is because a dattetime.timedelta can only return either its `.days` (integer, rounded down) or\n",
    "    its `total_seconds()`.\n",
    "    \n",
    "    This helper also resolves some complications that can arise in other datasets when the age is measured in years,\n",
    "    or if the the age is a range.\n",
    "    \"\"\"\n",
    "    if \"/\" in age_iso:  # Some ages can be have upper and lower ranges due to uncertainty\n",
    "        return  # Skip\n",
    "\n",
    "    age_duration = parse_duration(datestring=age_iso)\n",
    "\n",
    "    if isinstance(age_duration, Duration):\n",
    "        experiment_datetime = datetime.fromisoformat(experiment_date)\n",
    "        time_delta = age_duration.totimedelta(end=experiment_datetime)\n",
    "    else:\n",
    "        time_delta = age_duration\n",
    "\n",
    "    return time_delta.total_seconds() / (  # Evaluate using the total number of seconds\n",
    "        60 *  # 60 seconds per minute\n",
    "        60 *  # 60 minutes per hour\n",
    "        24  # 24 hours per day (ignoring daylight savings time)\n",
    "    )\n",
    "\n",
    "\n",
    "all_subject_ages_in_days = []\n",
    "\n",
    "dandiset = client.get_dandiset(\"000398\")\n",
    "assets = list(dandiset.get_assets())\n",
    "for asset in tqdm(assets):\n",
    "    raw_metadata = asset.get_raw_metadata()\n",
    "    subjects = raw_metadata[\"wasAttributedTo\"]\n",
    "\n",
    "    for subject_metadata in subjects:\n",
    "        if \"age\" in subject_metadata:\n",
    "            age_in_days = iso_to_fractional_days(\n",
    "                age_iso=subject_metadata[\"age\"][\"value\"],\n",
    "                experiment_date=raw_metadata[\"wasGeneratedBy\"][0][\"startDate\"]\n",
    "            )\n",
    "\n",
    "            if age_in_days:  # Skip if the age is null\n",
    "                all_subject_ages_in_days.append(age_in_days)\n",
    "print(f\"The average age of the subjects in dandiset #398 is: {np.mean(all_subject_ages_in_days)} days\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb82bd5c-cbf1-4312-81a3-da0120a0d154",
   "metadata": {},
   "source": [
    "# Count the number of spiking units across all sessions in an experiment\n",
    "\n",
    "The number of units identified from spike sorting is not something that DANDI extracts automatically during upload...\n",
    "\n",
    "But we can calculate it ourselves without downloading an entire dandiset!\n",
    "\n",
    "We do this by streaming directly from the archive, which requires us to retrieve the asset path on the S3 backend of the DANDI archive and then set the `driver` argument to `ros3` (Read-Only S3).\n",
    "\n",
    "There are several ways to retrieve the S3 path, but the easiest is to use the NWB Inspector helper function `nwbinspector.tools.get_s3_urls_and_dandi_paths`, which will format the path in the way `ros3` expects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2edbd010-1c2c-469c-a492-5e96d6e98a36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting remfile\n",
      "  Downloading remfile-0.1.13-py3-none-any.whl.metadata (3.7 kB)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.11/site-packages (from remfile) (1.26.3)\n",
      "Requirement already satisfied: h5py in /opt/conda/lib/python3.11/site-packages (from remfile) (3.10.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.11/site-packages (from remfile) (2.31.0)\n",
      "Requirement already satisfied: mpi4py>=3.1.1 in /opt/conda/lib/python3.11/site-packages (from h5py->remfile) (3.1.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests->remfile) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests->remfile) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests->remfile) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests->remfile) (2024.2.2)\n",
      "Downloading remfile-0.1.13-py3-none-any.whl (11 kB)\n",
      "Installing collected packages: remfile\n",
      "Successfully installed remfile-0.1.13\n"
     ]
    }
   ],
   "source": [
    "!pip install remfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e23e50af-678f-4517-b0dc-ba10ba85d529",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from warnings import simplefilter\n",
    "simplefilter(\"ignore\")  # Suppress namespace warnings from reading older NWB files\n",
    "\n",
    "from nwbinspector.tools import get_s3_urls_and_dandi_paths\n",
    "from pynwb import NWBHDF5IO\n",
    "import remfile\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "85ac164c-5613-4eaa-8a20-0e2ae870583b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ad4fa87e6fb4bf29cef775cff1b0dd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s3_urls = get_s3_urls_and_dandi_paths(dandiset_id=\"000059\")\n",
    "\n",
    "num_units_per_asset = dict()\n",
    "for s3_url in tqdm(s3_urls):\n",
    "\n",
    "    rem_file = remfile.File(s3_url)\n",
    "    h5py_file = h5py.File(rem_file, \"r\")\n",
    "    io = NWBHDF5IO(file=h5py_file, load_namespaces=True)\n",
    "    nwbfile = io.read()\n",
    "    \n",
    "    if nwbfile.units:\n",
    "        num_units_per_asset.update({s3_url: len(nwbfile.units)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7ba55e2e-12ae-4626-842a-bb42c18e3686",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'https://dandiarchive.s3.amazonaws.com/blobs/093/2c2/0932c245-ac35-4dfd-be76-20ae328f43a4': 395,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/dac/f24/dacf2486-4b1c-4b7f-b3a6-43453670e436': 381,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/fd9/600/fd9600d5-3623-48fb-b84b-796b7e00b9cf': 365,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/f4f/37e/f4f37e2f-5b44-4ab9-a8a0-9fbdc218b357': 344,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/235/5bd/2355bdc4-394a-4a9e-bbd7-fbfcc4d9ec7b': 361,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/a19/13b/a1913b51-ab11-42af-a5ca-7b346d59ef37': 915,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/927/c68/927c68bb-82c2-4cdb-bde8-af836d0d1664': 1114,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/8ea/05c/8ea05cea-0159-4b78-9058-30de1348e7c7': 468,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/824/ef1/824ef138-56b9-4abc-bb3f-233d623c9f3e': 928,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/cd1/1ae/cd11ae52-a90d-48cc-bc16-c90113fd094b': 1024,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/45e/204/45e20453-213b-4dd2-ab89-fcaa92b2f891': 1730,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/fa7/9ee/fa79ee85-df07-403b-8c30-89bd2f2d7684': 854,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/9f8/0d9/9f80d963-413e-4c17-a18b-30aff27e0e56': 1827,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/f2f/f6b/f2ff6b52-63da-46a8-ac45-c70e792835a9': 855,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/503/121/50312181-7b3f-4456-bf12-3aa891b9ab31': 1384,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/4d4/da2/4d4da28c-15c9-4cd7-8aea-f09771f70c98': 1191,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/075/b32/075b32be-1c44-4e2f-8a91-3eeb182af587': 1509,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/85b/a21/85ba21dd-bafb-4523-a697-6b6342346e29': 1183,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/70f/91c/70f91c46-5ed4-4577-9fec-8d616ddd1364': 1664,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/dd2/649/dd264957-d469-4a1c-b303-ebb3a6a06be3': 1344,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/626/ac4/626ac469-e53b-4c83-a5db-8baf977dd62c': 584,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/af4/1b2/af41b25a-510c-4bad-abe3-d3fb6aec76b0': 476,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/4c4/f6f/4c4f6ff5-199b-4f5a-b241-4e5d6cf8a5ad': 409,\n",
       " 'https://dandiarchive.s3.amazonaws.com/blobs/d6e/dc0/d6edc032-a51e-4890-a621-06e168fa788f': 1014}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_units_per_asset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "46e23b08-4a86-42c2-8ffc-5e3161900cc9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dandiset #59 has a total of 22319 identified spiking units!\n"
     ]
    }
   ],
   "source": [
    "print(f\"Dandiset #59 has a total of {sum(num_units_per_asset.values())} identified spiking units!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56643220-eef1-4dbb-a104-c378897649c4",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Going beyond\n",
    "These examples show a few types of queries, but since the metadata structures are quite rich on both the dandiset and asset levels, they enable many complex queries beyond the examples here.\n",
    "\n",
    "These metadata structures are also expanding over time as DANDI becomes more strict about what counts as essential metadata.\n",
    "\n",
    "The `.get_raw_metadata` method of both `client.get_dandiset(...)` and `client.get_dandiset(...).get_assets()` provides a nice view into the available fields.\n",
    "\n",
    "Note: for any attribute, it is recommended to first check that it is not `None` before checking for its value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ff061749-dd12-4cc8-a70a-ad42b8ed862c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"id\": \"dandiasset:11c25674-6eff-43a8-8dba-7dea2e8c76c4\",\n",
      "    \"path\": \"sub-San4/sub-San4_ses-20200302T142114_ecephys.nwb\",\n",
      "    \"access\": [\n",
      "        {\n",
      "            \"status\": \"dandi:OpenAccess\",\n",
      "            \"schemaKey\": \"AccessRequirements\"\n",
      "        }\n",
      "    ],\n",
      "    \"digest\": {\n",
      "        \"dandi:dandi-etag\": \"4c907ae8685aea1bfbe57316942b881f-4\",\n",
      "        \"dandi:sha2-256\": \"b770e3ac3f75f40618de2ba2a81e996429d5fc01dd530e9d826acc7a1ad0853c\"\n",
      "    },\n",
      "    \"@context\": \"https://raw.githubusercontent.com/dandi/schema/master/releases/0.6.3/context.json\",\n",
      "    \"approach\": [\n",
      "        {\n",
      "            \"name\": \"electrophysiological approach\",\n",
      "            \"schemaKey\": \"ApproachType\"\n",
      "        }\n",
      "    ],\n",
      "    \"schemaKey\": \"Asset\",\n",
      "    \"contentUrl\": [\n",
      "        \"https://api.dandiarchive.org/api/assets/11c25674-6eff-43a8-8dba-7dea2e8c76c4/download/\",\n",
      "        \"https://dandiarchive.s3.amazonaws.com/blobs/429/baa/429baaad-a057-411d-8957-8460947aef73\"\n",
      "    ],\n",
      "    \"identifier\": \"11c25674-6eff-43a8-8dba-7dea2e8c76c4\",\n",
      "    \"contentSize\": 237267068,\n",
      "    \"publishedBy\": {\n",
      "        \"id\": \"urn:uuid:f9fc60e7-3126-4912-8222-4de0c7d2cd7a\",\n",
      "        \"name\": \"DANDI publish\",\n",
      "        \"endDate\": \"2022-12-08T18:03:14.228709+00:00\",\n",
      "        \"schemaKey\": \"PublishActivity\",\n",
      "        \"startDate\": \"2022-12-08T18:03:14.228709+00:00\",\n",
      "        \"wasAssociatedWith\": [\n",
      "            {\n",
      "                \"id\": \"urn:uuid:2a080a04-80f8-4d25-835b-77aa87eb4b81\",\n",
      "                \"name\": \"DANDI API\",\n",
      "                \"version\": \"0.1.0\",\n",
      "                \"schemaKey\": \"Software\",\n",
      "                \"identifier\": \"RRID:SCR_017571\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"dateModified\": \"2022-12-02T20:15:00.997018-08:00\",\n",
      "    \"datePublished\": \"2022-12-08T18:03:14.228709+00:00\",\n",
      "    \"schemaVersion\": \"0.6.3\",\n",
      "    \"encodingFormat\": \"application/x-nwb\",\n",
      "    \"wasGeneratedBy\": [\n",
      "        {\n",
      "            \"name\": \"Acquisition session\",\n",
      "            \"schemaKey\": \"Session\",\n",
      "            \"startDate\": \"2020-03-02T14:21:14-08:00\",\n",
      "            \"description\": \"Fig 3i, S10\"\n",
      "        },\n",
      "        {\n",
      "            \"id\": \"urn:uuid:a698c09b-ca57-4cf3-a523-dfbb00a8f524\",\n",
      "            \"name\": \"Metadata generation\",\n",
      "            \"schemaKey\": \"Activity\",\n",
      "            \"description\": \"Metadata generated by DANDI cli\",\n",
      "            \"wasAssociatedWith\": [\n",
      "                {\n",
      "                    \"url\": \"https://github.com/dandi/dandi-cli\",\n",
      "                    \"name\": \"DANDI Command Line Interface\",\n",
      "                    \"version\": \"0.46.6\",\n",
      "                    \"schemaKey\": \"Software\",\n",
      "                    \"identifier\": \"RRID:SCR_019009\"\n",
      "                }\n",
      "            ]\n",
      "        }\n",
      "    ],\n",
      "    \"wasAttributedTo\": [\n",
      "        {\n",
      "            \"age\": {\n",
      "                \"value\": \"P209DT55274S\",\n",
      "                \"unitText\": \"ISO-8601 duration\",\n",
      "                \"schemaKey\": \"PropertyValue\",\n",
      "                \"valueReference\": {\n",
      "                    \"value\": \"dandi:BirthReference\",\n",
      "                    \"schemaKey\": \"PropertyValue\"\n",
      "                }\n",
      "            },\n",
      "            \"sex\": {\n",
      "                \"name\": \"Male\",\n",
      "                \"schemaKey\": \"SexType\",\n",
      "                \"identifier\": \"http://purl.obolibrary.org/obo/PATO_0000384\"\n",
      "            },\n",
      "            \"species\": {\n",
      "                \"name\": \"Mus musculus - House mouse\",\n",
      "                \"schemaKey\": \"SpeciesType\",\n",
      "                \"identifier\": \"http://purl.obolibrary.org/obo/NCBITaxon_10090\"\n",
      "            },\n",
      "            \"genotype\": \"Emx1-Cre[tg/wt];Ai32[tg/wt]\",\n",
      "            \"schemaKey\": \"Participant\",\n",
      "            \"identifier\": \"San4\"\n",
      "        }\n",
      "    ],\n",
      "    \"blobDateModified\": \"2022-12-02T17:21:54.708718-08:00\",\n",
      "    \"variableMeasured\": [\n",
      "        {\n",
      "            \"value\": \"ElectrodeGroup\",\n",
      "            \"schemaKey\": \"PropertyValue\"\n",
      "        },\n",
      "        {\n",
      "            \"value\": \"ElectricalSeries\",\n",
      "            \"schemaKey\": \"PropertyValue\"\n",
      "        }\n",
      "    ],\n",
      "    \"measurementTechnique\": [\n",
      "        {\n",
      "            \"name\": \"multi electrode extracellular electrophysiology recording technique\",\n",
      "            \"schemaKey\": \"MeasurementTechniqueType\"\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"surgical technique\",\n",
      "            \"schemaKey\": \"MeasurementTechniqueType\"\n",
      "        }\n",
      "    ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(assets[0].get_raw_metadata(), indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21f004d3-1b3d-496c-b71f-2f4061cc95c8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}