measurements/million_scale_measurements.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Probing part 1\n",
                "\n",
                "Vantage points will probe either the targets themselves (step 2) or other addresses in the same /24 prefix (step 2).\n",
                "\n",
                "Vantage points are only the anchors.  \n",
                "As always, targets are the anchors.  \n",
                "\n",
                "This notebook is an implementation of the million scale method. Check the paper for more information.  \n",
                "To do after create_datasets.ipynb"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import uuid\n",
                "\n",
                "from logger import logger\n",
                "from scripts.utils.file_utils import load_json\n",
                "from scripts.utils.measurement_utils import (\n",
                "    load_targets, \n",
                "    load_vps,\n",
                "    get_measurement_config,\n",
                "    save_measurement_config,\n",
                "    get_target_prefixes,\n",
                "    ping_prefixes,\n",
                "    ping_targets,\n",
                "    retrieve_results,\n",
                "    get_latest_measurements,\n",
                "    insert_prefix_results,\n",
                "    insert_target_results,\n",
                ")\n",
                "from default import (\n",
                "    USER_ANCHORS_FILE, \n",
                "    USER_HITLIST_FILE, \n",
                "    PREFIX_MEASUREMENT_RESULTS,\n",
                "    TARGET_MEASUREMENT_RESULTS,\n",
                "    MEASUREMENT_CONFIG_PATH,\n",
                ")\n",
                "\n",
                "# will define the number of vps and targets to use\n",
                "NB_TARGETS = 2\n",
                "NB_VPS = 4"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Load targets and vps dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2023-09-20 22:49:46::INFO:root:2498612375:: nb targets: 2\n",
                        "2023-09-20 22:49:46::INFO:root:2498612375:: nb_vps : 4\n"
                    ]
                }
            ],
            "source": [
                "\n",
                "targets = load_targets(USER_ANCHORS_FILE, nb_target=NB_TARGETS)\n",
                "vps = load_vps(USER_ANCHORS_FILE, nb_vps=NB_VPS)\n",
                "\n",
                "# all target prefixes\n",
                "target_prefixes = get_target_prefixes(targets)\n",
                "\n",
                "# responsive IP addresses in all /24 prefixes\n",
                "targets_per_prefix = load_json(USER_HITLIST_FILE)\n",
                "\n",
                "logger.info(f\"nb targets: {len(targets)}\")\n",
                "logger.info(f\"nb_vps : {len(vps)}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Generate measurement config\n",
                "\n",
                "This configuration is used to retrieve all measurements results from RIPE Atlas using their API."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [],
            "source": [
                "# measurement configuration for retrieval\n",
                "experiment_uuid = uuid.uuid4()\n",
                "target_measurement_uuid = uuid.uuid4()\n",
                "prefix_measurement_uuid = uuid.uuid4()\n",
                "\n",
                "logger.info(f\"Starting experiment with uuid : {experiment_uuid}\")\n",
                "\n",
                "measurement_config = get_measurement_config(\n",
                "    experiment_uuid=experiment_uuid,\n",
                "    target_measurement_uuid=target_measurement_uuid,\n",
                "    prefix_measurement_uuid=prefix_measurement_uuid,\n",
                "    targets=targets, \n",
                "    target_prefixes=target_prefixes,\n",
                "    vps=vps,\n",
                ")\n",
                "\n",
                "save_measurement_config(measurement_config, MEASUREMENT_CONFIG_PATH)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Step 1: probing each target prefixes"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Probe target prefixes\n",
                "WARNING : Time consumming section"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2023-09-20 22:49:46::INFO:root:measurement_utils:: Starting measurements 6fa82de2-507c-43eb-8b6f-80a5f6391f90 with parameters: dry_run=False; nb_targets=2; nb_vps=4.\n",
                        "2023-09-20 22:49:47::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195879\n",
                        "2023-09-20 22:49:47::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195880\n",
                        "2023-09-20 22:49:48::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195881\n",
                        "2023-09-20 22:49:48::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195882\n",
                        "2023-09-20 22:49:49::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195883\n",
                        "2023-09-20 22:49:50::INFO:root:ping_and_traceroute_classes:: measurement tag: 6fa82de2-507c-43eb-8b6f-80a5f6391f90 : started measurement id : 60195884\n",
                        "2023-09-20 22:49:50::INFO:root:ping_and_traceroute_classes:: measurement : 6fa82de2-507c-43eb-8b6f-80a5f6391f90 done\n"
                    ]
                }
            ],
            "source": [
                "ping_prefixes(\n",
                "    measurement_uuid=prefix_measurement_uuid,\n",
                "    measurement_config=measurement_config,\n",
                "    target_prefixes=target_prefixes,\n",
                "    targets_per_prefix=targets_per_prefix,\n",
                "    vps=vps,\n",
                ")\n",
                "\n",
                "save_measurement_config(measurement_config, MEASUREMENT_CONFIG_PATH)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Step 2: probing each target"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Probe targets\n",
                "WARNING : Time consumming section"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2023-09-20 22:49:50::INFO:root:measurement_utils:: Starting measurements 47b8f0db-a9f2-4886-8b5c-0ad5ba667c9f with parameters: dry_run=False; nb_targets=2; nb_vps=4.\n",
                        "2023-09-20 22:49:50::INFO:root:ping_and_traceroute_classes:: measurement tag: 47b8f0db-a9f2-4886-8b5c-0ad5ba667c9f : started measurement id : 60195885\n",
                        "2023-09-20 22:49:51::INFO:root:ping_and_traceroute_classes:: measurement tag: 47b8f0db-a9f2-4886-8b5c-0ad5ba667c9f : started measurement id : 60195886\n",
                        "2023-09-20 22:49:51::INFO:root:ping_and_traceroute_classes:: measurement : 47b8f0db-a9f2-4886-8b5c-0ad5ba667c9f done\n"
                    ]
                }
            ],
            "source": [
                "# measurement configuration for retrieval\n",
                "ping_targets(\n",
                "    measurement_uuid=target_measurement_uuid,\n",
                "    measurement_config=measurement_config,\n",
                "    targets=targets,\n",
                "    vps=vps,\n",
                "    use_cache=False\n",
                ")\n",
                "save_measurement_config(measurement_config, MEASUREMENT_CONFIG_PATH)\n"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Retrieve prefix results\n",
                "WARNING : Time consuming section\n",
                "\n",
                "Note: it might take some time before measurement results are available through RIPE API. If no results are available, retry after a few minutes (or hours, it might really depends on the probe itself)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2023-09-20 22:49:51::INFO:root:2416656981:: {'experiment_uuid': 'e05b7508-bb3f-406e-a839-e2178687425a', 'status': 'finished', 'start_time': '2023-09-20 22:49:46.595547', 'end_time': '2023-09-20 22:49:51.081318', 'is_dry_run': False, 'nb_targets': 2, 'nb_vps': 4, 'description': 'measurements from a set of vps towards all targets/target prefixes', 'af': 4, 'target_measurements': {'measurement_uuid': '47b8f0db-a9f2-4886-8b5c-0ad5ba667c9f', 'targets': ['185.159.159.9', '200.132.1.32'], 'vps': {'45.129.95.8': {'id': 7132, 'address_v4': '45.129.95.8', 'asn_v4': 207960, 'country_code': 'GB', 'geometry': {'type': 'Point', 'coordinates': [-3.1615, 51.4985]}}, '193.201.40.210': {'id': 6235, 'address_v4': '193.201.40.210', 'asn_v4': 24796, 'country_code': 'IT', 'geometry': {'type': 'Point', 'coordinates': [12.5085, 41.8995]}}, '93.90.135.201': {'id': 7096, 'address_v4': '93.90.135.201', 'asn_v4': 47295, 'country_code': 'DE', 'geometry': {'type': 'Point', 'coordinates': [14.3515, 51.1905]}}, '103.126.52.180': {'id': 6764, 'address_v4': '103.126.52.180', 'asn_v4': 138466, 'country_code': 'AU', 'geometry': {'type': 'Point', 'coordinates': [151.1905, -33.9205]}}}, 'ids': [60195885, 60195886], 'start_time': 1695242990.064615, 'end_time': 1695242991.081204}, 'prefix_measurements': {'measurement_uuid': '6fa82de2-507c-43eb-8b6f-80a5f6391f90', 'targets': ['185.159.159.0', '200.132.1.0'], 'vps': {'45.129.95.8': {'id': 7132, 'address_v4': '45.129.95.8', 'asn_v4': 207960, 'country_code': 'GB', 'geometry': {'type': 'Point', 'coordinates': [-3.1615, 51.4985]}}, '193.201.40.210': {'id': 6235, 'address_v4': '193.201.40.210', 'asn_v4': 24796, 'country_code': 'IT', 'geometry': {'type': 'Point', 'coordinates': [12.5085, 41.8995]}}, '93.90.135.201': {'id': 7096, 'address_v4': '93.90.135.201', 'asn_v4': 47295, 'country_code': 'DE', 'geometry': {'type': 'Point', 'coordinates': [14.3515, 51.1905]}}, '103.126.52.180': {'id': 6764, 'address_v4': '103.126.52.180', 'asn_v4': 138466, 'country_code': 'AU', 'geometry': {'type': 'Point', 'coordinates': [151.1905, -33.9205]}}}, 'ids': [60195879, 60195880, 60195881, 60195882, 60195883, 60195884], 'start_time': 1695242986.5993469, 'end_time': 1695242990.055087}}\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "60195885\n",
                        "60195886\n"
                    ]
                },
                {
                    "data": {
                        "text/plain": [
                            "'https://atlas.ripe.net/api/v2/measurements/60195539/'"
                        ]
                    },
                    "execution_count": 6,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "measurement_config = get_latest_measurements(MEASUREMENT_CONFIG_PATH)\n",
                "logger.info(measurement_config)\n",
                "\n",
                "for measurement_id in measurement_config[\"target_measurements\"][\"ids\"]:\n",
                "    print(measurement_id)\n",
                "\n",
                "\"https://atlas.ripe.net/api/v2/measurements/60195539/\"\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import uuid\n",
                "\n",
                "from logger import logger\n",
                "from scripts.utils.file_utils import load_json\n",
                "from scripts.utils.measurement_utils import (\n",
                "    load_targets, \n",
                "    load_vps,\n",
                "    get_measurement_config,\n",
                "    save_measurement_config,\n",
                "    get_target_prefixes,\n",
                "    ping_prefixes,\n",
                "    ping_targets,\n",
                "    retrieve_results,\n",
                "    get_latest_measurements,\n",
                "    insert_prefix_results,\n",
                "    insert_target_results,\n",
                ")\n",
                "from default import (\n",
                "    USER_ANCHORS_FILE, \n",
                "    USER_HITLIST_FILE, \n",
                "    PREFIX_MEASUREMENT_RESULTS,\n",
                "    TARGET_MEASUREMENT_RESULTS,\n",
                "    MEASUREMENT_CONFIG_PATH,\n",
                ")\n",
                "\n",
                "# will define the number of vps and targets to use\n",
                "NB_TARGETS = 2\n",
                "NB_VPS = 4"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "measurement_config = get_latest_measurements(MEASUREMENT_CONFIG_PATH)\n",
                "logger.info(measurement_config)\n",
                "\n",
                "prefix_measurement_uuid = measurement_config[\"prefix_measurements\"]['measurement_uuid']\n",
                "\n",
                "logger.info(f\"retrieving results for measurement ids: {prefix_measurement_uuid}\")\n",
                "\n",
                "# sometimes, not all probes give output, reduce timeout if you do not want to wait for too long\n",
                "response = retrieve_results(prefix_measurement_uuid,PREFIX_MEASUREMENT_RESULTS, NB_TARGETS, NB_VPS, timeout= 5)\n",
                "\n",
                "insert_prefix_results(response)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Retrieve traget results\n",
                "WARNING : Time consumming section\n",
                "\n",
                "Note: it might take some time before measurement results are available through RIPE API. If no results are available, retry after a few minutes (or hours, it might really depends on the probe itself)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2023-09-13 15:41:07::INFO:root:measurement_utils:: nb measurements retrieved: 8\n",
                        "2023-09-13 15:41:07::INFO:root:measurement_utils:: Target measurements successfully inserted in table : user_vps_to_target\n"
                    ]
                }
            ],
            "source": [
                "measurement_config = get_latest_measurements(MEASUREMENT_CONFIG_PATH)\n",
                "\n",
                "target_measurement_uuid = measurement_config[\"target_measurements\"][\"measurement_uuid\"]\n",
                "\n",
                "response = retrieve_results(target_measurement_uuid, TARGET_MEASUREMENT_RESULTS, NB_TARGETS, NB_VPS)\n",
                "\n",
                "insert_target_results(response)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "geoloc-imc-2023-GZT64Hva-py3.10",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.9"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}