From 3ab85f089744cfc023444212435d45e1bfaf360c Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 17:39:18 +0200 Subject: [PATCH 1/2] get sequence data --- ...oad_Sequences_from_a_DistantAlertAPI.ipynb | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb b/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb index af39784..b039bfd 100644 --- a/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb +++ b/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb @@ -24,7 +24,6 @@ "metadata": {}, "outputs": [], "source": [ - "SEQUENCE_ID_LIST = [13802, 9456]\n", "BASE_DIRECTORY = \"alerts\" # directory where to put sequences data" ] }, @@ -70,6 +69,40 @@ "os.makedirs(base_dir, exist_ok=True)\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "512ce6f1-76db-43b6-adb3-131942e8a95e", + "metadata": {}, + "outputs": [], + "source": [ + "response = api_client.fetch_sequences_from_date(\"2025-08-26\", limit=50)\n", + "api_sequences = pd.DataFrame(response.json())\n", + "api_sequences.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e414559-ae99-4467-b7cb-4324b9b8d0a8", + "metadata": {}, + "outputs": [], + "source": [ + "SEQUENCE_ID_LIST = [] #fullfill this with alertes id" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1fa5e0df-a090-4f96-ae9a-cd54f365d2c2", + "metadata": {}, + "outputs": [], + "source": [ + "# you can also reproduce all alerts from this day\n", + "if len(SEQUENCE_ID_LIST)==0:\n", + " SEQUENCE_ID_LIST = list(api_sequences[\"id\"])" + ] + }, { "cell_type": "code", "execution_count": null, From 408e4d71f893acd4eb54511a2912efc694c8874b Mon Sep 17 00:00:00 2001 From: fe51 <55736935+fe51@users.noreply.github.com> Date: Sat, 4 Oct 2025 19:44:55 +0200 Subject: [PATCH 2/2] docs and small refacto --- ...oad_Sequences_from_a_DistantAlertAPI.ipynb | 482 ++++++++---------- 1 file changed, 219 insertions(+), 263 deletions(-) diff --git a/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb b/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb index 8251758..b38b3a1 100644 --- a/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb +++ b/containers/notebooks/app/Download_Sequences_from_a_DistantAlertAPI.ipynb @@ -1,266 +1,222 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "a1c721ae-469b-48d7-a1df-3cd24623b3ee", - "metadata": {}, - "source": [ - "# Intro\n", - "\n", - "This notebook aims at downloading sequences infos(images and bboxes) from a Distant/Real Pyronear Alert API isntance in order to recreate alerts/sequences with ease in local dev environnement\n", - "\n", - "You need to have accesses to a distant api and provide ```DISTANT_API_URL```, ```DISTANT_ALERT_API_LOGIN```, ```DISTANT_ALERT_API_PASSWORD```\n", - "\n", - "And obvisouly, fill ```SEQUENCE_ID_LIST``` with the sequence id you want in order to download related data\n", - "\n", - "\n", - "If you do not have anything from above, do not worry, you can use the notebook \"send_real_alerts\" to send some alerts to local env dev" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "649157c6-0ce3-4324-a7b6-15c911ddaf36", - "metadata": {}, - "outputs": [], - "source": [ - "BASE_DIRECTORY = \"alerts\" # directory where to put sequences data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4e511d97-245c-45cc-9216-a16d31b663aa", - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from dotenv import load_dotenv\n", - "import os\n", - "\n", - "from pyroclient import Client\n", - "\n", - "load_dotenv(\"../.env\")\n", - "DISTANT_API_URL = os.environ.get(\"DISTANT_API_URL\")\n", - "DISTANT_ALERT_API_LOGIN = os.environ.get(\"DISTANT_ALERT_API_LOGIN\")\n", - "DISTANT_ALERT_API_PASSWORD = os.environ.get(\"DISTANT_ALERT_API_PASSWORD\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ac07ba29-68d0-42d2-bafe-e94d961ec6c2", - "metadata": {}, - "outputs": [], - "source": [ - "token = requests.post(\n", - " f\"{DISTANT_API_URL}api/v1/login/creds\",\n", - " data={\"username\": DISTANT_ALERT_API_LOGIN, \"password\": DISTANT_ALERT_API_PASSWORD},\n", - " timeout=5,\n", - ").json()['access_token']\n", - "\n", - "api_client = Client(token, DISTANT_API_URL)\n", - "\n", - "# get cameras -> in order to get cam name from cam id\n", - "cameras = api_client.fetch_cameras().json()\n", - "\n", - "# define base directory\n", - "base_dir = os.path.join(BASE_DIRECTORY)\n", - "os.makedirs(base_dir, exist_ok=True)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "512ce6f1-76db-43b6-adb3-131942e8a95e", - "metadata": {}, - "outputs": [], - "source": [ - "response = api_client.fetch_sequences_from_date(\"2025-08-26\", limit=50)\n", - "api_sequences = pd.DataFrame(response.json())\n", - "api_sequences.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2e414559-ae99-4467-b7cb-4324b9b8d0a8", - "metadata": {}, - "outputs": [], - "source": [ - "SEQUENCE_ID_LIST = [] #fullfill this with alertes id" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1fa5e0df-a090-4f96-ae9a-cd54f365d2c2", - "metadata": {}, - "outputs": [], - "source": [ - "# you can also reproduce all alerts from this day\n", - "if len(SEQUENCE_ID_LIST)==0:\n", - " SEQUENCE_ID_LIST = list(api_sequences[\"id\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3c12b1d-54f1-4d8d-b591-dc04ecc35db2", - "metadata": {}, - "outputs": [], - "source": [ - "def dl_seqs_in_target_dir(sequence_id_list, target_dir, api_client):\n", - " \"\"\"\n", - " Download sequences from sequence_id_list to target_dir, using an instanciated api_client (of distant Pyronear alert API)\n", - "\n", - " \"\"\"\n", - " \n", - " for seq_id in sequence_id_list: \n", - " sequences = api_client.fetch_sequences_detections(sequence_id=seq_id, limit=10, desc=False).json()\n", - " \n", - " cam_name = [item['name'] for item in cameras if item['id'] == sequences[0]['camera_id']][0]\n", - " created_at_rounded = sequences[0][\"created_at\"].split('.')[0].replace(':', '-').replace('T', '_')\n", - " cam_id_distant_api = sequences[0][\"camera_id\"]\n", - " print(f\"== Download Alerts data for sequence ID {seq_id} - camera {cam_name} at {created_at_rounded}\")\n", - " \n", - " alert_dir = os.path.join(f\"{cam_id_distant_api}_{cam_name}_{created_at_rounded}\")\n", - " image_dir = os.path.join(target_dir,alert_dir, \"images\")\n", - " pred_dir = os.path.join(target_dir, alert_dir, \"labels_predictions\")\n", - " os.makedirs(image_dir, exist_ok=True)\n", - " os.makedirs(pred_dir, exist_ok=True)\n", - " \n", - " for seq in sequences:\n", - " \n", - " # bbox\n", - " #yolo_format_bbox = ' '.join(map(str,ast.literal_eval(seq[\"bboxes\"])[0]))\n", - " bboxes = seq[\"bboxes\"]\n", - " \n", - " bbox_file_name = seq[\"bucket_key\"][:-4] + \".txt\"\n", - " \n", - " with open(os.path.join(target_dir, alert_dir, \"labels_predictions\",bbox_file_name), 'w') as f:\n", - " f.write(bboxes)\n", - " \n", - " url = seq['url']\n", - " nom_fichier = seq[\"bucket_key\"]\n", - " # image\n", - " response = requests.get(url, stream=True)\n", - " if response.status_code == 200:\n", - " full_img_path = os.path.join(image_dir, nom_fichier)\n", - " with open(full_img_path, 'wb') as f:\n", - " for chunk in response.iter_content(1024):\n", - " f.write(chunk)\n", - " else:\n", - " print(f\"Error during download.\")\n", - " print(\"Download complete\")\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "9503b5d5-9930-4c9e-9fbe-1eb1e563fd44", - "metadata": {}, - "source": [ - "# Download sequences from SEQUENCE_ID_LIST list" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "b2d033e9-7ce2-4cf4-995e-01ea2c15e235", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "== Download Alerts data for sequence ID 15462 - camera serre-de-barre-01 at 2025-09-29_13-56-58\n", - "== Download Alerts data for sequence ID 15463 - camera brison-03 at 2025-09-29_13-58-36\n", - "== Download Alerts data for sequence ID 15561 - camera pouncho-agast-01 at 2025-10-01_06-18-57\n", - "== Download Alerts data for sequence ID 15526 - camera pouncho-agast-01 at 2025-09-30_15-29-49\n", - "== Download Alerts data for sequence ID 13880 - camera fontaneilles-01 at 2025-09-04_07-34-32\n", - "== Download Alerts data for sequence ID 13572 - camera pouncho-agast-01 at 2025-08-29_06-59-39\n", - "== Download Alerts data for sequence ID 13537 - camera pouncho-agast-01 at 2025-08-28_15-31-17\n", - "== Download Alerts data for sequence ID 12997 - camera pouncho-agast-01 at 2025-08-17_15-52-27\n", - "== Download Alerts data for sequence ID 12562 - camera pouncho-agast-01 at 2025-08-06_12-32-09\n", - "== Download Alerts data for sequence ID 15376 - camera pouncho-agast-02 at 2025-09-28_10-52-39\n", - "== Download Alerts data for sequence ID 15526 - camera pouncho-agast-01 at 2025-09-30_15-29-49\n", - "Download complete\n" - ] - } - ], - "source": [ - "single_alerts = os.path.join(BASE_DIRECTORY, \"single_sequences\")\n", - "os.makedirs(single_alerts, exist_ok=True)\n", - "\n", - "dl_seqs_in_target_dir(SEQUENCE_ID_LIST, single_alerts, api_client)" - ] - }, - { - "cell_type": "markdown", - "id": "e9d109db-324c-4268-bae6-27c5960def55", - "metadata": {}, - "source": [ - "## Download triangulated alerts\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "68a2cff8-f847-4e6c-8c74-37b7ecffc9af", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "== Download Alerts data for sequence ID 10535 - camera moret-sur-loing-01 at 2025-07-14_11-22-05\n", - "== Download Alerts data for sequence ID 10537 - camera nemours-02 at 2025-07-14_11-24-10\n", - "== Download Alerts data for sequence ID 10538 - camera croix-augas-02 at 2025-07-14_11-24-31\n", - "== Download Alerts data for sequence ID 15462 - camera serre-de-barre-01 at 2025-09-29_13-56-58\n", - "== Download Alerts data for sequence ID 15463 - camera brison-03 at 2025-09-29_13-58-36\n", - "Download complete\n" - ] - } - ], - "source": [ - "\n", - "TRIANGULATED_SEQUENCE_LIST = [10535, 10537, 10538, 15462, 15463]\n", - "TRIANGULATED_DIRECTORY= \"triangulated_sequences\"\n", - "triangulated_dir = os.path.join(BASE_DIRECTORY, TRIANGULATED_DIRECTORY)\n", - "os.makedirs(triangulated_dir, exist_ok=True)\n", - "\n", - "dl_seqs_in_target_dir(TRIANGULATED_SEQUENCE_LIST, triangulated_dir, api_client)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ffd56f84-ec77-4843-837a-06de85232c94", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.7" - } + "cells": [ + { + "cell_type": "markdown", + "id": "a1c721ae-469b-48d7-a1df-3cd24623b3ee", + "metadata": {}, + "source": [ + "# Intro\n", + "\n", + "This notebook aims at downloading sequences data(images and bboxes) from a Distant/Real Pyronear Alert API isntance in order to recreate alerts/sequences with ease in local dev environnement\n", + "\n", + "You need to have accesses to a distant api and provide ```DISTANT_API_URL```, ```DISTANT_ALERT_API_LOGIN```, ```DISTANT_ALERT_API_PASSWORD```\n", + "\n", + "Then, \n", + "- either fill ```SEQUENCE_ID_LIST``` with the sequence ids you want in order to download related data (pick api admin credentialsi if you intend to download alert from various organisation)\n", + "- or let ```SEQUENCE_ID_LIST``` empty, pick a day to download by filling ```DAY_TO_DOWNLOAD``` (in this case, yyou need credentials from the organisation you want to download seq data)\n", + "\n", + "Also, you can fill ```TRIANGULATED_SEQUENCE_LIST``` with sequences id that triangulates\n", + "\n", + "\n", + "If you do not have anything from above, do not worry, you can use the notebook \"send_real_alerts\" to send some alerts to local env dev" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "649157c6-0ce3-4324-a7b6-15c911ddaf36", + "metadata": {}, + "outputs": [], + "source": [ + "BASE_DIRECTORY = \"../data/alert_samples\" # directory where to put sequences data\n", + "DAY_TO_DOWNLOAD = \"2025-08-26\" #YYYY-MM-DD\n", + "SEQUENCE_ID_LIST = [] #fullfill this with alerts id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e511d97-245c-45cc-9216-a16d31b663aa", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from dotenv import load_dotenv\n", + "import os\n", + "import pandas as pd\n", + "from pyroclient import Client\n", + "\n", + "load_dotenv(\"../.env\")\n", + "DISTANT_API_URL = os.environ.get(\"DISTANT_API_URL\")\n", + "DISTANT_ALERT_API_LOGIN = os.environ.get(\"DISTANT_ALERT_API_LOGIN\")\n", + "DISTANT_ALERT_API_PASSWORD = os.environ.get(\"DISTANT_ALERT_API_PASSWORD\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac07ba29-68d0-42d2-bafe-e94d961ec6c2", + "metadata": {}, + "outputs": [], + "source": [ + "token = requests.post(\n", + " f\"{DISTANT_API_URL}api/v1/login/creds\",\n", + " data={\"username\": DISTANT_ALERT_API_LOGIN, \"password\": DISTANT_ALERT_API_PASSWORD},\n", + " timeout=5,\n", + ").json()['access_token']\n", + "\n", + "api_client = Client(token, DISTANT_API_URL)\n", + "\n", + "# get cameras -> in order to get cam name from cam id\n", + "cameras = api_client.fetch_cameras().json()\n", + "\n", + "# define base directory\n", + "base_dir = os.path.join(BASE_DIRECTORY)\n", + "os.makedirs(base_dir, exist_ok=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "512ce6f1-76db-43b6-adb3-131942e8a95e", + "metadata": {}, + "outputs": [], + "source": [ + "response = api_client.fetch_sequences_from_date(DAY_TO_DOWNLOAD, limit=50)\n", + "api_sequences = pd.DataFrame(response.json())\n", + "api_sequences.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fa5e0df-a090-4f96-ae9a-cd54f365d2c2", + "metadata": {}, + "outputs": [], + "source": [ + "# you can also reproduce all alerts from this day\n", + "if len(SEQUENCE_ID_LIST)==0:\n", + " SEQUENCE_ID_LIST = list(api_sequences[\"id\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3c12b1d-54f1-4d8d-b591-dc04ecc35db2", + "metadata": {}, + "outputs": [], + "source": [ + "def dl_seqs_in_target_dir(sequence_id_list, target_dir, api_client):\n", + " \"\"\"\n", + " Download sequences from sequence_id_list to target_dir, using an instanciated api_client (of distant Pyronear alert API)\n", + "\n", + " \"\"\"\n", + " \n", + " for seq_id in sequence_id_list: \n", + " sequences = api_client.fetch_sequences_detections(sequence_id=seq_id, limit=10, desc=False).json()\n", + " \n", + " cam_name = [item['name'] for item in cameras if item['id'] == sequences[0]['camera_id']][0]\n", + " created_at_rounded = sequences[0][\"created_at\"].split('.')[0].replace(':', '-').replace('T', '_')\n", + " cam_id_distant_api = sequences[0][\"camera_id\"]\n", + " print(f\"== Download Alerts data for sequence ID {seq_id} - camera {cam_name} at {created_at_rounded}\")\n", + " \n", + " alert_dir = os.path.join(f\"{cam_id_distant_api}_{cam_name}_{created_at_rounded}\")\n", + " image_dir = os.path.join(target_dir,alert_dir, \"images\")\n", + " pred_dir = os.path.join(target_dir, alert_dir, \"labels_predictions\")\n", + " os.makedirs(image_dir, exist_ok=True)\n", + " os.makedirs(pred_dir, exist_ok=True)\n", + " \n", + " for seq in sequences:\n", + " \n", + " # bbox\n", + " #yolo_format_bbox = ' '.join(map(str,ast.literal_eval(seq[\"bboxes\"])[0]))\n", + " bboxes = seq[\"bboxes\"]\n", + " \n", + " bbox_file_name = seq[\"bucket_key\"][:-4] + \".txt\"\n", + " \n", + " with open(os.path.join(target_dir, alert_dir, \"labels_predictions\",bbox_file_name), 'w') as f:\n", + " f.write(bboxes)\n", + " \n", + " url = seq['url']\n", + " nom_fichier = seq[\"bucket_key\"]\n", + " # image\n", + " response = requests.get(url, stream=True)\n", + " if response.status_code == 200:\n", + " full_img_path = os.path.join(image_dir, nom_fichier)\n", + " with open(full_img_path, 'wb') as f:\n", + " for chunk in response.iter_content(1024):\n", + " f.write(chunk)\n", + " else:\n", + " print(f\"Error during download.\")\n", + " print(\"Download complete\")\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "9503b5d5-9930-4c9e-9fbe-1eb1e563fd44", + "metadata": {}, + "source": [ + "# Download sequences from SEQUENCE_ID_LIST list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2d033e9-7ce2-4cf4-995e-01ea2c15e235", + "metadata": {}, + "outputs": [], + "source": [ + "single_alerts = os.path.join(BASE_DIRECTORY, \"single_sequences\")\n", + "os.makedirs(single_alerts, exist_ok=True)\n", + "\n", + "dl_seqs_in_target_dir(SEQUENCE_ID_LIST, single_alerts, api_client)" + ] + }, + { + "cell_type": "markdown", + "id": "e9d109db-324c-4268-bae6-27c5960def55", + "metadata": {}, + "source": [ + "## Download triangulated alerts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "68a2cff8-f847-4e6c-8c74-37b7ecffc9af", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "TRIANGULATED_SEQUENCE_LIST = [10535, 10537, 10538, 15462, 15463]\n", + "TRIANGULATED_DIRECTORY= \"triangulated_sequences\"\n", + "triangulated_dir = os.path.join(BASE_DIRECTORY, TRIANGULATED_DIRECTORY)\n", + "os.makedirs(triangulated_dir, exist_ok=True)\n", + "\n", + "dl_seqs_in_target_dir(TRIANGULATED_SEQUENCE_LIST, triangulated_dir, api_client)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 }, - "nbformat": 4, - "nbformat_minor": 5 + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 }