From 2dd66c2b18224dc683e1892438aa65cc99810f42 Mon Sep 17 00:00:00 2001 From: "maximilian.jesch" Date: Mon, 13 Nov 2023 05:29:04 +0000 Subject: [PATCH 1/5] amazing change for demo --- 03-train_models.ipynb | 111 +++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/03-train_models.ipynb b/03-train_models.ipynb index e955cb6..07d9743 100644 --- a/03-train_models.ipynb +++ b/03-train_models.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "c0e6d4bf-e9ac-4724-8780-609e6c6c0798", + "tags": [] + }, "source": [ "![Alt text](images/banner.png)" ] @@ -10,7 +13,7 @@ { "cell_type": "markdown", "metadata": { - "id": "a7db81b2-a955-436b-a4a9-c4e31ff1351e", + "id": "1a43dd74-8805-4c4a-bdd6-8b331bd6b055", "tags": [] }, "source": [ @@ -19,7 +22,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "8b34fe3c-2b61-41e6-a230-50716bc6cf31" + }, "source": [ "### Initial Setup\n", "\n", @@ -30,7 +35,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "d17c866e-cab4-4283-bb67-d2f11d838a37" + "id": "426f8780-363a-42b0-9218-d373c052935a" }, "outputs": [], "source": [ @@ -43,7 +48,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "a671034f-cb0b-4be3-8000-80e1791fa28b" + "id": "c90d6049-8716-4ece-ba4f-051a4df73467" }, "outputs": [], "source": [ @@ -59,7 +64,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "8ad76681-6bd5-496f-a646-4befff7f530c" + }, "source": [ "## Imports" ] @@ -68,7 +75,7 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "id": "54418317-5b6f-4d8f-8f01-3f8fcb012975", + "id": "db95f430-80de-400e-9c3e-25511a634210", "tags": [] }, "outputs": [], @@ -110,7 +117,7 @@ { "cell_type": "markdown", "metadata": { - "id": "e721c290-5509-412a-83ce-573935d14365" + "id": "277383c1-2f15-492b-96da-ee0916aac233" }, "source": [ "## Utility Functions" @@ -119,7 +126,7 @@ { "cell_type": "markdown", "metadata": { - "id": "189ac0c4-ddf1-4e51-b4f2-8478c1d8d4f2" + "id": "13533a52-f97e-44ff-8cb5-d0008bfdbe3c" }, "source": [ "## Load the Saved Transformer from IBM COS " @@ -129,7 +136,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "f3edea10-3dd6-4572-8b67-0caf36a9b7ca" + "id": "8a547cb8-d3a3-4349-aeda-4959f7e812cf" }, "outputs": [], "source": [ @@ -140,7 +147,7 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "id": "97a6f8fa-3fa3-4310-84f7-435a544fdb42", + "id": "d5dfef69-ef74-4f1a-9800-21b3b5df9b57", "tags": [] }, "outputs": [], @@ -151,7 +158,7 @@ { "cell_type": "markdown", "metadata": { - "id": "22fd1435-a5a1-430a-8347-96498617dc05" + "id": "a9f4de87-187a-449e-ae43-c56f176f46d2" }, "source": [ "## Instantiate FactSheets Client" @@ -161,7 +168,7 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "id": "601c230e-8608-46a4-a84b-d8c522fa699f", + "id": "69c97777-cdbe-47cf-9808-9c0274bd971a", "tags": [] }, "outputs": [], @@ -205,7 +212,7 @@ { "cell_type": "markdown", "metadata": { - "id": "364b13f5-d3da-49c6-a07f-6edeace9f425" + "id": "cf488463-f80f-40f3-840b-0b208ccb6767" }, "source": [ "## Load Train Data and Test Data " @@ -215,7 +222,7 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "id": "c31c5352-efa4-40be-b1c0-20a11c092b62", + "id": "796fda8b-dee9-4d67-b108-c42a39d903eb", "tags": [] }, "outputs": [ @@ -420,7 +427,7 @@ { "cell_type": "markdown", "metadata": { - "id": "5f58d554-bb87-4ff4-843a-a496d82110a2" + "id": "88cfea83-bb59-4bfc-a420-715d966b658c" }, "source": [ "## Load train and test set" @@ -430,7 +437,7 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "id": "824e8d47-f471-431c-958f-2412802fd91f", + "id": "cf8bc456-5dcd-47d6-a798-d49dafe18086", "tags": [] }, "outputs": [], @@ -447,7 +454,7 @@ { "cell_type": "markdown", "metadata": { - "id": "e81fae16-dbb4-4d01-b538-b61f71d13c90" + "id": "07d6e6c4-815f-49ef-8482-7fff11bed30a" }, "source": [ "## Make validation set" @@ -457,7 +464,7 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "id": "d3af40a2-1e07-4187-bfcb-c12b4b7208cc", + "id": "254d4567-14c1-4d4e-aa10-3d6f7c286937", "tags": [] }, "outputs": [], @@ -469,7 +476,7 @@ { "cell_type": "markdown", "metadata": { - "id": "f79dd02f-22ad-468c-b6a2-235a342c0b48" + "id": "0fdba6cc-171c-442f-a65b-b2ca7cb37888" }, "source": [ "## Instantiate a Classifier " @@ -479,20 +486,20 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "id": "be29da66-e1b4-4802-94bd-c47255df3f3b", + "id": "f5ec1d84-4fe3-4fb3-a50f-a887478c3ea1", "tags": [] }, "outputs": [], "source": [ "from lightgbm import LGBMClassifier\n", "\n", - "model_lgb = LGBMClassifier(learning_rate=0.1,max_depth=5,random_state=42)" + "model_lgb = LGBMClassifier(learning_rate=0.09,max_depth=5,random_state=42)" ] }, { "cell_type": "markdown", "metadata": { - "id": "a305e5d9-4750-43f2-8484-85aff909c846" + "id": "29c55a85-e89e-4899-a28f-49dda521fe1d" }, "source": [ "## Append the Pipeline" @@ -502,7 +509,7 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "id": "392e164d-948c-49be-adca-1f66b0a72752", + "id": "bffd937d-7a50-4718-a4d0-5c27466c8a2b", "scrolled": true, "tags": [] }, @@ -516,7 +523,7 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "id": "80f0fb0b-4131-4357-8bdf-3352f2acab94", + "id": "85119fa8-fcf4-46b8-973c-332e8d7ae6e5", "scrolled": true, "tags": [] }, @@ -575,7 +582,7 @@ { "cell_type": "markdown", "metadata": { - "id": "7a1f5215-9233-4087-9315-439632632074" + "id": "9fbc4586-231f-4033-8196-2070023d2fbf" }, "source": [ "## Baseline Model " @@ -585,7 +592,7 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "id": "f0119280-a0e5-4e74-92cc-520352a85aba", + "id": "6517c2f4-e0c6-418e-aa88-b1d3f0758e52", "tags": [] }, "outputs": [ @@ -626,7 +633,7 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "id": "06f4e2c0-8cf3-4752-8d50-f605014e9756", + "id": "ac254e0a-ea2e-4e85-a2ae-133efe1c99d8", "scrolled": true, "tags": [] }, @@ -660,7 +667,7 @@ { "cell_type": "markdown", "metadata": { - "id": "6a765b3b-46df-4832-998d-f6aa88549744" + "id": "6b475de0-704d-4aa9-b671-76847c08f26a" }, "source": [ "## Log the Train and Val loss" @@ -670,7 +677,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "ae34e2b6-e9fd-401b-b1e2-8de878db2338", + "id": "e66d86ca-c61d-4fa8-a5ff-4cd702da5364", "tags": [] }, "outputs": [], @@ -682,7 +689,7 @@ { "cell_type": "markdown", "metadata": { - "id": "3ea87773-790a-48fa-a4b1-5c8134747c9c" + "id": "f63d5ae7-df9b-4a80-a3f2-3d10b0bd9689" }, "source": [ "## Save train and val loss to COS" @@ -692,7 +699,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "79517f43-a037-418d-88da-e872f5507d8b", + "id": "82dce204-9179-4ab4-b6fc-c3666c1b2ba3", "scrolled": true, "tags": [] }, @@ -708,7 +715,7 @@ { "cell_type": "markdown", "metadata": { - "id": "4c267c68-d6ab-4283-b157-27791c612f46" + "id": "7383bf5c-dd5a-4553-9be3-f4f5c82a9cc1" }, "source": [ "## Check if the files are copied in COS\n" @@ -718,7 +725,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "6261ab04-eec2-4ba7-aeba-f4f1b2e0c355", + "id": "7acdafe5-99fe-4fb0-a9fa-047c8eb447a5", "tags": [] }, "outputs": [], @@ -730,7 +737,7 @@ { "cell_type": "markdown", "metadata": { - "id": "a6bb0c1d-b884-4a7a-bd50-e6fb0646be16" + "id": "523501c9-fab6-41a5-a5e2-56eee748b7e0" }, "source": [ "## Baseline Results of the Model" @@ -740,7 +747,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "7804bdbb-2029-467e-8584-1a15bc90d10b", + "id": "1a085202-ac52-4ea5-b296-511640706167", "tags": [] }, "outputs": [], @@ -754,7 +761,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "5d51f2fa-4deb-45c8-bad9-3bc489d5d739", + "id": "863d2f3f-fafe-43ec-b192-0dc9bcaf4e82", "tags": [] }, "outputs": [], @@ -771,7 +778,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "6a28c746-bd73-4080-933a-9ae887c77b31", + "id": "63a2911c-df05-4361-b0ed-486b291595e3", "tags": [] }, "outputs": [], @@ -787,7 +794,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "69df87eb-9213-497b-987a-0cb95330e0fe", + "id": "cf0040d6-75e4-4f47-878e-66128d605731", "tags": [] }, "outputs": [], @@ -810,7 +817,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "b26c707b-15e5-4eb2-8fb8-8badb2f9ceba", + "id": "11ed16b7-7c2a-4c22-9d24-746fbe34bba3", "tags": [] }, "outputs": [], @@ -836,7 +843,7 @@ { "cell_type": "markdown", "metadata": { - "id": "dac2a1f6-84c0-41c3-85ba-8bbdb574fd27" + "id": "a0bb90bd-5ab8-4aac-8441-85fe84e9b71a" }, "source": [ "## Serialize Model locally" @@ -846,7 +853,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "2ec81eb2-18be-4045-8265-dd57a269e6cb", + "id": "4b76e623-29b2-48ea-ba6e-6a83f9998e4f", "tags": [] }, "outputs": [], @@ -859,7 +866,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "b1ebe5c0-e8d9-47ed-94d8-2098f582d2b1", + "id": "2aa0857f-a656-478c-b225-b043cad82802", "tags": [] }, "outputs": [], @@ -872,7 +879,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "16bc9347-e945-4f0d-b108-1a12dd384667", + "id": "f2cde0f0-4420-46f4-9c2b-5bf8c19d1625", "tags": [] }, "outputs": [], @@ -889,7 +896,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "e2bab309-8819-45ab-9baa-c796258c7704", + "id": "68910d40-e7a0-4a94-aa20-f9bf6a5efd51", "tags": [] }, "outputs": [], @@ -900,7 +907,7 @@ { "cell_type": "markdown", "metadata": { - "id": "0b945b0d-2a2a-4415-bb1d-aad42e553958" + "id": "537607ce-a83a-4d79-af52-c2db0855c024" }, "source": [ "## Save and Log Models in AI Factsheets." @@ -910,7 +917,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "b814e00a-f777-4fb2-8f8b-58b17f6855c1" + "id": "4af60e2b-0bd9-4aee-a285-c02afb00f4bb" }, "outputs": [], "source": [ @@ -920,7 +927,7 @@ { "cell_type": "markdown", "metadata": { - "id": "9341e59f-1bea-4325-9371-e4317034e99e" + "id": "277ad990-c8fb-4cae-8da4-4abd087ae28b" }, "source": [ "## Custom succes check: Check if training and test score are above a (very low) threshold" @@ -930,7 +937,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "9e6439d9-be5e-4f7e-aa75-77f288b7d100" + "id": "3bad54ba-d738-408d-8467-655928d551d2" }, "outputs": [], "source": [ @@ -948,7 +955,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "91dce01e-d426-40e2-88bd-901010d64a96" + }, "source": [ "## Register the output variables for the next pipeine stage\n", "every notebook outputs a \"was_successful\" boolean variable. The logic behind this is different for every notebook and can be altered to fit the needs of the project.\n", @@ -959,7 +968,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "7ac6de9b-1f91-4e95-a3c2-409c0b065ce6", + "id": "81d2941c-2515-4a10-a8c9-78c1f22410f5", "tags": [] }, "outputs": [], From 0bfba96f0ab6d09c30c70e2c6e60d47313677ea1 Mon Sep 17 00:00:00 2001 From: "maximilian.jesch" Date: Mon, 13 Nov 2023 08:07:44 +0000 Subject: [PATCH 2/5] amazing lr change for demo --- 03-train_models.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/03-train_models.ipynb b/03-train_models.ipynb index 07d9743..e89eb72 100644 --- a/03-train_models.ipynb +++ b/03-train_models.ipynb @@ -493,7 +493,7 @@ "source": [ "from lightgbm import LGBMClassifier\n", "\n", - "model_lgb = LGBMClassifier(learning_rate=0.09,max_depth=5,random_state=42)" + "model_lgb = LGBMClassifier(learning_rate=0.1,max_depth=5,random_state=42)" ] }, { From 64a8c5bc400015a1f157f6ae729daca7ad977b60 Mon Sep 17 00:00:00 2001 From: "maximilian.jesch" Date: Fri, 15 Dec 2023 11:16:24 +0000 Subject: [PATCH 3/5] this is a test --- 03-train_models.ipynb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/03-train_models.ipynb b/03-train_models.ipynb index e89eb72..06b370d 100644 --- a/03-train_models.ipynb +++ b/03-train_models.ipynb @@ -980,6 +980,15 @@ "pipelines_client = WSPipelines.from_token(TOKEN)\n", "pipelines_client.store_results(train_params)" ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3c49e8d0-9ea0-4a52-bbae-ee24f5bbcaa2" + }, + "source": [ + "test123" + ] } ], "metadata": { From 535bfe6ad3a16d02a8904f337cf964d4e5f306e3 Mon Sep 17 00:00:00 2001 From: "maximilian.jesch" Date: Fri, 15 Dec 2023 12:17:33 +0000 Subject: [PATCH 4/5] this is an awesome change! for demo! --- 03-train_models.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/03-train_models.ipynb b/03-train_models.ipynb index 06b370d..3010366 100644 --- a/03-train_models.ipynb +++ b/03-train_models.ipynb @@ -493,7 +493,7 @@ "source": [ "from lightgbm import LGBMClassifier\n", "\n", - "model_lgb = LGBMClassifier(learning_rate=0.1,max_depth=5,random_state=42)" + "model_lgb = LGBMClassifier(learning_rate=0.09,max_depth=5,random_state=42)" ] }, { From 08a381f6c8730967784630fe3490ac772768c209 Mon Sep 17 00:00:00 2001 From: "maximilian.jesch" Date: Fri, 15 Dec 2023 12:18:53 +0000 Subject: [PATCH 5/5] this is another awesome change! second try --- 03-train_models.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/03-train_models.ipynb b/03-train_models.ipynb index 3010366..bcc304e 100644 --- a/03-train_models.ipynb +++ b/03-train_models.ipynb @@ -493,7 +493,7 @@ "source": [ "from lightgbm import LGBMClassifier\n", "\n", - "model_lgb = LGBMClassifier(learning_rate=0.09,max_depth=5,random_state=42)" + "model_lgb = LGBMClassifier(learning_rate=0.1,max_depth=5,random_state=42)" ] }, { @@ -987,7 +987,7 @@ "id": "3c49e8d0-9ea0-4a52-bbae-ee24f5bbcaa2" }, "source": [ - "test123" + "test123asdf" ] } ],