{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3605a507",
   "metadata": {},
   "source": [
    "1.2 Unik dat pri predspracovani"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e1329b13",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "# fetch data from OpenML\n",
    "sonar = fetch_openml(name=\"sonar\", version=1, as_frame=True)\n",
    "X_sonar = sonar.data\n",
    "y_sonar = LabelEncoder().fit_transform(sonar.target)   # \"Mine\"/\"Rock\" -> 0/1\n",
    "\n",
    "# shuffle data\n",
    "X_sonar, y_sonar = shuffle(X_sonar, y_sonar, random_state=42)\n",
    "X_sonar = X_sonar.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2c55828",
   "metadata": {},
   "source": [
    "shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "830609ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with incorrect preprocessing: 0.8095\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# incorrect: the scaling parameters are estimated on the whole dataset\n",
    "scaler = StandardScaler()\n",
    "X_sonar_scaled = scaler.fit_transform(X_sonar)\n",
    "\n",
    "X_sonar_train, X_sonar_test, y_sonar_train, y_sonar_test = train_test_split(X_sonar_scaled, y_sonar, test_size=0.2, random_state=42, stratify=y_sonar)\n",
    "\n",
    "model = LogisticRegression(max_iter=1000).fit(X_sonar_train, y_sonar_train)\n",
    "acc_sonar_leaky = accuracy_score(y_sonar_test, model.predict(X_sonar_test))\n",
    "print(f\"Accuracy with incorrect preprocessing: {acc_sonar_leaky:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "640a70c1",
   "metadata": {},
   "source": [
    "Úloha 1.3 - Predspracovanie bez úniku pomocou Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cddacc19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with correct preprocessing: 0.8095\n",
      "Difference compared to the incorrect procedure: +0.0000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# TODO 1: split the original, unscaled X_sonar and y_sonar with the same parameters as in Task 1.2\n",
    "X_sonar_train, X_sonar_test, y_sonar_train, y_sonar_test = train_test_split(X_sonar, y_sonar, test_size=0.2, random_state=42, stratify=y_sonar)\n",
    "\n",
    "# TODO 2: create a Pipeline with the steps StandardScaler and LogisticRegression(max_iter=1000)\n",
    "pipe = Pipeline([\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"clf\", LogisticRegression(max_iter=1000)),\n",
    "])\n",
    "\n",
    "# TODO 3: fit the whole Pipeline on the training set only\n",
    "pipe.fit(X_sonar_train, y_sonar_train)\n",
    "\n",
    "# TODO 4: compute the accuracy on the test set\n",
    "acc_sonar_clean = accuracy_score(y_sonar_test, pipe.predict(X_sonar_test))\n",
    "\n",
    "print(f\"Accuracy with correct preprocessing: {acc_sonar_clean:.4f}\")\n",
    "print(f\"Difference compared to the incorrect procedure: {acc_sonar_leaky - acc_sonar_clean:+.4f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f527b513",
   "metadata": {},
   "source": [
    "## Úloha 1.4 – Únik dát pri výbere príznakov\n",
    "### Nesprávny postup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d534dafe",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'ellipsis' object has no attribute 'get_support'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mAttributeError\u001b[39m                            Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m      2\u001b[39m \n\u001b[32m      3\u001b[39m \u001b[38;5;66;03m# INCORRECT procedure: feature selection on the whole dataset, split afterwards\u001b[39;00m\n\u001b[32m      4\u001b[39m \u001b[38;5;66;03m# TODO 1: fit SelectKBest(score_func=f_classif, k=10) on the whole X_sonar and y_sonar\u001b[39;00m\n\u001b[32m      5\u001b[39m selector_leaky = ...\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m features_leaky = set(X_sonar.columns[selector_leaky.get_support()])\n\u001b[32m      7\u001b[39m X_sonar_selected = selector_leaky.transform(X_sonar)\n\u001b[32m      8\u001b[39m \n\u001b[32m      9\u001b[39m \u001b[38;5;66;03m# TODO 2: split X_sonar_selected and y_sonar with the same parameters as in Task 1.2,\u001b[39;00m\n",
      "\u001b[31mAttributeError\u001b[39m: 'ellipsis' object has no attribute 'get_support'"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "# INCORRECT procedure: feature selection on the whole dataset, split afterwards\n",
    "# TODO 1: fit SelectKBest(score_func=f_classif, k=10) on the whole X_sonar and y_sonar\n",
    "selector_leaky = ...\n",
    "features_leaky = set(X_sonar.columns[selector_leaky.get_support()])\n",
    "X_sonar_selected = selector_leaky.transform(X_sonar)\n",
    "\n",
    "# TODO 2: split X_sonar_selected and y_sonar with the same parameters as in Task 1.2,\n",
    "#         train LogisticRegression(max_iter=1000) and compute the accuracy on the test set\n",
    "acc_sonar_leaky_fs = ...\n",
    "\n",
    "# CORRECT procedure: split first, feature selection is a step of the Pipeline\n",
    "X_sonar_train, X_sonar_test, y_sonar_train, y_sonar_test = train_test_split(X_sonar, y_sonar, test_size=0.2, random_state=42, stratify=y_sonar)\n",
    "\n",
    "# TODO 3: create a Pipeline with the steps \"select\" (SelectKBest), \"scaler\" (StandardScaler)\n",
    "#         and \"clf\" (LogisticRegression), fit it on the training set only and compute the test accuracy\n",
    "pipe_select = ...\n",
    "acc_sonar_correct_fs = ...\n",
    "features_correct = set(X_sonar.columns[pipe_select.named_steps[\"select\"].get_support()])\n",
    "\n",
    "print(f\"Accuracy (selection on the whole dataset): {acc_sonar_leaky_fs:.4f}\")\n",
    "print(f\"Accuracy (selection on the training part): {acc_sonar_correct_fs:.4f}\")\n",
    "print(f\"Differing features: {sorted(features_leaky ^ features_correct)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5954ae20",
   "metadata": {},
   "source": [
    "### Korektný postup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8f31c61a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected features (correct): ['attribute_9', 'attribute_10', 'attribute_11', 'attribute_12', 'attribute_36', 'attribute_45', 'attribute_46', 'attribute_47', 'attribute_48', 'attribute_49']\n",
      "Accuracy (correct feature selection): 0.8095\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'features_leaky' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 23\u001b[39m\n\u001b[32m     19\u001b[39m acc_select_clean = accuracy_score(y_test, pipe_select.predict(X_test))\n\u001b[32m     20\u001b[39m print(f\"Accuracy (correct feature selection): {acc_select_clean:.4f}\")\n\u001b[32m     21\u001b[39m \n\u001b[32m     22\u001b[39m \u001b[38;5;66;03m# porovnanie\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m23\u001b[39m print(\u001b[33m\"\\nOnly in leaky selection:\"\u001b[39m, sorted(set(features_leaky) - set(features_clean)))\n\u001b[32m     24\u001b[39m print(\u001b[33m\"Only in correct selection:\"\u001b[39m, sorted(set(features_clean) - set(features_leaky)))\n",
      "\u001b[31mNameError\u001b[39m: name 'features_leaky' is not defined"
     ]
    }
   ],
   "source": [
    "# 1. NAJPRV rozdelenie pôvodných dát\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_sonar, y_sonar, test_size=0.2, random_state=42, stratify=y_sonar)\n",
    "\n",
    "# 2. Pipeline: výber príznakov -> škálovanie -> model\n",
    "pipe_select = Pipeline([\n",
    "    (\"select\", SelectKBest(score_func=f_classif, k=10)),\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"clf\", LogisticRegression(max_iter=1000)),\n",
    "])\n",
    "\n",
    "# 3. tréning IBA na trénovacích dátach (výber sa robí len z nich)\n",
    "pipe_select.fit(X_train, y_train)\n",
    "\n",
    "# 4. vybrané príznaky a presnosť\n",
    "features_clean = X_sonar.columns[pipe_select.named_steps[\"select\"].get_support()]\n",
    "print(\"Selected features (correct):\", list(features_clean))\n",
    "\n",
    "acc_select_clean = accuracy_score(y_test, pipe_select.predict(X_test))\n",
    "print(f\"Accuracy (correct feature selection): {acc_select_clean:.4f}\")\n",
    "\n",
    "# porovnanie\n",
    "print(\"\\nOnly in leaky selection:\", sorted(set(features_leaky) - set(features_clean)))\n",
    "print(\"Only in correct selection:\", sorted(set(features_clean) - set(features_leaky)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba5d7c2d",
   "metadata": {},
   "source": [
    "# Krok 2: Vnorená krížová validácia"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f370771",
   "metadata": {},
   "source": [
    "### Úloha 2.1 - Odhad výkonu pomocou vnorenej krížovej validácie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5647eb08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested CV accuracy: 0.7740 ± 0.0451\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=1)\n",
    "outer_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=2)\n",
    "\n",
    "param_grid = {\"clf__C\": [0.01, 0.1, 1, 10, 100]}\n",
    "pipe = Pipeline([(\"scaler\", StandardScaler()), (\"clf\", LogisticRegression(max_iter=1000))])\n",
    "\n",
    "grid_search = GridSearchCV(pipe, param_grid, cv=inner_cv, scoring=\"accuracy\")\n",
    "nested_scores = cross_val_score(grid_search, X_sonar, y_sonar, cv=outer_cv, scoring=\"accuracy\")\n",
    "\n",
    "print(f\"Nested CV accuracy: {nested_scores.mean():.4f} ± {nested_scores.std():.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54861284",
   "metadata": {},
   "source": [
    "### 2.2 Úloha 2.2 - Porovnanie s nevnorenou krížovou validáciou"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eecc2dcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# TODO: create GridSearchCV with the same pipe and param_grid as in Task 2.1, but with cv=5, so that\n",
    "#       the same folds are used to select C and to report the score; fit it on the whole X_sonar, y_sonar\n",
    "naive_grid = ...\n",
    "\n",
    "print(f\"Non-nested CV (best_score_): {naive_grid.best_score_:.4f}\")\n",
    "print(f\"Selected C: {naive_grid.best_params_}\")\n",
    "print(f\"Nested CV accuracy:          {nested_scores.mean():.4f}\")\n",
    "print(f\"Difference:                  {naive_grid.best_score_ - nested_scores.mean():+.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7721d002",
   "metadata": {},
   "source": [
    "# Krok 3: Postupná validácia časových dát"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c76ad2d0",
   "metadata": {},
   "source": [
    "### Úloha 3.1 - Vyhodnotenie modelu pomocou TimeSeriesSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bdfb31fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0: train= 85, test= 83, MAE=12.410\n",
      "Fold 1: train=168, test= 83, MAE= 3.777\n",
      "Fold 2: train=251, test= 83, MAE= 5.093\n",
      "Fold 3: train=334, test= 83, MAE=10.448\n",
      "Fold 4: train=417, test= 83, MAE= 2.751\n",
      "\n",
      "MAE across folds: 6.896 ± 3.826\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "# synthetic time series combining random increments with a linear trend\n",
    "rng = np.random.default_rng(42)\n",
    "trend = np.linspace(0, 10, 500)\n",
    "y_ts = np.cumsum(rng.normal(scale=1.0, size=500)) + trend\n",
    "X_ts = np.arange(len(y_ts)).reshape(-1, 1)\n",
    "\n",
    "tscv = TimeSeriesSplit(n_splits=5)\n",
    "maes = []\n",
    "for fold, (train_idx, test_idx) in enumerate(tscv.split(X_ts)):\n",
    "    # the whole point of walk-forward validation: training uses the past only\n",
    "    assert train_idx.max() < test_idx.min(), \"the training set reaches into the future\"\n",
    "\n",
    "    # fit on the past, predict the future, evaluate\n",
    "    model_ts = LinearRegression().fit(X_ts[train_idx], y_ts[train_idx])\n",
    "    mae = mean_absolute_error(y_ts[test_idx], model_ts.predict(X_ts[test_idx]))\n",
    "    maes.append(mae)\n",
    "    print(f\"Fold {fold}: train={len(train_idx):3d}, test={len(test_idx):3d}, MAE={mae:6.3f}\")\n",
    "\n",
    "print(f\"\\nMAE across folds: {np.mean(maes):.3f} ± {np.std(maes):.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e78c6b7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pok-isi (3.14.5)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.5rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
