{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Experiment 2: Model Evaluating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.models import load_model\n",
    "from matplotlib import ticker\n",
    "from scipy.io import loadmat\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "In this experiment, we load model weights from the experiment1 and evaluate them on test dataset."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of data:\n",
      "x_train: (5728, 1, 102), y_train: (5728, 1),\n",
      "x_val: (2455, 1, 102), y_val: (2455, 1)\n",
      "x_test: (3508, 1, 102), y_test: (3508, 1)\n"
     ]
    }
   ],
   "source": [
    "data = loadmat('./preprocess/dataset/mango/mango_dm_split.mat')\n",
    "x_train, y_train, x_test, y_test = data['x_train'], data['y_train'], data['x_test'], data['y_test']\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.3, random_state=12, shuffle=True)\n",
    "x_train, x_val, x_test = x_train[:, np.newaxis, :], x_val[:, np.newaxis, :], x_test[:, np.newaxis, :]\n",
    "print(f\"shape of data:\\n\"\n",
    "      f\"x_train: {x_train.shape}, y_train: {y_train.shape},\\n\"\n",
    "      f\"x_val: {x_val.shape}, y_val: {y_val.shape}\\n\"\n",
    "      f\"x_test: {x_test.shape}, y_test: {y_test.shape}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "## Build model and load weights\n",
    "plain_5, plain_11 = load_model('./checkpoints/plain5.hdf5'), load_model('./checkpoints/plain11.hdf5')\n",
    "shortcut5, shortcut11 = load_model('./checkpoints/shortcut5.hdf5'), load_model('./checkpoints/shortcut11.hdf5')\n",
    "models = {'plain 5': plain_5, 'plain 11': plain_11, 'shortcut 5': shortcut5, 'shortcut11': shortcut11}\n",
    "results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
    "for model_name, model_result in results.items():\n",
    "      print(model_name, \" : \", mean_squared_error(y_test, model_result)*100, \"%\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "plain 5  :  0.2707851525589865 %\n",
      "plain 11  :  0.26240810192725905 %\n",
      "shortcut 5  :  0.28330442301217196 %\n",
      "shortcut11  :  0.25743312483685266 %\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}