mirror of
https://github.com/NanjingForestryUniversity/SCNet.git
synced 2025-11-08 14:24:03 +00:00
363 lines
13 KiB
Plaintext
Executable File
363 lines
13 KiB
Plaintext
Executable File
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"collapsed": true,
|
|
"pycharm": {
|
|
"name": "#%% md\n"
|
|
}
|
|
},
|
|
"source": [
|
|
"# Experiment 2: Model Evaluating"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"from keras.models import load_model\n",
|
|
"from matplotlib import ticker\n",
|
|
"from scipy.io import loadmat\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.metrics import mean_squared_error\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% md\n"
|
|
}
|
|
},
|
|
"source": [
|
|
"In this experiment, we load model weights from the experiment1 and evaluate them on test dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%% md\n"
|
|
}
|
|
},
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"shape of data:\n",
|
|
"x_train: (5728, 1, 102), y_train: (5728, 1),\n",
|
|
"x_val: (2455, 1, 102), y_val: (2455, 1)\n",
|
|
"x_test: (3508, 1, 102), y_test: (3508, 1)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"data = loadmat('./dataset/mango/mango_dm_split.mat')\n",
|
|
"\n",
|
|
"min_value, max_value = data['min_y'][-1][-1], data['max_y'][-1][-1]\n",
|
|
"retransform = lambda x: x * (max_value - min_value)\n",
|
|
"x_train, y_train, x_test, y_test = data['x_train'], data['y_train'], data['x_test'], data['y_test']\n",
|
|
"x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.3, random_state=12, shuffle=True)\n",
|
|
"x_train, x_val, x_test = x_train[:, np.newaxis, :], x_val[:, np.newaxis, :], x_test[:, np.newaxis, :]\n",
|
|
"print(f\"shape of data:\\n\"\n",
|
|
" f\"x_train: {x_train.shape}, y_train: {y_train.shape},\\n\"\n",
|
|
" f\"x_val: {x_val.shape}, y_val: {y_val.shape}\\n\"\n",
|
|
" f\"x_test: {x_test.shape}, y_test: {y_test.shape}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"plain 5 mse: 0.007162414257423199\n",
|
|
"plain 5 rmse : 0.08463104783365971\n",
|
|
"plain 5 Dry matter content error 1.278990434152287\n",
|
|
"plain 5 r^2 : 0.7362122841198271\n",
|
|
"plain 11 mse: 0.027140651722534437\n",
|
|
"plain 11 rmse : 0.16474420087679698\n",
|
|
"plain 11 Dry matter content error 2.4897039844954327\n",
|
|
"plain 11 r^2 : 0.0004249589491984729\n",
|
|
"shortcut 5 mse: 0.007429169596940547\n",
|
|
"shortcut 5 rmse : 0.08619263075774254\n",
|
|
"shortcut 5 Dry matter content error 1.3025899248021375\n",
|
|
"shortcut 5 r^2 : 0.7263878339859644\n",
|
|
"shortcut11 mse: 0.007825262774295792\n",
|
|
"shortcut11 rmse : 0.08846051534043757\n",
|
|
"shortcut11 Dry matter content error 1.3368634303450377\n",
|
|
"shortcut11 r^2 : 0.7117999435379954\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from sklearn.metrics import r2_score\n",
|
|
"\n",
|
|
"## Build model and load weights\n",
|
|
"plain_5, plain_11 = load_model('./checkpoints/plain5.hdf5'), load_model('./checkpoints/plain11.hdf5')\n",
|
|
"shortcut5, shortcut11 = load_model('./checkpoints/shortcut5.hdf5'), load_model('./checkpoints/shortcut11.hdf5')\n",
|
|
"models = {'plain 5': plain_5, 'plain 11': plain_11, 'shortcut 5': shortcut5, 'shortcut11': shortcut11}\n",
|
|
"results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
|
|
"for model_name, model_result in results.items():\n",
|
|
" mse = mean_squared_error(y_test, model_result)\n",
|
|
" rmse = np.sqrt(mse)\n",
|
|
" print(model_name, \"mse: \", mse)\n",
|
|
" print(model_name, \"rmse : \", rmse)\n",
|
|
" print(model_name, \"Dry matter content error\", retransform(rmse))\n",
|
|
" print(model_name, \"r^2 :\", r2_score(y_test, model_result))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"plain 5 nadam mse: 0.0025042022303828344\n",
|
|
"plain 5 nadam rmse : 0.0500420046599138\n",
|
|
"plain 5 nadam Dry matter content error 0.7562619972711523\n",
|
|
"plain 5 nadam r^2 : 0.9077716308058986\n",
|
|
"plain 11 nadam mse: 0.0027148425657891745\n",
|
|
"plain 11 nadam rmse : 0.05210415113778531\n",
|
|
"plain 11 nadam Dry matter content error 0.7874262766524306\n",
|
|
"plain 11 nadam r^2 : 0.900013864925283\n",
|
|
"shortcut 5 nadam mse: 0.0026949613632585197\n",
|
|
"shortcut 5 nadam rmse : 0.051913017281396\n",
|
|
"shortcut 5 nadam Dry matter content error 0.7845377578378575\n",
|
|
"shortcut 5 nadam r^2 : 0.9007460785080253\n",
|
|
"shortcut11 nadam mse: 0.002492666414101387\n",
|
|
"shortcut11 nadam rmse : 0.049926610280504595\n",
|
|
"shortcut11 nadam Dry matter content error 0.754518094634978\n",
|
|
"shortcut11 nadam r^2 : 0.9081964884751603\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"plain_5, plain_11 = load_model('./checkpoints/plain5_nadam.hdf5'), load_model('./checkpoints/plain11_nadam.hdf5')\n",
|
|
"shortcut5, shortcut11 = load_model('./checkpoints/shortcut5_nadam.hdf5'), load_model('./checkpoints/shortcut11_nadam.hdf5')\n",
|
|
"models = {'plain 5 nadam': plain_5, 'plain 11 nadam': plain_11, 'shortcut 5 nadam': shortcut5, 'shortcut11 nadam': shortcut11}\n",
|
|
"results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
|
|
"for model_name, model_result in results.items():\n",
|
|
" mse = mean_squared_error(y_test, model_result)\n",
|
|
" rmse = np.sqrt(mse)\n",
|
|
" print(model_name, \"mse: \", mse)\n",
|
|
" print(model_name, \"rmse : \", rmse)\n",
|
|
" print(model_name, \"Dry matter content error\", retransform(rmse))\n",
|
|
" print(model_name, \"r^2 :\", r2_score(y_test, model_result))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"plain 5 rmsprop mse: 0.0025762880890505646\n",
|
|
"plain 5 rmsprop rmse : 0.05075714815718634\n",
|
|
"plain 5 rmsprop Dry matter content error 0.7670696348399975\n",
|
|
"plain 5 rmsprop r^2 : 0.905116748901307\n",
|
|
"plain 11 rmsprop mse: 0.0026627965381971945\n",
|
|
"plain 11 rmsprop rmse : 0.051602291985891426\n",
|
|
"plain 11 rmsprop Dry matter content error 0.7798419081376315\n",
|
|
"plain 11 rmsprop r^2 : 0.9019306910464329\n",
|
|
"shortcut 5 rmsprop mse: 0.002591777512557128\n",
|
|
"shortcut 5 rmsprop rmse : 0.0509095031654909\n",
|
|
"shortcut 5 rmsprop Dry matter content error 0.7693721066066205\n",
|
|
"shortcut 5 rmsprop r^2 : 0.9045462820865926\n",
|
|
"shortcut11 rmsprop mse: 0.0025523285716201384\n",
|
|
"shortcut11 rmsprop rmse : 0.050520575725343214\n",
|
|
"shortcut11 rmsprop Dry matter content error 0.7634944235545812\n",
|
|
"shortcut11 rmsprop r^2 : 0.9059991645434926\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"plain_5, plain_11 = load_model('./checkpoints/plain5_rmsprop.hdf5'), load_model('./checkpoints/plain11_rmsprop.hdf5')\n",
|
|
"shortcut5, shortcut11 = load_model('./checkpoints/shortcut5_rmsprop.hdf5'), load_model('./checkpoints/shortcut11_rmsprop.hdf5')\n",
|
|
"models = {'plain 5 rmsprop': plain_5, 'plain 11 rmsprop': plain_11, 'shortcut 5 rmsprop': shortcut5, 'shortcut11 rmsprop': shortcut11}\n",
|
|
"results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
|
|
"for model_name, model_result in results.items():\n",
|
|
" mse = mean_squared_error(y_test, model_result)\n",
|
|
" rmse = np.sqrt(mse)\n",
|
|
" print(model_name, \"mse: \", mse)\n",
|
|
" print(model_name, \"rmse : \", rmse)\n",
|
|
" print(model_name, \"Dry matter content error\", retransform(rmse))\n",
|
|
" print(model_name, \"r^2 :\", r2_score(y_test, model_result))"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"plain 5 sgd mse : 0.11134143767715826\n",
|
|
"plain 5 sgd Dry matter content error 1.682652375919312\n",
|
|
"plain 5 sgd r^2 : 0.543428518577538\n",
|
|
"plain 11 sgd mse : 0.1281311162291065\n",
|
|
"plain 11 sgd Dry matter content error 1.936387131781486\n",
|
|
"plain 11 sgd r^2 : 0.3953495916351124\n",
|
|
"shortcut 5 sgd mse : 0.07824195777158978\n",
|
|
"shortcut 5 sgd Dry matter content error 1.1824350294692925\n",
|
|
"shortcut 5 sgd r^2 : 0.7745373801958391\n",
|
|
"shortcut11 sgd mse : 0.09167697720606416\n",
|
|
"shortcut11 sgd Dry matter content error 1.3854723518136416\n",
|
|
"shortcut11 sgd r^2 : 0.690460767243821\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"plain_5, plain_11 = load_model('./checkpoints/plain5_sgd.hdf5'), load_model('./checkpoints/plain11_sgd.hdf5')\n",
|
|
"shortcut5, shortcut11 = load_model('./checkpoints/shortcut5_sgd.hdf5'), load_model('./checkpoints/shortcut11_sgd.hdf5')\n",
|
|
"models = {'plain 5 sgd': plain_5, 'plain 11 sgd': plain_11, 'shortcut 5 sgd': shortcut5, 'shortcut11 sgd': shortcut11}\n",
|
|
"results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
|
|
"for model_name, model_result in results.items():\n",
|
|
" rmse = np.sqrt(mean_squared_error(y_test, model_result))\n",
|
|
" print(model_name, \"mse : \", rmse)\n",
|
|
" print(model_name, \"Dry matter content error\", retransform(rmse))\n",
|
|
" print(model_name, \"r^2 :\", r2_score(y_test, model_result))"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"plain 5 adagrad mse : 0.09175815616893311\n",
|
|
"plain 5 adagrad Dry matter content error 1.3866991724618731\n",
|
|
"plain 5 adagrad r^2 : 0.6899123373115493\n",
|
|
"plain 11 adagrad mse : 0.10080308998502606\n",
|
|
"plain 11 adagrad Dry matter content error 1.5233911327346656\n",
|
|
"plain 11 adagrad r^2 : 0.6257663620335936\n",
|
|
"shortcut 5 adagrad mse : 0.07867384574842859\n",
|
|
"shortcut 5 adagrad Dry matter content error 1.18896195552234\n",
|
|
"shortcut 5 adagrad r^2 : 0.7720414471534031\n",
|
|
"shortcut11 adagrad mse : 0.08402036572248298\n",
|
|
"shortcut11 adagrad Dry matter content error 1.2697614738771157\n",
|
|
"shortcut11 adagrad r^2 : 0.7400054367232816\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"plain_5, plain_11 = load_model('./checkpoints/plain5_adagrad.hdf5'), load_model('./checkpoints/plain11_adagrad.hdf5')\n",
|
|
"shortcut5, shortcut11 = load_model('./checkpoints/shortcut5_adagrad.hdf5'), load_model('./checkpoints/shortcut11_adagrad.hdf5')\n",
|
|
"models = {'plain 5 adagrad': plain_5, 'plain 11 adagrad': plain_11, 'shortcut 5 adagrad': shortcut5, 'shortcut11 adagrad': shortcut11}\n",
|
|
"results = {model_name: model.predict(x_test).reshape((-1, )) for model_name, model in models.items()}\n",
|
|
"for model_name, model_result in results.items():\n",
|
|
" rmse = np.sqrt(mean_squared_error(y_test, model_result))\n",
|
|
" print(model_name, \"mse : \", rmse)\n",
|
|
" print(model_name, \"Dry matter content error\", retransform(rmse))\n",
|
|
" print(model_name, \"r^2 :\", r2_score(y_test, model_result))"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"metadata": {
|
|
"interpreter": {
|
|
"hash": "7f619fc91ee8bdab81d49e7c14228037474662e3f2d607687ae505108922fa06"
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3.9.7 ('base')",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |