Multivariate LSTM Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multivariate Time Series Forecasting with LSTM\n",
    "\n",
    "Neural networks like Long Short-Term Memory (LSTM) recurrent neural networks are able to almost seamlessly model problems with multiple input variables.\n",
    "This is a great benefit in time series forecasting, where classical linear methods can be difficult to adapt to multivariate or multiple input forecasting problems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import numpy as numpy\n",
    "import pandas as pd\n",
    "import pylab\n",
    "import calendar\n",
    "from scipy import stats\n",
    "import seaborn as sns\n",
    "from sklearn import model_selection, preprocessing\n",
    "from scipy.stats import kendalltau\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas\n",
    "## Keras comes here\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dropout, BatchNormalization\n",
    "from keras.layers.advanced_activations import PReLU\n",
    "from keras.optimizers import Adam\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras import callbacks\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading train, prop and sample data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Reshma.nspr\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2785: DtypeWarning: Columns (22,32,34,49,55) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "print('Loading train, prop and sample data')\n",
    "train = pd.read_csv(\"train_2016_v2.csv\", parse_dates=[\"transactiondate\"])\n",
    "prop = pd.read_csv('properties_2016.csv')\n",
    "sample = pd.read_csv('sample_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Label Encoder\n",
    "LabelEncoder is a utility class to help normalize labels categorical values and to encode such that they contain only values between 0 and n_classes-1.\n",
    "\n",
    "Here, we LabelEncode the properties dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Label Encoder on properties\n"
     ]
    }
   ],
   "source": [
    "print('Fitting Label Encoder on properties')\n",
    "for c in prop.columns:\n",
    "    prop[c]=prop[c].fillna(-1)\n",
    "    if prop[c].dtype == 'object':\n",
    "        lbl = LabelEncoder()\n",
    "        lbl.fit(list(prop[c].values))\n",
    "        prop[c] = lbl.transform(list(prop[c].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating training set:\n",
      "Creating df_test  :\n",
      "Merge Sample with property data :\n"
     ]
    }
   ],
   "source": [
    "print('Creating training set:')\n",
    "df_train = train.merge(prop, how='left', on='parcelid')\n",
    "\n",
    "print('Creating df_test  :')\n",
    "sample['parcelid'] = sample['ParcelId']\n",
    "\n",
    "print(\"Merge Sample with property data :\")\n",
    "df_test = sample.merge(prop, on='parcelid', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[\"transactiondate\"] = pd.to_datetime(df_train[\"transactiondate\"])\n",
    "df_train['transactiondate_quarter'] = df_train['transactiondate'].dt.quarter\n",
    "\n",
    "basedate = pd.to_datetime('2015-11-15').toordinal()\n",
    "df_train['cos_season'] = \\\n",
    "        ( (pd.to_datetime(df_train['transactiondate']).apply(lambda x: x.toordinal()-basedate)) * \\\n",
    "          (2*np.pi/365.25) ).apply(np.cos)\n",
    "df_train['sin_season'] = \\\n",
    "        ( (pd.to_datetime(df_train['transactiondate']).apply(lambda x: x.toordinal()-basedate)) * \\\n",
    "          (2*np.pi/365.25) ).apply(np.sin)\n",
    "    \n",
    "#test dataset\n",
    "df_test[\"transactiondate\"] = pd.to_datetime('2016-11-15')\n",
    "df_test['transactiondate_quarter'] = df_test['transactiondate'].dt.quarter\n",
    "\n",
    "df_test['cos_season'] = np.cos( (pd.to_datetime('2016-11-15').toordinal() - basedate) * \\\n",
    "                                    (2*np.pi/365.25) )\n",
    "df_test['sin_season'] = np.sin( (pd.to_datetime('2016-11-15').toordinal() - basedate) * \\\n",
    "                                    (2*np.pi/365.25) )\n",
    "\n",
    "df_train_x = df_train.drop(['logerror','parcelid', 'transactiondate', 'propertyzoningdesc', 'propertycountylandusecode',\n",
    "                             'fireplacecnt', 'fireplaceflag'],axis=1)    \n",
    "df_train = df_train.drop(['parcelid', 'transactiondate', 'propertyzoningdesc', 'propertycountylandusecode',\n",
    "                             'fireplacecnt', 'fireplaceflag'], axis=1)\n",
    "\n",
    "train_columns = df_train_x.columns\n",
    "df_test=df_test[train_columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LSTM Data Preparation\n",
    "The first step is to prepare properties dataset for LSTM.This involves framing the dataset as a supervised learning problem and normalizing the input variables.\n",
    "We will frame the supervised learning problem as predicting the Log Error for a particular parcel Id given other features and conditions at the prior time step.\n",
    "\n",
    "We can transform the dataset using the series_to_supervised() function that is developed below;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):\n",
    "    n_vars = 1 if type(data) is list else data.shape[1]\n",
    "    df = pd.DataFrame(data)\n",
    "    cols, names = list(), list()\n",
    "    # input sequence (t-n, ... t-1)\n",
    "    for i in range(n_in, 0, -1):\n",
    "        cols.append(df.shift(i))\n",
    "        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\n",
    "    # forecast sequence (t, t+1, ... t+n)\n",
    "    for i in range(0, n_out):\n",
    "        cols.append(df.shift(-i))\n",
    "        if i == 0:\n",
    "            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]\n",
    "        else:\n",
    "            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\n",
    "    # put it all together\n",
    "    agg = pd.concat(cols, axis=1)\n",
    "    agg.columns = names\n",
    "    # drop rows with NaN values\n",
    "    if dropnan:\n",
    "        agg.dropna(inplace=True)\n",
    "    return agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "Values = df_train.values\n",
    "test_values = df_test.values\n",
    "values = Values.astype('float32')\n",
    "test_values = test_values.astype('float32')\n",
    "scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "scaled = scaler.fit_transform(values)\n",
    "scaled_test = scaler.fit_transform(test_values)\n",
    "reframed = series_to_supervised(scaled, 1, 1)\n",
    "reframed_test = series_to_supervised(scaled,1,1)\n",
    "reframed.drop(reframed.columns[58:116], axis=1, inplace=True)\n",
    "reframed_test.drop(reframed_test.columns[56:111],axis =1,inplace=True)\n",
    "\n",
    "train_X, train_Y = reframed.iloc[:80000,:-1], reframed.iloc[:80000,-1]\n",
    "valid_X, valid_Y = reframed.iloc[80000:,:-1], reframed.iloc[80000:,-1]\n",
    "\n",
    "train_X = np.array(train_X)\n",
    "train_Y = np.array(train_Y)\n",
    "\n",
    "valid_X = np.array(valid_X)\n",
    "valid_Y = np.array(valid_Y)\n",
    "test_X = np.array(reframed_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80000, 1, 57) (80000,) (10274, 1, 57) (10274,) (90274, 1, 59)\n"
     ]
    }
   ],
   "source": [
    "train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))\n",
    "valid_X = valid_X.reshape((valid_X.shape[0], 1, valid_X.shape[1]))\n",
    "test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))\n",
    "print(train_X.shape, train_Y.shape, valid_X.shape, valid_Y.shape,test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 80000 samples, validate on 10274 samples\n",
      "Epoch 1/50\n",
      " - 6s - loss: 0.0152 - val_loss: 0.0321\n",
      "Epoch 2/50\n",
      " - 3s - loss: 0.0083 - val_loss: 0.0320\n",
      "Epoch 3/50\n",
      " - 3s - loss: 0.0082 - val_loss: 0.0279\n",
      "Epoch 4/50\n",
      " - 3s - loss: 0.0080 - val_loss: 0.0225\n",
      "Epoch 5/50\n",
      " - 3s - loss: 0.0079 - val_loss: 0.0193\n",
      "Epoch 6/50\n",
      " - 3s - loss: 0.0078 - val_loss: 0.0163\n",
      "Epoch 7/50\n",
      " - 3s - loss: 0.0078 - val_loss: 0.0137\n",
      "Epoch 8/50\n",
      " - 4s - loss: 0.0077 - val_loss: 0.0115\n",
      "Epoch 9/50\n",
      " - 4s - loss: 0.0076 - val_loss: 0.0097\n",
      "Epoch 10/50\n",
      " - 4s - loss: 0.0076 - val_loss: 0.0082\n",
      "Epoch 11/50\n",
      " - 4s - loss: 0.0076 - val_loss: 0.0078\n",
      "Epoch 12/50\n",
      " - 4s - loss: 0.0076 - val_loss: 0.0077\n",
      "Epoch 13/50\n",
      " - 4s - loss: 0.0076 - val_loss: 0.0075\n",
      "Epoch 14/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 15/50\n",
      " - 3s - loss: 0.0076 - val_loss: 0.0077\n",
      "Epoch 16/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 17/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0076\n",
      "Epoch 18/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 19/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0075\n",
      "Epoch 20/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 21/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0079\n",
      "Epoch 22/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 23/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0076\n",
      "Epoch 24/50\n",
      " - 3s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 25/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 26/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 27/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0080\n",
      "Epoch 28/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0076\n",
      "Epoch 29/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 30/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 31/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0076\n",
      "Epoch 32/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 33/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0076\n",
      "Epoch 34/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0077\n",
      "Epoch 35/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0078\n",
      "Epoch 36/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0079\n",
      "Epoch 37/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0079\n",
      "Epoch 38/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0080\n",
      "Epoch 39/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0081\n",
      "Epoch 40/50\n",
      " - 5s - loss: 0.0075 - val_loss: 0.0082\n",
      "Epoch 41/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0082\n",
      "Epoch 42/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0083\n",
      "Epoch 43/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0084\n",
      "Epoch 44/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0083\n",
      "Epoch 45/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0084\n",
      "Epoch 46/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0084\n",
      "Epoch 47/50\n",
      " - 4s - loss: 0.0074 - val_loss: 0.0084\n",
      "Epoch 48/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0085\n",
      "Epoch 49/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0085\n",
      "Epoch 50/50\n",
      " - 4s - loss: 0.0075 - val_loss: 0.0085\n",
      "Test RMSE: 0.017\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))\n",
    "model.add(Dense(1))\n",
    "model.compile(loss='mae', optimizer='adam')\n",
    "# fit network\n",
    "model.fit(train_X, train_Y, epochs=50, batch_size=72, validation_data=(valid_X, valid_Y), verbose=2, shuffle=False)\n",
    "yhat = model.predict(valid_X)\n",
    "valid_X = valid_X.reshape((valid_X.shape[0], valid_X.shape[2]))\n",
    "\n",
    "rmse = np.sqrt(mean_squared_error(valid_Y, yhat))\n",
    "print('Test RMSE: %.3f' % rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary:\n",
    "\n",
    "In this notebook, we have implemented Simple neural networks with 5 layers for prediction of LogError = (log(Zestimate)-log(salesprice)) using 2016 property dataset and its corresponding log error values provided by zillow for home value prediction in Python using Keras and tensorflow deep learning libraries.\n",
    "\n",
    "Finally, we have predicted logerror values of 2016 and 2017 for the last quarter (from November to December) in the test dataset.\n",
    "Calculated RMSE for the Network built can be seen as 0.017 which is improved from that which we got by simple neural network. we can aslo infer that there is minimal error in the logerror gives us the better predictions.Further, the model can improvised by add more layers or changing the backpropagation parameters.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Referrence:\n",
    "https://www.sciencedirect.com/science/article/pii/S0377221703005484"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}