A semi-comprehensive tour for Deep Learning with Tensorflow

import sys
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from data import Data
import os
import pandas as pd
import numpy as np
from sklearn.metrics import auc, confusion_matrix
from matplotlib import pyplot as plt
import datetime
import plot_function # just loads some pyplot parameters to make the plots up to the scientific standards.

import tarfile
import tensorboard

Set-up of notebook

• how to create a model
• how to train a model
• how to measure performance
• a simple regression example
• a complex regression example
• a simple classification example
• a complex classification example
• a "real-world" classification example with data from Lending Club

A. Didisheim, S. Scheidegger, 02/21

def create_model(
        data,
        learning_rate=0.005,
        opti='adam', # you can put 'adam', 'sgd', or 'rms_prop'
        batch_normalization=False,
        activation='relu', # 'sigmoid', or 'relu' are the main two, but others are coded in keras (see documentation)
        architecture=[64, 32, 16], # each number correspond to a number of neurons in a layer
        verbose=0):

    # check that the input are correct
    assert type(architecture) == list, 'architecture must be a list of integers (e.g. [64,32,16], each representing a number of neurones in a layer'
    assert opti in ['adam','sigmoid','rms_prop'], "invalid optimizer, please chose among ['adam','sigmoid','rms_prop']"
    assert type(data) == Data, 'the data object must come from the "Data" class'

    # extract the input shape and output shape from the data
    input_dim = data.X_te.shape[1]
    output_dim = data.y_te.shape[1]

    # build the network
    L = []
    for i, l in enumerate(architecture):
        if i == 0:
            L.append(layers.Dense(l, activation=activation, input_shape=[input_dim])) # add the first layer
        else:
            L.append(layers.Dense(l, activation=activation)) # add a layer
        if batch_normalization:
            # add batch normalization if specified
            L.append(layers.BatchNormalization())

    # add the final layer
    if output_dim == 1:
        L.append(layers.Dense(output_dim))
    else:
        # if we are doing classification, we wish to normalize the ouput between 0 and 1, hence the softmax
        L.append(layers.Dense(output_dim, activation='softmax'))

    # keras command to build a simple feed forward network with the parameters defined above
    model = keras.Sequential(L)

    # select the optimizer
    if opti == 'sgd':
        optimizer = tf.keras.optimizers.SGD(learning_rate)
    if opti == 'rms_prop':
        optimizer = tf.keras.optimizers.RMSprop(learning_rate)
    if opti == 'adam':
        optimizer = tf.keras.optimizers.Adam(learning_rate)

    ##################
    # define an additional metrics
    ##################
    if output_dim == 1:
        # here we do a regression---that, is we predict one continous thing, hence we can add the R squared as a metric
        def r_square(y_true, y_pred):
            y_true = tf.cast(y_true, y_pred.dtype)
            SS_res = tf.reduce_sum(tf.square(y_true - y_pred))
            SS_tot = tf.reduce_sum(tf.square(y_true - tf.reduce_mean(y_true)))
            return (1 - SS_res / (SS_tot + tf.keras.backend.epsilon()))

        # we have three metrics:
        # mean absolute error -> mae
        # mean square error -> mse
        # and our custom r-square.
        model.compile(loss='mae', optimizer=optimizer, metrics=['mae', 'mse', r_square])
    if output_dim > 1:
        # here we do a classifier ---that, is we predict a percentage of being in one dimension instead of another
        # we have two  metrics:
        # mean absolute error -> mae
        # accracy
        model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=False), optimizer=optimizer, metrics=['accuracy', 'mae'])
    if verbose:
        # if specified, print the network architecture
        print(model.summary())
    return model


def train_model(model, data, epoch=10, bs=256, verbose=0, tensor_board_name = None):
    assert type(data) == Data, 'the data object must come from the "Data" class'
    tf.random.set_seed(1234)
    np.random.seed(1234)
    print('### start training for', epoch, 'epochs')
    # Prepare the validation dataset
    val_dataset = tf.data.Dataset.from_tensor_slices((data.X_va, data.y_va))
    val_dataset = val_dataset.batch(256)

    if tensor_board_name is not None:
        # set_up the tensorboard name

        log_dir = "logs/"+tensor_board_name#+"/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
        tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
        history_training = model.fit(x=data.X_tr, y=data.y_tr, batch_size=bs, epochs=epoch, validation_data=val_dataset, verbose=verbose, callbacks=tensorboard_callback)

    else:
        # the keras command to launch the training routine
        history_training = model.fit(x=data.X_tr, y=data.y_tr, batch_size=bs, epochs=epoch, validation_data=val_dataset, verbose=verbose)
    print('### training finish \n')

    # return the history of training process
    return pd.DataFrame(history_training.history)


def show_performance(label_, model, history_training, data):
    assert type(data) == Data, 'the data object must come from the "Data" class'

    # check the output_dim to calibrate the plot
    output_dim = data.y_te.shape[1]

    # use the pandas function to start the plot (history training is a PD data frame).
    history_training[['loss', 'val_loss']].plot()
    plt.grid(True)  # add a grid for visibiliy
    plt.xlabel('epochs')
    if output_dim == 1:
        plt.ylabel('mean absolute error')  # if its a regression, we plot the mae
    else:
        plt.ylabel('cross entropy loss')   # if its a classification, we plot the cross entropy loss
    plt.title(label_)
    plt.savefig(label_+'.png')
    plt.show()

    if output_dim == 1:
        # if regression, compute the out of sample performance measure
        print('=' * 50)
        print('Out of sample performance:')
        model.evaluate(data.X_te, data.y_te, verbose=2)
        print('=' * 50)

    else:
        # if classification  print the  confusion matrix
        pred = model.predict(data.X_te)
        cf = confusion_matrix(y_true=np.argmax(data.y_te, 1), y_pred=np.argmax(pred, 1))
        cf = pd.DataFrame(cf)
        index = ['True ' + str(x) for x in cf.index]
        col = ['Predicted ' + str(x) for x in cf.columns]
        cf.index = index
        cf.columns = col
        print(cf)

Simple Regression

• N is tha sample size.
• D is the dimension of input.
• L is a measure of complexity of the function to estimate, output_dim is the output dimension.
• output_dim = 1 means we do a regression, output_dim >1 means we do a classification exercise.

data = Data()

n = 'Simple Regression'
print('\n' + '=' * 50)
print(n)
print('=' * 50, '\n')

data.gen_random_data(N=10 * 1000, D=10, LL=[20,20], output_dim=1, sigma=0.1)

model = create_model(data=data, learning_rate=0.005, opti='adam', batch_normalization=False, activation='relu', architecture=[64, 32], verbose=0)
history = train_model(model, data, epoch=20, verbose=0)
show_performance(n, model, history, data)

==================================================
Simple Regression
================================================== 

### start training for 20 epochs
### training finish 

==================================================
Out of sample performance:
16/16 - 0s - loss: 0.0821 - mae: 0.0821 - mse: 0.0106 - r_square: 0.9887
==================================================

Complex Regression

• N is tha sample size.
• D is the dimension of input.
• L is a measure of complexity of the function to estimate.
• output_dim is the output dimension. output_dim = 1 means we do a regression, output_dim >1 means we do a classification exercise.

n = 'Complex Regression'
print('\n' + '=' * 50)
print(n)
print('=' * 50, '\n')

data.gen_random_data(N=10 * 1000, D=10, LL=[200,200,200,200], output_dim=1, sigma=0.1)
# TODO try to add one layer and see if performance improve
model = create_model(data=data, learning_rate=0.005, opti='adam', batch_normalization=False, activation='relu', architecture=[64,32], verbose=0)
history = train_model(model, data, epoch=20, verbose=0)
show_performance(n, model, history, data)

==================================================
Complex Regression
================================================== 

### start training for 20 epochs
### training finish 

==================================================
Out of sample performance:
16/16 - 0s - loss: 0.4537 - mae: 0.4537 - mse: 0.4969 - r_square: 0.5014
==================================================

Simple Classification

• N is tha sample size.
• D is the dimension of input.
• L is a measure of complexity of the function to estimate.
• output_dim is the output dimension. output_dim = 1 means we do a regression, output_dim >1 means we do a classification exercise.

n = 'Simple Classification'
print('\n' + '=' * 50)
print(n)
print('=' * 50, '\n')


data.gen_random_data(N=10 * 1000, D=10, LL=[20,20], output_dim=3, sigma=0.1)

model = create_model(data=data, learning_rate=0.005, opti='adam', batch_normalization=False, activation='relu', architecture=[64,32], verbose=0)
history = train_model(model, data, epoch=20, verbose=0)
show_performance(n, model, history, data)

==================================================
Simple Classification
================================================== 

### start training for 20 epochs
### training finish 

        Predicted 0  Predicted 1  Predicted 2
True 0          153           14            3
True 1           12          155            6
True 2           11            7          139

Complex Classification

• N is tha sample size.
• D is the dimension of input.
• L is a measure of complexity of the function to estimate.
• output_dim is the output dimension
• output_dim = 1 means we do a regression, output_dim >1 means we do a classification exercise

n = 'Complex  Classification'
print('\n' + '=' * 50)
print(n)
print('=' * 50, '\n')

data.gen_random_data(N=10 * 1000, D=10, LL=[200,200,200,200], output_dim=3, sigma=0.1)
# TODO try re-runing the same result with a three layer architecture [64,16,32] and a larger sample
model = create_model(data=data, learning_rate=0.005, opti='adam', batch_normalization=False, activation='relu', architecture=[64,32], verbose=0)
history = train_model(model, data, epoch=20, verbose=0)
show_performance(n, model, history, data)

==================================================
Complex  Classification
================================================== 

### start training for 20 epochs
### training finish 

        Predicted 0  Predicted 1  Predicted 2
True 0          155           10            8
True 1            8          155            4
True 2           11           17          132

Lending Club data

• The Lending Club (https://www.lendingclub.com/) is a peer-to-peer lending website. On this platform, an individual borrower can ask for loans, and the individual lender (anyone with some money) can decide or not to lend them money.

• The interest rate charged by the lender to the borrower is defined by a rating (from A to G). This grade is assigned by a secret algorithm which takes as input the lender's characteristics (history of credit, revenue, assets, etc.) and give as output a grade.

• The LendingClub published data containing the rating and some of the input characteristics of the lender and the data have been collected by the website Kaggle (https://www.kaggle.com/wendykan/lending-club-loan-data).

• The example below uses a subset of the full data set to make the model training quicker. The aim is to use a neural network to estimate the rating from the lender's input. In other words, use a neural network to estimate the proprietary algorithm used by the lending club.

# the Lending Club data ship as data.tar.gz next to this notebook; untar them once
if not os.path.exists('./data'):
    with tarfile.open('data.tar.gz') as tar:
        tar.extractall('.')

data_untared= False
if os.path.exists('./data'):
    if ('LCy.csv' in os.listdir('./data')) & ('LCx.csv' in os.listdir('./data')) :
        data_untared = True

if data_untared:
    ##################
    # Lending club data
    ##################
    # Here we take a real example using data from the lending club https://www.lendingclub.com/
    # The X contains characteristics of prospective borrowers (FICO score, number of credit cards etc)
    # The data have already been simplified and normalized for you
    # The y contains data binary variable with the grades of each loan. The first columns correspond to a grade A (best) and the last to a grade of G (worst)

    n = 'Lending Club data'
    print('\n' + '=' * 50)
    print(n)
    print('=' * 50, '\n')

    data.load_lending_club_data()
    # TODO, can a different architecture and number of epoch significantly improve performance with these real data? If you find something great, I am interrested ;)
    model = create_model(data=data, learning_rate=0.005, opti='adam', batch_normalization=False, activation='relu', architecture=[64,32], verbose=0)
    history = train_model(model, data, epoch=5,bs=1024, verbose=1, tensor_board_name='LC')
    show_performance(n, model, history, data)
else:
    print('!!!! To test the lending club data, first untar the data folder')
    print('To do so, run the follwing line in the terminal')
    print('tar -xzvf data.tar.gz')

==================================================
Lending Club data
================================================== 

### start training for 5 epochs
Epoch 1/5
528/528 [==============================] - 4s 7ms/step - loss: 0.3372 - accuracy: 0.3977 - mae: 0.1959 - val_loss: 0.3066 - val_accuracy: 0.4255 - val_mae: 0.1871
Epoch 2/5
528/528 [==============================] - 4s 7ms/step - loss: 0.3067 - accuracy: 0.4291 - mae: 0.1874 - val_loss: 0.3050 - val_accuracy: 0.4300 - val_mae: 0.1857
Epoch 3/5
528/528 [==============================] - 4s 7ms/step - loss: 0.3052 - accuracy: 0.4323 - mae: 0.1865 - val_loss: 0.3048 - val_accuracy: 0.4304 - val_mae: 0.1868
Epoch 4/5
528/528 [==============================] - 3s 7ms/step - loss: 0.3044 - accuracy: 0.4335 - mae: 0.1861 - val_loss: 0.3039 - val_accuracy: 0.4329 - val_mae: 0.1861
Epoch 5/5
528/528 [==============================] - 3s 7ms/step - loss: 0.3041 - accuracy: 0.4343 - mae: 0.1859 - val_loss: 0.3036 - val_accuracy: 0.4337 - val_mae: 0.1858
### training finish 

        Predicted 0  Predicted 1  Predicted 2  Predicted 3  Predicted 4  \
True 0         3081         2291          476            4            0   
True 1         1342         4862         3105           81            0   
True 2          408         2836         4486          433            0   
True 3           86          846         2689          545            0   
True 4           21          305         1111          292            0   
True 5            5           58          355          139            0   
True 6            2            8           79           54            0   

        Predicted 5  Predicted 6  
True 0            0            0  
True 1            0            0  
True 2            0            0  
True 3            0            0  
True 4            0            0  
True 5            0            0  
True 6            0            0  

Once the model is trained, we can use it to make predictions.

if data_untared:
    pred = model.predict(data.X_te)
    print(pred)
    

[[7.30891526e-01 1.82356134e-01 6.69683665e-02 ... 5.07308496e-03
  1.16718956e-03 1.67310049e-04]
 [6.50813207e-02 7.16606319e-01 1.86565116e-01 ... 4.49264562e-03
  4.44616278e-04 6.37986086e-05]
 [4.38472658e-01 4.48170364e-01 9.76380929e-02 ... 2.67589744e-03
  4.84692107e-04 1.02519523e-04]
 ...
 [4.27131820e-03 1.01918414e-01 5.11173546e-01 ... 8.74381438e-02
  2.26727203e-02 6.59079244e-03]
 [3.58375162e-01 4.28610653e-01 1.72514096e-01 ... 9.53811966e-03
  1.40877522e-03 3.09120252e-04]
 [4.61091608e-01 3.62147510e-01 1.39067963e-01 ... 8.35596956e-03
  1.39642856e-03 2.14188665e-04]]

• The output matrix contains the nnet predicted probabilities that each observation belong to each possible rating.

• For some, a matrix of probability may be hard to read or interpet.

• In the following cell we look at the first 10 observations in the test sample. We plot the nnet-predicted probabilities that each observation belong to a particular rating. The red columns indicate the true grades obtained by the borrower.

if data_untared:
    def bar_plot(i):
        pr_i = pred[i,:]
        true_i = data.y_te[i,:]
        col=pd.Series(true_i).map({0:'b', 1:'r'}).values
        rating = ['A','B','C','D','E','F','G']
        plt.bar(rating, pr_i,color=col)
    bar_plot(0)
    
    plt.figure(figsize=(5*3,2*5))
    for i in range(10):
        plt.subplot(5,2,i+1)
        bar_plot(i)
    
    plt.show()

Tensorboard

To improve performance, the first thing to do would be to use the full sample. In addition, the model should probably be tuned. Proper tuning requires access to some information about the training process.

Tensorboard is a tool that saves information about the model during the training process and allows you to visualize it while the model run and after.

When creating the model we used the line: history = train_model(model, data, epoch=5,bs=1024, verbose=1, tensor_board_name='LC')

The last parameter of the train_model function gave a name to the directory to store the Tensorboard data.

To launch Tensorboard use the command: tensorboard --logdir logs/LC

Tensorboard on Nuvolos

On Nuvolos, those additional commands need to be

jupyter tensorboard enable --user

There is the possibility that the tensorboard launcher is throwing "404: Not Found" errors. If this is the case, hitting the command

jupyter tensorboard enable --user

and restarting the application should solve the issue.

tensorboard.notebook.list()

No known TensorBoard instances running.

%reload_ext tensorboard

%tensorboard --logdir logs/LC --host "127.0.0.1"