.. DO NOT EDIT. .. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. .. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: .. "auto_examples/misc/plot_learning_curves_lr.py" .. LINE NUMBERS ARE GIVEN BELOW. .. only:: html .. note:: :class: sphx-glr-download-link-note Click :ref:`here ` to download the full example code .. rst-class:: sphx-glr-example-title .. _sphx_glr_auto_examples_misc_plot_learning_curves_lr.py: Learning curves (learning rate) =============================== Hello world .. GENERATED FROM PYTHON SOURCE LINES 8-27 .. code-block:: default import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from tensorflow.keras.optimizers import Adam from tensorflow.keras.initializers import GlorotUniform from sklearn.datasets import load_boston from sklearn.model_selection import train_test_split from matplotlib.colors import LogNorm from mpl_toolkits.mplot3d import Axes3D from etudes.metrics import nmse .. GENERATED FROM PYTHON SOURCE LINES 29-42 .. code-block:: default num_index_points = 64 num_layers = 3 num_units = 32 num_epochs = 100 batch_size = 64 val_rate = 0.2 seed = 8888 # set random seed for reproducibility random_state = np.random.RandomState(seed) .. GENERATED FROM PYTHON SOURCE LINES 43-50 .. code-block:: default t_grid = np.arange(num_epochs) # lr_grid = np.logspace(-5, -0.5, num_index_points) log_lr_grid = np.linspace(-5.0, -1.0, num_index_points) lr_grid = 10**log_lr_grid .. GENERATED FROM PYTHON SOURCE LINES 51-52 .. code-block:: default dataset = load_boston() .. GENERATED FROM PYTHON SOURCE LINES 53-57 .. code-block:: default X_train, X_val, y_train, y_val = train_test_split(dataset.data, dataset.target, test_size=val_rate, random_state=random_state) .. GENERATED FROM PYTHON SOURCE LINES 58-79 .. code-block:: default frames = [] for i, lr in enumerate(lr_grid): optimizer = Adam(learning_rate=lr) model = Sequential() for _ in range(num_layers): model.add(Dense(num_units, activation="relu", kernel_initializer=GlorotUniform(seed=seed))) model.add(Dense(1, kernel_initializer=GlorotUniform(seed=seed))) model.compile(optimizer=optimizer, loss="mean_squared_error", metrics=[nmse]) hist = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=num_epochs, batch_size=batch_size, verbose=False) frame = pd.DataFrame(hist.history).assign(log_lr=log_lr_grid[i], seed=seed) frame.index.name = "epoch" frame.reset_index(inplace=True) frames.append(frame) .. GENERATED FROM PYTHON SOURCE LINES 80-82 .. code-block:: default data = pd.concat(frames, axis="index", ignore_index=True, sort=True) data.rename(lambda s: s.replace('_', ' '), axis="columns", inplace=True) .. GENERATED FROM PYTHON SOURCE LINES 83-94 .. code-block:: default fig, ax = plt.subplots() sns.lineplot(x="epoch", y="val nmse", hue="log lr", units="seed", estimator=None, palette="viridis_r", linewidth=0.4, data=data, ax=ax) ax.set_xscale("log") ax.set_yscale("log") plt.show() .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_001.png :alt: plot learning curves lr :class: sphx-glr-single-img .. GENERATED FROM PYTHON SOURCE LINES 95-105 .. code-block:: default fig, ax = plt.subplots() sns.lineplot(x="log lr", y="val nmse", hue="epoch", units="seed", estimator=None, palette="viridis_r", linewidth=0.4, data=data, ax=ax) ax.set_yscale("log") plt.show() .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_002.png :alt: plot learning curves lr :class: sphx-glr-single-img .. GENERATED FROM PYTHON SOURCE LINES 106-108 .. code-block:: default new_data = data.pivot(index="log lr", columns="epoch", values="val nmse") Z = new_data.to_numpy() .. GENERATED FROM PYTHON SOURCE LINES 109-120 .. code-block:: default fig, ax = plt.subplots() ax.contour(*np.broadcast_arrays(lr_grid.reshape(-1, 1), t_grid), Z, levels=np.logspace(0, 4, 25), norm=LogNorm(), cmap="viridis") ax.set_xscale("log") ax.set_xlabel(r"learning rate") ax.set_ylabel(r"epoch") plt.show() .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_003.png :alt: plot learning curves lr :class: sphx-glr-single-img .. GENERATED FROM PYTHON SOURCE LINES 121-131 .. code-block:: default fig, ax = plt.subplots(subplot_kw=dict(projection="3d", azim=50)) ax.plot_surface(log_lr_grid.reshape(-1, 1), t_grid, np.log(Z), alpha=0.8, edgecolor='k', linewidth=0.4, cmap="Spectral_r") ax.set_xlabel(r"$\log_{10}$ learning rate") ax.set_ylabel("epoch") ax.set_zlabel(r"$\log_{10}$ val nmse") plt.show() .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_004.png :alt: plot learning curves lr :class: sphx-glr-single-img .. GENERATED FROM PYTHON SOURCE LINES 132-134 .. code-block:: default new_data.rename(lambda s: s + 1, axis="columns", inplace=True) columns = list(np.minimum(3**np.arange(6), num_epochs)) .. GENERATED FROM PYTHON SOURCE LINES 135-137 .. code-block:: default g = sns.PairGrid(new_data[columns], corner=True) g = g.map_lower(plt.plot) .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_005.png :alt: plot learning curves lr :class: sphx-glr-single-img .. GENERATED FROM PYTHON SOURCE LINES 138-141 .. code-block:: default g = sns.PairGrid(new_data[columns].reset_index(), hue="log lr", palette="Spectral", corner=True) g = g.map_lower(plt.scatter, facecolor="none", alpha=0.8) .. image:: /auto_examples/misc/images/sphx_glr_plot_learning_curves_lr_006.png :alt: plot learning curves lr :class: sphx-glr-single-img .. rst-class:: sphx-glr-timing **Total running time of the script:** ( 2 minutes 55.205 seconds) .. _sphx_glr_download_auto_examples_misc_plot_learning_curves_lr.py: .. only :: html .. container:: sphx-glr-footer :class: sphx-glr-footer-example .. container:: sphx-glr-download sphx-glr-download-python :download:`Download Python source code: plot_learning_curves_lr.py ` .. container:: sphx-glr-download sphx-glr-download-jupyter :download:`Download Jupyter notebook: plot_learning_curves_lr.ipynb ` .. only:: html .. rst-class:: sphx-glr-signature `Gallery generated by Sphinx-Gallery `_