update docs at GitHub Pages

.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: f9b42652519873cd8f19081c1600598f
+tags: 645f666f9bcd5a90fca523b33c5a78b7

_downloads/0462c80a242cab8cc5e6434efa932671/plot_svrg.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Sensitivity to hyper-parameters in SVRG\n\nThis example shows the sensitivity of SVRG with respect\nto different hyperparameters.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "print(__doc__)\n\nimport sys\nimport time\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.datasets import make_classification\nfrom sklearn.datasets import fetch_20newsgroups_vectorized\n\nfrom lightning.classification import SVRGClassifier\n\n\nclass Callback(object):\n\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n        self.obj = []\n        self.times = []\n        self.start_time = time.time()\n        self.test_time = 0\n\n    def __call__(self, clf):\n        test_time = time.time()\n        clf._finalize_coef()\n        y_pred = clf.decision_function(self.X).ravel()\n        loss = (np.maximum(1 - self.y * y_pred, 0) ** 2).mean()\n        coef = clf.coef_.ravel()\n        regul = 0.5 * clf.alpha * np.dot(coef, coef)\n        self.obj.append(loss + regul)\n        self.test_time += time.time() - test_time\n        self.times.append(time.time() -  self.start_time - self.test_time)\n\ntry:\n    dataset = sys.argv[1]\nexcept:\n    dataset = \"synthetic\"\n\nif dataset == \"news20\":\n    bunch = fetch_20newsgroups_vectorized(subset=\"all\")\n    X = bunch.data\n    y = bunch.target\n    y[y >= 1] = 1\n\n    etas = (0.5, 1e-1, 1e-2)\n    n_inners = (1.0, 2.0, 3.0)\nelse:\n    X, y = make_classification(n_samples=10000,\n                               n_features=100,\n                               n_classes=2,\n                               random_state=0)\n    etas = (1e-3, 1e-4, 1e-5)\n    n_inners = (0.25, 0.5, 1.0, 1.5)\n\ny = y * 2 - 1\n\n\nplt.figure()\n\nfor eta in etas:\n    print(\"eta =\", eta)\n    cb = Callback(X, y)\n    clf = SVRGClassifier(loss=\"squared_hinge\", alpha=1e-5, eta=eta,\n                         n_inner=1.0, max_iter=20, random_state=0, callback=cb)\n    clf.fit(X, y)\n    plt.plot(cb.times, cb.obj, label=\"eta=\" + str(eta))\n\nplt.xlabel(\"CPU time\")\nplt.ylabel(\"Objective value\")\nplt.legend()\n\nplt.figure()\n\nfor n_inner in n_inners:\n    print(\"n_inner =\", n_inner)\n    cb = Callback(X, y)\n    clf = SVRGClassifier(loss=\"squared_hinge\", alpha=1e-5, eta=1e-4,\n                         n_inner=n_inner, max_iter=20, random_state=0,\n                         callback=cb)\n    clf.fit(X, y)\n    plt.plot(cb.times, cb.obj, label=\"n_inner=\" + str(n_inner))\n\nplt.xlabel(\"CPU time\")\nplt.ylabel(\"Objective value\")\nplt.legend()\n\nplt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

_downloads/10e9cd1dfc67c24a4cf7a41eabda0e2b/plot_l2_solvers.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# L2 solver comparison\n\nThis example compares different solvers with L2 regularization.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "print(__doc__)\n\nimport sys\nimport time\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.datasets import make_classification\nfrom sklearn.datasets import fetch_20newsgroups_vectorized\n\nfrom lightning.classification import SVRGClassifier\nfrom lightning.classification import SDCAClassifier\nfrom lightning.classification import CDClassifier\nfrom lightning.classification import AdaGradClassifier\nfrom lightning.classification import SAGAClassifier, SAGClassifier\n\nfrom lightning.impl.adagrad_fast import _proj_elastic_all\n\nclass Callback(object):\n\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n        self.obj = []\n        self.times = []\n        self.start_time = time.time()\n        self.test_time = 0\n\n    def __call__(self, clf, t=None):\n        test_time = time.time()\n\n        if hasattr(clf, \"_finalize_coef\"):\n            clf._finalize_coef()\n\n        if t is not None:\n            _proj_elastic_all(clf.eta, t, clf.g_sum_[0], clf.g_norms_[0],\n                              alpha1=0, alpha2=clf.alpha, delta=0,\n                              w=clf.coef_[0])\n\n\n        y_pred = clf.decision_function(self.X).ravel()\n        loss = (np.maximum(1 - self.y * y_pred, 0) ** 2).mean()\n        coef = clf.coef_.ravel()\n        regul = 0.5 * clf.alpha * np.dot(coef, coef)\n        self.obj.append(loss + regul)\n        self.test_time += time.time() - test_time\n        self.times.append(time.time() -  self.start_time - self.test_time)\n\ntry:\n    dataset = sys.argv[1]\nexcept:\n    dataset = \"synthetic\"\n\nif dataset == \"news20\":\n    bunch = fetch_20newsgroups_vectorized(subset=\"all\")\n    X = bunch.data\n    y = bunch.target\n    y[y >= 1] = 1\n    alpha = 1e-4\n    eta_svrg = 1e-1\n    eta_adagrad = 1\n    xlim = (0, 20)\n\nelse:\n    X, y = make_classification(n_samples=10000,\n                               n_features=100,\n                               n_classes=2,\n                               random_state=0)\n    alpha = 1e-2\n    eta_svrg = 1e-3\n    eta_adagrad = 1e-2\n    xlim = [0, 2]\n\ny = y * 2 - 1\n\n# make sure the method does not stop prematurely, we want to see\n# the full convergence path\ntol = 1e-24\n\nclf1 = SVRGClassifier(loss=\"squared_hinge\", alpha=alpha, eta=eta_svrg,\n                      n_inner=1.0, max_iter=100, random_state=0, tol=1e-24)\nclf2 = SDCAClassifier(loss=\"squared_hinge\", alpha=alpha,\n                      max_iter=100, n_calls=X.shape[0]/2, random_state=0, tol=tol)\nclf3 = CDClassifier(loss=\"squared_hinge\", alpha=alpha, C=1.0/X.shape[0],\n                    max_iter=50, n_calls=X.shape[1]/3, random_state=0, tol=tol)\nclf4 = AdaGradClassifier(loss=\"squared_hinge\", alpha=alpha, eta=eta_adagrad,\n                    n_iter=100, n_calls=X.shape[0]/2, random_state=0)\nclf5 = SAGAClassifier(loss=\"squared_hinge\", alpha=alpha,\n                    max_iter=100, random_state=0, tol=tol)\nclf6 = SAGClassifier(loss=\"squared_hinge\", alpha=alpha,\n                    max_iter=100, random_state=0, tol=tol)\n\nplt.figure()\n\ndata = {}\nfor clf, name in ((clf1, \"SVRG\"),\n                  (clf2, \"SDCA\"),\n                  (clf3, \"PCD\"),\n                  (clf4, \"AdaGrad\"),\n                  (clf5, \"SAGA\"),\n                  (clf6, \"SAG\")\n                  ):\n    print(name)\n    cb = Callback(X, y)\n    clf.callback = cb\n\n    if name == \"PCD\" and hasattr(X, \"tocsc\"):\n        clf.fit(X.tocsc(), y)\n    else:\n        clf.fit(X, y)\n    data[name] = (cb.times, np.array(cb.obj))\n\n# get best value\nfmin = min([np.min(a[1]) for a in data.values()])\nfor name in data:\n    plt.plot(data[name][0], data[name][1] - fmin, label=name, lw=3)\n\nplt.xlim(xlim)\nplt.yscale('log')\nplt.xlabel(\"CPU time\")\nplt.ylabel(\"Objective value minus optimum\")\nplt.legend()\nplt.grid()\n\nplt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

_downloads/plot_1d_total_variation.py → _downloads/415a37556be4083e8ecf64d9b2ea2a7f/plot_1d_total_variation.py

@@ -21,7 +21,7 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from lightning.classification import FistaClassifier
-from sklearn.grid_search import GridSearchCV
+from sklearn.model_selection import GridSearchCV
 
 # generate some synthetic data
 n_samples = 200
@@ -31,7 +31,7 @@
 np.random.seed(0)  # for reproducibility
 X = np.random.rand(n_samples, n_features)
 # generate y as a linear model, y = sign(X w + noise)
-y = np.sign(X.dot(ground_truth) + 0.5 * np.random.randn(n_samples)).astype(np.int)
+y = np.sign(X.dot(ground_truth) + 0.5 * np.random.randn(n_samples)).astype(int)
 
 
 for penalty in ('l1', 'tv1d'):
@@ -45,4 +45,4 @@
 plt.grid()
 plt.legend()
 plt.ylim((-1.5, 1.5))
-plt.show()
+plt.show()

_downloads/48b881a16efa1f8954906b7f7fa1ee97/plot_sample_weight.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# SAGA: Weighted samples\n\nPlot decision function of a weighted dataset, where the size of points\nis proportional to its weight.\n\nAdapted from scikit-learn's plot_sgd_weighted_samples.py\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "print(__doc__)\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom lightning.impl.sag import SAGAClassifier\n\n# we create 20 points\nnp.random.seed(0)\nX = np.r_[np.random.randn(10, 2) + [1, 1], np.random.randn(10, 2)]\ny = np.array([1] * 10 + [-1] * 10)\nsample_weight = 100 * np.abs(np.random.randn(20))\n# and assign a bigger weight to the last 10 samples\nsample_weight[:10] *= 10\n\n# plot the weighted data points\nxx, yy = np.meshgrid(np.linspace(-4, 5, 500), np.linspace(-4, 5, 500))\nplt.figure()\nplt.scatter(X[:, 0], X[:, 1], c=y, s=sample_weight, alpha=0.9,\n            cmap=plt.cm.bone)\n\n# fit the unweighted model\nclf = SAGAClassifier(alpha=0.01, loss='log')\nclf.fit(X, y)\nZ = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\nZ = Z.reshape(xx.shape)\nno_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['solid'])\n\n# fit the weighted model\nclf = SAGAClassifier(alpha=0.01, loss='log')\nclf.fit(X, y, sample_weight=sample_weight)\nZ = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\nZ = Z.reshape(xx.shape)\nsamples_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['dashed'])\n\nplt.legend([no_weights.collections[0], samples_weights.collections[0]],\n           [\"no weights\", \"with weights\"], loc=\"lower left\")\n\nplt.xticks(())\nplt.yticks(())\nplt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

_downloads/4c3d9734ffe68a5cd5b658ef4f9fc398/document_classification_news20.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Classification of text documents\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n\nfrom sklearn.datasets import fetch_20newsgroups_vectorized\n\ntry:\n    from sklearn.model_selection import train_test_split\nexcept ImportError:\n    from sklearn.cross_validation import train_test_split\n    \nfrom lightning.classification import CDClassifier\nfrom lightning.classification import LinearSVC\nfrom lightning.classification import SGDClassifier\n\n# Load News20 dataset from scikit-learn.\nbunch = fetch_20newsgroups_vectorized(subset=\"all\")\nX = bunch.data\ny = bunch.target\n\n# Select a subset of the classes for faster training.\nind = np.arange(X.shape[0])\nsubset = y < 5\nX = X[ind[subset]]\ny = y[subset]\n\n# Train / test split.\nX_tr, X_te, y_tr, y_te = train_test_split(X, y,\n                                          train_size=0.75,\n                                          test_size=0.25,\n                                          random_state=0)\n\nclfs = (CDClassifier(loss=\"squared_hinge\",\n                     penalty=\"l2\",\n                     max_iter=20,\n                     random_state=0),\n\n        LinearSVC(max_iter=20,\n                  random_state=0),\n\n        SGDClassifier(learning_rate=\"constant\",\n                      alpha=1e-3,\n                      max_iter=20,\n                      random_state=0))\n\nfor clf in clfs:\n    print(clf.__class__.__name__)\n    clf.fit(X_tr, y_tr)\n    print(clf.score(X_te, y_te))"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

_downloads/84ae69625d80a9a50c846b70ea180dec/plot_sparse_non_linear.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Sparse non-linear classification\n\nThis examples demonstrates how to use `CDClassifier` with L1 penalty to do\nsparse non-linear classification. The trick simply consists in fitting the\nclassifier with a kernel matrix (e.g., using an RBF kernel).\n\nThere are a few interesting differences with standard kernel SVMs:\n\n1. the kernel matrix does not need to be positive semi-definite (hence the\nexpression \"kernel matrix\" above is an abuse of terminology)\n\n2. the number of \"support vectors\" will be typically smaller thanks to L1\nregularization and can be adjusted by the regularization parameter C (the\nsmaller C, the fewer the support vectors)\n\n3. the \"support vectors\" need not be located at the margin\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\nimport pylab as pl\n\nfrom sklearn.metrics.pairwise import rbf_kernel\n\nfrom lightning.classification import CDClassifier\nfrom lightning.classification import KernelSVC\n\nnp.random.seed(0)\n\nclass SparseNonlinearClassifier(CDClassifier):\n\n    def __init__(self, gamma=1e-2, C=1, alpha=1):\n        self.gamma = gamma\n        super().__init__(C=C,\n                         alpha=alpha,\n                         loss=\"squared_hinge\",\n                         penalty=\"l1\")\n\n    def fit(self, X, y):\n        K = rbf_kernel(X, gamma=self.gamma)\n        self.X_train_ = X\n        super().fit(K, y)\n        return self\n\n    def decision_function(self, X):\n        K = rbf_kernel(X, self.X_train_, gamma=self.gamma)\n        return super().decision_function(K)\n\n\ndef gen_non_lin_separable_data():\n    mean1 = [-1, 2]\n    mean2 = [1, -1]\n    mean3 = [4, -4]\n    mean4 = [-4, 4]\n    cov = [[1.0,0.8], [0.8, 1.0]]\n    X1 = np.random.multivariate_normal(mean1, cov, 50)\n    X1 = np.vstack((X1, np.random.multivariate_normal(mean3, cov, 50)))\n    y1 = np.ones(len(X1))\n    X2 = np.random.multivariate_normal(mean2, cov, 50)\n    X2 = np.vstack((X2, np.random.multivariate_normal(mean4, cov, 50)))\n    y2 = np.ones(len(X2)) * -1\n    return X1, y1, X2, y2\n\ndef plot_contour(X, X1, X2, clf, title):\n    pl.figure()\n    pl.title(title)\n\n    # Plot instances of class 1.\n    pl.plot(X1[:,0], X1[:,1], \"ro\")\n    # Plot instances of class 2.\n    pl.plot(X2[:,0], X2[:,1], \"bo\")\n\n    # Select \"support vectors\".\n    if hasattr(clf, \"support_vectors_\"):\n        sv = clf.support_vectors_\n    else:\n        sv = X[clf.coef_.ravel() != 0]\n\n    # Plot support vectors.\n    pl.scatter(sv[:, 0], sv[:, 1], s=100, c=\"g\")\n\n    # Plot decision surface.\n    A, B = np.meshgrid(np.linspace(-6,6,50), np.linspace(-6,6,50))\n    C = np.array([[x1, x2] for x1, x2 in zip(np.ravel(A), np.ravel(B))])\n    Z = clf.decision_function(C).reshape(A.shape)\n    pl.contour(A, B, Z, [0.0], colors='k', linewidths=1, origin='lower')\n\n    pl.axis(\"tight\")\n\n# Generate synthetic data from 2 classes.\nX1, y1, X2, y2 = gen_non_lin_separable_data()\n\n# Combine them to form a training set.\nX = np.vstack((X1, X2))\ny = np.hstack((y1, y2))\n\n# Train the classifiers.\nclf = SparseNonlinearClassifier(gamma=0.1, alpha=1./0.05)\nclf.fit(X, y)\n\nclf2 = KernelSVC(gamma=0.1, kernel=\"rbf\", alpha=1e-2)\nclf2.fit(X, y)\n\n# Plot contours.\nplot_contour(X, X1, X2, clf, \"Sparse\")\nplot_contour(X, X1, X2, clf2, \"Kernel SVM\")\n\npl.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

_downloads/plot_sparse_non_linear.py → _downloads/8e93a4e8b69a7bf144ff3d244bcbfd2e/plot_sparse_non_linear.py

@@ -33,20 +33,20 @@ class SparseNonlinearClassifier(CDClassifier):
 
     def __init__(self, gamma=1e-2, C=1, alpha=1):
         self.gamma = gamma
-        super(SparseNonlinearClassifier, self).__init__(C=C,
-                                                        alpha=alpha,
-                                                        loss="squared_hinge",
-                                                        penalty="l1")
+        super().__init__(C=C,
+                         alpha=alpha,
+                         loss="squared_hinge",
+                         penalty="l1")
 
     def fit(self, X, y):
         K = rbf_kernel(X, gamma=self.gamma)
         self.X_train_ = X
-        super(SparseNonlinearClassifier, self).fit(K, y)
+        super().fit(K, y)
         return self
 
     def decision_function(self, X):
         K = rbf_kernel(X, self.X_train_, gamma=self.gamma)
-        return super(SparseNonlinearClassifier, self).decision_function(K)
+        return super().decision_function(K)
 
 
 def gen_non_lin_separable_data():