3  Clojure with Smile Algorithm

(ns assignment.scicloj
  (:require
    [assignment.eda :refer [liver-disease]]
    [calc-metric.patch]                                     ;eval from milliseconds to nanoseconds
    [scicloj.ml.core :as ml]
    [scicloj.ml.dataset :as ds]
    [scicloj.ml.metamorph :as mm]
    [tech.v3.datatype.functional :as dfn]
    [utils.helpful-extracts
     :refer [best-models evaluate-pipe extract-params model->ds]]))

Define regressors and response

(def response :drinks)

(def regressors
  (remove #{response} (ds/column-names liver-disease)))

3.1 Build pipelines

(def pipeline-fn
  (ml/pipeline
    (mm/remove-column :selector)
    (mm/std-scale regressors {})
    (mm/set-inference-target response)))

I’m building three different pipelines because smile’s implementation of elastic net, the parameters are lambda1 and lambda2. I cannot make either of those parameters 0. If I do, the model throws and errors and says to try the explicit model. That is, if \(lambda1 = 0\), elastic net throws an error saying, “try using a ridge regression model.”

(defn ridge-pipe-fn [params]                                ;alpha = 0
  (ml/pipeline
    pipeline-fn
    {:metamorph/id :model}
    (mm/model (merge {:model-type :smile.regression/ridge}
                     params))))

(defn lasso-pipe-fn [params]                                ;alpha = 1
  (ml/pipeline
    pipeline-fn
    {:metamorph/id :model}
    (mm/model (merge {:model-type :smile.regression/lasso}
                     params))))

(defn elastic-net-pipe-fn [params]                          ;0 < alpha < 1
  (ml/pipeline
    pipeline-fn
    {:metamorph/id :model}
    (mm/model (merge {:model-type :smile.regression/elastic-net}
                     params))))

3.2 Partition data

(def ds-split
  (ds/split->seq
    liver-disease
    :kfold {:seed 123 :k 5 :ratio [0.8 0.2]}))

:split-names [:train-val :test]

(def train-val-splits
  (ds/split->seq
    (:train (first ds-split))
    :kfold {:seed 123 :k 5}))

3.3 Build models

3.3.1 Ridge

(def ridge-pipelines
  (->> (ml/sobol-gridsearch {:lambda (ml/linear 0 1000 250)})
       (map ridge-pipe-fn)))

(def eval-ridge-val
  (evaluate-pipe ridge-pipelines train-val-splits))

3.3.2 Lasso

(def lasso-pipelines
  (->> (ml/sobol-gridsearch {:lambda (ml/linear 0 1000 250)})
       (map lasso-pipe-fn)))

(def eval-lasso-val
  (evaluate-pipe lasso-pipelines train-val-splits))

3.3.3 Elastic Net

(def elastic-pipelines
  (->> (ml/sobol-gridsearch {:lambda1 (ml/linear 0.0001 100 16)
                             :lambda2 (ml/linear 0.0001 100 16)})
       (map elastic-net-pipe-fn)))

(comment
  (def elastic-pipelines
    (->> (ml/sobol-gridsearch
           (dissoc
             (ml/hyperparameters :smile.regression/elastic-net)
             :tolerance :max-iterations))
         (take 500)
         (map elastic-net-pipe-fn))))

(def eval-enet-val
  (evaluate-pipe elastic-pipelines train-val-splits))

3.4 Extract models

(def models-ridge-val
  (-> (best-models eval-ridge-val)
      reverse))

(def models-lasso-val
  (-> (best-models eval-lasso-val)
      reverse))

(def models-enet-val
  (-> (best-models eval-enet-val)
      reverse))

3.4.1 Best model for each pipeline

(-> models-ridge-val first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -6.7453     -2.4532     -0.3884      1.6303     14.5707

Coefficients:
Intercept           3.5679
mcv                 0.4123
alkphos             0.1640
sgpt                0.0857
sgot                0.3015
gammagt             0.3779

Residual standard error: 3.1737 on 216 degrees of freedom
Multiple R-squared: 0.1375,    Adjusted R-squared: 0.1216
F-statistic: 8.6103 on 5 and 216 DF,  p-value: 1.823e-06

(-> models-lasso-val first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -3.2421     -2.7421     -1.2421      1.7579     16.7579

Coefficients:
Intercept           3.2421
mcv                 0.0000
alkphos             0.0000
sgpt                0.0000
sgot                0.0000
gammagt             0.0000

Residual standard error: 3.3179 on 216 degrees of freedom
Multiple R-squared: 0.0000,    Adjusted R-squared: -0.0185
F-statistic: 0.0000 on 5 and 216 DF,  p-value: 1.000

(-> models-enet-val first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -5.9860     -2.5215     -0.3551      1.5392     15.0635

Coefficients:
Intercept           3.5679
mcv                 0.3809
alkphos             0.0016
sgpt                0.0002
sgot                0.2526
gammagt             0.3754

Residual standard error: 3.1991 on 216 degrees of freedom
Multiple R-squared: 0.1236,    Adjusted R-squared: 0.1074
F-statistic: 7.6179 on 5 and 216 DF,  p-value: 9.270e-06

The summary for our best lasso model looks wrong. A negative Adjusted R\(2\)? We can see an issue derives from not calculating the Adjusted R\(^2\), i.e the :metric.

(-> models-lasso-val first :metric)

##NaN

Instead, we will collect the best lasso models according to the lowest mae, i.e. :other-metric-1.

(def models-lasso-val-2
  (->> (best-models eval-lasso-val)
       (sort-by :other-metric-1)))

(-> models-lasso-val-2 first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -7.5909     -2.2128     -0.4783      1.8028     13.6481

Coefficients:
Intercept           3.5679
mcv                 0.6996
alkphos             0.2249
sgpt               -0.2766
sgot                0.6109
gammagt             0.6564

Residual standard error: 3.1301 on 216 degrees of freedom
Multiple R-squared: 0.1610,    Adjusted R-squared: 0.1455
F-statistic: 10.3659 on 5 and 216 DF,  p-value: 1.067e-07

3.5 Build final models for evaluation

3.5.1 Ridge

(def eval-ridge
  (evaluate-pipe
    (->> (extract-params models-ridge-val 3)                ;use best 3 lambdas
         (map ridge-pipe-fn))
    ds-split))

(def models-ridge
  (->> (best-models eval-ridge)
       reverse))

3.5.2 Lasso

(def eval-lasso
  (evaluate-pipe
    (->> (extract-params models-lasso-val-2 3)              ;use best 3 lambdas
         (map lasso-pipe-fn))
    ds-split))

(def models-lasso
  (->> (best-models eval-lasso)
       reverse))

3.5.3 Elastic net

(def eval-enet
  (evaluate-pipe
    (->> (extract-params models-enet-val 3)                 ;use best 3 lambda1s and lambda2s
         (map elastic-net-pipe-fn))
    ds-split))

(def models-enet
  (-> (best-models eval-enet)
      reverse))

3.6 Build final models for evaluation

(def ds-ridge
  (-> (model->ds models-ridge 3)
      (ds/rename-columns {:lambda :lambda2})
      (ds/add-columns {:lambda1 0 :alpha 0})))

(def ds-lasso
  (-> (model->ds models-lasso 3)
      (ds/rename-columns {:lambda :lambda1})
      (ds/add-columns {:lambda2 0 :alpha 1})))

(def ds-elastic
  (-> (model->ds models-enet 3)
      (ds/add-columns {:alpha (fn [ds]
                                (dfn// (:lambda1 ds) (dfn/+ (:lambda1 ds) (:lambda2 ds))))})))

(def col-order
  [:model-type :compute-time-ns :alpha :lambda1 :lambda2 :adj-r2 :mae :rmse])

3.7 Final comparisons

(def top-scicloj
  (-> (model->ds (concat (ds/rows ds-ridge :as-maps) (ds/rows ds-lasso :as-maps) (ds/rows ds-elastic :as-maps)))
      (ds/reorder-columns col-order)
      (ds/order-by :adj-r2 :desc)))

top-scicloj

_unnamed [9 8]:

:model-type	:compute-time-ns	:alpha	:lambda1	:lambda2	:adj-r2	:mae	:rmse
:smile.regression/ridge	1365481	0.00000000	0.00000000	196.78714859	0.27138367	2.19656820	2.88244828
:smile.regression/ridge	1467237	0.00000000	0.00000000	192.77108434	0.27131830	2.19552025	2.88102276
:smile.regression/ridge	985196	0.00000000	0.00000000	188.75502008	0.27124938	2.19445469	2.87959384
:smile.regression/lasso	1664485	1.00000000	4.01606426	0.00000000	0.26141892	2.73072125	3.40320579
:smile.regression/elastic-net	1229910	0.62499984	100.00000000	60.00004000	0.26059398	2.22496112	2.92246056
:smile.regression/elastic-net	1176776	0.65217371	100.00000000	53.33338000	0.26021738	2.22204401	2.91718016
:smile.regression/lasso	1151502	1.00000000	8.03212851	0.00000000	0.25986847	2.73272023	3.40683967
:smile.regression/elastic-net	1237148	0.68181793	100.00000000	46.66672000	0.25984550	2.21890789	2.91171010
:smile.regression/lasso	1225671	1.00000000	12.04819277	0.00000000	0.25806319	2.73472124	3.41060937

(comment
  ((-> models-ridge first :pipe-fn)
   (merge (-> models-ridge first :fit-ctx)
          {:metamorph/data (ds/tail (:test (second ds-split)))
           :metamorph/mode :transform})))

source: src/assignment/scicloj.clj