3  Clojure with Smile Algorithms

(ns assignment.scicloj
  (:require
    [assignment.eda :refer [boston boston-transformed]]
    [calc-metric.patch]
    [fastmath.stats :as stats]
    [scicloj.ml.core :as ml]
    [scicloj.ml.dataset :as ds]
    [scicloj.ml.metamorph :as mm]))

3.1 Define regressors and response

(def response :medv)

(def regressors
  (ds/column-names boston (complement #{response})))

3.2 Convert Boston column types

(ds/info boston)

data/boston.csv: descriptive-stats [14 11]:

:col-name	:datatype	:n-valid	:n-missing	:min	:mean	:max	:standard-deviation	:skew	:first	:last
:crim	:float64	506	0	0.00632	3.61352356	88.9762	8.60154511	5.22314880	0.00632	0.04741
:zn	:float64	506	0	0.00000	11.36363636	100.0000	23.32245299	2.22566632	18.00000	0.00000
:indus	:float64	506	0	0.46000	11.13677866	27.7400	6.86035294	0.29502157	2.31000	11.93000
:chas	:int16	506	0	0.00000	0.06916996	1.0000	0.25399404	3.40590417	0.00000	0.00000
:nox	:float64	506	0	0.38500	0.55469506	0.8710	0.11587768	0.72930792	0.53800	0.57300
:rm	:float64	506	0	3.56100	6.28463439	8.7800	0.70261714	0.40361213	6.57500	6.03000
:age	:float64	506	0	2.90000	68.57490119	100.0000	28.14886141	-0.59896264	65.20000	80.80000
:dis	:float64	506	0	1.12960	3.79504269	12.1265	2.10571013	1.01178058	4.09000	2.50500
:rad	:int16	506	0	1.00000	9.54940711	24.0000	8.70725938	1.00481465	1.00000	1.00000
:tax	:float64	506	0	187.00000	408.23715415	711.0000	168.53711605	0.66995594	296.00000	273.00000
:ptratio	:float64	506	0	12.60000	18.45553360	22.0000	2.16494552	-0.80232493	15.30000	21.00000
:b	:float64	506	0	0.32000	356.67403162	396.9000	91.29486438	-2.89037371	396.90000	396.90000
:lstat	:float64	506	0	1.73000	12.65306324	37.9700	7.14106151	0.90646009	4.98000	7.88000
:medv	:float64	506	0	5.00000	22.53280632	50.0000	9.19710409	1.10809841	24.00000	11.90000

Right now, boston has too informative columns, viz. type :float64. Normally, I’d prefer to have more information per entry, however, trying to run this notebook without converting to :float32 breaks the JVM.

(-> boston
    (ds/convert-types :type/float64 :float32)
    ds/info)

data/boston.csv: descriptive-stats [14 11]:

:col-name	:datatype	:n-valid	:n-missing	:min	:mean	:max	:standard-deviation	:skew	:first	:last
:crim	:float32	506	0	0.00632000	3.61352356	88.97619629	8.60154509	5.22314872	0.00632000	0.04741000
:zn	:float32	506	0	0.00000000	11.36363636	100.00000000	23.32245299	2.22566632	18.00000000	0.00000000
:indus	:float32	506	0	0.46000001	11.13677875	27.73999977	6.86035298	0.29502153	2.30999994	11.93000031
:chas	:int16	506	0	0.00000000	0.06916996	1.00000000	0.25399404	3.40590417	0.00000000	0.00000000
:nox	:float32	506	0	0.38499999	0.55469506	0.87099999	0.11587768	0.72930787	0.53799999	0.57300001
:rm	:float32	506	0	3.56100011	6.28463439	8.77999973	0.70261715	0.40361212	6.57499981	6.03000021
:age	:float32	506	0	2.90000010	68.57490120	100.00000000	28.14886153	-0.59896263	65.19999695	80.80000305
:dis	:float32	506	0	1.12960005	3.79504270	12.12650013	2.10571014	1.01178059	4.09000015	2.50500011
:rad	:int16	506	0	1.00000000	9.54940711	24.00000000	8.70725938	1.00481465	1.00000000	1.00000000
:tax	:float32	506	0	187.00000000	408.23715415	711.00000000	168.53711605	0.66995594	296.00000000	273.00000000
:ptratio	:float32	506	0	12.60000038	18.45553383	22.00000000	2.16494578	-0.80232477	15.30000019	21.00000000
:b	:float32	506	0	0.31999999	356.67402960	396.89999390	91.29486340	-2.89037375	396.89999390	396.89999390
:lstat	:float32	506	0	1.73000002	12.65306323	37.97000122	7.14106150	0.90646009	4.98000002	7.88000011
:medv	:float32	506	0	5.00000000	22.53280636	50.00000000	9.19710411	1.10809839	24.00000000	11.89999962

(def boston-32
  (-> boston
      (ds/convert-types :type/float64 :float32)))

3.3 Setup Pipelines

(def pipeline-fn
  (ml/pipeline
    (mm/set-inference-target response)))

3.3.1 Generic pipeline function

(defn create-model-pipeline
  [model-type params]
  (ml/pipeline
    pipeline-fn
    {:metamorph/id :model}
    (mm/model (merge {:model-type model-type} params))))

3.3.1.1 Ridge context

(defn ridge-pipe-fn
  [params]
  (create-model-pipeline :smile.regression/ridge params))

3.3.1.2 Lasso context

(defn lasso-pipe-fn
  [params]
  (create-model-pipeline :smile.regression/lasso params))

3.3.1.3 Best-subset context

Best subset has a different pattern/logic of pipelining. Whereas Ridge and Lasso have hyperparameters, the best subset algorithm, as I am implementing it, is nothing but ordinary least square models iterated over all combinations of regressors.

(defn all-combinations [coll]
  (letfn [(comb [coll]
            (if (empty? coll)
              [[]]
              (let [rest (comb (rest coll))]
                (concat rest (map #(cons (first coll) %) rest)))))]
    (rest (comb coll))))

(Math/pow 2 (count regressors))

8192.0

Above, we see the number of combination possibilities with the number of regressors we have, 14. 2 to the power 13 is 8192. Below, we see the count of the all-combinations function on the list of regressors. The number is 2 to the power 13 minus 1. The difference of 1 is that I did not include a null model, i.e. no regressors.

(count (all-combinations regressors))

8191

(defn best-subset-pipe-fn
  [dataset y regrs]
  (let [combinations (all-combinations regrs)]
    (pmap (fn [Xs]                                          ; test `map` vs `pmap` vs `mapv`
            (ml/pipeline
              (mm/select-columns (cons y Xs))
              (mm/set-inference-target y)
              {:metamorph/id :model}
              (mm/model {:model-type :smile.regression/ordinary-least-square})))
          combinations)))

3.4 Pipeline Functions

3.4.1 Evaluate pipeline

(defn evaluate-pipe [pipe data]
  (ml/evaluate-pipelines
    pipe
    data
    stats/omega-sq
    :accuracy
    {:other-metrices                   [{:name :mae :metric-fn ml/mae}
                                        {:name :rmse :metric-fn ml/rmse}]
     :return-best-pipeline-only        false
     :return-best-crossvalidation-only true}))

3.4.2 Generate hyperparameters for models

(defn generate-hyperparams [model-type]
  (case model-type
    :ridge (ml/sobol-gridsearch
             (assoc-in (ml/hyperparameters :smile.regression/ridge) [:lambda :n-steps] 500))
    :lasso (take 500 (ml/sobol-gridsearch (ml/hyperparameters :smile.regression/lasso)))))

3.4.3 Evaluate a single model

(defn evaluate-model [dataset split-fn model-type model-fn]
  (let [data-split (split-fn dataset)
        pipelines (cond
                    (= model-type :best-subset)
                    (model-fn dataset response regressors)
                    :else (map model-fn (generate-hyperparams model-type)))]
    (evaluate-pipe pipelines data-split)))

3.4.4 Split functions

(defn train-test [dataset]
  (ds/split->seq dataset :bootstrap {:seed 123 :repeats 30}))

(defn train-val [dataset]
  (let [ds-split (train-test dataset)]
    (ds/split->seq (:train (first ds-split)) :kfold {:seed 123 :k 5})))

3.4.5 Define model types and corresponding functions as a vector of vectors

(def model-type-fns
  {:ridge ridge-pipe-fn
   :lasso lasso-pipe-fn})

3.4.6 Evaluate models for a dataset

(defn evaluate-models [dataset split-fn]
  (mapv (fn [[model-type model-fn]]
          (evaluate-model dataset split-fn model-type model-fn))
        model-type-fns))

3.4.7 Evaluate separately

(def ridge-lasso-models (evaluate-models boston-32 train-val))

(comment
  (def best-subset-model
    (evaluate-model boston-32 train-val :best-subset best-subset-pipe-fn)))

3.5 Extract Useable Models

(defn best-models [eval]
  (->> eval
       flatten
       (map
         #(hash-map :summary (ml/thaw-model (get-in % [:fit-ctx :model]))
                    :fit-ctx (:fit-ctx %)
                    :timing-fit (:timing-fit %)
                    :metric ((comp :metric :test-transform) %)
                    :other-metrices ((comp :other-metrices :test-transform) %)
                    :other-metric-1 ((comp :metric first) ((comp :other-metrices :test-transform) %))
                    :other-metric-2 ((comp :metric second) ((comp :other-metrices :test-transform) %))
                    :params ((comp :options :model :fit-ctx) %)
                    :pipe-fn (:pipe-fn %)))
       (sort-by :metric)))

(def best-val-ridge
  (-> (first ridge-lasso-models)
      best-models
      reverse))

(-> best-val-ridge first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
  -13.6318     -2.9084     -1.2797      1.4830     27.6106

Coefficients:
Intercept          23.4641
crim               -0.0713
zn                  0.0169
indus              -0.0672
chas                2.8096
nox                -7.3539
rm                  3.6681
age                -0.0073
dis                -0.7498
rad                 0.0721
tax                -0.0041
ptratio            -0.7041
b                   0.0088
lstat              -0.3975

Residual standard error: 4.9576 on 392 degrees of freedom
Multiple R-squared: 0.7074,    Adjusted R-squared: 0.6985
F-statistic: 78.9944 on 13 and 392 DF,  p-value: 1.106e-96

(-> best-val-ridge first :metric)

0.8173950200512978

(-> best-val-ridge first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 2.65714420138291}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 3.6179796771850925})

(-> best-val-ridge first :params)

{:model-type :smile.regression/ridge, :lambda 100.2004997995992}

(def best-val-lasso
  (-> (second ridge-lasso-models)
      best-models
      reverse))

(-> best-val-lasso first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
  -14.9960     -2.8211     -0.6650      1.5758     24.7895

Coefficients:
Intercept          36.2838
crim               -0.1133
zn                  0.0323
indus               0.0341
chas                1.9610
nox               -19.1848
rm                  3.8042
age                 0.0029
dis                -1.4452
rad                 0.3103
tax                -0.0124
ptratio            -0.9352
b                   0.0106
lstat              -0.5243

Residual standard error: 4.7062 on 392 degrees of freedom
Multiple R-squared: 0.7364,    Adjusted R-squared: 0.7283
F-statistic: 91.2444 on 13 and 392 DF,  p-value: 1.860e-105

(-> best-val-lasso first :metric)

0.7956543000071373

(-> best-val-lasso first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 2.8223327247370036}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 3.683061864448507})

(-> best-val-lasso first :params)

{:model-type :smile.regression/lasso,
 :lambda 10.0,
 :tolerance 1.0E-6,
 :max-iterations 3138181}

(comment
  (def best-val-subset
    (-> best-subset-model
        best-models
        reverse))

  (-> best-val-subset first :summary)
    ;=>
    ;#object[smile.regression.LinearModel
    ;        0x7a721884
    ;        "Linear Model:
    ;
    ;       Residuals:
    ;              Min          1Q      Median          3Q         Max
    ;         -18.3055     -3.3507     -0.9813      2.3172     31.4060
    ;
    ;       Coefficients:
    ;                         Estimate Std. Error    t value   Pr(>|t|)
    ;       Intercept          -2.4171     4.7582    -0.5080     0.6118
    ;       crim               -0.0852     0.0381    -2.2372     0.0258 *
    ;       zn                 -0.0117     0.0161    -0.7239     0.4696
    ;       chas                3.7276     1.1892     3.1345     0.0018 **
    ;       rm                  6.5272     0.4104    15.9034     0.0000 ***
    ;       age                -0.0486     0.0134    -3.6262     0.0003 ***
    ;       ptratio            -0.9594     0.1583    -6.0608     0.0000 ***
    ;       b                   0.0149     0.0033     4.4582     0.0000 ***
    ;       ---------------------------------------------------------------------
    ;       Significance codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
    ;
    ;       Residual standard error: 5.6289 on 397 degrees of freedom
    ;       Multiple R-squared: 0.6181,    Adjusted R-squared: 0.6113
    ;       F-statistic: 91.7731 on 8 and 397 DF,  p-value: 5.506e-79
    ;       "])
  (-> best-val-subset first :metric)
    ;=> 0.8303967218087499)
  (-> best-val-subset first :other-metrices)
    ;=>
    ;({:name :mae, :metric-fn #object[scicloj.ml.core$mae 0x798eacab "scicloj.ml.core$mae@798eacab"], :metric 2.770744533104}
    ; {:name :rmse,
    ;  :metric-fn #object[scicloj.ml.core$rmse 0x4350b8e7 "scicloj.ml.core$rmse@4350b8e7"],
    ;  :metric 3.7412435449760975}))
  (-> best-val-subset first :fit-ctx :model :feature-columns)
    ;=> [:crim :zn :chas :rm :age :ptratio :b]

  (def best-subset-regressors
    (-> best-val-subset first :fit-ctx :model :feature-columns)))

3.6 Build final models for evaluation

3.6.1 Ridge

(def final-model-ridge
  (-> (evaluate-pipe
        (map ridge-pipe-fn
             (-> best-val-ridge first :params))
        (train-test boston-32))
      last
      best-models))

(-> final-model-ridge first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
  -10.6398     -2.6735     -0.7566      1.5185     26.5333

Coefficients:
Intercept          29.7249
crim               -0.1092
zn                  0.0312
indus              -0.0330
chas                2.6792
nox                -9.6526
rm                  2.8383
age                 0.0005
dis                -0.8172
rad                 0.1132
tax                -0.0025
ptratio            -0.7403
b                   0.0090
lstat              -0.4848

Residual standard error: 5.0037 on 493 degrees of freedom
Multiple R-squared: 0.6779,    Adjusted R-squared: 0.6700
F-statistic: 86.4592 on 13 and 493 DF,  p-value: 6.176e-113

(-> final-model-ridge first :metric)

0.774634935015279

(-> final-model-ridge first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 3.2571196365539317}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 4.428692066973044})

(-> final-model-ridge first :params)

{:model-type :smile.regression/ridge, :lambda 100.2004997995992}

3.6.2 Lasso

(def final-model-lasso
  (-> (evaluate-pipe
        (map lasso-pipe-fn
             (-> best-val-lasso first :params))
        (train-test boston-32))
      last
      best-models))

(-> final-model-lasso first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
  -10.6707     -2.6755     -0.3740      1.4763     23.6708

Coefficients:
Intercept          47.4587
crim               -0.1512
zn                  0.0564
indus               0.0563
chas                2.2097
nox               -21.7640
rm                  2.2966
age                 0.0170
dis                -1.4559
rad                 0.3811
tax                -0.0124
ptratio            -0.9734
b                   0.0092
lstat              -0.6154

Residual standard error: 4.7684 on 493 degrees of freedom
Multiple R-squared: 0.7075,    Adjusted R-squared: 0.7004
F-statistic: 99.3584 on 13 and 493 DF,  p-value: 3.708e-123

(-> final-model-lasso first :metric)

0.760624548957392

(-> final-model-lasso first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 3.346427974017279}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 4.461146349347001})

(-> final-model-lasso first :params)

{:model-type :smile.regression/lasso, :lambda 10.0}

3.6.3 Best Subset

(defn pipeline-best-subset-fn [y Xs]
  (ml/pipeline
    (mm/select-columns (cons y Xs))
    (mm/set-inference-target y)))

(defn ols-pipe-fn [y Xs]
  (ml/pipeline
    (pipeline-best-subset-fn y Xs)
    {:metamorph/id :model}
    (mm/model {:model-type :smile.regression/ordinary-least-square})))

(comment
  (def final-best-subset
    (-> (evaluate-pipe
          [(ols-pipe-fn response best-subset-regressors)]
          (train-test boston-32))
        best-models))

 (-> final-best-subset first :summary)
   ;=>
   ;#object[smile.regression.LinearModel
   ;        0x9334757
   ;        "Linear Model:
   ;
   ;         Residuals:
   ;                Min          1Q      Median          3Q         Max
   ;           -11.7343     -3.2668     -0.9311      1.8266     38.1650
   ;
   ;         Coefficients:
   ;                           Estimate Std. Error    t value   Pr(>|t|)
   ;         Intercept          -2.6685     4.5855    -0.5819     0.5609
   ;         crim               -0.1507     0.0337    -4.4762     0.0000 ***
   ;         zn                  0.0059     0.0152     0.3913     0.6958
   ;         chas                3.7895     1.2637     2.9989     0.0028 **
   ;         rm                  5.9341     0.4373    13.5685     0.0000 ***
   ;         age                -0.0350     0.0126    -2.7764     0.0057 **
   ;         ptratio            -0.8448     0.1477    -5.7189     0.0000 ***
   ;         b                   0.0168     0.0030     5.6008     0.0000 ***
   ;         ---------------------------------------------------------------------
   ;         Significance codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
   ;
   ;         Residual standard error: 6.0676 on 498 degrees of freedom
   ;         Multiple R-squared: 0.5215,    Adjusted R-squared: 0.5148
   ;         F-statistic: 77.5462 on 8 and 498 DF,  p-value: 1.140e-75
   ;         "])
 (-> final-best-subset first :metric)
   ;=> 0.7419895970972805)
 (-> final-best-subset first :other-metrices)
   ;=>
   ;({:name :mae,
   ;  :metric-fn #object[scicloj.ml.core$mae 0x798eacab "scicloj.ml.core$mae@798eacab"],
   ;  :metric 3.266730104211804}
   ; {:name :rmse,
   ;  :metric-fn #object[scicloj.ml.core$rmse 0x4350b8e7 "scicloj.ml.core$rmse@4350b8e7"],
   ;  :metric 4.641450683867226}))
 (-> final-best-subset first :fit-ctx :model :feature-columns))

=> [:crim :zn :chas :rm :age :ptratio :b])

4 Transformation Data

(def boston-trans-32
  (-> boston-transformed
      (ds/convert-types :type/float64 :float32)))

4.0.1 Evaluate separately

(def ridge-lasso-trans-models (evaluate-models boston-trans-32 train-val))

(def best-val-trans-ridge
  (-> (first ridge-lasso-trans-models)
      best-models
      reverse))

(-> best-val-trans-ridge first :params)

{:model-type :smile.regression/ridge, :lambda 100.2004997995992}

(def best-val-trans-lasso
  (-> (second ridge-lasso-trans-models)
      best-models
      reverse))

(-> best-val-trans-lasso first :params)

{:model-type :smile.regression/lasso,
 :lambda 0.20211818181818184,
 :tolerance 0.007980000000000001,
 :max-iterations 3642727}

4.1 Build final models for evaluation

4.1.1 Ridge

(def final-model-trans-ridge
  (-> (evaluate-pipe
        (map ridge-pipe-fn
             (-> best-val-trans-ridge first :params))
        (train-test boston-trans-32))
      last
      best-models))

(-> final-model-trans-ridge first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -0.9000     -0.1096      0.0155      0.1133      0.9480

Coefficients:
Intercept           8.5042
crim                0.0413
zn                 -0.0190
indus              -0.0010
chas                0.1201
nox                -1.4973
rm                  0.1746
age                -0.0000
dis                -0.0697
rad                 0.0624
tax                -2.5175
ptratio            -0.0000
b                   0.0000
lstat              -0.3482

Residual standard error: 0.2284 on 493 degrees of freedom
Multiple R-squared: 0.6898,    Adjusted R-squared: 0.6823
F-statistic: 91.3789 on 13 and 493 DF,  p-value: 5.981e-117

(-> final-model-trans-ridge first :metric)

0.7931317852935841

(-> final-model-trans-ridge first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 0.13891291081695156}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 0.18699913565581375})

4.1.2 Lasso

(def final-model-trans-lasso
  (-> (evaluate-pipe
        (map lasso-pipe-fn
             (-> best-val-trans-lasso first :params))
        (train-test boston-trans-32))
      last
      best-models))

(-> final-model-trans-lasso first :summary)

Linear Model:

Residuals:
       Min          1Q      Median          3Q         Max
   -0.8486     -0.1196      0.0096      0.1225      0.8573

Coefficients:
Intercept          11.8059
crim                0.1365
zn                 -0.0356
indus               0.0110
chas                0.0979
nox                -2.7298
rm                  0.0776
age                 0.0000
dis                -0.0934
rad                 0.1471
tax                -3.8350
ptratio            -0.0000
b                   0.0000
lstat              -0.4730

Residual standard error: 0.2203 on 493 degrees of freedom
Multiple R-squared: 0.7114,    Adjusted R-squared: 0.7044
F-statistic: 101.2904 on 13 and 493 DF,  p-value: 1.314e-124

(-> final-model-trans-lasso first :metric)

0.7806508779969971

(-> final-model-trans-lasso first :other-metrices)

({:name :mae,
  :metric-fn
  #object[scicloj.ml.core$mae 0x1f2c3690 "scicloj.ml.core$mae@1f2c3690"],
  :metric 0.14498282692619485}
 {:name :rmse,
  :metric-fn
  #object[scicloj.ml.core$rmse 0x6a325167 "scicloj.ml.core$rmse@6a325167"],
  :metric 0.19074069330281707})

source: src/assignment/scicloj.clj