2  Exploratory Data Analysis

(ns assignment.eda
  (:require
    [clojure.math.combinatorics :as combo]
    [fastmath.stats :as stats]
    [scicloj.kindly.v4.kind :as kind]
    [scicloj.ml.dataset :as ds]))

Load data

(defonce concrete-data
         (ds/dataset "data/Concrete_Data.csv"
                     {:key-fn (fn [colname]
                                (-> colname
                                    (clojure.string/replace #"\.|\s" "-")
                                    clojure.string/lower-case
                                    (clojure.string/split #"\-\(|\--\(|\(")
                                    first
                                    keyword))}))

(ds/info concrete-data)

data/Concrete_Data.csv: descriptive-stats [9 11]:

:col-name	:datatype	:n-valid	:n-missing	:min	:mean	:max	:standard-deviation	:skew	:first	:last
:cement	:float64	1030	0	102.00	281.16786408	540.0	104.50636449	0.50948118	540.00	260.9
:blast-furnace-slag	:float64	1030	0	0.00	73.89582524	359.4	86.27934175	0.80071690	0.00	100.5
:fly-ash	:float64	1030	0	0.00	54.18834951	200.1	63.99700415	0.53735391	0.00	78.3
:water	:float64	1030	0	121.80	181.56728155	247.0	21.35421857	0.07462838	162.00	200.6
:superplasticizer	:float64	1030	0	0.00	6.20466019	32.2	5.97384139	0.90720257	2.50	8.6
:coarse-aggregate	:float64	1030	0	801.00	972.91893204	1145.0	77.75395397	-0.04021974	1040.00	864.5
:fine-aggregate	:float64	1030	0	594.00	773.58048544	992.6	80.17598014	-0.25300960	676.00	761.5
:age	:int16	1030	0	1.00	45.66213592	365.0	63.16991158	3.26917740	28.00	28.0
:concrete-compressive-strength	:float64	1030	0	2.33	35.81796117	82.6	16.70574196	0.41697729	79.99	32.4

(def response :concrete-compressive-strength)

(def regressors
  (ds/column-names concrete-data (complement #{response})))

^kind/vega
(let [data (ds/rows concrete-data :as-maps)
      column-names (ds/column-names concrete-data)]
  {:data   {:values data}
   :repeat {:column column-names}
   :spec   {:mark     "bar"
            :encoding {:x {:field {:repeat "column"} :type "quantitative"}
                       :y {:aggregate "count"}}}})

^kind/vega
(let [data (ds/rows concrete-data :as-maps)
      column-names (ds/column-names concrete-data)]
  {:data   {:values data}
   :repeat {:column column-names}
   :spec   {:width    60 :mark "boxplot"
            :encoding {:y {:field {:repeat "column"} :type "quantitative" :scale {:zero false}}}}})

(let [columns (ds/column-names concrete-data)]
  (for [column columns]
    (vector column (count (stats/outliers (get concrete-data column))))))

([:cement 0]
 [:blast-furnace-slag 2]
 [:fly-ash 0]
 [:water 9]
 [:superplasticizer 10]
 [:coarse-aggregate 0]
 [:fine-aggregate 5]
 [:age 59]
 [:concrete-compressive-strength 4])

2.0.1 Pairs-plots.

^kind/vega
(let [data (ds/rows concrete-data :as-maps)
      column-names (ds/column-names concrete-data)]
  {:data   {:values data}
   :repeat {:column column-names
            :row    column-names}
   :spec   {:height   100 :width 100
            :mark     "circle"
            :encoding {:x {:field {:repeat "column"} :type "quantitative" :scale {:zero false}}
                       :y {:field {:repeat "row"} :type "quantitative" :scale {:zero false}}}}})

(let [combos (combo/combinations regressors 2)]
  (for [[x y] combos]
    (assoc {} [x y] (stats/correlation (get concrete-data x) (get concrete-data y)))))

({[:cement :blast-furnace-slag] -0.2752159110303216}
 {[:cement :fly-ash] -0.3974673408068984}
 {[:cement :water] -0.081586748464108}
 {[:cement :superplasticizer] 0.09238617275378022}
 {[:cement :coarse-aggregate] -0.10934899412490102}
 {[:cement :fine-aggregate] -0.22271784866558753}
 {[:cement :age] 0.08194602387182141}
 {[:blast-furnace-slag :fly-ash] -0.3235799005063863}
 {[:blast-furnace-slag :water] 0.10725202721109249}
 {[:blast-furnace-slag :superplasticizer] 0.0432704187610415}
 {[:blast-furnace-slag :coarse-aggregate] -0.2839986119592143}
 {[:blast-furnace-slag :fine-aggregate] -0.28160267061419436}
 {[:blast-furnace-slag :age] -0.044246019304454105}
 {[:fly-ash :water] -0.2569840228651458}
 {[:fly-ash :superplasticizer] 0.37750314590136597}
 {[:fly-ash :coarse-aggregate] -0.009960827842262435}
 {[:fly-ash :fine-aggregate] 0.07910849096499861}
 {[:fly-ash :age] -0.15437051606792704}
 {[:water :superplasticizer] -0.6575329076284498}
 {[:water :coarse-aggregate] -0.1822936018614345}
 {[:water :fine-aggregate] -0.4506611741300146}
 {[:water :age] 0.2776182215210028}
 {[:superplasticizer :coarse-aggregate] -0.26599914818628756}
 {[:superplasticizer :fine-aggregate] 0.2226912299131872}
 {[:superplasticizer :age] -0.192700028043472}
 {[:coarse-aggregate :fine-aggregate] -0.17848095742332457}
 {[:coarse-aggregate :age] -0.0030158803467434966}
 {[:fine-aggregate :age] -0.15609470264758582})

(for [[x y] (mapv (fn [r] [response r]) regressors)]
  (assoc {} [x y] (stats/correlation (get concrete-data x) (get concrete-data y))))

({[:concrete-compressive-strength :cement] 0.4978319193241562}
 {[:concrete-compressive-strength :blast-furnace-slag]
  0.13482926149740534}
 {[:concrete-compressive-strength :fly-ash] -0.1057549162973129}
 {[:concrete-compressive-strength :water] -0.2896333849853036}
 {[:concrete-compressive-strength :superplasticizer]
  0.36607882718852003}
 {[:concrete-compressive-strength :coarse-aggregate]
  -0.1649346144601096}
 {[:concrete-compressive-strength :fine-aggregate]
  -0.16724124729005832}
 {[:concrete-compressive-strength :age] 0.3288730007799831})