Messing around with Babashka

source: babashka.org

Premise

test:
runs-on: ubuntu-22.04
name: "Runner #${{ matrix.ci_node_index }}: Run test suite in parallel"
strategy:
fail-fast: false
matrix:
ci_node_total: [3]
ci_node_index: [0,1,2]
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Test
run: |
echo "a test task";

Dive-in

circleci tests glob "test/**/*.clj"
circleci tests split - split-by=timings - timings-type=classname
  • Read and parse previous test results
  • Parse and determine what files are to be considered as containing tests and are not just utility files
  • Split the test based on previous test run times and pack them in their respective workers, in a reasonable and efficient manner.

What the ancients called a clever fighter is one who not only wins, but excels in winning with ease — Sun Tzu

{:paths          ["."]
:min-bb-version "1.0.165"
:tasks
{:requires ([ci.test-helper :as cth])
split {:docs "Split tests based on timings"
:task (cth/run-split *command-line-args*)}}}
(def cli-options
;; Options with a required argument
[["-w" "--workers WORKERS" "Number of Workers/Runners"
:default 30
:parse-fn parse-long]
["-i" "--index INDEX" "Runner Index"
:parse-fn parse-long]
["-b" "--test-results-base-path TEST_RESULTS_BASE_PATH" "Test results base path"
:default "./target/test2junit/"
:parse-fn str]
["-p" "--test-results-pattern TEST_RESULTS_PATTERN" "Test results pattern"
:default "**/*.xml"
:parse-fn str]
["-t" "--tests-path TESTS_PATH" "Tests path"
:default "test/"
:parse-fn str]
["-h" "--help"]
["-p" "--plan" "Execution plan"
:default false]])
;; This is to ensure we parse the test times correctly in spite of execution locale
(defn parse-to-double [string]
"Decimal seperator varies depending on system locale, string replacement standardizes this"
(-> string
(str/replace #"," ".")
parse-double))
;; when we parse the files as using the glob function, we get them as unix objects
(defn read-file [file]
(-> file
str
slurp))
;; parse the test files and determine if it actually has tests
(defn parse-test-files [file test-results]
(let [file-content (read-file file)]
(when (str/includes? file-content "deftest")
{:filename (str file)
:line-count (count (str/split-lines file-content))
:test-time ((keyword (filename-to-classname file)) test-results 1)
})))
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="org.class-name" errors="0" failures="0" tests="2" time="21.3709" timestamp="2022-11-15_18:31:22+0000">
<testcase name="test-1" classname="org.class-name" time="0.0132">
</testcase>
<testcase name="test-2" classname="org.class-name" time="21.3577">
</testcase>
<system-err>
</system-err>
<system-out>
</system-out>
<properties>
... way too many properties ...
</properties>
</testsuite>
;; If the test file was not properly constructed, populate the times with default values
(defn try-parse-xml [xml-content filename]
(try
(parse-str xml-content)
(catch Exception _
{:tag :testsuite
:attrs {:name (filename-to-classname(str filename))
:errors 1
:failures 1
:tests 0
:time 1
:timestamp 0}
:content []}
)))
;; Parse the test file and extract the test class name and the timing
(defn parse-test-results [file]
(-> (read-file file)
(try-parse-xml file)
:attrs
((juxt :name :time))))
  • Minimizing the largest sum
  • Maximizing the smallest sum
  • Maximizing the sum of products
;; appending to a bin
(defn add-to-bin [bin [_ size :as item]]
{:size (+ (:size bin) size)
:items (conj (:items bin) item)})

;; searching for the smallest bin
(defn select-smallest-bin [bins]
(first (apply min-key
(fn [[_ bin]] (:size bin))
(map-indexed vector bins))))
(defn pack
"Simple first fit packing algorithm."
[items no-fit-fn]
(let [init-bins [{:size 0 :items []}]]
(reduce no-fit-fn init-bins items)))

(defn grow-bins [bins item]
(conj bins (add-to-bin {:size 0 :items []} item)))

;; searching for and adding to the smallest bin
(defn add-to-smallest-bin [n bins item]
(if (< (count bins) n)
(grow-bins bins item)
(update-in bins [(select-smallest-bin bins)]
#(add-to-bin % item))))
(defn pack-n-bins
[items n]
(pack items (partial add-to-smallest-bin n)))
#!/usr/bin/env bb

(ns ci.test-helper
(:require
[babashka.fs :as fs]
[clojure.data.xml :refer :all]
[clojure.java.shell :refer :all]
[clojure.string :as str]
[clojure.tools.cli :as cli]
[clojure.repl :as repl]))

(def cli-options
;; Options with a required argument
[["-w" "--workers WORKERS" "Number of Workers/Runners"
:default 30
:parse-fn parse-long]
["-i" "--index INDEX" "Runner Index"
:parse-fn parse-long]
["-b" "--test-results-base-path TEST_RESULTS_BASE_PATH" "Test results base path"
:default "./target/test2junit/"
:parse-fn str]
["-p" "--test-results-pattern TEST_RESULTS_PATTERN" "Test results pattern"
:default "**/*.xml"
:parse-fn str]
["-t" "--tests-path TESTS_PATH" "Tests path"
:default "test/"
:parse-fn str]
["-h" "--help"]
["-p" "--plan" "Execution plan"
:default false]])

(defn filename-to-classname [filename]
(-> filename
(str/replace #"^target/test2junit/xml/(.+?)\.xml$" "$1")
(str/replace #"^test/(.+?)\.clj$" "$1")
(str/replace #"/" ".")
repl/demunge))

(defn add-to-bin [bin [_ size :as item]]
{:size (+ (:size bin) size)
:items (conj (:items bin) item)})

(defn select-smallest-bin [bins]
(first (apply min-key
(fn [[_ bin]] (:size bin))
(map-indexed vector bins))))
(defn pack
"Simple first fit packing algorithm."
[items no-fit-fn]
(let [init-bins [{:size 0 :items []}]]
(reduce no-fit-fn init-bins items)))

(defn grow-bins [bins item]
(conj bins (add-to-bin {:size 0 :items []} item)))

(defn add-to-smallest-bin [n bins item]
(if (< (count bins) n)
(grow-bins bins item)
(update-in bins [(select-smallest-bin bins)]
#(add-to-bin % item))))

(defn pack-n-bins
[items n]
(pack items (partial add-to-smallest-bin n)))

(defn parse-to-double [string]
"Decimal seperator varies depending on system locale, string replacement standardizes this"
(-> string
(str/replace #"," ".")
parse-double))

(defn try-parse-xml [xml-content filename]
(try
(parse-str xml-content)
(catch Exception _
{:tag :testsuite
:attrs {:name (filename-to-classname(str filename))
:errors 1
:failures 1
:tests 0
:time 1
:timestamp 0}
:content []}
)))

(defn read-file [file]
(-> file
str
slurp))

(defn parse-test-results [file]
(-> (read-file file)
(try-parse-xml file)
:attrs
((juxt :name :time))))

(defn parse-test-files [file test-results]
(let [file-content (read-file file)]
(when (str/includes? file-content "deftest")
{:filename (str file)
:line-count (count (str/split-lines file-content))
:test-time ((keyword (filename-to-classname file)) test-results 1)
})))

(defn run-split [arglist]
(if-let [errors (:errors (cli/parse-opts arglist cli-options))]
(print errors)
(let [options (:options (cli/parse-opts arglist cli-options))
{:keys [workers index tests-path test-results-base-path test-results-pattern]} options
test-results (when (fs/exists? test-results-base-path)
(->> (fs/glob test-results-base-path test-results-pattern {:recursive true})
(map parse-test-results)
(into {})
(#(update-vals % parse-to-double))
(#(update-keys % keyword))))
aggregated-results (->> (fs/glob tests-path "**/*.clj" {:recursive true})
(keep #(parse-test-files % test-results))
(sort-by :test-time #(compare %2 %1)))
items (->> (pack-n-bins
(map vector
(map (comp filename-to-classname :filename) aggregated-results)
(map :test-time aggregated-results))
workers))
runner-spec (-> items
(get index))
runner-items (->> runner-spec
(:items)
(into {})
keys)]
(if (:plan options)
(do
(println (str "Number of workers: " workers))
(println (str "Runner Index: " index))
(println (str "Estimated runtime: " (:size runner-spec)))
(println (str "Planned classes: " runner-items)))
(doseq [item runner-items]
(print (str item " ")))))))
bb split --workers ${{ matrix.ci_node_total }} --index ${{ matrix.ci_node_index }} --plan
bb split --workers ${{ matrix.ci_node_total }} --index ${{ matrix.ci_node_index }} | xargs lein test2junit

TL;DR

source: tenor.com
  • Approximate proper timing defaults based on test file complexity and line count

References

--

--

If I have to do it more than twice, I am automating it. #StayLazy

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Ian Muge

If I have to do it more than twice, I am automating it. #StayLazy