From 3b8b9537a5e84083d5ec7a7dbff651ae621894c7 Mon Sep 17 00:00:00 2001 From: Manu Phatak Date: Mon, 18 Jan 2021 10:51:09 -0800 Subject: [PATCH] Practice with benchmarks --- AdventOfCode2020.cabal | 25 +- bench/Bench.hs | 22 ++ fibber2.html | 589 +++++++++++++++++++++++++++++++++++++++++ hie.yaml | 6 + package.yaml | 20 ++ 5 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 bench/Bench.hs create mode 100644 fibber2.html diff --git a/AdventOfCode2020.cabal b/AdventOfCode2020.cabal index 24ca7f1..2a1d96d 100644 --- a/AdventOfCode2020.cabal +++ b/AdventOfCode2020.cabal @@ -62,6 +62,7 @@ library Paths_AdventOfCode2020 hs-source-dirs: src + ghc-options: -Wall -O2 build-tool-depends: hlint:hlint , hpc-lcov:hpc-lcov @@ -112,7 +113,7 @@ test-suite AdventOfCode2020-test Paths_AdventOfCode2020 hs-source-dirs: test - ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall + ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N -Wall -O2 build-tool-depends: hlint:hlint , hpc-lcov:hpc-lcov @@ -128,3 +129,25 @@ test-suite AdventOfCode2020-test , parsec , unordered-containers default-language: Haskell2010 + +benchmark AdventOfCode2020-bench + type: exitcode-stdio-1.0 + main-is: Bench.hs + other-modules: + Paths_AdventOfCode2020 + hs-source-dirs: + bench + ghc-options: -Wall -O2 -threaded -rtsopts -with-rtsopts=-N -Wall -O2 + build-tool-depends: + hlint:hlint + , hpc-lcov:hpc-lcov + , implicit-hie:implicit-hie + build-depends: + AdventOfCode2020 + , base >=4.7 && <5 + , containers + , criterion + , hashable + , parsec + , unordered-containers + default-language: Haskell2010 diff --git a/bench/Bench.hs b/bench/Bench.hs new file mode 100644 index 0000000..68633c2 --- /dev/null +++ b/bench/Bench.hs @@ -0,0 +1,22 @@ +import Criterion.Main + +fib :: Integer -> Integer +fib m + | m < 0 = undefined + | otherwise = go m + where + go 0 = 0 + go 1 = 1 + go n = go (n + 1) + go (n -2) + +main :: IO () +main = + defaultMain + [ bgroup + "fib" + [ bench "1" $ whnf fib 1, + bench "5" $ whnf fib 5, + bench "9" $ whnf fib 9, + bench "11" $ whnf fib 11 + ] + ] diff --git a/fibber2.html b/fibber2.html new file mode 100644 index 0000000..885d144 --- /dev/null +++ b/fibber2.html @@ -0,0 +1,589 @@ + + + + + criterion report + + + + + + + + + +
+
+

criterion performance measurements

+ +

overview

+ +

want to understand this report?

+ +
+ +

fib/1

+ + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time5.879709339745211e-95.92292549569551e-95.9761350265666524e-9
Standard deviation1.26985788507779e-101.6753188349691544e-102.220229367302624e-10
+ + +

Outlying measurements have moderate + (0.4815796186753411%) + effect on estimated standard deviation.

+ + +

understanding this report

+ +

In this report, each function benchmarked by criterion is assigned + a section of its own. The charts in each section are active; if + you hover your mouse over data points and annotations, you will see + more details.

+ +
    +
  • The chart on the left is a + kernel + density estimate (also known as a KDE) of time + measurements. This graphs the probability of any given time + measurement occurring. A spike indicates that a measurement of a + particular time occurred; its height indicates how often that + measurement was repeated.
    • + +
  • The chart on the right is the raw data from which the kernel + density estimate is built. The x axis indicates the + number of loop iterations, while the y axis shows measured + execution time for the given number of loop iterations. The + line behind the values is the linear regression prediction of + execution time for a given number of iterations. Ideally, all + measurements will be on (or very near) this line.
    • +
+ +

Under the charts is a small table. + The first two rows are the results of a linear regression run + on the measurements displayed in the right-hand chart.

+ +
    +
  • OLS regression indicates the + time estimated for a single loop iteration using an ordinary + least-squares regression model. This number is more accurate + than the mean estimate below it, as it more effectively + eliminates measurement overhead and other constant factors.
    • +
  • R² goodness-of-fit is a measure of how + accurately the linear regression model fits the observed + measurements. If the measurements are not too noisy, R² + should lie between 0.99 and 1, indicating an excellent fit. If + the number is below 0.99, something is confounding the accuracy + of the linear model.
    • +
  • Mean execution time and standard deviation are + statistics calculated from execution time + divided by number of iterations.
    • +
+ +

We use a statistical technique called + the bootstrap + to provide confidence intervals on our estimates. The + bootstrap-derived upper and lower bounds on estimates let you see + how accurate we believe those estimates to be. (Hover the mouse + over the table headers to see the confidence levels.)

+ +

A noisy benchmarking environment can cause some or many + measurements to fall far from the mean. These outlying + measurements can have a significant inflationary effect on the + estimate of the standard deviation. We calculate and display an + estimate of the extent to which the standard deviation has been + inflated by outliers.

+ + + +
+
+ + + diff --git a/hie.yaml b/hie.yaml index 0e92854..2454a56 100644 --- a/hie.yaml +++ b/hie.yaml @@ -5,3 +5,9 @@ cradle: - path: "./test" component: "AdventOfCode2020:test:AdventOfCode2020-test" + + - path: "./bench/Bench.hs" + component: "AdventOfCode2020:bench:AdventOfCode2020-bench" + + - path: "./bench/Paths_AdventOfCode2020.hs" + component: "AdventOfCode2020:bench:AdventOfCode2020-bench" diff --git a/package.yaml b/package.yaml index 047d3cb..05ea72e 100644 --- a/package.yaml +++ b/package.yaml @@ -26,6 +26,10 @@ dependencies: - parsec - unordered-containers +ghc-options: +- -Wall +- -O2 + library: source-dirs: src @@ -38,6 +42,7 @@ tests: - -rtsopts - -with-rtsopts=-N - -Wall + - -O2 dependencies: - AdventOfCode2020 @@ -45,6 +50,21 @@ tests: - hspec - hspec-golden +benchmarks: + AdventOfCode2020-bench: + main: Bench.hs + source-dirs: bench + ghc-options: + - -threaded + - -rtsopts + - -with-rtsopts=-N + - -Wall + - -O2 + + dependencies: + - AdventOfCode2020 + - criterion + build-tools: - implicit-hie - hpc-lcov