Classification-only metrics: drop rank metrics, richer aggregation, F-beta everywhere (0.3.0)

.github/dependabot.yml

@@ -1,20 +1,20 @@
 version: 2
 updates:
-    - package-ecosystem: "maven"
-      directory: "/"
-      schedule:
-          interval: "monthly"
-          time: "11:00"
-          timezone: "Europe/Berlin"
-      commit-message:
-          prefix: "Dependency"
-          include: "scope"
-    - package-ecosystem: "docker"
-      directory: "/"
-      schedule:
-          interval: "monthly"
-          time: "11:00"
-          timezone: "Europe/Berlin"
-      commit-message:
-          prefix: "Dependency"
-          include: "scope"
+    -   package-ecosystem: "maven"
+        directory: "/"
+        schedule:
+            interval: "monthly"
+            time: "11:00"
+            timezone: "Europe/Berlin"
+        commit-message:
+            prefix: "Dependency"
+            include: "scope"
+    -   package-ecosystem: "docker"
+        directory: "/"
+        schedule:
+            interval: "monthly"
+            time: "11:00"
+            timezone: "Europe/Berlin"
+        commit-message:
+            prefix: "Dependency"
+            include: "scope"

.github/workflows/docs.yml

@@ -1,36 +1,36 @@
 name: Documentation
 
 on:
-  push:
-    branches:
-      - main
-    tags: "v**"
-    paths:
-      - 'docs/**'
-      - '.github/workflows/docs.yml'
-  workflow_dispatch:
+    push:
+        branches:
+            - main
+        tags: "v**"
+        paths:
+            - 'docs/**'
+            - '.github/workflows/docs.yml'
+    workflow_dispatch:
 
 jobs:
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.SDQ_TOKEN }}
-      - uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}.wiki
-          path: wiki
-          token: ${{ secrets.SDQ_TOKEN }}
+    docs:
+        runs-on: ubuntu-latest
+        steps:
+            -   uses: actions/checkout@v4
+                with:
+                    token: ${{ secrets.SDQ_TOKEN }}
+            -   uses: actions/checkout@v4
+                with:
+                    repository: ${{ github.repository }}.wiki
+                    path: wiki
+                    token: ${{ secrets.SDQ_TOKEN }}
 
-      - name: Remove contents in Wiki
-        working-directory: wiki
-        run: ls -A1 | grep -v '.git' | xargs rm -r
+            -   name: Remove contents in Wiki
+                working-directory: wiki
+                run: ls -A1 | grep -v '.git' | xargs rm -r
 
-      - name: Copy Wiki from Docs folder
-        run: cp -r ./docs/. ./wiki
+            -   name: Copy Wiki from Docs folder
+                run: cp -r ./docs/. ./wiki
 
-      - name: Deploy 🚀
-        uses: stefanzweifel/git-auto-commit-action@v5
-        with:
-          repository: wiki
+            -   name: Deploy 🚀
+                uses: stefanzweifel/git-auto-commit-action@v5
+                with:
+                    repository: wiki

.github/workflows/format.yml

@@ -12,21 +12,21 @@ jobs:
         runs-on: ubuntu-latest
 
         steps:
-            - uses: actions/checkout@v4
-              with:
-                  submodules: true
-                  fetch-depth: 0
-            - uses: joshlong/java-version-export-github-action@v28
-              id: jve
-            - name: Set up JDK
-              uses: actions/setup-java@v4
-              with:
-                  distribution: 'temurin'
-                  java-version: ${{ steps.jve.outputs.java_major_version }}
-                  cache: 'maven'
-            - name: Format with Spotless in Maven
-              run: mvn -B spotless:apply --file pom.xml
-            - name: Commit changes
-              uses: stefanzweifel/git-auto-commit-action@v5
-              with:
-                  commit_message: Apply formatting changes
+            -   uses: actions/checkout@v4
+                with:
+                    submodules: true
+                    fetch-depth: 0
+            -   uses: joshlong/java-version-export-github-action@v28
+                id: jve
+            -   name: Set up JDK
+                uses: actions/setup-java@v4
+                with:
+                    distribution: 'temurin'
+                    java-version: ${{ steps.jve.outputs.java_major_version }}
+                    cache: 'maven'
+            -   name: Format with Spotless in Maven
+                run: mvn -B spotless:apply --file pom.xml
+            -   name: Commit changes
+                uses: stefanzweifel/git-auto-commit-action@v5
+                with:
+                    commit_message: Apply formatting changes

.github/workflows/publish.yml

@@ -1,27 +1,27 @@
 name: Deploy to GitHub
 on:
-  workflow_dispatch:
-  release:
-    types: [created]
+    workflow_dispatch:
+    release:
+        types: [ created ]
 jobs:
-  publish-release-artifact:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: joshlong/java-version-export-github-action@v28
-        id: jve
+    publish-release-artifact:
+        runs-on: ubuntu-latest
+        steps:
+            -   uses: actions/checkout@v4
+            -   uses: joshlong/java-version-export-github-action@v28
+                id: jve
 
-      - name: Java without Cache
-        uses: actions/setup-java@v4
-        with:
-            java-version: ${{ steps.jve.outputs.java_major_version }}
-            distribution: 'temurin'
+            -   name: Java without Cache
+                uses: actions/setup-java@v4
+                with:
+                    java-version: ${{ steps.jve.outputs.java_major_version }}
+                    distribution: 'temurin'
 
-      - name: Build Metrics
-        run: mvn -U -B clean package
+            -   name: Build Metrics
+                run: mvn -U -B clean package
 
-      - name: Attach CLI to Release on GitHub
-        uses: softprops/action-gh-release@v2
-        with:
-          files: cli/target/metrics-cli.jar
-          fail_on_unmatched_files: true
+            -   name: Attach CLI to Release on GitHub
+                uses: softprops/action-gh-release@v2
+                with:
+                    files: cli/target/metrics-cli.jar
+                    fail_on_unmatched_files: true

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020-2024 ArDoCo
+Copyright (c) 2020-2024 ARDoCo
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,17 +1,17 @@
-# ArDoCo: Metrics Calculator
-Welcome to the **ArDoCo Metrics Calculator** project! This tool provides functionality to calculate and aggregate **classification** and **rank metrics** for various machine learning and ranking tasks.
+# ARDoCo: Metrics Calculator
+Welcome to the **ARDoCo Metrics Calculator** project! This tool provides functionality to calculate and aggregate **classification metrics** for various machine learning tasks.
 
-The [Wiki](https://github.com/ArDoCo/Metrics/wiki) contains all the necessary information to use the **ArDoCo Metrics Calculator** via multiple interfaces, including a library, REST API, and command-line interface (CLI).
+The [Wiki](https://github.com/ardoco/metrics/wiki) contains all the necessary information to use the **ARDoCo Metrics Calculator** via multiple interfaces, including a library, REST API, and command-line interface (CLI).
 
 ## Quickstart
 
 To use this project as a Maven dependency, you need to include the following dependency in your `pom.xml` file:
 
 ```xml
 <dependency>
-    <groupId>io.github.ardoco</groupId>
-    <artifactId>metrics</artifactId>
-    <version>${revision}</version>
+	<groupId>io.github.ardoco</groupId>
+	<artifactId>metrics</artifactId>
+	<version>${revision}</version>
 </dependency>
 ```
 

calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/ClassificationMetricsCalculator.kt

@@ -1,7 +1,7 @@
 package edu.kit.kastel.mcse.ardoco.metrics
 
 import edu.kit.kastel.mcse.ardoco.metrics.internal.ClassificationMetricsCalculatorImpl
-import edu.kit.kastel.mcse.ardoco.metrics.result.AggregatedClassificationResult
+import edu.kit.kastel.mcse.ardoco.metrics.result.ClassificationAggregationResult
 import edu.kit.kastel.mcse.ardoco.metrics.result.SingleClassificationResult
 
 /**
@@ -12,10 +12,45 @@ interface ClassificationMetricsCalculator {
         /** A default instance of the classification metrics calculator. */
         @JvmStatic
         val Instance: ClassificationMetricsCalculator = ClassificationMetricsCalculatorImpl()
+
+        /** The betas of the F-beta scores that are calculated if no betas are requested explicitly. */
+        @JvmStatic
+        val DefaultBetas: Set<Double> = setOf(1.0)
     }
 
+    /**
+     * Calculates the metrics for the given classification, including only the F1-score.
+     * @param T the type of classified elements
+     * @param classification the classification
+     * @param groundTruth the ground truth
+     * @param confusionMatrixSum the sum of the confusion matrix. If not provided, some metrics can't be calculated.
+     * @return the classification result
+     */
+    fun <T> calculateMetrics(
+        classification: Set<T>,
+        groundTruth: Set<T>,
+        confusionMatrixSum: Int?
+    ): SingleClassificationResult<T> = calculateMetrics(classification, groundTruth, confusionMatrixSum, DefaultBetas)
+
     /**
      * Calculates the metrics for the given classification.
+     * @param T the type of classified elements
+     * @param classification the classification
+     * @param groundTruth the ground truth
+     * @param confusionMatrixSum the sum of the confusion matrix. If not provided, some metrics can't be calculated.
+     * @param betas the betas of the F-beta scores to calculate. Beta 1.0 is always calculated. Each beta must be finite and greater than 0.
+     * @return the classification result
+     */
+    fun <T> calculateMetrics(
+        classification: Set<T>,
+        groundTruth: Set<T>,
+        confusionMatrixSum: Int?,
+        betas: Collection<Double>
+    ): SingleClassificationResult<T>
+
+    /**
+     * Calculates the metrics for the given classification, including only the F1-score.
+     * @param T the type of classified elements
      * @param classification the classification
      * @param groundTruth the ground truth
      * @param stringProvider a function to convert the classification and ground truth to strings
@@ -27,36 +62,71 @@ interface ClassificationMetricsCalculator {
         groundTruth: Set<T>,
         stringProvider: (T) -> String,
         confusionMatrixSum: Int?
-    ): SingleClassificationResult<String> {
-        return calculateMetrics(
-            classification.map { stringProvider(it) }.toSet(),
-            groundTruth.map { stringProvider(it) }.toSet(),
-            confusionMatrixSum
-        )
-    }
+    ): SingleClassificationResult<String> = calculateMetrics(classification, groundTruth, stringProvider, confusionMatrixSum, DefaultBetas)
 
     /**
      * Calculates the metrics for the given classification.
      * @param T the type of classified elements
      * @param classification the classification
      * @param groundTruth the ground truth
+     * @param stringProvider a function to convert the classification and ground truth to strings
      * @param confusionMatrixSum the sum of the confusion matrix. If not provided, some metrics can't be calculated.
+     * @param betas the betas of the F-beta scores to calculate. Beta 1.0 is always calculated. Each beta must be finite and greater than 0.
      * @return the classification result
      */
     fun <T> calculateMetrics(
         classification: Set<T>,
         groundTruth: Set<T>,
-        confusionMatrixSum: Int?
-    ): SingleClassificationResult<T>
+        stringProvider: (T) -> String,
+        confusionMatrixSum: Int?,
+        betas: Collection<Double>
+    ): SingleClassificationResult<String> =
+        calculateMetrics(
+            classification.map { stringProvider(it) }.toSet(),
+            groundTruth.map { stringProvider(it) }.toSet(),
+            confusionMatrixSum,
+            betas
+        )
+
+    /**
+     * Aggregates the given classification results, weighting each result by the size of its ground truth.
+     * @param T the type of classified elements
+     * @param singleClassificationResults the classification results
+     * @return the aggregation of the classification results
+     */
+    fun <T> calculateAverages(singleClassificationResults: List<SingleClassificationResult<out T>>): ClassificationAggregationResult<T> =
+        calculateAverages(singleClassificationResults, null)
 
     /**
-     * Calculates the averages of the given classification results.
+     * Aggregates the given classification results.
+     * @param T the type of classified elements
+     * @param singleClassificationResults the classification results
+     * @param weights the weights for the classification results. If not provided, the size of the gold standard is used as weight. There must be
+     *                exactly one weight per result.
+     * @return the aggregation of the classification results
+     */
+    fun <T> calculateAverages(
+        singleClassificationResults: List<SingleClassificationResult<out T>>,
+        weights: List<Int>?
+    ): ClassificationAggregationResult<T> = calculateAverages(singleClassificationResults, weights, null)
+
+    /**
+     * Aggregates the given classification results.
+     *
+     * The macro and the weighted average of an F-beta score are the (weighted) mean of the F-beta scores of the single results, whereas the micro
+     * average is recalculated from the pooled confusion matrix.
+     *
+     * @param T the type of classified elements
      * @param singleClassificationResults the classification results
-     * @param weights the weights for the classification results. If not provided, the size of the gold standard is used as weight.
-     * @return the aggregated classification results
+     * @param weights the weights for the classification results. If not provided, the size of the gold standard is used as weight. There must be
+     *                exactly one weight per result.
+     * @param betas the betas of the F-beta scores to aggregate. If not provided, all betas of the given results are used. Beta 1.0 is always
+     *              included. A beta that is missing from a single result is recalculated from its precision and recall.
+     * @return the aggregation of the classification results
      */
-    fun calculateAverages(
-        singleClassificationResults: List<SingleClassificationResult<*>>,
-        weights: List<Int>? = null
-    ): List<AggregatedClassificationResult>
+    fun <T> calculateAverages(
+        singleClassificationResults: List<SingleClassificationResult<out T>>,
+        weights: List<Int>?,
+        betas: Collection<Double>?
+    ): ClassificationAggregationResult<T>
 }