From 381920526c0e7592a267a7f081f8e1f1554f9091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anna=20Hedstr=C3=B6m?= Date: Fri, 7 Feb 2025 17:21:54 +0000 Subject: [PATCH] gef starter --- README.md | 2 +- quantus/helpers/constants.py | 3 + quantus/helpers/enums.py | 1 + quantus/metrics/__init__.py | 1 + quantus/metrics/unified/__init__.py | 0 .../generalised_explanation_faithfulness.py | 58 +++++++++++++++++++ tox.ini | 6 +- 7 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 quantus/metrics/unified/__init__.py create mode 100644 quantus/metrics/unified/generalised_explanation_faithfulness.py diff --git a/README.md b/README.md index 0e37207c7..88f55a499 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ If you want to contribute/ improve/ extend Quantus, join our [Discord](https://d - 🐼 For **training data attribution** evaluation, check out [quanda](https://github.com/dilyabareeva/quanda)! - New [batch implementation](https://github.com/understandable-machine-intelligence-lab/Quantus/pull/351) for 12X speedup of existing faithfulness metrics (!) -- New metrics added: [EfficientMPRT](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/quantus/metrics/randomisation/efficient_mprt.py) and [SmoothMPRT](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/quantus/metrics/randomisation/smooth_mprt.py) by [Hedström et al., (2023)](https://openreview.net/pdf?id=vVpefYmnsG) +- New metric added: [Generalised Explanation Faithfulness (GEF)](https://github.com/understandable-machine-intelligence-lab/Quantus/blob/main/quantus/metrics/unified/generalised_explanation_faithfulness.py) by [Hedström et al., (2025]https://openreview.net/pdf?id=ukLxqA8zXj) - Accepted to Journal of Machine Learning Research (MLOSS), read the [paper](https://jmlr.org/papers/v24/22-0142.html) - Offers more than **35+ metrics in 6 categories** for XAI evaluation - Supports different data types (image, time-series, tabular, NLP next up!) and models (PyTorch, TensorFlow) diff --git a/quantus/helpers/constants.py b/quantus/helpers/constants.py index 5c1d68b3f..6597e860e 100644 --- a/quantus/helpers/constants.py +++ b/quantus/helpers/constants.py @@ -72,6 +72,9 @@ "NonSensitivity": NonSensitivity, "InputInvariance": InputInvariance, }, + "Unified": { + "Generalised Explanation Faithfulness": GeneralisedExplanationFaithfulness + } } diff --git a/quantus/helpers/enums.py b/quantus/helpers/enums.py index ca260cb15..cc285f77d 100644 --- a/quantus/helpers/enums.py +++ b/quantus/helpers/enums.py @@ -58,4 +58,5 @@ class EvaluationCategory(Enum): COMPLEXITY = "Complexity" LOCALISATION = "Localisation" AXIOMATIC = "Axiomatic" + UNIFIED = "Unified" NONE = "None" diff --git a/quantus/metrics/__init__.py b/quantus/metrics/__init__.py index f8003d017..98acaae92 100644 --- a/quantus/metrics/__init__.py +++ b/quantus/metrics/__init__.py @@ -11,3 +11,4 @@ from quantus.metrics.localisation import * from quantus.metrics.randomisation import * from quantus.metrics.robustness import * +from quantus.metrics.unified import * \ No newline at end of file diff --git a/quantus/metrics/unified/__init__.py b/quantus/metrics/unified/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/quantus/metrics/unified/generalised_explanation_faithfulness.py b/quantus/metrics/unified/generalised_explanation_faithfulness.py new file mode 100644 index 000000000..249844b40 --- /dev/null +++ b/quantus/metrics/unified/generalised_explanation_faithfulness.py @@ -0,0 +1,58 @@ +"""This module contains the implementation of the Completeness metric.""" + +# This file is part of Quantus. +# Quantus is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Quantus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +# You should have received a copy of the GNU Lesser General Public License along with Quantus. If not, see . +# Quantus project URL: . + +import sys +from typing import Any, Callable, Dict, List, Optional + +import numpy as np + +from quantus.functions.perturb_func import baseline_replacement_by_indices +from quantus.helpers import warn +from quantus.helpers.enums import ( + DataType, + EvaluationCategory, + ModelType, + ScoreDirection, +) +from quantus.helpers.model.model_interface import ModelInterface +from quantus.helpers.perturbation_utils import make_perturb_func +from quantus.metrics.base import Metric +from quantus.helpers.utils import identity + +if sys.version_info >= (3, 8): + from typing import final +else: + from typing_extensions import final + + +@final +class GeneralisedExplanationFaithfulness(Metric[List[float]]): + """ + Implementation of Generalised Explanation Faithfulness test by Hedström et al., 2025. + + Insert desription + + References: + 1) Hedström et al., "Evaluating Interpretable Methods via Geometric Alignment of Functional Distortions" + Transactions of Machine Learning Research, 2025. + + Attributes: + - _name: The name of the metric. + - _data_applicability: The data types that the metric implementation currently supports. + - _models: The model types that this metric can work with. + - score_direction: How to interpret the scores, whether higher/ lower values are considered better. + """ + + name = "GEF" + data_applicability = {DataType.IMAGE, DataType.TIMESERIES, DataType.TABULAR} + model_applicability = {ModelType.TORCH} + score_direction = ScoreDirection.HIGHER + evaluation_category = EvaluationCategory.UNIFIED + + def __init__() + pass \ No newline at end of file diff --git a/tox.ini b/tox.ini index aa678a730..cf8cf42f8 100644 --- a/tox.ini +++ b/tox.ini @@ -27,14 +27,14 @@ commands = [testenv:build] description = Build environment -base_python = py310 +base_python = python3.10 deps = . build twine commands = - python3 -m build . - python3 -m twine check ./dist/* --strict + python3.10 -m build . + python3.10 -m twine check ./dist/* --strict [testenv:lint] description = Check the code style