diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 40897b08..c21e320f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -10,9 +10,10 @@ on:
jobs:
build:
runs-on: ubuntu-latest
- timeout-minutes: 30 # minutes
+ timeout-minutes: 10 # minutes
if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
strategy:
+ max-parallel: 1
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
@@ -35,6 +36,12 @@ jobs:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
+ - name: Cache Hugging Face models
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/huggingface
+ key: hf-${{ runner.os }}-${{ hashFiles('.github/workflows/build.yml', 'pyproject.toml') }}
+
- name: Recreate virtual environment if cache is invalid
run: |
if [ ! -f .venv/bin/python ]; then
@@ -63,6 +70,9 @@ jobs:
AutoTokenizer,
)
+ # A test also needs a real tokenizer
+ AutoTokenizer.from_pretrained("bert-base-uncased")
+
# List all of the hf-internal-testing model IDs that your tests need:
model_ids = [
"hf-internal-testing/tiny-random-bert",
@@ -73,14 +83,14 @@ jobs:
"textattack/bert-base-uncased-imdb",
]
- # We will call .from_pretrained(...) once per model_id,
- # and also download its tokenizer. Because HF_HUB_TOKEN
- # is set in the environment, these calls are authenticated
+ # We will call .from_pretrained(...) once per model_id,
+ # and also download its tokenizer. Because HF_HUB_TOKEN
+ # is set in the environment, these calls are authenticated
# and will not exhaust the low unauthenticated rate limit.
- for m in set(model_ids):
- # Sequence classification or masked-LM vs causal-LM
- # is determined by your conftest/test code.
- # We can try all possible AutoModel types here, but
+ for m in set(model_ids):
+ # Sequence classification or masked-LM vs causal-LM
+ # is determined by your conftest/test code.
+ # We can try all possible AutoModel types here, but
# since your conftest already splits them out, we just do:
try:
AutoModelForSequenceClassification.from_pretrained(m)
@@ -107,8 +117,6 @@ jobs:
- name: Run style checks
run: |
source .venv/bin/activate
- make update-deps
- uv pip install -r requirements-dev.txt
make lint
- name: Run fast tests
diff --git a/.github/workflows/sync-overcomplete.yml b/.github/workflows/sync-overcomplete.yml
index 50214495..3808e0fc 100644
--- a/.github/workflows/sync-overcomplete.yml
+++ b/.github/workflows/sync-overcomplete.yml
@@ -26,6 +26,10 @@ jobs:
- name: Apply vendored patch
run: |
+ if [[ -z "$(git status --porcelain)" ]]; then
+ echo "No vendored changes to patch."
+ exit 0
+ fi
git apply --check interpreto/_vendor/overcomplete/overcomplete.patch
git apply --whitespace=fix interpreto/_vendor/overcomplete/overcomplete.patch
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 45d16752..b33e3109 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,7 +24,7 @@ repos:
exclude: LICENSE
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.14.14
+ rev: v0.15.10
hooks:
- id: ruff-format
- id: ruff
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000..845b23ff
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,214 @@
+# AGENTS.md
+
+## Goal
+
+`interpreto` is a modular interpretability toolkit for transformer models. The repository aims to provide:
+
+- an easy-to-use public API for attribution and concept-based explanations,
+- detailed documentation with concrete examples,
+- precise internal representations for tensors, targets, and activations,
+- reusable building blocks that can be combined without rewriting the whole pipeline.
+
+The main product surface is:
+
+- attribution methods for classification and generation,
+- concept discovery and concept interpretation workflows,
+- evaluation metrics,
+- HTML visualizations,
+- docs and notebooks showing real usage.
+
+## Repository Map
+
+- `interpreto/__init__.py`
+ - Curated public API. If a feature is meant to be user-facing, it usually belongs here too.
+- `interpreto/concepts.splitters/`
+ - Bridges raw Hugging Face models to Interpreto internals.
+ - `inference_wrapper.py`: shared batching, device handling, logits/gradient access, padding helpers.
+ - `classification_inference_wrapper.py`: targeted scoring for classification tasks.
+ - `generation_inference_wrapper.py`: targeted scoring for generation tasks.
+ - `model_with_split_points.py`: `nnsight`-based model splitting and activation extraction for concept methods.
+ - `llm_interface.py`: abstraction layer for LLM-based concept labeling.
+- `interpreto/attributions/`
+ - Attribution framework.
+ - `base.py`: shared explainers, normalization, output dataclasses, classification/generation glue.
+ - `methods/`: LIME, KernelShap, Occlusion, Sobol, Saliency, Integrated Gradients, SmoothGrad, etc.
+ - `perturbations/`: perturbation generators used by attribution methods.
+ - `aggregations/`: score aggregation logic.
+ - `metrics/`: insertion/deletion evaluation.
+- `interpreto/concepts/`
+ - Concept-based interpretability framework.
+ - `base.py`: base concept explainer interfaces.
+ - `methods/`: neurons-as-concepts, overcomplete/SAE methods, sklearn-based methods, Cockatiel.
+ - `interpretations/`: `TopKInputs`, `LLMLabels`, and related interpretation utilities.
+ - `metrics/`: reconstruction, sparsity, stability, and ConSim.
+- `interpreto/commons/`
+ - Shared utilities such as granularity handling, generator helpers, and distances.
+- `interpreto/typing.py`
+ - Central typing aliases and protocols. This file expresses the intended normalized internal shapes and interfaces.
+- `interpreto/visualizations/`
+ - HTML/CSS/JS renderers for attribution and concept outputs.
+ - Visualizations should consume normalized outputs, not recompute model logic.
+- `interpreto/_vendor/overcomplete/`
+ - Vendored dependency for concept learning backends. Avoid touching it unless the change really belongs there.
+- `tests/`
+ - Pytest suite. Reuse fixtures from `tests/conftest.py` whenever possible.
+- `docs/`
+ - MkDocs source, API pages, and notebooks.
+- `site/`
+ - Generated documentation output. Prefer editing `docs/`, not `site/`.
+
+## Key Dependencies
+
+- `torch`
+ - Core tensor and model execution backend.
+- `transformers`
+ - Main model/tokenizer interface and public compatibility target.
+- `nnsight`
+ - Used by `ModelWithSplitPoints` for split points and activation capture.
+- `jaxtyping` and `beartype`
+ - Preferred tools for explicit tensor typing and shape contracts.
+- `scikit-learn`, `scipy`, `einops`, `matplotlib`, `nltk`
+ - Supporting libraries for methods, metrics, preprocessing, and visualization.
+- `bitsandbytes`
+ - Compatibility with quantized transformer loading.
+- `mkdocs` stack
+ - Documentation build system.
+
+## How The Pieces Interact
+
+### Attribution pipeline
+
+User inputs can arrive in several formats: strings, tokenized mappings, tensors, or iterables of those. The code should normalize them early, then keep core computations on one internal format.
+
+Typical flow:
+
+1. User input and targets enter an attribution explainer from `interpreto.attributions`.
+2. The explainer normalizes inputs/targets in `attributions/base.py`.
+3. A perturbator or gradient path generates the computation stream.
+4. A task-specific inference wrapper computes targeted logits or gradients.
+5. An aggregator converts raw scores into final attribution values.
+6. The result is packaged as `AttributionOutput`.
+7. Metrics and visualizations consume `AttributionOutput`.
+
+Important style point: attribution code is intentionally generator-friendly. Many paths are designed to work sample by sample or batch by batch instead of materializing everything eagerly. Preserve that when making changes, especially for generation and prompt construction logic.
+
+### Concept pipeline
+
+Typical flow:
+
+1. `ModelWithSplitPoints` wraps a transformer model and exposes split points.
+2. `get_activations()` extracts latent activations at a chosen granularity.
+3. A concept explainer from `interpreto.concepts.methods` fits or applies a concept model on those activations.
+4. Interpretation methods such as `TopKInputs` or `LLMLabels` map concept dimensions to human-readable descriptions.
+5. Metrics and visualizations operate on the resulting concept-space artifacts.
+
+`ModelWithSplitPoints` is the bridge between the transformer world and concept methods. Most concept changes should respect that layering instead of bypassing it.
+
+### Granularity and normalization
+
+Granularity is a core abstraction shared across attribution and concept code. The code often accepts flexible user inputs, but should converge quickly toward:
+
+- normalized `TensorMapping`-style model inputs,
+- normalized target tensors,
+- normalized activation tensors,
+- normalized output dataclasses.
+
+This repository prefers a flexible public API and a stricter internal core.
+
+## Repository Vibe
+
+- Keep the public API easy to use.
+ - Users may provide several input formats.
+ - Internal computations should still be normalized into a single clear format as early as possible.
+- Prefer precise typing.
+ - `jaxtyping` is valuable here because tensor shapes matter a lot for readability and debugging.
+ - Be pragmatic at boundaries with `transformers` and `nnsight`; do not make the code worse just to force shape annotations through awkward external APIs.
+- Documentation matters.
+ - Detailed docstrings, examples, file-level comments, and inline comments are a feature of the repository, not noise.
+ - When adding or changing logic, explain the shape conventions and the intent, especially around generators, token alignment, split points, and concept encoding.
+- The repository is modular.
+ - Prefer plug-and-play building blocks over special-purpose monoliths.
+ - Reuse wrappers, perturbators, aggregators, metrics, and visualization outputs rather than duplicating logic.
+- Prefer one place for validation.
+ - Do not add repeated guardrails in every layer if the check already belongs at the public boundary or is already enforced by typing/contracts.
+ - Re-check only if a lower-level function can be called independently or if the invariant genuinely changes.
+- Smaller changes are usually better.
+ - Do not refactor by default.
+ - If a minimal patch would conflict with the method/class/repository design, then do the slightly larger coherent refactor instead of adding a local hack.
+- Keep implementations efficient but simple.
+ - Prefer straightforward Torch code.
+ - If a much faster version would add a lot of complexity, it is often better to land the clean version first and leave a focused `TODO`.
+- In attribution code, preserve the generator-based pipeline mindset.
+ - The repository often processes attribution sample by sample, while trying to construct good prompts and avoid unnecessary materialization.
+
+## Coding Expectations
+
+- Write docstrings and the important inline comments at the same time as the code change, or before.
+- Prefer file-level comments when the whole module has a specific role or subtle invariant.
+- Keep internal data formats explicit.
+- If adding a new public class or function, check whether it should be re-exported in a package `__init__.py` and documented in `docs/`.
+- Use the existing module boundaries.
+ - New attribution methods usually belong in `interpreto/attributions/methods/`.
+ - New perturbation logic belongs in `interpreto/attributions/perturbations/`.
+ - New concept methods belong in `interpreto/concepts/methods/`.
+ - New interpretation strategies should use the existing concept explainer interfaces.
+
+## Tests
+
+Testing style in this repository is usually a mix of:
+
+- method-level tests for specific algorithmic behavior,
+- class-level tests for API and integration behavior,
+- sanity checks for end-to-end invariants.
+
+Guidelines:
+
+- For a new feature, test-driven development is preferred when practical.
+- Keep tests reviewable. Do not add large numbers of nearly identical tests.
+- Be very clear in test comments/docstrings about what the test is proving.
+- Reuse `tests/conftest.py`, `tests/fixtures/`, and existing helpers before inventing new scaffolding.
+- Prefer `hf-internal-testing/*` tiny models over large custom placeholders or long fake model definitions.
+- Do not test the same invariant in many places unless it protects distinct call paths.
+
+## Change Workflow For Agents
+
+1. Think first.
+ - Understand which layer should change.
+ - Prefer the smallest coherent modification.
+ - If the design tradeoff is uncertain, it is better to ask for an opinion than to guess.
+2. Add or update tests.
+ - For new features or bug fixes, start from the behavior you want to lock in.
+ - Reuse fixtures and tiny test models whenever possible.
+3. Implement the change.
+ - Keep the code aligned with existing abstractions.
+ - Avoid clever one-off tricks that only satisfy the immediate patch.
+4. Update documentation if needed.
+ - Public API changes usually need docstring and docs updates.
+ - Example-driven documentation is part of the repository style.
+5. Verify with targeted commands first.
+
+Useful commands:
+
+- `make install-dev`
+- `make lint`
+- `make fast-test`
+- `make test-cpu`
+- `python -m pytest -n auto -c pyproject.toml -v path/to/test_file.py`
+
+## Practical Do / Don't
+
+Do:
+
+- Normalize flexible user inputs into one internal format early.
+- Use `jaxtyping` where it improves shape clarity.
+- Preserve generator-based or streaming-friendly flows.
+- Add comments where tensor shapes, batching, or prompt construction are non-obvious.
+- Favor small coherent patches.
+
+Don't:
+
+- Add redundant guardrails in every layer.
+- Materialize huge intermediate lists if the existing pipeline is intentionally iterable/generator-based.
+- Refactor broadly without a concrete design reason.
+- Fight external library APIs just to satisfy an idealized typing style.
+- Edit generated docs in `site/` when the real source lives in `docs/`.
diff --git a/README.md b/README.md
index 95e081c9..9991f7a2 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,7 @@
📚 Explore Interpreto docs >>
🖼️ Checkout our explanation gallery >>
+ 📜 Read our paper >>
## 🚀 Quick Start
@@ -58,41 +59,51 @@ They all work seamlessly for both classification (`...ForSequenceClassification`
Concept-based explanations aim to provide high-level interpretations of latent model representations.
-Interpreto generalizes these methods through four core steps:
+We propose both supervised (probes and CAVs) and unsupervised (dictionary learning) approaches.
+
+Interpreto generalizes these methods through four core steps, the two first are common between both approaches:
1. Split a model in two and obtain a dataset of activations
-2. Concept Discovery (e.g., from latent embeddings)
-3. Concept Interpretation (mapping discovered concepts to human-understandable elements)
-4. Concept-to-Output Attribution (assessing concept relevance to model outputs)
+2. Learn concepts (e.g., from latent embeddings)
+3. Interpret concepts (mapping discovered concepts to human-understandable elements)
+4. Estimate concepts importance (assessing concept relevance to model outputs)
**1. Split a model in two and obtain a dataset of activations:** (mainly via [`nnsight`](https://github.com/ndif-team/nnsight)):
Choose any layer in any HuggingFace language model with our `ModelWithSplitPoints` based on `nnsight`. Then pass a dataset through it to obtain a dataset of activations.
-**2. Dictionary Learning for Concept Discovery** (mainly via [`overcomplete`](https://github.com/KempnerInstitute/overcomplete)):
+**2. (supervised) Train probe** with the [`ProbeExplainer`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/)
+
+We differentiate two families of probes:
+
+- Linear probes: [`LinearRegressionProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LinearRegressionProbe), [`LogisticRegressionProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LogisticRegressionProbe), [`LinearSVMProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LinearSVMProbe), [`MeansDiffProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.MeansDiffProbe)
+- Centroid-based probes: [`CosineCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.CosineCentroidProbe), [`DotProductCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.DotProductCentroidProbe), [`SqL2CentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.SqL2CentroidProbe), [`SVDDCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.SVDDCentroidProbe), [`DiagonalMahalanobisCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.DiagonalMahalanobisCentroidProbe)
+
+Both can be tuned with `bias_calibrator` and `normalization` parameters.
+
+**2. (unsupervised) Dictionary Learning for Concept Discovery** (mainly via [`overcomplete`](https://github.com/KempnerInstitute/overcomplete)):
- Interpret neurons directly via [`NeuronsAsConcepts`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/neurons_as_concepts/)
- [`NMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.NMFConcepts), [`Semi-NMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.SemiNMFConcepts), [`ConvexNMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.ConvexNMFConcepts)
- [`ICA`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.ICAConcepts), [`SVD`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.SVDConcepts), [`PCA`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.PCAConcepts), [`KMeans`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.KMeansConcepts)
- SAE variants: [`Vanilla SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.VanillaSAEConcepts), [`TopK SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.TopKSAEConcepts), [`JumpReLU SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.JumpReLUSAEConcepts), [`BatchTopK SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.BatchTopKSAEConcepts)
-**3. Available Concept Interpretation Techniques:**
+**3. (unsupervised) Available Concept Interpretation Techniques:**
- Top-k tokens from tokenizer vocabulary via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.TopKInputs) and `use_vocab=True`
- Top-k tokens/words/sentences/samples from specific datasets via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.TopKInputs)
- Label concepts via LLMs with [`LLMLabels`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.LLMLabels) ([Bills et al. 2023](https://openai.com/index/language-models-can-explain-neurons-in-language-models/))
+- Input-to-concept attribution from dataset examples ([Concept Attributions](https://for-sight-ai.github.io/interpreto/api/concepts/interpretations/concept_attributions/)) ([Jourdan et al. 2023](https://aclanthology.org/2023.findings-acl.317/))
Concept Interpretation Techniques Added in the future:
-- Input-to-concept attribution from dataset examples ([Jourdan et al. 2023](https://aclanthology.org/2023.findings-acl.317/))
-- Theme prediction via LLMs from top-k tokens/sentences
- Aligning concepts with human labels ([Sajjad et al. 2022](https://aclanthology.org/2022.naacl-main.225/))
- Word cloud visualizations of concepts ([Dalvi et al. 2022](https://arxiv.org/abs/2205.07237))
- VocabProj & TokenChange ([Gur-Arieh et al. 2025](https://arxiv.org/abs/2501.08319))
🖼️ Checkout our explanation gallery >>
+ 📜 Read our paper >>
## 🚀 Quick Start
@@ -62,43 +63,53 @@ They all work seamlessly for both classification (`...ForSequenceClassification`
Concept-based explanations aim to provide high-level interpretations of latent model representations.
-Interpreto generalizes these methods through four core steps:
+We propose both supervised (probes and CAVs) and unsupervised (dictionary learning) approaches.
+
+Interpreto generalizes these methods through four core steps, the two first are common between both approaches:
1. Split a model in two and obtain a dataset of activations
-2. Concept Discovery (e.g., from latent embeddings)
-3. Concept Interpretation (mapping discovered concepts to human-understandable elements)
-4. Concept-to-Output Attribution (assessing concept relevance to model outputs)
+2. Learn concepts (e.g., from latent embeddings)
+3. Interpret concepts (mapping discovered concepts to human-understandable elements)
+4. Estimate concepts importance (assessing concept relevance to model outputs)
**1. Split a model in two and obtain a dataset of activations:** (mainly via [`nnsight`](https://github.com/ndif-team/nnsight)):
Choose any layer in any HuggingFace language model with our `ModelWithSplitPoints` based on `nnsight`. Then pass a dataset through it to obtain a dataset of activations.
-**2. Dictionary Learning for Concept Discovery** (mainly via [`overcomplete`](https://github.com/KempnerInstitute/overcomplete)):
+**2. (supervised) Train probe** with the [`ProbeExplainer`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/)
+
+We differentiate two families of probes:
+
+- Linear probes: [`LinearRegressionProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LinearRegressionProbe), [`LogisticRegressionProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LogisticRegressionProbe), [`LinearSVMProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.LinearSVMProbe), [`MeansDiffProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.MeansDiffProbe)
+- Centroid-based probes: [`CosineCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.CosineCentroidProbe), [`DotProductCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.DotProductCentroidProbe), [`SqL2CentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.SqL2CentroidProbe), [`SVDDCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.SVDDCentroidProbe), [`DiagonalMahalanobisCentroidProbe`](https://for-sight-ai.github.io/interpreto/api/concepts/probes/#interpreto.concepts.probes.DiagonalMahalanobisCentroidProbe)
+
+Both can be tuned with `bias_calibrator` and `normalization` parameters.
+
+**2. (unsupervised) Dictionary Learning for Concept Discovery** (mainly via [`overcomplete`](https://github.com/KempnerInstitute/overcomplete)):
- Interpret neurons directly via [`NeuronsAsConcepts`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/neurons_as_concepts/)
- [`NMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.NMFConcepts), [`Semi-NMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.SemiNMFConcepts), [`ConvexNMF`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.ConvexNMFConcepts)
- [`ICA`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.ICAConcepts), [`SVD`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.SVDConcepts), [`PCA`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.PCAConcepts), [`KMeans`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/optim/#interpreto.concepts.KMeansConcepts)
- SAE variants: [`Vanilla SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.VanillaSAEConcepts), [`TopK SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.TopKSAEConcepts), [`JumpReLU SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.JumpReLUSAEConcepts), [`BatchTopK SAE`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/sae/#interpreto.concepts.BatchTopKSAEConcepts)
-**3. Available Concept Interpretation Techniques:**
+**3. (unsupervised) Available Concept Interpretation Techniques:**
-- Top-k tokens from tokenizer vocabulary via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.TopKInputs) and `use_vocab=True`
-- Top-k tokens/words/sentences/samples from specific datasets via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.TopKInputs)
-- Label concepts via LLMs with [`LLMLabels`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.LLMLabels) ([Bills et al. 2023](https://openai.com/index/language-models-can-explain-neurons-in-language-models/))
+- Top-k tokens from tokenizer vocabulary via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/interpretations/topk_inputs/#interpreto.concepts.interpretations.TopKInputs) and `use_vocab=True`
+- Top-k tokens/words/sentences/samples from specific datasets via [`TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/interpretations/topk_inputs/#interpreto.concepts.interpretations.TopKInputs)
+- Label concepts via LLMs with [`LLMLabels`](https://for-sight-ai.github.io/interpreto/api/concepts/interpretations/llm_labels/#interpreto.concepts.interpretations.LLMLabels) ([Bills et al. 2023](https://openai.com/index/language-models-can-explain-neurons-in-language-models/))
+- Input-to-concept attributions via perturbation methods ([Concept Attributions](https://for-sight-ai.github.io/interpreto/api/concepts/interpretations/concept_attributions/)) ([Jourdan et al. 2023](https://aclanthology.org/2023.findings-acl.317/))
Concept Interpretation Techniques Added in the future:
-- Input-to-concept attribution from dataset examples ([Jourdan et al. 2023](https://aclanthology.org/2023.findings-acl.317/))
-- Theme prediction via LLMs from top-k tokens/sentences
- Aligning concepts with human labels ([Sajjad et al. 2022](https://aclanthology.org/2022.naacl-main.225/))
- Word cloud visualizations of concepts ([Dalvi et al. 2022](https://arxiv.org/abs/2205.07237))
- VocabProj & TokenChange ([Gur-Arieh et al. 2025](https://arxiv.org/abs/2501.08319))
-**4. Concept-to-Output Attribution:**
+**4. (unsupervised) Concept-to-Output Attribution:**
Estimate the contribution of each concept to the model output.
-Can be obtained with any concept-based explainer via [`MethodConcepts.concept_output_gradient()`](https://for-sight-ai.github.io/interpreto/api/concepts/methods/base/#interpreto.concepts.ConceptAutoEncoderExplainer.concept_output_gradient).
+Can be obtained with any concept-based explainer via [`MethodConcepts.concept_output_gradient()`](https://for-sight-ai.github.io/interpreto/api/concepts/concept_spaces/base/#interpreto.concepts.ConceptAutoEncoderExplainer.concept_output_gradient).
**Specific methods:**
@@ -106,7 +117,6 @@ Thanks to this generalization encompassing all concept-based methods and our hig
The following list will **soon be available**:
-- CAV and TCAV: [Kim et al. 2018, Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV)](http://proceedings.mlr.press/v80/kim18d.html)
- ConceptSHAP: [Yeh et al. 2020, On Completeness-aware Concept-Based Explanations in Deep Neural Networks](https://proceedings.neurips.cc/paper/2020/hash/ecb287ff763c169694f682af52c1f309-Abstract.html)
- COCKATIEL: [Jourdan et al. 2023, COCKATIEL: COntinuous Concept ranKed ATtribution with Interpretable ELements for explaining neural net classifiers on NLP](https://aclanthology.org/2023.findings-acl.317/)
- Yun et al. 2021, [Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors](https://arxiv.org/abs/2103.15949)
@@ -161,7 +171,7 @@ Interpreto 🪄 is a project of the [FOR](https://www.irt-saintexupery.com/fr/fo
## 🗞️ Citation
-If you use Interpreto 🪄 as part of your workflow in a scientific publication, please consider citing 🗞️ our paper:
+If you use Interpreto 🪄 as part of your workflow in a scientific publication, please consider citing 🗞️ [our paper](https://arxiv.org/abs/2512.09730):
```bibtex
@article{poche2025interpreto,
diff --git a/docs/notebooks/attribution_walkthrough.ipynb b/docs/notebooks/attribution_tutorial.ipynb
similarity index 99%
rename from docs/notebooks/attribution_walkthrough.ipynb
rename to docs/notebooks/attribution_tutorial.ipynb
index 3b5c674e..ab67d840 100644
--- a/docs/notebooks/attribution_walkthrough.ipynb
+++ b/docs/notebooks/attribution_tutorial.ipynb
@@ -31365,7 +31365,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": ".venv (3.12.3)",
+ "display_name": ".venv",
"language": "python",
"name": "python3"
},
diff --git a/docs/notebooks/classification_concept_tutorial.ipynb b/docs/notebooks/classification_concept_tutorial.ipynb
index c696f966..4b58a909 100644
--- a/docs/notebooks/classification_concept_tutorial.ipynb
+++ b/docs/notebooks/classification_concept_tutorial.ipynb
@@ -48,7 +48,11 @@
"\n",
"We choose a `DistilBERT` fine-tuned on the `AG-News` dataset and split it just before the classification head.\n",
"\n",
- "To split the model, we use the [`interpreto.ModelWithSplitPoints`](https://for-sight-ai.github.io/interpreto/api/concepts/model_with_split_points/) which wraps around the `transformers` model and allows the computation of activations at the specified `split_points`."
+ "To split the model, we use the [`interpreto.SplitSequenceClassification`](https://for-sight-ai.github.io/interpreto/api/concepts/splitter_for_classification/) which wraps around the `transformers` model.\n",
+ "\n",
+ "It splits the model at the [CLS] token, thus the first part is the encoder and the second the classification head. Which is the setup we highly recommend for interpretability.\n",
+ "\n",
+ "For other settings, we invite the user to use the [`interpreto.ModelWithSplitPoints`](https://for-sight-ai.github.io/interpreto/api/concepts/model_with_split_points/) instead (parent of the above class)."
]
},
{
@@ -57,16 +61,12 @@
"metadata": {},
"outputs": [],
"source": [
- "from transformers import AutoModelForSequenceClassification\n",
+ "from interpreto import SplitterForClassification\n",
"\n",
- "from interpreto import ModelWithSplitPoints\n",
- "\n",
- "model_with_split_points = ModelWithSplitPoints(\n",
+ "splitter = SplitterForClassification(\n",
" model_or_repo_id=\"textattack/distilbert-base-uncased-ag-news\",\n",
- " automodel=AutoModelForSequenceClassification,\n",
- " split_points=[5], # split at the sixth layer\n",
" device_map=\"cuda\",\n",
- " batch_size=1024,\n",
+ " batch_size=64,\n",
")"
]
},
@@ -76,43 +76,57 @@
"source": [
"## 2. 🚦 Compute a datasets of **activations** \n",
"\n",
- "We load the first 10000 documents of the `AG-News` train set.\n",
+ "We load the `AG-News` train set.\n",
"\n",
"Then we extract the activations of the [CLS] token of each document.\n",
"\n",
- "> ➡️ **Common practice**\n",
- ">\n",
- "> In the literature, to train concepts for classification it is common to use the [CLS] just before the classification head.\n",
- ">\n",
- "> In fact, at this layer, it makes no sense to use other elements.\n",
- "\n",
- "> ⚠️ **Warning**\n",
- ">\n",
- "> In this notebook, many things are specific to the use of the [CLS] token.\n",
- "\n",
- "[`interpreto.ModelWithSplitPoints.get_activations()`](https://for-sight-ai.github.io/interpreto/api/concepts/model_with_split_points/#interpreto.ModelWithSplitPoints.get_activations)"
+ "[`interpreto.SplitSequenceClassification.get_activations()`](https://for-sight-ai.github.io/interpreto/api/concepts/split_sequence_classification/#interpreto.SplitSequenceClassification.get_activations)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/1875 [00:00 ⚠️ **Warning**\n",
- ">\n",
- "> If the `granularity` specified to the interpretation method is not the same as the one used for activations, the results will be wrong."
+ "In this case, we will use the [`interpreto.concepts.interpretations.TopKInputs`](https://for-sight-ai.github.io/interpreto/api/concepts/concepts_interpretations/#interpreto.concepts.interpretations.TopKInputs) to find the 8 words which activates the most our concepts."
]
},
{
@@ -178,8 +188,7 @@
"topk_inputs_method = TopKInputs(\n",
" concept_explainer=concept_explainer,\n",
" k=5,\n",
- " activation_granularity=granularity,\n",
- " use_unique_words=True, # with the [CLS] token granularity, we are forced to use unique words\n",
+ " use_unique_words=3, # for classification we are force to use unique words or ngrams\n",
" unique_words_kwargs={\n",
" \"count_min_threshold\": round(\n",
" len(inputs) * 0.002\n",
@@ -231,38 +240,45 @@
"execution_count": 7,
"metadata": {},
"outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "100%|██████████| 1875/1875 [00:02<00:00, 710.88it/s]\n"
+ ]
+ },
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Class: World:\n",
- "\tconcept id: 27,\timportance: 0.091,\ttopk words: ['serbia-montenegro', 'nato', 'liechtenstein', 'nikkei', 'ossetia']\n",
- "\tconcept id: 45,\timportance: 0.078,\ttopk words: ['separatist', 'militia', 'militiaman', 'usatoday.com', 'inquirer']\n",
- "\tconcept id: 11,\timportance: 0.07,\ttopk words: ['betting', 'saudi', 'gambler', 'shark', 'kidnapper']\n",
- "\tconcept id: 31,\timportance: 0.064,\ttopk words: ['betting', 'fraud', 'holy', 'kmart', 'p.m.']\n",
- "\tconcept id: 48,\timportance: 0.041,\ttopk words: ['afp', 'armed', 'hue', 'anarchist', 'naval']\n",
+ "\tconcept id: 4,\timportance: 0.085,\ttopk words: ['baghdad ,', 'hamid karzai', 'al-sadr', 'of baghdad', 'baghdad']\n",
+ "\tconcept id: 42,\timportance: 0.049,\ttopk words: ['foreign minister', 'west bank', 'iraq and', 'darfur', 'in darfur']\n",
+ "\tconcept id: 12,\timportance: 0.049,\ttopk words: ['security council', 'the un', 'european union', 'european commission', 'china # 39']\n",
+ "\tconcept id: 11,\timportance: 0.045,\ttopk words: ['al-sadr', 'cbs.mw ) --', 'of iraq', 'iraq', 'iraq -']\n",
+ "\tconcept id: 31,\timportance: 0.045,\ttopk words: ['vodafone', 'verizon', 'oracle corp.', '( nasdaq :', 'google inc.']\n",
"\n",
"Class: Sports:\n",
- "\tconcept id: 7,\timportance: 0.11,\ttopk words: ['phelps', '200-meter', 'batter', 'inning', 'homered']\n",
- "\tconcept id: 49,\timportance: 0.092,\ttopk words: ['heat', '100-meter', 'fastest', '200-meter', '200m']\n",
- "\tconcept id: 30,\timportance: 0.087,\ttopk words: ['200-meter', '100-meter', 'breaststroke', '400-meter', 'heat']\n",
- "\tconcept id: 33,\timportance: 0.069,\ttopk words: ['phillies', 'mets', 'baltimore', 'sox', 'nl']\n",
- "\tconcept id: 24,\timportance: 0.062,\ttopk words: ['stryker', 'nfl', 'armadillo', 'homer', 'autodesk']\n",
+ "\tconcept id: 8,\timportance: 0.087,\ttopk words: [', fla.', 'gt ; ...', 'west bank', 'on wednesday .', 'on saturday .']\n",
+ "\tconcept id: 41,\timportance: 0.084,\ttopk words: ['nl', 'ap ) ap', '; a href=', 'pacer', 'real madrid']\n",
+ "\tconcept id: 21,\timportance: 0.072,\ttopk words: [', fla.', 'west bank', 'cbs.mw ) --', ', quot ;', '. quot ;']\n",
+ "\tconcept id: 19,\timportance: 0.067,\ttopk words: ['real madrid', 'manchester united', 'arsenal', 'chelsea', ', quot ;']\n",
+ "\tconcept id: 20,\timportance: 0.066,\ttopk words: ['. quot ;', ', quot ;', 'third-quarter', 'on tuesday ,', 'the third quarter']\n",
"\n",
"Class: Business:\n",
- "\tconcept id: 13,\timportance: 0.084,\ttopk words: ['pharmacare', 'procurement', 'costly', 'sector', 'marketer']\n",
- "\tconcept id: 28,\timportance: 0.068,\ttopk words: ['vodafone', 'ipo.google.com', 'antitrust', 'verizon', 'lenovo']\n",
- "\tconcept id: 19,\timportance: 0.046,\ttopk words: [';', 'ott', 'atp', 'fcc', '3g']\n",
- "\tconcept id: 7,\timportance: 0.042,\ttopk words: ['phelps', '200-meter', 'batter', 'inning', 'homered']\n",
- "\tconcept id: 37,\timportance: 0.04,\ttopk words: ['eurozone', 'finance', 'imf', 'economy', 'balance']\n",
+ "\tconcept id: 33,\timportance: 0.053,\ttopk words: ['shopper', 'wal-mart', 'oracle corp.', 'store inc.', 'prime minister tony']\n",
+ "\tconcept id: 16,\timportance: 0.052,\ttopk words: ['# 151 ;', 'pfizer', 'energy', 'gt ; ,', '; ,']\n",
+ "\tconcept id: 31,\timportance: 0.052,\ttopk words: ['vodafone', 'verizon', 'oracle corp.', '( nasdaq :', 'google inc.']\n",
+ "\tconcept id: 8,\timportance: 0.045,\ttopk words: [', fla.', 'gt ; ...', 'west bank', 'on wednesday .', 'on saturday .']\n",
+ "\tconcept id: 20,\timportance: 0.042,\ttopk words: ['. quot ;', ', quot ;', 'third-quarter', 'on tuesday ,', 'the third quarter']\n",
"\n",
"Class: Sci/Tech:\n",
- "\tconcept id: 7,\timportance: 0.057,\ttopk words: ['phelps', '200-meter', 'batter', 'inning', 'homered']\n",
- "\tconcept id: 49,\timportance: 0.053,\ttopk words: ['heat', '100-meter', 'fastest', '200-meter', '200m']\n",
- "\tconcept id: 13,\timportance: 0.052,\ttopk words: ['pharmacare', 'procurement', 'costly', 'sector', 'marketer']\n",
- "\tconcept id: 27,\timportance: 0.05,\ttopk words: ['serbia-montenegro', 'nato', 'liechtenstein', 'nikkei', 'ossetia']\n",
- "\tconcept id: 30,\timportance: 0.046,\ttopk words: ['200-meter', '100-meter', 'breaststroke', '400-meter', 'heat']\n"
+ "\tconcept id: 8,\timportance: 0.054,\ttopk words: [', fla.', 'gt ; ...', 'west bank', 'on wednesday .', 'on saturday .']\n",
+ "\tconcept id: 41,\timportance: 0.048,\ttopk words: ['nl', 'ap ) ap', '; a href=', 'pacer', 'real madrid']\n",
+ "\tconcept id: 21,\timportance: 0.043,\ttopk words: [', fla.', 'west bank', 'cbs.mw ) --', ', quot ;', '. quot ;']\n",
+ "\tconcept id: 19,\timportance: 0.041,\ttopk words: ['real madrid', 'manchester united', 'arsenal', 'chelsea', ', quot ;']\n",
+ "\tconcept id: 39,\timportance: 0.041,\ttopk words: ['peoplesoft inc.', 'saturn', 'microsoft corp.', 'hewlett-packard', 'oracle corp.']\n"
]
}
],
@@ -271,11 +287,9 @@
"\n",
"# estimate the importance of concepts for each class using the gradient\n",
"gradients = concept_explainer.concept_output_gradient(\n",
- " inputs=inputs,\n",
+ " inputs=activations,\n",
" targets=None, # None means all classes\n",
- " activation_granularity=granularity,\n",
- " concepts_x_gradients=True, # the concept to output gradients are multiplied by the concepts values, this is common practice in the literature\n",
- " batch_size=64,\n",
+ " tqdm_bar=True,\n",
")\n",
"\n",
"# stack gradients on samples and average them over samples\n",
@@ -2848,15 +2862,15 @@
"})();\n",
"\n",
"\n",
- "Classes
\n",
- "\n",
+ "Classes
\n",
+ "\n",
"\n",
" \n",
@@ -2936,7 +2950,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 19,
"metadata": {},
"outputs": [
{
@@ -2945,40 +2959,40 @@
"text": [
"\n",
"Class: World\n",
- "\timportance: 0.129,\tConcise pattern: Formal, informational language with emphasis on factual reporting and descriptive detail.\n",
- "\timportance: 0.111,\tEvent-focused, concise, multi-word summaries highlighting key entities, actions, or dynamics.\n",
- "\timportance: 0.086,\tEvent-focused, descriptive summaries emphasizing struggle, health, and rituals.\n",
- "\timportance: 0.081,\tRepetitive political and event reporting\n",
- "\timportance: 0.072,\tHighly satirical, exaggerated, humorous patterns.\n",
+ "\timportance: 0.081,\tEvent-focused, third-person news reporting.\n",
+ "\timportance: 0.076,\tMilitary reporting patterns.\n",
+ "\timportance: 0.07,\tAggressive, crime-related, global or political themes, and focus on terrorism or security.\n",
+ "\timportance: 0.069,\tEvent emphasis on hostage releases and geopolitical tensions.\n",
+ "\timportance: 0.066,\tData reflects political, social, and ethnic themes with patterns of elections, protests, and minority issues.\n",
"\n",
"Class: Sports\n",
- "\timportance: 0.094,\tEvent reporting emphasizes action and outcomes, often highlighting surprises, victories, or controversies, with a focus on dynamics and conflicts.\n",
- "\timportance: 0.088,\tConsistent references to named entities, events, and competitions with minimal descriptive language.\n",
- "\timportance: 0.085,\tEvent summaries with specific details and recent developments.\n",
- "\timportance: 0.084,\tConcise pattern: Event descriptions with structured reporting, including participants, outcomes, and sometimes specific details or quotes.\n",
- "\timportance: 0.076,\tEvent-focused, sports-related, specific terminology.\n",
+ "\timportance: 0.075,\tRepetitive sports and news reporting language, with frequent mention of names, scores, and events.\n",
+ "\timportance: 0.073,\tSports reporting focuses on events, scores, and player movements.\n",
+ "\timportance: 0.072,\tStructured sports and news reports with embedded factual details.\n",
+ "\timportance: 0.07,\tScores and events.\n",
+ "\timportance: 0.069,\tConcise race and sports event descriptions, utilizing brief summaries and key details.\n",
"\n",
"Class: Business\n",
- "\timportance: 0.117,\tConcise pattern: Economic focus on data-driven, factual reporting.\n",
- "\timportance: 0.105,\tDiverse topics centered on socio-cultural, economic, and urban themes, with emphasis on local identities and modernization.\n",
- "\timportance: 0.099,\tDiverse topics and sources, variable detail, consistent news reporting style.\n",
- "\timportance: 0.094,\tTopic-specific keywords.\n",
- "\timportance: 0.085,\tConsistent focus on finance, investments, or economic topics.\n",
+ "\timportance: 0.094,\tFinancial and corporate events center on numerical data, specific terminology, and formal reporting.\n",
+ "\timportance: 0.084,\tStructured, financial news summaries with embedded hyperlinks.\n",
+ "\timportance: 0.078,\tConsistent use of non-standard character sequences, especially \"#39;\", replacing apostrophes, and irregular formatting.\n",
+ "\timportance: 0.076,\tConsistent mentions of acquisitions, bids, and mergers involving corporations and assets, often linked to specific countries and financial figures.\n",
+ "\timportance: 0.073,\tGlobal commodity and energy market shifts; industry and corporate financials.\n",
"\n",
"Class: Sci/Tech\n",
- "\timportance: 0.13,\tData-driven scientific advancements and technical developments\n",
- "\timportance: 0.116,\tTechnology focus on communication, regulation, security.\n",
- "\timportance: 0.102,\tConcise, global phenomena or scientific explanations.\n",
- "\timportance: 0.102,\tDiverse technical and scientific descriptions with consistent focus on innovations, mechanisms, and discoveries.\n",
- "\timportance: 0.064,\tConcise pattern: Commercial language with factual tone.\n"
+ "\timportance: 0.098,\tTechnical innovation focus\n",
+ "\timportance: 0.085,\tTechnological, informative language with frequent references to innovation, products, and research.\n",
+ "\timportance: 0.075,\tConsistent focus on handheld gaming devices; mentions of release dates, prices, sales, and market rivalry.\n",
+ "\timportance: 0.067,\tConcise style, technical vocabulary, objectivity\n",
+ "\timportance: 0.062,\tFact-based, formal reporting style; frequent use of specific details and proper nouns; consistent IMF-style sentence structure; focus on significant events and official statements.\n"
]
}
],
"source": [
"import os\n",
"\n",
+ "from interpreto.commons.llm_interface import OpenAILLM\n",
"from interpreto.concepts import LLMLabels, SemiNMFConcepts\n",
- "from interpreto.model_wrapping.llm_interface import OpenAILLM\n",
"\n",
"# Load API key from environment variable\n",
"api_key = os.getenv(\"OPENAI_API_KEY\")\n",
@@ -3000,21 +3014,22 @@
"for target, class_name in enumerate(classes_names):\n",
" # ----------------------------------------------------------------------------------------------\n",
" # 2. construct the dataset of activations (extract the ones related to the class)\n",
- " indices = (activations[\"predictions\"] == target).nonzero(as_tuple=True)[0]\n",
+ " indices = (predictions == target).nonzero(as_tuple=True)[0]\n",
+ " if not len(indices):\n",
+ " continue\n",
" class_wise_inputs = [inputs[i] for i in indices]\n",
- " class_wise_activations = {k: v[indices] for k, v in activations.items()}\n",
+ " class_wise_activations = activations[indices]\n",
"\n",
" # ----------------------------------------------------------------------------------------------\n",
" # 3. train concept model\n",
- " concept_explainers[target] = SemiNMFConcepts(model_with_split_points, nb_concepts=20, device=\"cuda\")\n",
+ " concept_explainers[target] = SemiNMFConcepts(splitter, nb_concepts=20, device=\"cuda\")\n",
" concept_explainers[target].fit(class_wise_activations)\n",
"\n",
" # ----------------------------------------------------------------------------------------------\n",
" # 5. compute concepts importance (before interpretations to limit the number of concepts interpreted)\n",
" gradients = concept_explainers[target].concept_output_gradient(\n",
- " inputs=class_wise_inputs,\n",
+ " inputs=class_wise_activations,\n",
" targets=[target],\n",
- " activation_granularity=granularity,\n",
" concepts_x_gradients=True,\n",
" batch_size=64,\n",
" )\n",
@@ -3029,7 +3044,6 @@
" # 4. interpret the important concepts concepts\n",
" llm_labels_method = LLMLabels(\n",
" concept_explainer=concept_explainers[target],\n",
- " activation_granularity=granularity,\n",
" llm_interface=llm_interface,\n",
" k_examples=20,\n",
" )\n",
@@ -3049,7 +3063,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
@@ -5601,15 +5615,15 @@
"})();\n",
"\n",
"\n",
- "Classes
\n",
- "\n",
+ "Classes
\n",
+ "\n",
"\n",
" \n",
@@ -5638,6 +5652,8 @@
"source": [
"## 6. 📍 **Locally** important concepts \n",
"\n",
+ "### 6.1 Local concepts-to-ouputs attributions\n",
+ "\n",
"We got which concept are important for the classes globally. However, the concepts are not all present in each sample and the model might rely on a specific concept for a specific sample. Let's look at locally important concepts, meaning, the concepts the model used in a specific sample.\n",
"\n",
"> ➡️ Note\n",
@@ -5647,7 +5663,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -8207,17 +8223,15 @@
"})();\n",
"\n",
"\n",
- "Classes
\n",
- "\n",
- "Sample
\n",
+ "Classes
\n",
+ "\n",
"\n",
" \n",
@@ -8235,14 +8249,10 @@
"source": [
"example = \"Bio-engineered shoes will revolutionized running throughout the world, as they cost only 50 dollars.\"\n",
"\n",
- "activations_dict = model_with_split_points.get_activations(\n",
- " inputs=[example],\n",
- " activation_granularity=granularity,\n",
- " include_predicted_classes=True,\n",
- ")\n",
- "pred = activations_dict.pop(\"predictions\").item()\n",
- "local_activations = next(iter(activations_dict.values()))\n",
- "concepts_activations = concept_explainers[pred].encode_activations(local_activations)\n",
+ "local_activations, local_predictions = splitter.get_activations([example])\n",
+ "\n",
+ "pred = local_predictions.item()\n",
+ "concepts_activations = concept_explainers[pred].activations_to_concepts(local_activations)\n",
"\n",
"print(f\"Example: {example}\")\n",
"print(f\"Predicted class: {classes_names[pred]}\")\n",
@@ -8251,15 +8261,13 @@
"# we use the class-wise\n",
"local_importance = concept_explainers[pred].concept_output_gradient(\n",
" inputs=[example],\n",
- " activation_granularity=granularity,\n",
" concepts_x_gradients=True,\n",
- " tqdm_bar=False,\n",
")[0] # there is only one sample\n",
"\n",
"plot_concepts(\n",
- " sample=[example],\n",
+ " # sample=[example],\n",
" classes_names=classes_names,\n",
- " concepts_activations=concepts_activations,\n",
+ " # concepts_activations=concepts_activations,\n",
" concepts_importances=local_importance.squeeze(), # importance of shape (t, g, c) -> (t, c)\n",
" concepts_labels=concept_interpretations,\n",
")"
@@ -8269,237 +8277,2868 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## 7. ⚖️ **Evaluate** concept-based explanations \n",
- "\n",
- "We take back the `ICAConcepts` explainer and evaluate it on new samples."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [],
- "source": [
- "test_inputs = dataset[\"test\"][\"text\"][:1000] # let's take one thousand test samples\n",
- "\n",
- "# Compute the [CLS] token activations\n",
- "test_activations = model_with_split_points.get_activations(\n",
- " inputs=test_inputs,\n",
- " activation_granularity=granularity,\n",
- " include_predicted_classes=True,\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 7.1 🌐 Evaluate the concept-space from the [third part](#fit)\n",
- "\n",
- "> ⚠️ Warning:\n",
- ">\n",
- "> These metrics should only be used to compare the concept-space trained in similar contexts, same model, split point, activation dataset..."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### Reconstruction error\n",
- "\n",
- "- [`interpreto.concepts.metrics.MSE`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/reconstruction_metrics/#interpreto.concepts.metrics.MSE)\n",
- "- [`interpreto.concepts.metrics.FID`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/reconstruction_metrics/#interpreto.concepts.metrics.FID)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "MSE: 165.919, FID: 0.041\n"
- ]
- }
- ],
- "source": [
- "from interpreto.concepts.metrics import FID, MSE\n",
- "\n",
- "mse = MSE(concept_explainer).compute(test_activations)\n",
- "fid = FID(concept_explainer).compute(test_activations)\n",
- "\n",
- "print(f\"MSE: {round(mse, 3)}, FID: {round(fid, 3)}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "> ➡️ Note\n",
- ">\n",
- "> Alone these values are useless, they should be compared between several concept explainers."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### Sparsity\n",
- "\n",
- "- [`interpreto.concepts.metrics.Sparsity`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/sparsity_metrics/#interpreto.concepts.metrics.Sparsity)\n",
- "- [`interpreto.concepts.metrics.SparsityRatio`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/sparsity_metrics/#interpreto.concepts.metrics.SparsityRatio)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Sparsity: 1.0, Sparsity ratio: 0.02\n"
- ]
- }
- ],
- "source": [
- "from interpreto.concepts.metrics import Sparsity, SparsityRatio\n",
- "\n",
- "sparsity = Sparsity(concept_explainer).compute(test_activations)\n",
- "ratio = SparsityRatio(concept_explainer).compute(test_activations)\n",
- "\n",
- "print(f\"Sparsity: {round(sparsity, 3)}, Sparsity ratio: {round(ratio, 3)}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "#### Dictionary metrics\n",
- "\n",
- "- [`interpreto.concepts.metrics.Stability`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/dictionary_metrics/#interpreto.concepts.metrics.Stability)\n",
- "\n",
- "The `Stability` metric requires two concept explainer, hence our first step step will be to train a new `ICAConcepts` with the same model, split, dataset, and hyper-parameters as the original `ICAConcepts`. However, to get a statistically robust metric score, one should compare more than just two instances of the same explainer."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Stability: 1.0\n"
- ]
- }
- ],
- "source": [
- "from interpreto.concepts.metrics import Stability\n",
- "\n",
- "# instantiate and train a second concept explainer\n",
- "second_explainer = ICAConcepts(model_with_split_points, nb_concepts=50, device=\"cuda\")\n",
- "second_explainer.fit(activations)\n",
- "\n",
- "stability = Stability(concept_explainer, second_explainer).compute()\n",
- "del second_explainer\n",
- "\n",
- "print(f\"Stability: {round(stability, 3)}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 7.2 💭 Evaluate the concepts-interpretations from the [fourth step](#important)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Work in progress, coming soon"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 7.3 ↔️ Evaluate the whole concept-based explanations with `ConSim`\n",
- "\n",
- "`ConSim` is a metric evaluating the whole concept-based explanations in an end-to-end manner. Indeed, this metric, evaluates to which extend the provided concept-based explanations help a meta-predictor to predict what the studied model would have predicted. The idea is that is a meta-predictor understands the model, it is able to predict what the model would have predicted on new samples.\n",
+ "### 6.2 Inputs-to-concepts attributions\n",
"\n",
- "- [`interpreto.concepts.metrics.ConSim`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/consim/#interpreto.concepts.metrics.ConSim)\n",
+ "This can be seen as an interpretation, but it works even better when concepts have labels as defined earlier. (Note that these concepts are not absolute.)\n",
"\n",
- "> ➡️ Note\n",
- ">\n",
- "> For significant scores, we iterate on 10 different seeds. (5 were used in the paper)."
+ "To obtain inputs-to-concepts attributions, we use the `get_inputs_to_concepts_model()` method from concepts explainers and give it to perturbation-based attribution methods."
]
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Baseline: 0.29, ICA: 0.28\n"
- ]
- }
- ],
- "source": [
- "from interpreto.concepts.metrics.consim import ConSim, PromptTypes\n",
- "\n",
- "# convert step 5 global concept importances to a dictionary\n",
- "global_importances = {\n",
- " class_name: dict(enumerate(importances))\n",
- " for class_name, importances in zip(classes_names, mean_gradients, strict=True)\n",
- "}\n",
- "\n",
- "# Load API key from environment variable\n",
- "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
- "if api_key is None:\n",
- " raise ValueError(\n",
- " \"An API key is required to use `OpenAILLM` interface. \",\n",
- " \"Cannot use LLMLabels without an LLM interface. \",\n",
- " \"See last section for an example of how to branch one.\",\n",
- " )\n",
- "\n",
- "# set the LLM interface used to generate labels based on the constructed prompts\n",
- "llm_interface = OpenAILLM(api_key=api_key, model=\"gpt-4.1-nano\")\n",
- "\n",
- "# Initialize the ConSim with the model with split points and the user-llm\n",
- "# Therefore, a given ConSim metric can be used on different explainers for cleaner comparison\n",
- "con_sim = ConSim(model_with_split_points, llm_interface, classes=classes_names, activation_granularity=granularity)\n",
- "\n",
- "baseline_list = []\n",
- "ica_score_list = []\n",
- "for seed in range(10):\n",
- " # Select examples for evaluation\n",
- " samples, labels, predictions = con_sim.select_examples(\n",
- " inputs=dataset[\"train\"][\"text\"][:5000],\n",
- " labels=torch.tensor(dataset[\"train\"][\"label\"][:5000]).cuda(),\n",
- " seed=seed,\n",
- " )\n",
- "\n",
- " # Compute a baseline and ConSim score to give sense to the explainer ConSim score\n",
- " baseline = con_sim.evaluate(\n",
- " interesting_samples=samples, predictions=predictions, prompt_type=PromptTypes.L2_baseline_with_lp\n",
- " )\n",
- "\n",
- " if baseline is None:\n",
- " continue\n",
- "\n",
- " # Compute the ConSim score for an explainer\n",
- " ica_score = con_sim.evaluate(\n",
+ "data": {
+ "text/html": [
+ "Classes
\n",
+ "\n",
+ "Sample
\n",
+ "\n",
+ " \n",
+ " \n",
+ "\n"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from interpreto import KernelShap\n",
+ "\n",
+ "attributions = {}\n",
+ "# Iterate on explainers because we did class-wise explanations\n",
+ "for class_id, cpt_explainer in concept_explainers.items():\n",
+ " attribution_explainer = KernelShap(\n",
+ " model=cpt_explainer.get_inputs_to_concepts_model(),\n",
+ " tokenizer=splitter.tokenizer,\n",
+ " )\n",
+ "\n",
+ " # Compute the attributions\n",
+ " attribution_outputs = attribution_explainer.explain(\n",
+ " example,\n",
+ " targets=None, # we want to explain all concepts\n",
+ " )[0]\n",
+ " attributions[class_id] = attribution_outputs.attributions.T\n",
+ "\n",
+ "# Visualization\n",
+ "plot_concepts(\n",
+ " sample=attribution_outputs.elements,\n",
+ " classes_names=classes_names, # TODO WARNING THE CLASS IS NOT CORRECT\n",
+ " concepts_activations=attributions, # argument name is miss-leading because we use concepts activations for generation\n",
+ " concepts_importances=local_importance.squeeze(), # importance of shape (t, g, c) -> (t, c)\n",
+ " concepts_labels=concept_interpretations,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 7. ⚖️ **Evaluate** concept-based explanations \n",
+ "\n",
+ "We take back the `ICAConcepts` explainer and evaluate it on new samples."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_inputs = dataset[\"test\"][\"text\"][:1000] # let's take one thousand test samples\n",
+ "test_labels = torch.tensor(dataset[\"test\"][\"label\"][:1000])\n",
+ "\n",
+ "# Compute the [CLS] token activations\n",
+ "test_activations, test_predictions = splitter.get_activations(inputs=test_inputs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 7.1 🌐 Evaluate the concept-space from the [third part](#fit)\n",
+ "\n",
+ "> ⚠️ Warning:\n",
+ ">\n",
+ "> These metrics should only be used to compare the concept-space trained in similar contexts, same model, split point, activation dataset..."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Reconstruction error\n",
+ "\n",
+ "- [`interpreto.concepts.metrics.MSE`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/reconstruction_metrics/#interpreto.concepts.metrics.MSE)\n",
+ "- [`interpreto.concepts.metrics.FID`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/reconstruction_metrics/#interpreto.concepts.metrics.FID)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MSE: 71.467, FID: 0.026\n"
+ ]
+ }
+ ],
+ "source": [
+ "from interpreto.concepts.metrics import FID, MSE\n",
+ "\n",
+ "mse = MSE(concept_explainer).compute(test_activations)\n",
+ "fid = FID(concept_explainer).compute(test_activations)\n",
+ "\n",
+ "print(f\"MSE: {round(mse, 3)}, FID: {round(fid, 3)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "> ➡️ Note\n",
+ ">\n",
+ "> Alone these values are useless, they should be compared between several concept explainers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Sparsity\n",
+ "\n",
+ "- [`interpreto.concepts.metrics.Sparsity`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/sparsity_metrics/#interpreto.concepts.metrics.Sparsity)\n",
+ "- [`interpreto.concepts.metrics.SparsityRatio`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/sparsity_metrics/#interpreto.concepts.metrics.SparsityRatio)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Sparsity: 1.0, Sparsity ratio: 0.02\n"
+ ]
+ }
+ ],
+ "source": [
+ "from interpreto.concepts.metrics import Sparsity, SparsityRatio\n",
+ "\n",
+ "sparsity = Sparsity(concept_explainer).compute(test_activations)\n",
+ "ratio = SparsityRatio(concept_explainer).compute(test_activations)\n",
+ "\n",
+ "print(f\"Sparsity: {round(sparsity, 3)}, Sparsity ratio: {round(ratio, 3)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Dictionary metrics\n",
+ "\n",
+ "- [`interpreto.concepts.metrics.Stability`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/dictionary_metrics/#interpreto.concepts.metrics.Stability)\n",
+ "\n",
+ "The `Stability` metric requires two concept explainer, hence our first step step will be to train a new `ICAConcepts` with the same model, split, dataset, and hyper-parameters as the original `ICAConcepts`. However, to get a statistically robust metric score, one should compare more than just two instances of the same explainer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Stability: 1.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "from interpreto.concepts.metrics import Stability\n",
+ "\n",
+ "# instantiate and train a second concept explainer\n",
+ "second_explainer = ICAConcepts(splitter, nb_concepts=50, device=\"cuda\")\n",
+ "second_explainer.fit(activations)\n",
+ "\n",
+ "stability = Stability(concept_explainer, second_explainer).compute()\n",
+ "del second_explainer\n",
+ "\n",
+ "print(f\"Stability: {round(stability, 3)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 7.2 💭 Evaluate the concepts-interpretations from the [fourth step](#important)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Work in progress, coming soon"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 7.3 ↔️ Evaluate the whole concept-based explanations with `ConSim`\n",
+ "\n",
+ "`ConSim` is a metric evaluating the whole concept-based explanations in an end-to-end manner. Indeed, this metric, evaluates to which extend the provided concept-based explanations help a meta-predictor to predict what the studied model would have predicted. The idea is that is a meta-predictor understands the model, it is able to predict what the model would have predicted on new samples.\n",
+ "\n",
+ "- [`interpreto.concepts.metrics.ConSim`](https://for-sight-ai.github.io/interpreto/api/concepts/metrics/consim/#interpreto.concepts.metrics.ConSim)\n",
+ "\n",
+ "> ➡️ Note\n",
+ ">\n",
+ "> For significant scores, we iterate on 10 different seeds. (5 were used in the paper)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/antonin.poche/interpreto/interpreto/concepts/metrics/consim.py:603: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.\n",
+ "Consider using tensor.detach() first. (Triggered internally at /pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:836.)\n",
+ " importances = torch.abs(torch.Tensor(list(concepts_importance.values())))\n",
+ "/home/antonin.poche/interpreto/interpreto/concepts/metrics/consim.py:1288: UserWarning: The user-llm responses are empty or the format is not respected. Returning None. The response was: 'Sample_20: Sports'\n",
+ " return self._compute_score(\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Baseline: 0.22, ICA: 0.23\n"
+ ]
+ }
+ ],
+ "source": [
+ "from interpreto.concepts.metrics.consim import ConSim, PromptTypes\n",
+ "\n",
+ "# convert step 5 global concept importances to a dictionary\n",
+ "global_importances = {\n",
+ " class_name: dict(enumerate(importances))\n",
+ " for class_name, importances in zip(classes_names, mean_gradients, strict=True)\n",
+ "}\n",
+ "\n",
+ "# Load API key from environment variable\n",
+ "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+ "if api_key is None:\n",
+ " raise ValueError(\n",
+ " \"An API key is required to use `OpenAILLM` interface. \",\n",
+ " \"Cannot use LLMLabels without an LLM interface. \",\n",
+ " \"See last section for an example of how to branch one.\",\n",
+ " )\n",
+ "\n",
+ "# set the LLM interface used to generate labels based on the constructed prompts\n",
+ "llm_interface = OpenAILLM(api_key=api_key, model=\"gpt-4.1-nano\")\n",
+ "\n",
+ "# Initialize the ConSim with the split model and the user LLM\n",
+ "# Therefore, a given ConSim metric can be used on different explainers for cleaner comparison\n",
+ "con_sim = ConSim(\n",
+ " splitter,\n",
+ " llm_interface,\n",
+ " classes=classes_names,\n",
+ ")\n",
+ "\n",
+ "baseline_list = []\n",
+ "ica_score_list = []\n",
+ "for seed in range(10):\n",
+ " # Select examples for evaluation\n",
+ " samples, labels, predictions = con_sim._extract_interesting_elements(\n",
+ " inputs=test_inputs,\n",
+ " labels=test_labels,\n",
+ " predictions=test_predictions,\n",
+ " seed=seed,\n",
+ " )\n",
+ "\n",
+ " # Compute a baseline and ConSim score to give sense to the explainer ConSim score\n",
+ " baseline = con_sim.evaluate(\n",
+ " interesting_samples=samples, predictions=predictions, prompt_type=PromptTypes.L2_baseline_with_lp\n",
+ " )\n",
+ "\n",
+ " if baseline is None:\n",
+ " continue\n",
+ "\n",
+ " # Compute the ConSim score for an explainer\n",
+ " ica_score = con_sim.evaluate(\n",
" interesting_samples=samples,\n",
" predictions=predictions,\n",
" concept_explainer=concept_explainer,\n",
@@ -8535,11 +11174,11 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "from interpreto.model_wrapping.llm_interface import LLMInterface, Role\n",
+ "from interpreto.commons.llm_interface import LLMInterface, Role\n",
"\n",
"\n",
"class GeminiLLM(LLMInterface):\n",
@@ -8599,8 +11238,22 @@
}
],
"metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
"language_info": {
- "name": "python"
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
}
},
"nbformat": 4,
diff --git a/docs/notebooks/examples/classification_demonstration.ipynb b/docs/notebooks/examples/classification_demonstration.ipynb
index 95449dd8..3c230b43 100644
--- a/docs/notebooks/examples/classification_demonstration.ipynb
+++ b/docs/notebooks/examples/classification_demonstration.ipynb
@@ -1,8217 +1,10803 @@
{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "{ style=\"display:block; max-width:100%; height:auto; margin:0 auto;\" }\n",
- "\n",
- "# Classification Demonstration\n",
- "\n",
- "This notebook show case what can be done with `interpreto` for classification.\n",
- "\n",
- "*author: Antonin Poché*"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 0. Imports and model loading"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "\n",
- "import datasets\n",
- "import torch\n",
- "import transformers\n",
- "\n",
- "import interpreto\n",
- "from interpreto import Lime, plot_attributions, plot_concepts\n",
- "from interpreto.attributions.metrics import Insertion\n",
- "from interpreto.concepts import LLMLabels, SemiNMFConcepts\n",
- "from interpreto.concepts.interpretations import TopKInputs\n",
- "from interpreto.concepts.metrics import FID, SparsityRatio\n",
- "from interpreto.model_wrapping.llm_interface import OpenAILLM"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "# load dataset examples\n",
- "dataset = datasets.load_dataset(\"dair-ai/emotion\", \"split\")[\"train\"][\"text\"][:1000]\n",
- "\n",
- "classes_names = [\"sadness\", \"joy\", \"love\", \"anger\", \"fear\", \"surprise\"]\n",
- "# class_names={0: \"sadness\", 1: \"joy\", 2: \"love\", 3: \"anger\", 4: \"fear\", 5: \"surprise\"}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "# load the model and tokenizer\n",
- "tokenizer = transformers.AutoTokenizer.from_pretrained(\"nateraw/bert-base-uncased-emotion\", use_fast=True)\n",
- "model = transformers.AutoModelForSequenceClassification.from_pretrained(\"nateraw/bert-base-uncased-emotion\").cuda()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 1. Attribution Demonstration"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### 1.1 Obtain the attributions"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "The new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`\n"
- ]
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "{ style=\"display:block; max-width:100%; height:auto; margin:0 auto;\" }\n",
+ "\n",
+ "# Classification Demonstration\n",
+ "\n",
+ "This notebook show case what can be done with `interpreto` for classification.\n",
+ "\n",
+ "*author: Antonin Poché*"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 0. Imports and model loading"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import datasets\n",
+ "import torch\n",
+ "import transformers\n",
+ "\n",
+ "import interpreto\n",
+ "from interpreto import Lime, plot_attributions, plot_concepts\n",
+ "from interpreto.attributions.metrics import Insertion\n",
+ "from interpreto.commons.llm_interface import OpenAILLM\n",
+ "from interpreto.concepts import LLMLabels, SemiNMFConcepts\n",
+ "from interpreto.concepts.interpretations import TopKInputs\n",
+ "from interpreto.concepts.metrics import FID, SparsityRatio"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load dataset examples\n",
+ "dataset = datasets.load_dataset(\"dair-ai/emotion\", \"split\")[\"train\"][\"text\"][:1000]\n",
+ "\n",
+ "classes_names = [\"sadness\", \"joy\", \"love\", \"anger\", \"fear\", \"surprise\"]\n",
+ "# class_names={0: \"sadness\", 1: \"joy\", 2: \"love\", 3: \"anger\", 4: \"fear\", 5: \"surprise\"}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "d3e5c6a1b3b748a1811d03b58f0fce62",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Loading weights: 0%| | 0/201 [00:00Classes
\n",
+ "Inputs
\n",
+ "\n",
+ " \n",
+ "