Skip to content

Qualcomm AI Engine Direct - Test Framework Refactor #2 #20735

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
haowhsu-quic wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Qualcomm AI Engine Direct - Test Framework Refactor #2 #20735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions backends/qualcomm/tests/rework/common/pass/test.py
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions backends/qualcomm/tests/rework/common/utils/test.py
View file
Open in desktop

This file was deleted.

11 changes: 11 additions & 0 deletions backends/qualcomm/tests/rework/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from abc import ABC, abstractmethod
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from typing import Any, List, Tuple

Expand Down Expand Up @@ -53,6 +54,7 @@
# et framework messages
EXCEPTION_EXIR_PROGRAM = "exir/program"
EXCEPTION_FROM_PASSES = "backends/qualcomm/_passes"
EXCEPTION_FROM_PREPROCESS = "backends/qualcomm/qnn_preprocess"


def check_exception(msg):
Expand All @@ -62,6 +64,15 @@ def _check(msg, _: Exception):
return partial(_check, msg)


# extend this for backend agnostic tests
def default_property():
@dataclass
class Property:
soc_model: str = "SM8750"

return Property()


class Metrics(ABC):
@abstractmethod
def __init__(self):
Expand Down
66 changes: 66 additions & 0 deletions backends/qualcomm/tests/rework/htp/op/v68/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,24 @@ def test_add(request, kwargs):
Add.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype(
[
Tolerance(),
pytest.raises(AssertionError, match=EXPECT_NOT_FULLY_DELEGATED),
Tolerance(rtol=1e-1),
]
)
@with_htp_context
def test_addmm(request, kwargs):
AddMM.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_alias(request, kwargs):
Alias.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_amax(request, kwargs):
Expand Down Expand Up @@ -464,6 +482,10 @@ def test_conv2d_transpose(request, kwargs):
},
id="16a4w_lpbq",
),
pytest.param(
{"act": "fp16", "param": 8, "pcq": True, "expected": Tolerance()},
id="fp16a8w_pcq",
),
],
)
@with_htp_context
Expand Down Expand Up @@ -527,6 +549,12 @@ def test_div(request, kwargs):
Div.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_div_with_rounding_mode(request, kwargs):
DivWithRoundingMode.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype(
[
Tolerance(),
Expand Down Expand Up @@ -595,6 +623,12 @@ def test_expm1(request, kwargs):
ExpM1.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_fill(request, kwargs):
Fill.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_flip(request, kwargs):
Expand Down Expand Up @@ -857,6 +891,14 @@ def test_linear_block_quant(request, kwargs):
{"act": 16, "param": 8, "pcq": True, "expected": Tolerance()},
id="16a8w_pcq",
),
pytest.param(
{"act": "fp16", "param": 8, "pcq": True, "expected": Tolerance()},
id="fp16a8w_pcq",
),
pytest.param(
{"act": 16, "param": 2, "pcq": True, "expected": CosineSimilarity(0.9)},
id="16a2w_pcq",
),
pytest.param(
{
"act": None,
Expand Down Expand Up @@ -1177,12 +1219,24 @@ def test_sdpa(request, kwargs):
ScaledDotProductAttention.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_scatter_src(request, kwargs):
ScatterSrc.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_select_copy(request, kwargs):
SelectCopy.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_select_scatter(request, kwargs):
SelectScatter.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_sigmoid(request, kwargs):
Expand Down Expand Up @@ -1255,6 +1309,12 @@ def test_swapaxes(request, kwargs):
SwapAxes.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_tan(request, kwargs):
Tan.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_tanh(request, kwargs):
Expand Down Expand Up @@ -1355,3 +1415,9 @@ def test_view_5d_flatten_last_two_dims(request, kwargs):
@with_htp_context
def test_where(request, kwargs):
Where.test(request, kwargs) # noqa: F405


@enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)])
@with_htp_context
def test_var(request, kwargs):
Var.test(request, kwargs) # noqa: F405
Loading
Loading