Skip to content

Commit

Permalink
[MAINTENANCE] Rename legacy batch definitions (great-expectations#9629)
Browse files Browse the repository at this point in the history
  • Loading branch information
tyler-hoffman authored Mar 15, 2024
1 parent a30f073 commit 09cc0b2
Show file tree
Hide file tree
Showing 80 changed files with 717 additions and 691 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
IDDict,
)
from great_expectations.checkpoint import Checkpoint
from great_expectations.core.batch import Batch, BatchDefinition
from great_expectations.core.batch import Batch
from great_expectations.execution_engine import PandasExecutionEngine
from great_expectations.util import filter_properties_dict
from great_expectations.validator.validator import Validator
Expand Down
8 changes: 4 additions & 4 deletions great_expectations/core/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def _get_metrics_calculator_class() -> Type[MetricsCalculator]:


@public_api
class BatchDefinition(SerializableDictDot):
class LegacyBatchDefinition(SerializableDictDot):
"""Precisely identifies a set of data from a data source.
More concretely, a BatchDefinition includes all the information required to precisely
Expand Down Expand Up @@ -739,7 +739,7 @@ def __init__( # noqa: PLR0913
self,
data: BatchDataType | None = None,
batch_request: BatchRequestBase | dict | None = None,
batch_definition: BatchDefinition | None = None,
batch_definition: LegacyBatchDefinition | None = None,
batch_spec: BatchSpec | None = None,
batch_markers: BatchMarkers | None = None,
# The remaining parameters are for backward compatibility.
Expand Down Expand Up @@ -838,7 +838,7 @@ def to_dict(self) -> dict:
"data": str(self.data),
"batch_request": self.batch_request.to_dict(),
"batch_definition": self.batch_definition.to_json_dict()
if isinstance(self.batch_definition, BatchDefinition)
if isinstance(self.batch_definition, LegacyBatchDefinition)
else {},
"batch_spec": self.batch_spec,
"batch_markers": self.batch_markers,
Expand All @@ -863,7 +863,7 @@ def to_json_dict(self) -> dict[str, JSONValues]:
@property
def id(self):
batch_definition = self._batch_definition
if isinstance(batch_definition, BatchDefinition):
if isinstance(batch_definition, LegacyBatchDefinition):
return batch_definition.id

if isinstance(batch_definition, IDDict):
Expand Down
4 changes: 2 additions & 2 deletions great_expectations/core/batch_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
from great_expectations.core.batch import (
Batch,
BatchDataUnion,
BatchDefinition,
BatchMarkers,
LegacyBatchDefinition,
_get_fluent_batch_class,
)

Expand Down Expand Up @@ -131,7 +131,7 @@ def active_batch_markers(self) -> Optional[BatchMarkers]:
return self.active_batch.batch_markers

@property
def active_batch_definition(self) -> Optional[BatchDefinition]:
def active_batch_definition(self) -> Optional[LegacyBatchDefinition]:
"""Getter for the active batch's batch definition"""
if not self.active_batch:
return None
Expand Down
7 changes: 5 additions & 2 deletions great_expectations/core/expectation_validation_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
from great_expectations._docs_decorators import public_api
from great_expectations.alias_types import JSONValues # noqa: TCH001
from great_expectations.compatibility.typing_extensions import override
from great_expectations.core.batch import BatchDefinition, BatchMarkers # noqa: TCH001
from great_expectations.core.batch import ( # noqa: TCH001
BatchMarkers,
LegacyBatchDefinition,
)
from great_expectations.core.id_dict import BatchSpec # noqa: TCH001
from great_expectations.core.run_identifier import RunIdentifier # noqa: TCH001
from great_expectations.core.util import (
Expand Down Expand Up @@ -457,7 +460,7 @@ def make_expectation_validation_result(self, data, **kwargs):


class ExpectationSuiteValidationResultMeta(TypedDict):
active_batch_definition: BatchDefinition
active_batch_definition: LegacyBatchDefinition
batch_markers: BatchMarkers
batch_spec: BatchSpec
checkpoint_id: Optional[str]
Expand Down
8 changes: 4 additions & 4 deletions great_expectations/datasource/data_connector/batch_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
if TYPE_CHECKING:
from typing_extensions import TypeAlias

from great_expectations.core.batch import BatchDefinition
from great_expectations.core.batch import LegacyBatchDefinition

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -299,16 +299,16 @@ def __repr__(self) -> str:
return str(doc_fields_dict)

def select_from_data_connector_query(
self, batch_definition_list: Optional[List[BatchDefinition]] = None
) -> List[BatchDefinition]:
self, batch_definition_list: Optional[List[LegacyBatchDefinition]] = None
) -> List[LegacyBatchDefinition]:
if batch_definition_list is None:
return []
filter_function: Callable
if self.custom_filter_function:
filter_function = self.custom_filter_function
else:
filter_function = self.best_effort_batch_definition_matcher()
selected_batch_definitions: List[BatchDefinition]
selected_batch_definitions: List[LegacyBatchDefinition]
selected_batch_definitions = list(
filter(
lambda batch_definition: filter_function(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
from great_expectations.compatibility import aws
from great_expectations.compatibility.typing_extensions import override
from great_expectations.core.batch import (
BatchDefinition,
BatchRequestBase,
BatchSpec,
IDDict,
LegacyBatchDefinition,
)
from great_expectations.core.batch_spec import GlueDataCatalogBatchSpec
from great_expectations.datasource.data_connector import DataConnector
Expand Down Expand Up @@ -112,13 +112,13 @@ def partitions(self) -> Optional[List[str]]:

@override
def build_batch_spec(
self, batch_definition: BatchDefinition
self, batch_definition: LegacyBatchDefinition
) -> GlueDataCatalogBatchSpec:
"""
Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
BatchSpec built from batch_definition
Expand Down Expand Up @@ -192,7 +192,7 @@ def get_batch_definition_list_from_batch_request(
if len(self._data_references_cache) == 0:
self._refresh_data_references_cache()

batch_definition_list: List[BatchDefinition] = []
batch_definition_list: List[LegacyBatchDefinition] = []
try:
sub_cache = self._get_data_reference_list_from_cache_by_data_asset_name(
data_asset_name=batch_request.data_asset_name
Expand All @@ -203,7 +203,7 @@ def get_batch_definition_list_from_batch_request(
)

for batch_identifiers in sub_cache:
batch_definition = BatchDefinition(
batch_definition = LegacyBatchDefinition(
datasource_name=self.datasource_name,
data_connector_name=self.name,
data_asset_name=batch_request.data_asset_name,
Expand Down Expand Up @@ -359,10 +359,10 @@ def _update_data_asset_name_from_config(
@override
def _map_data_reference_to_batch_definition_list(
self, data_reference, data_asset_name: Optional[str] = None
) -> Optional[List[BatchDefinition]]:
) -> Optional[List[LegacyBatchDefinition]]:
# Note: data references *are* dictionaries, allowing us to invoke `IDDict(data_reference)`
return [
BatchDefinition(
LegacyBatchDefinition(
datasource_name=self.datasource_name,
data_connector_name=self.name,
data_asset_name=data_asset_name, # type: ignore[arg-type]
Expand All @@ -372,7 +372,7 @@ def _map_data_reference_to_batch_definition_list(

@override
def _generate_batch_spec_parameters_from_batch_definition(
self, batch_definition: BatchDefinition
self, batch_definition: LegacyBatchDefinition
) -> dict:
"""
Build BatchSpec parameters from batch_definition with the following components:
Expand All @@ -381,7 +381,7 @@ def _generate_batch_spec_parameters_from_batch_definition(
3. data_asset from data_connector
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
dict built from batch_definition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
)

if TYPE_CHECKING:
from great_expectations.core.batch import BatchDefinition
from great_expectations.core.batch import LegacyBatchDefinition
from great_expectations.datasource.data_connector.asset import Asset
from great_expectations.execution_engine import ExecutionEngine

Expand Down Expand Up @@ -109,12 +109,14 @@ def __init__( # noqa: PLR0913
)

@override
def build_batch_spec(self, batch_definition: BatchDefinition) -> AzureBatchSpec:
def build_batch_spec(
self, batch_definition: LegacyBatchDefinition
) -> AzureBatchSpec:
"""
Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
BatchSpec built from batch_definition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from great_expectations.datasource.data_connector.util import _build_asset_from_config

if TYPE_CHECKING:
from great_expectations.core.batch import BatchDefinition
from great_expectations.core.batch import LegacyBatchDefinition
from great_expectations.core.batch_spec import PathBatchSpec
from great_expectations.datasource.data_connector.asset.asset import (
Asset,
Expand Down Expand Up @@ -108,7 +108,7 @@ def _refresh_data_references_cache(self) -> None:
for data_reference in self._get_data_reference_list(
data_asset_name=data_asset_name
):
mapped_batch_definition_list: List[BatchDefinition] = (
mapped_batch_definition_list: List[LegacyBatchDefinition] = (
self._map_data_reference_to_batch_definition_list( # type: ignore[assignment]
data_reference=data_reference,
data_asset_name=data_asset_name,
Expand Down Expand Up @@ -167,8 +167,8 @@ def get_unmatched_data_references(self) -> List[str]:
return unmatched_data_references

@override
def _get_batch_definition_list_from_cache(self) -> List[BatchDefinition]:
batch_definition_list: List[BatchDefinition] = [
def _get_batch_definition_list_from_cache(self) -> List[LegacyBatchDefinition]:
batch_definition_list: List[LegacyBatchDefinition] = [
batch_definitions[0]
for data_reference_sub_cache in self._data_references_cache.values()
for batch_definitions in data_reference_sub_cache.values()
Expand Down Expand Up @@ -222,12 +222,14 @@ def _get_full_file_path_for_asset(self, path: str, asset: Optional[Asset]) -> st
raise NotImplementedError

@override
def build_batch_spec(self, batch_definition: BatchDefinition) -> PathBatchSpec:
def build_batch_spec(
self, batch_definition: LegacyBatchDefinition
) -> PathBatchSpec:
"""
Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
BatchSpec built from batch_definition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from great_expectations.datasource.data_connector.util import list_gcs_keys

if TYPE_CHECKING:
from great_expectations.core.batch import BatchDefinition
from great_expectations.core.batch import LegacyBatchDefinition
from great_expectations.datasource.data_connector.asset import Asset
from great_expectations.execution_engine import ExecutionEngine

Expand Down Expand Up @@ -108,12 +108,12 @@ def __init__( # noqa: PLR0913
)

@override
def build_batch_spec(self, batch_definition: BatchDefinition) -> GCSBatchSpec:
def build_batch_spec(self, batch_definition: LegacyBatchDefinition) -> GCSBatchSpec:
"""
Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
BatchSpec built from batch_definition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
)

if TYPE_CHECKING:
from great_expectations.core.batch import BatchDefinition
from great_expectations.core.batch import LegacyBatchDefinition
from great_expectations.datasource.data_connector.asset import Asset
from great_expectations.execution_engine import ExecutionEngine

Expand Down Expand Up @@ -90,12 +90,12 @@ def __init__( # noqa: PLR0913
)

@override
def build_batch_spec(self, batch_definition: BatchDefinition) -> S3BatchSpec:
def build_batch_spec(self, batch_definition: LegacyBatchDefinition) -> S3BatchSpec:
"""
Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
Args:
batch_definition (BatchDefinition): to be used to build batch_spec
batch_definition (LegacyBatchDefinition): to be used to build batch_spec
Returns:
BatchSpec built from batch_definition
Expand Down
Loading

0 comments on commit 09cc0b2

Please sign in to comment.