Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor workspaces into projects #3364

Draft
wants to merge 24 commits into
base: develop
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
f983d69
Add workspace consistency checks
schustmi Jan 16, 2025
e926075
Add base domain models for flexible scoping
stefannica Feb 12, 2025
5115913
Add flexible scoping to domain models for stack/component/flavor/serv…
stefannica Feb 12, 2025
04d9f93
Update ORM schemas to make workspace optional for flexible entities
stefannica Feb 12, 2025
0806c7f
Add DB migration to make workspace optional
stefannica Feb 13, 2025
60824bd
WIP
stefannica Feb 13, 2025
ba6317c
Moved all unrelated endpoints out of the workspaces_endpoints modult
stefannica Feb 14, 2025
315ba9b
Stop using workspace scoped endpoints in the REST zen store
stefannica Feb 14, 2025
63d476b
Move request user scoping to the SQLZenStore
stefannica Feb 15, 2025
0565bec
Remove unused user_id field in user scope filter model
stefannica Feb 15, 2025
d5ecd22
Remove unused workspace_id field in workspace scope base filter model
stefannica Feb 15, 2025
0d6cf32
Mark all workspace endpoints as deprecated
stefannica Feb 17, 2025
e3efd14
Remove un-necessary user fields from various filters
stefannica Feb 17, 2025
cbc0ba7
Implement RBAC scoping for workspaces
stefannica Feb 18, 2025
0c70fce
Aggregated request support in RBAC endpoint utils and unified all req…
stefannica Feb 19, 2025
d691f33
Make tags workspace scoped resources
stefannica Feb 19, 2025
a353524
Fix some linter errors
stefannica Feb 19, 2025
9a45ff7
Unified name uniqueness checks in the SQLZenStore and applied workspa…
stefannica Feb 19, 2025
4e6c8be
Fix the tagged filters to account for multiple inheritance
stefannica Feb 19, 2025
adfcfc0
Make artifacts workspace and user scoped
stefannica Feb 19, 2025
123a51d
SQL zen store scope reference verification and proper tags scoping
stefannica Feb 21, 2025
3e5d964
Remove unneeded fields from create/update models
stefannica Feb 21, 2025
e83139a
Make flexible-scoped resources global-scoped
stefannica Feb 21, 2025
b31ab41
Replace secret scope with private secret feature (+ fix all linter er…
stefannica Feb 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/zenml/artifacts/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@ def _store_artifact_data_and_prepare_request(
uri=materializer.uri,
materializer=source_utils.resolve(materializer.__class__),
data_type=source_utils.resolve(data_type),
user=Client().active_user.id,
workspace=Client().active_workspace.id,
artifact_store_id=artifact_store.id,
visualizations=visualizations,
Expand Down Expand Up @@ -350,7 +349,6 @@ def register_artifact(
uri=folder_or_file_uri,
materializer=source_utils.resolve(PreexistingDataMaterializer),
data_type=source_utils.resolve(Path),
user=Client().active_user.id,
workspace=Client().active_workspace.id,
artifact_store_id=artifact_store.id,
has_custom_name=has_custom_name,
Expand Down
91 changes: 47 additions & 44 deletions src/zenml/cli/secret.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from zenml.constants import SECRET_VALUES
from zenml.enums import (
CliCategories,
SecretScope,
)
from zenml.exceptions import EntityExistsError, ZenKeyError
from zenml.logger import get_logger
Expand All @@ -59,11 +58,12 @@ def secret() -> None:
)
@click.argument("name", type=click.STRING)
@click.option(
"--scope",
"-s",
"scope",
type=click.Choice([scope.value for scope in list(SecretScope)]),
default=SecretScope.WORKSPACE.value,
"--private",
"-p",
"private",
is_flag=True,
help="Whether the secret is private. A private secret is only accessible "
"to the user who creates it.",
)
@click.option(
"--interactive",
Expand All @@ -84,13 +84,13 @@ def secret() -> None:
)
@click.argument("args", nargs=-1, type=click.UNPROCESSED)
def create_secret(
name: str, scope: str, interactive: bool, values: str, args: List[str]
name: str, private: bool, interactive: bool, values: str, args: List[str]
) -> None:
"""Create a secret.

Args:
name: The name of the secret to create.
scope: The scope of the secret to create.
private: Whether the secret is private.
interactive: Whether to use interactive mode to enter the secret values.
values: Secret key-value pairs to be passed as JSON or YAML.
args: The arguments to pass to the secret.
Expand Down Expand Up @@ -152,7 +152,7 @@ def create_secret(
with console.status(f"Saving secret `{name}`..."):
try:
client.create_secret(
name=name, values=parsed_args, scope=SecretScope(scope)
name=name, values=parsed_args, private=private
)
declare(f"Secret '{name}' successfully created.")
except EntityExistsError as e:
Expand Down Expand Up @@ -186,7 +186,7 @@ def list_secrets(**kwargs: Any) -> None:
dict(
name=secret.name,
id=str(secret.id),
scope=secret.scope.value,
private=secret.private,
)
for secret in secrets.items
]
Expand All @@ -200,22 +200,26 @@ def list_secrets(**kwargs: Any) -> None:
type=click.STRING,
)
@click.option(
"--scope",
"-s",
type=click.Choice([scope.value for scope in list(SecretScope)]),
default=None,
"--private",
"-p",
"private",
type=click.BOOL,
required=False,
help="Use this flag to explicitly fetch a private secret or a public secret.",
)
def get_secret(name_id_or_prefix: str, scope: Optional[str] = None) -> None:
def get_secret(name_id_or_prefix: str, private: Optional[bool] = None) -> None:
"""Get a secret and print it to the console.

Args:
name_id_or_prefix: The name of the secret to get.
scope: The scope of the secret to get.
private: Private status of the secret to filter for.
"""
secret = _get_secret(name_id_or_prefix, scope)
secret = _get_secret(name_id_or_prefix, private)
scope = ""
if private is not None:
scope = "private " if private else "public "
declare(
f"Fetched secret with name `{secret.name}` and ID `{secret.id}` in "
f"scope `{secret.scope.value}`:"
f"Fetched {scope}secret with name `{secret.name}` and ID `{secret.id}`:"
)
if not secret.secret_values:
warning(f"Secret with name `{name_id_or_prefix}` is empty.")
Expand All @@ -224,25 +228,22 @@ def get_secret(name_id_or_prefix: str, scope: Optional[str] = None) -> None:


def _get_secret(
name_id_or_prefix: str, scope: Optional[str] = None
name_id_or_prefix: str, private: Optional[bool] = None
) -> SecretResponse:
"""Get a secret with a given name, prefix or id.

Args:
name_id_or_prefix: The name of the secret to get.
scope: The scope of the secret to get.
private: Private status of the secret to filter for.

Returns:
The secret response model.
"""
client = Client()
try:
if scope:
return client.get_secret(
name_id_or_prefix=name_id_or_prefix, scope=SecretScope(scope)
)
else:
return client.get_secret(name_id_or_prefix=name_id_or_prefix)
return client.get_secret(
name_id_or_prefix=name_id_or_prefix, private=private
)
except ZenKeyError as e:
error(
f"Error fetching secret with name id or prefix "
Expand All @@ -267,9 +268,12 @@ def _get_secret(
type=click.STRING,
)
@click.option(
"--new-scope",
"-s",
type=click.Choice([scope.value for scope in list(SecretScope)]),
"--private",
"-p",
"private",
type=click.BOOL,
required=False,
help="Update the private status of the secret.",
)
@click.option(
"--interactive",
Expand All @@ -293,7 +297,7 @@ def _get_secret(
def update_secret(
name_or_id: str,
extra_args: List[str],
new_scope: Optional[str] = None,
private: Optional[bool] = None,
remove_keys: List[str] = [],
interactive: bool = False,
values: str = "",
Expand All @@ -302,7 +306,7 @@ def update_secret(

Args:
name_or_id: The name or id of the secret to update.
new_scope: The new scope of the secret.
private: Private status of the secret to update.
extra_args: The arguments to pass to the secret.
interactive: Whether to use interactive mode to update the secret.
remove_keys: The keys to remove from the secret.
Expand Down Expand Up @@ -331,10 +335,7 @@ def update_secret(
except NotImplementedError as e:
error(f"Centralized secrets management is disabled: {str(e)}")

declare(
f"Updating secret with name '{secret.name}' and ID '{secret.id}' in "
f"scope '{secret.scope.value}:"
)
declare(f"Updating secret with name '{secret.name}' and ID '{secret.id}'")

if "name" in parsed_args:
error("The word 'name' cannot be used as a key for a secret.")
Expand Down Expand Up @@ -388,7 +389,7 @@ def update_secret(

client.update_secret(
name_id_or_prefix=secret.id,
new_scope=SecretScope(new_scope) if new_scope else None,
update_private=private,
add_or_update_values=secret_args_add_update,
remove_values=remove_keys,
)
Expand Down Expand Up @@ -492,10 +493,12 @@ def delete_secret(name_or_id: str, yes: bool = False) -> None:
type=click.STRING,
)
@click.option(
"--scope",
"-s",
type=click.Choice([scope.value for scope in list(SecretScope)]),
default=None,
"--private",
"-p",
"private",
type=click.BOOL,
required=False,
help="Use this flag to explicitly fetch a private secret or a public secret.",
)
@click.option(
"--filename",
Expand All @@ -509,7 +512,7 @@ def delete_secret(name_or_id: str, yes: bool = False) -> None:
)
def export_secret(
name_id_or_prefix: str,
scope: Optional[str] = None,
private: Optional[bool] = None,
filename: Optional[str] = None,
) -> None:
"""Export a secret as a YAML file.
Expand All @@ -519,12 +522,12 @@ def export_secret(

Args:
name_id_or_prefix: The name of the secret to export.
scope: The scope of the secret to export.
private: Private status of the secret to export.
filename: The name of the file to export the secret to.
"""
from zenml.utils.yaml_utils import write_yaml

secret = _get_secret(name_id_or_prefix=name_id_or_prefix, scope=scope)
secret = _get_secret(name_id_or_prefix=name_id_or_prefix, private=private)
if not secret.secret_values:
warning(f"Secret with name `{name_id_or_prefix}` is empty.")
return
Expand Down
1 change: 0 additions & 1 deletion src/zenml/cli/service_connectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1131,7 +1131,6 @@ def describe_service_connector(
)

connector = connector_client.to_response_model(
workspace=client.active_workspace,
user=client.active_user,
)
else:
Expand Down
1 change: 0 additions & 1 deletion src/zenml/cli/stack.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,6 @@ def register_stack(
try:
created_stack = client.zen_store.create_stack(
stack=StackRequest(
user=client.active_user.id,
workspace=client.active_workspace.id,
name=stack_name,
components=components,
Expand Down
5 changes: 4 additions & 1 deletion src/zenml/cli/tag.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ def register_tag(name: str, color: Optional[ColorVariants]) -> None:
color: The color variant for UI.
"""
request_dict = remove_none_values(dict(name=name, color=color))
client = Client()
try:
tag = Client().create_tag(TagRequest(**request_dict))
tag = client.create_tag(
TagRequest(workspace=client.active_workspace.id, **request_dict)
)
except (EntityExistsError, ValueError) as e:
cli_utils.error(str(e))

Expand Down
7 changes: 0 additions & 7 deletions src/zenml/cli/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1795,7 +1795,6 @@ def print_service_connector_configuration(
else "N/A"
),
"OWNER": user_name,
"WORKSPACE": connector.workspace.name,
"CREATED_AT": connector.created,
"UPDATED_AT": connector.updated,
}
Expand Down Expand Up @@ -2161,9 +2160,6 @@ def print_debug_stack() -> None:
console.print(f"ID: {str(stack.id)}")
if stack.user and stack.user.name and stack.user.id: # mypy check
console.print(f"User: {stack.user.name} / {str(stack.user.id)}")
console.print(
f"Workspace: {stack.workspace.name} / {str(stack.workspace.id)}"
)

for component_type, components in stack.components.items():
component = components[0]
Expand All @@ -2190,9 +2186,6 @@ def print_debug_stack() -> None:
console.print(
f"User: {component_response.user.name} / {str(component_response.user.id)}"
)
console.print(
f"Workspace: {component_response.workspace.name} / {str(component_response.workspace.id)}"
)


def _component_display_name(
Expand Down
Loading