Merge branch 'master' into feat/add-mlflow-scripts

.github/workflows/docker-unified.yml

@@ -44,6 +44,7 @@ jobs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}
       full_tag: ${{ steps.tag.outputs.full_tag }}
+      short_sha: ${{ steps.tag.outputs.short_sha }} # needed for auto-deploy
       unique_tag: ${{ steps.tag.outputs.unique_tag }}
       unique_slim_tag: ${{ steps.tag.outputs.unique_slim_tag }}
       unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }}
@@ -65,6 +66,8 @@ jobs:
       postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
       elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
       smoke_test_change: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
+      integrations_service_change: "false"
+      datahub_executor_change: "false"
     steps:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
@@ -864,7 +867,8 @@ jobs:
           context: .
           file: ./docker/datahub-ingestion/Dockerfile
           platforms: linux/amd64,linux/arm64/v8
-          depot-project: ${{ vars.DEPOT_PROJECT_ID }}
+          # Workaround 2025-01-25 - Depot publishing errors
+          depot-project: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'release') && '' || vars.DEPOT_PROJECT_ID }}
       - name: Compute Tag
         id: tag
         run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT"
@@ -963,7 +967,8 @@ jobs:
           context: .
           file: ./docker/datahub-ingestion/Dockerfile
           platforms: linux/amd64,linux/arm64/v8
-          depot-project: ${{ vars.DEPOT_PROJECT_ID }}
+          # Workaround 2025-01-25 - Depot publishing errors
+          depot-project: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'release') && '' || vars.DEPOT_PROJECT_ID }}
       - name: Compute Tag (Full)
         id: tag
         run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> "$GITHUB_OUTPUT"
@@ -1178,11 +1183,6 @@ jobs:
             docker pull '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head'
             docker tag '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
           fi
-          if [ '${{ needs.setup.outputs.integrations_service_change }}' == 'false' ]; then
-            echo 'datahub-integration-service head images'
-            docker pull '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:head'
-            docker tag '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:head' '${{ env.DATAHUB_INTEGRATIONS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
-          fi
       - name: CI Slim Head Images
         run: |
           if [ '${{ needs.setup.outputs.ingestion_change }}' == 'false' ]; then

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java

@@ -40,6 +40,12 @@ public MLModelGroupProperties apply(
       result.setCreated(
           TimeStampToAuditStampMapper.map(context, mlModelGroupProperties.getCreated()));
     }
+    if (mlModelGroupProperties.getName() != null) {
+      result.setName(mlModelGroupProperties.getName());
+    } else {
+      // backfill name from URN for backwards compatibility
+      result.setName(entityUrn.getEntityKey().get(1)); // indexed access is safe here
+    }
 
     if (mlModelGroupProperties.hasLastModified()) {
       result.setLastModified(

datahub-graphql-core/src/main/resources/lineage.graphql

@@ -25,3 +25,32 @@ input LineageEdge {
     """
     upstreamUrn: String!
 }
+
+"""
+Represents lineage information for ML entities.
+"""
+type MLModelLineageInfo {
+   """
+   List of jobs or processes used to train the model.
+   """
+   trainingJobs: [String!]
+
+   """
+   List of jobs or processes that use this model.
+   """
+   downstreamJobs: [String!]
+}
+
+extend type MLModelProperties {
+    """
+    Information related to lineage to this model group
+    """
+    mlModelLineageInfo: MLModelLineageInfo
+}
+
+extend type MLModelGroupProperties {
+    """
+    Information related to lineage to this model group
+    """
+    mlModelLineageInfo: MLModelLineageInfo
+}

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapperTest.java

@@ -19,6 +19,9 @@ public void testMapMLModelGroupProperties() throws URISyntaxException {
     // Set description
     input.setDescription("a ml trust model group");
 
+    // Set Name
+    input.setName("ML trust model group");
+
     // Create URN
     Urn groupUrn =
         Urn.createFromString(
@@ -31,6 +34,7 @@ public void testMapMLModelGroupProperties() throws URISyntaxException {
     // Verify mapped properties
     assertNotNull(result);
     assertEquals(result.getDescription(), "a ml trust model group");
+    assertEquals(result.getName(), "ML trust model group");
 
     // Verify lineage info is null as in the mock data
     assertNotNull(result.getMlModelLineageInfo());

datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import { ApiOutlined } from '@ant-design/icons';
-import { DataProcessInstance, EntityType, OwnershipType, SearchResult, DataJob } from '../../../types.generated';
+import { Entity as GraphQLEntity } from '@types';
+import { DataProcessInstance, EntityType, OwnershipType, SearchResult } from '../../../types.generated';
 import { Preview } from './preview/Preview';
 import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity';
 import { EntityProfile } from '../shared/containers/profile/EntityProfile';
@@ -19,12 +20,7 @@ import DataProductSection from '../shared/containers/profile/sidebar/DataProduct
 import { getDataProduct } from '../shared/utils';
 import SummaryTab from './profile/DataProcessInstanceSummary';
 
-type PreviewEntity = {
-    urn: string;
-    type: EntityType;
-};
-
-const getParentEntities = (data: DataProcessInstance): Entity<DataJob>[] => {
+const getParentEntities = (data: DataProcessInstance): GraphQLEntity[] => {
     const parentEntity = data?.relationships?.relationships?.find(
         (rel) => rel.type === 'InstanceOf' && rel.entity?.type === EntityType.DataJob,
     );
@@ -34,7 +30,7 @@ const getParentEntities = (data: DataProcessInstance): Entity<DataJob>[] => {
     }
 
     // First cast to unknown, then to Entity with proper type
-    return [parentEntity.entity as unknown as Entity<DataJob>];
+    return [parentEntity.entity];
 };
 
 /**

datahub-web-react/src/app/entity/dataProcessInstance/profile/DataProcessInstanceSummary.tsx

@@ -3,7 +3,8 @@ import styled from 'styled-components';
 import { Space, Table, Typography } from 'antd';
 import { formatDetailedDuration } from '@src/app/shared/time/timeUtils';
 import { capitalize } from 'lodash';
-import { MlHyperParam, MlMetric } from '../../../../types.generated';
+import moment from 'moment';
+import { MlHyperParam, MlMetric, DataProcessInstanceRunResultType } from '../../../../types.generated';
 import { useBaseEntity } from '../../shared/EntityContext';
 import { InfoItem } from '../../shared/components/styled/InfoItem';
 import { GetDataProcessInstanceQuery } from '../../../../graphql/dataProcessInstance.generated';
@@ -41,17 +42,11 @@ export default function MLModelSummary() {
     const baseEntity = useBaseEntity<GetDataProcessInstanceQuery>();
     const dpi = baseEntity?.dataProcessInstance;
 
-    const formatDate = (timestamp?: number) => {
-        if (!timestamp) return '-';
-        const milliseconds = timestamp < 10000000000 ? timestamp * 1000 : timestamp;
-        return new Date(milliseconds).toISOString().slice(0, 19).replace('T', ' ');
-    };
-
     const formatStatus = (state) => {
         if (!state || state.length === 0) return '-';
         const result = state[0]?.result?.resultType;
-        const statusColor = result === 'SUCCESS' ? 'green' : 'red';
-        return <Pill label={capitalize(result)} colorScheme={statusColor} />;
+        const statusColor = result === DataProcessInstanceRunResultType.Success ? 'green' : 'red';
+        return <Pill label={capitalize(result)} colorScheme={statusColor} clickable={false} />;
     };
 
     const formatDuration = (state) => {
@@ -65,7 +60,11 @@ export default function MLModelSummary() {
                 <Typography.Title level={3}>Details</Typography.Title>
                 <InfoItemContainer justifyContent="left">
                     <InfoItem title="Created At">
-                        <InfoItemContent>{formatDate(dpi?.properties?.created?.time)}</InfoItemContent>
+                        <InfoItemContent>
+                            {dpi?.properties?.created?.time
+                                ? moment(dpi.properties.created.time).format('YYYY-MM-DD HH:mm:ss')
+                                : '-'}
+                        </InfoItemContent>
                     </InfoItem>
                     <InfoItem title="Status">
                         <InfoItemContent>{formatStatus(dpi?.state)}</InfoItemContent>

datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx

@@ -68,6 +68,7 @@ export default function DataProductsTab() {
                 searchFlags: { skipCache: true },
             },
         },
+        fetchPolicy: 'no-cache',
     });
     const totalResults = data?.searchAcrossEntities?.total || 0;
     const searchResults = data?.searchAcrossEntities?.searchResults?.map((r) => r.entity) || [];

datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx

@@ -21,7 +21,8 @@ export const Preview = ({
     return (
         <DefaultPreviewCard
             url={entityRegistry.getEntityUrl(EntityType.Mlmodel, model.urn)}
-            name={model.name || ''}
+            // eslint-disable-next-line @typescript-eslint/dot-notation
+            name={model.properties?.['propertiesName'] || model.name || ''}
             urn={model.urn}
             description={model.description || ''}
             platformInstanceId={model.dataPlatformInstance?.instanceId}

datahub-web-react/src/app/entity/mlModel/profile/MLModelSummary.tsx

@@ -2,12 +2,15 @@ import React from 'react';
 import styled from 'styled-components';
 import { Space, Table, Typography } from 'antd';
 import { Link } from 'react-router-dom';
+import { colors } from '@src/alchemy-components/theme';
+import moment from 'moment';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import { MlHyperParam, MlMetric, EntityType } from '../../../../types.generated';
 import { useBaseEntity } from '../../shared/EntityContext';
 import { GetMlModelQuery } from '../../../../graphql/mlModel.generated';
 import { InfoItem } from '../../shared/components/styled/InfoItem';
 import { notEmpty } from '../../shared/utils';
+import { Pill } from '../../../../alchemy-components/components/Pills';
 
 const TabContent = styled.div`
     padding: 16px;
@@ -23,22 +26,13 @@ const InfoItemContainer = styled.div<{ justifyContent }>`
 const InfoItemContent = styled.div`
     padding-top: 8px;
     width: 100px;
-`;
-
-const VersionTagContainer = styled.span`
-    padding: 2px 8px;
-    display: inline-flex;
-    align-items: center;
-    border-radius: 4px;
-    border: 1px solid #d9d9d9;
-    color: #595959;
-    background: #fafafa;
-    margin-right: 8px;
-    margin-bottom: 4px;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 5px;
 `;
 
 const JobLink = styled(Link)`
-    color: #1890ff;
+    color: ${colors.blue[700]};
     &:hover {
         text-decoration: underline;
     }
@@ -61,12 +55,6 @@ export default function MLModelSummary() {
         },
     ];
 
-    const formatDate = (timestamp?: number) => {
-        if (!timestamp) return '-';
-        const milliseconds = timestamp < 10000000000 ? timestamp * 1000 : timestamp;
-        return new Date(milliseconds).toISOString().slice(0, 19).replace('T', ' ');
-    };
-
     const renderTrainingJobs = () => {
         const trainingJobs =
             model?.trainedBy?.relationships?.map((relationship) => relationship.entity).filter(notEmpty) || [];
@@ -75,19 +63,17 @@ export default function MLModelSummary() {
 
         return (
             <div>
-                {trainingJobs.map((job, index) => (
-                    <span key={(job as { urn: string }).urn}>
-                        <JobLink
-                            to={entityRegistry.getEntityUrl(
-                                EntityType.DataProcessInstance,
-                                (job as { urn: string }).urn,
-                            )}
-                        >
-                            {(job as { name?: string })?.name || (job as { urn: string }).urn}
-                        </JobLink>
-                        {index < trainingJobs.length - 1 && ', '}
-                    </span>
-                ))}
+                {trainingJobs.map((job, index) => {
+                    const { urn, name } = job as { urn: string; name?: string };
+                    return (
+                        <span key={urn}>
+                            <JobLink to={entityRegistry.getEntityUrl(EntityType.DataProcessInstance, urn)}>
+                                {name || urn}
+                            </JobLink>
+                            {index < trainingJobs.length - 1 && ', '}
+                        </span>
+                    );
+                })}
             </div>
         );
     };
@@ -101,10 +87,18 @@ export default function MLModelSummary() {
                         <InfoItemContent>{model?.versionProperties?.version?.versionTag}</InfoItemContent>
                     </InfoItem>
                     <InfoItem title="Registered At">
-                        <InfoItemContent>{formatDate(model?.properties?.created?.time)}</InfoItemContent>
+                        <InfoItemContent>
+                            {model?.properties?.created?.time
+                                ? moment(model.properties.created.time).format('YYYY-MM-DD HH:mm:ss')
+                                : '-'}
+                        </InfoItemContent>
                     </InfoItem>
                     <InfoItem title="Last Modified At">
-                        <InfoItemContent>{formatDate(model?.properties?.lastModified?.time)}</InfoItemContent>
+                        <InfoItemContent>
+                            {model?.properties?.lastModified?.time
+                                ? moment(model.properties.lastModified.time).format('YYYY-MM-DD HH:mm:ss')
+                                : '-'}
+                        </InfoItemContent>
                     </InfoItem>
                     <InfoItem title="Created By">
                         <InfoItemContent>{model?.properties?.created?.actor}</InfoItemContent>
@@ -114,7 +108,12 @@ export default function MLModelSummary() {
                     <InfoItem title="Aliases">
                         <InfoItemContent>
                             {model?.versionProperties?.aliases?.map((alias) => (
-                                <VersionTagContainer key={alias.versionTag}>{alias.versionTag}</VersionTagContainer>
+                                <Pill
+                                    label={alias.versionTag ?? '-'}
+                                    key={alias.versionTag}
+                                    colorScheme="blue"
+                                    clickable={false}
+                                />
                             ))}
                         </InfoItemContent>
                     </InfoItem>

datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx

@@ -139,7 +139,7 @@ export class MLModelGroupEntity implements Entity<MlModelGroup> {
     };
 
     displayName = (data: MlModelGroup) => {
-        return data.name || data.urn;
+        return data.properties?.name || data.name || data.urn;
     };
 
     getGenericEntityProperties = (mlModelGroup: MlModelGroup) => {

datahub-web-react/src/app/entity/mlModelGroup/preview/Preview.tsx

@@ -19,7 +19,8 @@ export const Preview = ({
     return (
         <DefaultPreviewCard
             url={entityRegistry.getEntityUrl(EntityType.MlmodelGroup, group.urn)}
-            name={group?.name || ''}
+            // eslint-disable-next-line @typescript-eslint/dot-notation
+            name={group?.properties?.['propertiesName'] || group?.name || ''}
             urn={group.urn}
             platformInstanceId={group.dataPlatformInstance?.instanceId}
             description={group?.description || ''}