Merge branch 'master' into oracle-thick-mode

.github/workflows/metadata-ingestion.yml

@@ -27,7 +27,7 @@ concurrency:
 jobs:
   metadata-ingestion:
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    timeout-minutes: 60
     env:
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-02-11 10:00:11 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -53,13 +53,6 @@ repos:
         files: ^metadata-dao-impl/kafka-producer/.*\.java$
         pass_filenames: false
 
-      - id: metadata-events-mxe-avro-spotless
-        name: metadata-events/mxe-avro Spotless Apply
-        entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
-        language: system
-        files: ^metadata-events/mxe-avro/.*\.java$
-        pass_filenames: false
-
       - id: metadata-events-mxe-registration-spotless
         name: metadata-events/mxe-registration Spotless Apply
         entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
@@ -312,6 +305,20 @@ repos:
         files: ^metadata-service/graphql-servlet-impl/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-iceberg-catalog-spotless
+        name: metadata-service/iceberg-catalog Spotless Apply
+        entry: ./gradlew :metadata-service:iceberg-catalog:spotlessApply
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-iceberg-catalog-lint-fix
+        name: metadata-service/iceberg-catalog Lint Fix
+        entry: ./gradlew :metadata-service:iceberg-catalog:lintFix
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.py$
+        pass_filenames: false
+
       - id: metadata-service-openapi-analytics-servlet-spotless
         name: metadata-service/openapi-analytics-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
@@ -410,6 +417,13 @@ repos:
         files: ^metadata-service/servlet/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-war-spotless
+        name: metadata-service/war Spotless Apply
+        entry: ./gradlew :metadata-service:war:spotlessApply
+        language: system
+        files: ^metadata-service/war/.*\.java$
+        pass_filenames: false
+
       - id: metadata-utils-spotless
         name: metadata-utils Spotless Apply
         entry: ./gradlew :metadata-utils:spotlessApply

README.md

@@ -154,6 +154,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Udemy](https://www.udemy.com/)
 - [Uphold](https://uphold.com)
 - [Viasat](https://viasat.com)
+- [Wealthsimple](https://www.wealthsimple.com)
 - [Wikimedia](https://www.wikimedia.org)
 - [Wolt](https://wolt.com)
 - [Zynga](https://www.zynga.com)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java

@@ -48,6 +48,8 @@ public Schema apply(
     result.setPrimaryKeys(input.getPrimaryKeys());
     result.setFields(
         input.getFields().stream()
+            // do not map the field if the field.getFieldPath() is null or empty
+            .filter(field -> field.getFieldPath() != null && !field.getFieldPath().isEmpty())
             .map(field -> SchemaFieldMapper.map(context, field, entityUrn))
             .collect(Collectors.toList()));
     result.setPlatformSchema(PlatformSchemaMapper.map(context, input.getPlatformSchema()));

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java

@@ -46,6 +46,8 @@ public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(
     result.setPrimaryKeys(input.getPrimaryKeys());
     result.setFields(
         input.getFields().stream()
+            // do not map the field if the field.getFieldPath() is null or empty
+            .filter(field -> field.getFieldPath() != null && !field.getFieldPath().isEmpty())
             .map(field -> SchemaFieldMapper.map(context, field, entityUrn))
             .collect(Collectors.toList()));
     result.setPlatformSchema(PlatformSchemaMapper.map(context, input.getPlatformSchema()));

datahub-web-react/package.json

@@ -95,7 +95,8 @@
         "uuid": "^8.3.2",
         "virtualizedtableforantd4": "^1.2.1",
         "web-vitals": "^0.2.4",
-        "yamljs": "^0.3.0"
+        "yamljs": "^0.3.0",
+        "country-data-list": "^1.3.4"
     },
     "scripts": {
         "analyze": "source-map-explorer 'dist/assets/*.js'",

datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx

@@ -183,9 +183,11 @@ export class DataProcessInstanceEntity implements Entity<DataProcessInstance> {
                 parentContainers={data.parentContainers}
                 parentEntities={parentEntities}
                 container={data.container || undefined}
-                duration={firstState?.durationMillis}
-                status={firstState?.result?.resultType}
-                startTime={firstState?.timestampMillis}
+                dataProcessInstanceProps={{
+                    startTime: firstState?.timestampMillis,
+                    duration: firstState?.durationMillis ?? undefined,
+                    status: firstState?.result?.resultType ?? undefined,
+                }}
             />
         );
     };

datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx

@@ -39,9 +39,7 @@ export const Preview = ({
     health,
     parentEntities,
     parentContainers,
-    duration,
-    status,
-    startTime,
+    dataProcessInstanceProps,
 }: {
     urn: string;
     name: string;
@@ -64,9 +62,11 @@ export const Preview = ({
     health?: Health[] | null;
     parentEntities?: Array<GeneratedEntity> | null;
     parentContainers?: ParentContainersResult | null;
-    duration?: number | null;
-    status?: string | null;
-    startTime?: number | null;
+    dataProcessInstanceProps?: {
+        startTime?: number;
+        duration?: number;
+        status?: string;
+    };
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
     return (
@@ -95,9 +95,7 @@ export const Preview = ({
             paths={paths}
             health={health || undefined}
             parentEntities={parentEntities}
-            duration={duration}
-            status={status}
-            startTime={startTime}
+            dataProcessInstanceProps={dataProcessInstanceProps}
         />
     );
 };

datahub-web-react/src/app/entity/shared/SidebarStyledComponents.tsx

@@ -64,6 +64,14 @@ export const Name = styled.div`
     margin: 13px 0 7px 0;
 `;
 
+export const UserDetails = styled.div`
+    font-size: 14px;
+    line-height: 35px;
+    font-weight: 400;
+    line-height: 28px;
+    text-align: center;
+`;
+
 export const TitleRole = styled.div`
     font-size: 14px;
     line-height: 22px;
@@ -74,6 +82,8 @@ export const TitleRole = styled.div`
 export const Team = styled.div`
     font-size: 12px;
     line-height: 20px;
+    font-weight: 400;
+    padding-bottom: 10px;
     color: #8c8c8c;
 `;
 
@@ -111,6 +121,23 @@ export const AboutSection = styled.div`
     color: #262626;
 `;
 
+export const LocationSection = styled.div`
+    text-align: left;
+    font-weight: bold;
+    font-size: 14px;
+    line-height: 26px;
+    color: #262626;
+`;
+
+export const LocationSectionText = styled.div`
+    text-align: left;
+    font-weight: normal;
+    font-size: 14px;
+    line-height: 26px;
+    margin-bottom: -10px;
+    color: #262626;
+`;
+
 export const AboutSectionText = styled.div`
     font-size: 12px;
     font-weight: 100;

datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx

@@ -2,44 +2,20 @@ import React from 'react';
 import * as QueryString from 'query-string';
 import { useHistory, useLocation } from 'react-router';
 import { ApolloError } from '@apollo/client';
-import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
+import { FacetFilterInput } from '../../../../../../types.generated';
 import useFilters from '../../../../../search/utils/useFilters';
 import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl';
 import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './types';
 import { useEntityQueryParams } from '../../../containers/profile/utils';
 import { EmbeddedListSearch } from './EmbeddedListSearch';
-import { UnionType } from '../../../../../search/utils/constants';
+import { EMBEDDED_LIST_SEARCH_ENTITY_TYPES, UnionType } from '../../../../../search/utils/constants';
 import {
     DownloadSearchResults,
     DownloadSearchResultsInput,
     DownloadSearchResultsParams,
 } from '../../../../../search/utils/types';
 
 const FILTER = 'filter';
-const SEARCH_ENTITY_TYPES = [
-    EntityType.Dataset,
-    EntityType.Dashboard,
-    EntityType.Chart,
-    EntityType.Mlmodel,
-    EntityType.MlmodelGroup,
-    EntityType.MlfeatureTable,
-    EntityType.Mlfeature,
-    EntityType.MlprimaryKey,
-    EntityType.DataFlow,
-    EntityType.DataJob,
-    EntityType.GlossaryTerm,
-    EntityType.GlossaryNode,
-    EntityType.Tag,
-    EntityType.Role,
-    EntityType.CorpUser,
-    EntityType.CorpGroup,
-    EntityType.Container,
-    EntityType.Domain,
-    EntityType.DataProduct,
-    EntityType.Notebook,
-    EntityType.BusinessAttribute,
-    EntityType.DataProcessInstance,
-];
 
 function getParamsWithoutFilters(params: QueryString.ParsedQuery<string>) {
     const paramsCopy = { ...params };
@@ -161,7 +137,7 @@ export const EmbeddedListSearchSection = ({
 
     return (
         <EmbeddedListSearch
-            entityTypes={SEARCH_ENTITY_TYPES}
+            entityTypes={EMBEDDED_LIST_SEARCH_ENTITY_TYPES}
             query={query || ''}
             page={page}
             unionType={unionType}

datahub-web-react/src/app/entity/shared/types.ts

@@ -45,6 +45,7 @@ import {
     Documentation,
     DisplayProperties,
     VersionProperties,
+    DataProcessRunEvent,
 } from '../../../types.generated';
 import { FetchedEntity } from '../../lineage/types';
 
@@ -135,6 +136,9 @@ export type GenericEntityProperties = {
     displayProperties?: Maybe<DisplayProperties>;
     notes?: Maybe<EntityRelationshipsResult>;
     versionProperties?: Maybe<VersionProperties>;
+
+    // Data process instance
+    lastRunEvent?: Maybe<DataProcessRunEvent>;
 };
 
 export type GenericEntityUpdate = {

datahub-web-react/src/app/entity/user/UserInfoSideBar.tsx

@@ -1,6 +1,8 @@
+import { getCountryName } from '@src/app/shared/sidebar/components';
 import { Divider, message, Space, Button, Typography, Tag } from 'antd';
 import React, { useEffect, useState } from 'react';
 import { EditOutlined, MailOutlined, PhoneOutlined, SlackOutlined } from '@ant-design/icons';
+import GlobeIcon from '../../../images/Globe.svg';
 import { useUpdateCorpUserPropertiesMutation } from '../../../graphql/user.generated';
 import { EntityRelationship, DataHubRole } from '../../../types.generated';
 import UserEditProfileModal from './UserEditProfileModal';
@@ -17,6 +19,9 @@ import {
     Name,
     TitleRole,
     Team,
+    LocationSection,
+    LocationSectionText,
+    UserDetails,
 } from '../shared/SidebarStyledComponents';
 import EntityGroups from '../shared/EntityGroups';
 import { mapRoleIcon } from '../../identity/user/UserUtils';
@@ -38,6 +43,8 @@ type SideBarData = {
     groupsDetails: Array<EntityRelationship>;
     urn: string | undefined;
     dataHubRoles: Array<EntityRelationship>;
+    countryCode: string | undefined;
+    username: string | undefined;
 };
 
 type Props = {
@@ -51,8 +58,22 @@ const AVATAR_STYLE = { marginTop: '14px' };
  * UserInfoSideBar- Sidebar section for users profiles.
  */
 export default function UserInfoSideBar({ sideBarData, refetch }: Props) {
-    const { name, aboutText, avatarName, email, groupsDetails, phone, photoUrl, role, slack, team, dataHubRoles, urn } =
-        sideBarData;
+    const {
+        name,
+        aboutText,
+        avatarName,
+        email,
+        groupsDetails,
+        phone,
+        photoUrl,
+        role,
+        slack,
+        team,
+        dataHubRoles,
+        urn,
+        countryCode,
+        username,
+    } = sideBarData;
 
     const [updateCorpUserPropertiesMutation] = useUpdateCorpUserPropertiesMutation();
 
@@ -114,12 +135,14 @@ export default function UserInfoSideBar({ sideBarData, refetch }: Props) {
     };
     const dataHubRoleName = dataHubRoles && dataHubRoles.length > 0 && (dataHubRoles[0]?.entity as DataHubRole).name;
 
+    const maybeCountryName = getCountryName(countryCode ?? '');
     return (
         <>
             <SideBar>
                 <SideBarSubSection className={isProfileOwner ? '' : 'fullView'}>
                     <CustomAvatar size={160} photoUrl={photoUrl} name={avatarName} style={AVATAR_STYLE} />
                     <Name>{name || <EmptyValue />}</Name>
+                    <UserDetails>{username || <EmptyValue />}</UserDetails>
                     {role && <TitleRole>{role}</TitleRole>}
                     {team && <Team>{team}</Team>}
                     {dataHubRoleName && <Tag icon={mapRoleIcon(dataHubRoleName)}>{dataHubRoleName}</Tag>}
@@ -143,6 +166,17 @@ export default function UserInfoSideBar({ sideBarData, refetch }: Props) {
                         </Space>
                     </SocialDetails>
                     <Divider className="divider-aboutSection" />
+                    {maybeCountryName != null ? (
+                        <LocationSection>
+                            Location
+                            <br />
+                            <LocationSectionText>
+                                <img src={GlobeIcon} alt="Manage Users" style={{ display: 'inline' }} /> &nbsp;
+                                {maybeCountryName}
+                            </LocationSectionText>
+                            <Divider className="divider-aboutSection" />
+                        </LocationSection>
+                    ) : null}
                     <AboutSection>
                         About
                         <AboutSectionText>

datahub-web-react/src/app/entity/user/UserProfile.tsx

@@ -107,13 +107,15 @@ export default function UserProfile() {
             undefined,
         role: data?.corpUser?.editableProperties?.title || data?.corpUser?.info?.title || undefined,
         team: data?.corpUser?.editableProperties?.teams?.join(',') || data?.corpUser?.info?.departmentName || undefined,
+        countryCode: data?.corpUser?.info?.countryCode || undefined,
         email: data?.corpUser?.editableProperties?.email || data?.corpUser?.info?.email || undefined,
         slack: data?.corpUser?.editableProperties?.slack || undefined,
         phone: data?.corpUser?.editableProperties?.phone || undefined,
         aboutText: data?.corpUser?.editableProperties?.aboutMe || undefined,
         groupsDetails: userGroups,
         dataHubRoles: userRoles,
         urn,
+        username: data?.corpUser?.username,
     };
 
     if (data?.corpUser?.exists === false) {