Kamu UI 453 query view (#471)

* Integrated monaco editor with load mpre button

* added search

* Added template for empty schema

* updated schema and graphql fragment

* Moved query and result sections in the separated component

* added separate component

* added feature with adding schemes after query

* Moved saved queries in the separate component

* fixed tests regression

* Modified css styles

* added custom placeholder for monaco

* Moved dataset schemas to the separate component

* created SqlQueryService

* Moved subscriptions into  SqlQueryService

* added ngonchanges lifecycle

* Fixed unit tests regression

* minor changes

* Added unit tests

* created shared-query module

* changed CHANGELOG.md

* Moved common components to shared folder

* removed commented code

---------

Co-authored-by: Dmitriy Borzenko <[email protected]>

Author: dmitriy-borzenko

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Added
+- Added a new page `Query` that allows you to make sql queries without being tied to a dataset
+
+
 ## [0.29.0] - 2024-11-12
 ### Fixed
 - Readme section refresh when navigating between datasets

resources/schema.graphql

@@ -384,6 +384,7 @@ enum DataQueryResultErrorKind {
 type DataQueryResultSuccess {
 	schema: DataSchema
 	data: DataBatch!
+	datasets: [DatasetState!]
 	limit: Int!
 }
 
@@ -706,6 +707,22 @@ type DatasetPermissions {
 
 scalar DatasetRef
 
+type DatasetState {
+	"""
+	Globally unique identity of the dataset
+	"""
+	id: DatasetID!
+	"""
+	Alias to be used in the query
+	"""
+	alias: String!
+	"""
+	Last block hash of the input datasets that was or should be considered
+	during the query planning
+	"""
+	blockHash: Multihash
+}
+
 enum DatasetVisibility {
 	PRIVATE
 	PUBLIC

src/app/api/gql/dataset-data-sql-run.graphql

@@ -4,7 +4,7 @@ query getDatasetDataSQLRun($query: String!, $limit: Int!, $skip: Int) {
       query: $query
       queryDialect: SQL_DATA_FUSION
       schemaFormat: PARQUET_JSON
-      dataFormat: JSON
+      dataFormat: JSON_AOS
       limit: $limit
       skip: $skip
     ) {

src/app/api/gql/fragments/fragment-data-query-result-success.graphql

@@ -7,4 +7,9 @@ fragment DataQueryResultSuccessView on DataQueryResultSuccess {
     format
     content
   }
+  datasets {
+    id
+    alias
+    blockHash
+  }
 }

src/app/api/kamu.graphql.interface.ts

@@ -446,6 +446,7 @@ export enum DataQueryResultErrorKind {
 export type DataQueryResultSuccess = {
     __typename?: "DataQueryResultSuccess";
     data: DataBatch;
+    datasets?: Maybe<Array<DatasetState>>;
     limit: Scalars["Int"];
     schema?: Maybe<DataSchema>;
 };
@@ -796,6 +797,19 @@ export type DatasetPermissions = {
     canView: Scalars["Boolean"];
 };
 
+export type DatasetState = {
+    __typename?: "DatasetState";
+    /** Alias to be used in the query */
+    alias: Scalars["String"];
+    /**
+     * Last block hash of the input datasets that was or should be considered
+     * during the query planning
+     */
+    blockHash?: Maybe<Scalars["Multihash"]>;
+    /** Globally unique identity of the dataset */
+    id: Scalars["DatasetID"];
+};
+
 export enum DatasetVisibility {
     Private = "PRIVATE",
     Public = "PUBLIC",
@@ -3663,6 +3677,7 @@ export type DataQueryResultSuccessViewFragment = {
     __typename?: "DataQueryResultSuccess";
     schema?: { __typename?: "DataSchema"; format: DataSchemaFormat; content: string } | null;
     data: { __typename?: "DataBatch"; format: DataBatchFormat; content: string };
+    datasets?: Array<{ __typename?: "DatasetState"; id: string; alias: string; blockHash?: string | null }> | null;
 };
 
 export type DatasetBasicsFragment = {
@@ -4648,6 +4663,11 @@ export const DataQueryResultSuccessViewFragmentDoc = gql`
             format
             content
         }
+        datasets {
+            id
+            alias
+            blockHash
+        }
     }
 `;
 export const DatasetDataFragmentDoc = gql`
@@ -5929,7 +5949,7 @@ export const GetDatasetDataSqlRunDocument = gql`
                 query: $query
                 queryDialect: SQL_DATA_FUSION
                 schemaFormat: PARQUET_JSON
-                dataFormat: JSON
+                dataFormat: JSON_AOS
                 limit: $limit
                 skip: $skip
             ) {

src/app/app-routing.module.ts

@@ -1,3 +1,4 @@
+import { GlobalQueryComponent } from "./query/global-query/global-query.component";
 import { AddPollingSourceComponent } from "./dataset-view/additional-components/metadata-component/components/source-events/add-polling-source/add-polling-source.component";
 import { MetadataBlockComponent } from "./dataset-block/metadata-block/metadata-block.component";
 import { AuthenticatedGuard } from "./auth/guards/authenticated.guard";
@@ -52,6 +53,11 @@ export const routes: Routes = [
         component: QueryExplainerComponent,
         loadChildren: () => import("./query-explainer/query-explainer.module").then((m) => m.QueryExplainerModule),
     },
+    {
+        path: ProjectLinks.URL_QUERY,
+        component: GlobalQueryComponent,
+        loadChildren: () => import("./query/query.module").then((m) => m.QueryModule),
+    },
     {
         path:
             `:${ProjectLinks.URL_PARAM_ACCOUNT_NAME}/:${ProjectLinks.URL_PARAM_DATASET_NAME}` +

src/app/app.component.spec.ts

@@ -26,6 +26,7 @@ import { LoginService } from "./auth/login/login.service";
 import { HttpClientTestingModule } from "@angular/common/http/testing";
 import { NotificationIndicatorComponent } from "./components/notification-indicator/notification-indicator.component";
 import { AngularSvgIconModule, SvgIconRegistryService } from "angular-svg-icon";
+import { MatIconModule } from "@angular/material/icon";
 
 describe("AppComponent", () => {
     let component: AppComponent;
@@ -46,6 +47,7 @@ describe("AppComponent", () => {
                 FormsModule,
                 HttpClientTestingModule,
                 RouterModule,
+                MatIconModule,
                 AngularSvgIconModule.forRoot(),
             ],
             declarations: [

src/app/common/app.values.ts

@@ -32,6 +32,7 @@ export default class AppValues {
     public static readonly HEADERS_AUTHORIZATION_KEY = "Authorization";
     public static readonly UPLOAD_FILE_IMAGE = "assets/images/upload-file-gear.gif";
     public static readonly DEFAULT_ADMIN_ACCOUNT_NAME = "kamu";
+    public static readonly DEFAULT_MONACO_EDITOR_PLACEHOLDER = "Please type your query here...";
 
     public static readonly MARKDOWN_CONTAIN = `## Markdown __rulez__!
 ---

src/app/common/data.helpers.ts

@@ -1,15 +1,21 @@
 import { MaybeNull } from "src/app/common/app.types";
 import { AbstractControl, ValidationErrors, ValidatorFn, Validators } from "@angular/forms";
-import { FlowSummaryDataFragment, MetadataBlockFragment, TimeUnit } from "../api/kamu.graphql.interface";
+import {
+    DataQueryResultSuccessViewFragment,
+    FlowSummaryDataFragment,
+    MetadataBlockFragment,
+    TimeUnit,
+} from "../api/kamu.graphql.interface";
 import { EventPropertyLogo } from "../dataset-block/metadata-block/components/event-details/supported.events";
 import { JsonFormValidators } from "../dataset-view/additional-components/metadata-component/components/source-events/add-polling-source/add-polling-source-form.types";
 import { MaybeUndefined } from "./app.types";
 import { RxwebValidators } from "@rxweb/reactive-form-validators";
 import { isValidCronExpression } from "./cron-expression-validator.helper";
 import { ErrorPolicy, WatchQueryFetchPolicy } from "@apollo/client";
 import moment from "moment";
-import { convertSecondsToHumanReadableFormat } from "./app.helpers";
+import { convertSecondsToHumanReadableFormat, removeAllLineBreaks } from "./app.helpers";
 import { SliceUnit } from "../dataset-view/additional-components/dataset-settings-component/tabs/compacting/dataset-settings-compacting-tab.types";
+import { DataRow, DatasetSchema } from "../interface/dataset.interface";
 
 export class DataHelpers {
     public static readonly BLOCK_DESCRIBE_SEED = "Dataset initialized";
@@ -357,3 +363,12 @@ export function sliceSizeMapperReverse(sizeInBytes: number): { size: number; uni
         return { size: sizeInBytes / Math.pow(2, 10), unit: SliceUnit.KB };
     }
 }
+
+export function parseSchema(schemaContent: string): DatasetSchema {
+    return JSON.parse(removeAllLineBreaks(schemaContent)) as DatasetSchema;
+}
+
+export function parseDataRows(successResult: DataQueryResultSuccessViewFragment): DataRow[] {
+    const content: string = successResult.data.content;
+    return JSON.parse(content) as DataRow[];
+}

src/app/components/app-header/app-header.component.html

@@ -107,24 +107,34 @@
                     >
                 </ng-template>
 
+                <a
+                    class="header-link d-flex align-items-center py-2 py-md-3 mr-4 mr-md-3 border-sm-top border-md-top-0 border-white-fade"
+                    [routerLink]="[URL_QUERY]"
+                >
+                    <mat-icon class="fs-4">query_stats</mat-icon>
+                    <span> Query</span>
+                </a>
+
                 <ng-template [ngIf]="isUserLoggedIn()">
                     <a
-                        class="header-link py-2 py-md-3 mr-0 mr-md-3 border-sm-top border-md-top-0 border-white-fade"
+                        class="header-link d-flex align-items-center py-2 py-md-3 mr-4 mr-md-3 border-sm-top border-md-top-0 border-white-fade"
                         data-ga-click="Header, click, Nav menu - item:dashboard:user"
                         data-test-id="userDatasetsHeader"
                         aria-label="Dashboard"
                         [routerLink]="['/', loggedAccount.accountName]"
                         [queryParams]="{ tab: AccountTabs.DATASETS }"
                     >
-                        Your Datasets
+                        <mat-icon class="fs-4">book</mat-icon>
+                        <span>Your Datasets </span>
                     </a>
 
                     <a
-                        class="header-link py-2 py-md-3 mr-0 mr-md-3 border-sm-top border-md-top-0 border-white-fade"
+                        class="header-link d-flex align-items-center py-2 py-md-3 mr-4 mr-md-3 border-sm-top border-md-top-0 border-white-fade"
                         (click)="onDashboard()"
                         *ngIf="isAdmin"
                     >
-                        Dashboard
+                        <mat-icon class="fs-4">dashboard</mat-icon>
+                        <span> Dashboard</span>
                     </a>
 
                     <a
@@ -203,7 +213,7 @@
                 </div>
                 <ng-container *ngIf="isUserLoggedIn()">
                     <a
-                        class="app-header__right-column__addnew-block d-block d-sm-none d-md-flex m-2"
+                        class="app-header__right-column__addnew-block header-link d-block d-sm-none d-md-flex m-2"
                         [routerLink]="[URL_DATASET_CREATE]"
                         data-test-id="addNewBlock"
                     >

src/app/components/app-header/app-header.component.spec.ts

@@ -31,6 +31,7 @@ import { AngularSvgIconModule } from "angular-svg-icon";
 import { HttpClientTestingModule } from "@angular/common/http/testing";
 import { BrowserAnimationsModule } from "@angular/platform-browser/animations";
 import { LoginMethod } from "src/app/app-config.model";
+import { MatIconModule } from "@angular/material/icon";
 
 describe("AppHeaderComponent", () => {
     let component: AppHeaderComponent;
@@ -55,6 +56,7 @@ describe("AppHeaderComponent", () => {
                 AngularSvgIconModule.forRoot(),
                 HttpClientTestingModule,
                 RouterModule,
+                MatIconModule,
             ],
             declarations: [AppHeaderComponent, NotificationIndicatorComponent],
             providers: [

src/app/components/app-header/app-header.component.ts

@@ -67,6 +67,7 @@ export class AppHeaderComponent extends BaseComponent implements OnInit {
     public readonly HOME_LINK = ProjectLinks.URL_SEARCH;
     public readonly URL_DATASET_CREATE = ProjectLinks.URL_DATASET_CREATE;
     public readonly URL_SETTINGS = ProjectLinks.URL_SETTINGS;
+    public readonly URL_QUERY = ProjectLinks.URL_QUERY;
 
     private appSearchAPI = inject(SearchApi);
     private route = inject(ActivatedRoute);

src/app/components/data-access-panel/data-access-modal/tabs/data-access-stream-tab/data-access-stream-tab.component.scss

@@ -0,0 +1,20 @@
+fieldset {
+    &.fieldset-border {
+        border: 1px solid #dee2e6;
+        padding: 0 1.4em 1em;
+        margin: 0 0 0.5em;
+        border-radius: 6px;
+    }
+
+    legend {
+        &.fieldset-border {
+            font-size: 12px;
+            font-weight: bold;
+            width: auto;
+            padding: 0 10px;
+            position: relative;
+            top: -10px;
+            background: white;
+        }
+    }
+}

src/app/components/data-access-panel/data-access-modal/tabs/data-access-stream-tab/data-access-stream-tab.component.ts

@@ -5,6 +5,7 @@ import { KafkaProtocolDesc, WebSocketProtocolDesc } from "src/app/api/kamu.graph
 @Component({
     selector: "app-data-access-stream-tab",
     templateUrl: "./data-access-stream-tab.component.html",
+    styleUrls: ["./data-access-stream-tab.component.scss"],
     changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class DataAccessStreamTabComponent extends DataAccessBaseTabComponent {

src/app/components/data-access-panel/data-access-panel.component.spec.ts

@@ -16,11 +16,13 @@ import { ApolloTestingModule } from "apollo-angular/testing";
 import { ProtocolsService } from "src/app/services/protocols.service";
 import { of } from "rxjs";
 import { mockDatasetEndPoints } from "./data-access-panel-mock.data";
+import { NgbModal } from "@ng-bootstrap/ng-bootstrap";
 
 describe("DataAccessPanelComponent", () => {
     let component: DataAccessPanelComponent;
     let fixture: ComponentFixture<DataAccessPanelComponent>;
     let protocolsService: ProtocolsService;
+    let ngbModalService: NgbModal;
 
     beforeEach(async () => {
         await TestBed.configureTestingModule({
@@ -44,6 +46,7 @@ describe("DataAccessPanelComponent", () => {
 
         fixture = TestBed.createComponent(DataAccessPanelComponent);
         protocolsService = TestBed.inject(ProtocolsService);
+        ngbModalService = TestBed.inject(NgbModal);
         component = fixture.componentInstance;
         component.datasetBasics = mockDatasetBasicsDerivedFragment;
         spyOn(protocolsService, "getProtocols").and.returnValue(of(mockDatasetEndPoints));
@@ -53,4 +56,10 @@ describe("DataAccessPanelComponent", () => {
     it("should create", () => {
         expect(component).toBeTruthy();
     });
+
+    it("should check open modal window", () => {
+        const ngbModalOpenSpy = spyOn(ngbModalService, "open").and.callThrough();
+        component.openDataAccessModal();
+        expect(ngbModalOpenSpy).toHaveBeenCalledTimes(1);
+    });
 });