From 9e1cecfd62bd5c7543172c84113509bc7c0b02f1 Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Thu, 23 Jan 2025 15:49:23 +0000 Subject: [PATCH 1/2] AB#10288 --- .../assets/v2/assets/service.proto | 85 ++++++++++++------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/datatrails-common-api/assets/v2/assets/service.proto b/datatrails-common-api/assets/v2/assets/service.proto index 7cf6c82..78d94f9 100644 --- a/datatrails-common-api/assets/v2/assets/service.proto +++ b/datatrails-common-api/assets/v2/assets/service.proto @@ -6,17 +6,16 @@ option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/ import "google/api/annotations.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; - -import "datatrails-common-api/assets/v2/assets/assetresponse.proto"; +import "datatrails-common-api/caps/v1/caps/caps.proto"; import "datatrails-common-api/assets/v2/assets/createasset.proto"; +import "datatrails-common-api/assets/v2/assets/assetresponse.proto"; +import "datatrails-common-api/assets/v2/assets/getasset.proto"; +import "datatrails-common-api/assets/v2/assets/listassets.proto"; import "datatrails-common-api/assets/v2/assets/createevent.proto"; import "datatrails-common-api/assets/v2/assets/eventresponse.proto"; -import "datatrails-common-api/assets/v2/assets/getasset.proto"; import "datatrails-common-api/assets/v2/assets/getevent.proto"; -import "datatrails-common-api/assets/v2/assets/listassets.proto"; import "datatrails-common-api/assets/v2/assets/listevents.proto"; import "datatrails-common-api/assets/v2/assets/miscmessages.proto"; -import "datatrails-common-api/caps/v1/caps/caps.proto"; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { info: { @@ -25,41 +24,28 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { version: "2.0" contact: { name: "DataTrails" - url: "https://www.datatrails.com" + url: "https://www.datatrails.ai" } }; - base_path: "/_api"; + base_path: "/archivist/v2/assets"; schemes: HTTPS; consumes: "application/json"; produces: "application/json"; tags: [ { - name: "Assets & Events", + name: "Assets and Events", description: "Primary API for Asset management" - }, - { - name: "Unsupported", - description: "Internal APIs that are unstable and should not be used in a production setting." } ]; }; -service Events { - rpc Create(CreateEventRequest) returns (EventResponse) { - option (google.api.http) = { - post: "/archivist/v2/assets/{uuid}/events" - body: "*" - }; - } -} - service Assets { rpc ListAssets(ListAssetsRequest) returns (ListAssetsResponse) { option (google.api.http) = { get: "/archivist/v2/assets" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "List Assets" description: "Retrieves a list of Assets" responses: { @@ -83,6 +69,12 @@ service Assets { description: "Returned when the user is not authenticated to the system." } } + responses: { + key: "402"; + value: { + description: "Returned when the user's quota of Assets for the given proof mechanism has been reached." + } + } responses: { key: "403"; value: { @@ -104,7 +96,7 @@ service Assets { body: "*" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "Create an Asset" description: "Creates an Asset" responses: { @@ -139,7 +131,7 @@ service Assets { get: "/archivist/v2/assets/{uuid}" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "Retrieves a specific Asset" description: "Retrieves a specific Asset" responses: { @@ -174,7 +166,7 @@ service Assets { get: "/archivist/v2/assets/{uuid}:publicurl" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "Retrieves the public url for a specific Asset." description: "Retrieves the public url for a specific Asset." responses: { @@ -209,7 +201,7 @@ service Assets { get: "/archivist/v2/assets/{asset_uuid}/events/{uuid}" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "Retrieves Event" description: "Retrieves a specific Event" responses: { @@ -244,7 +236,7 @@ service Assets { get: "/archivist/v2/assets/{asset_uuid}/events/{uuid}:publicurl" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "Retrieves the public url for a specific Event." description: "Retrieves the public url for a specific Event." responses: { @@ -279,7 +271,7 @@ service Assets { get: "/archivist/v2/assets/{uuid}/events" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events" ] + tags: [ "Assets and Events" ] summary: "List Events" description: "Lists Events" responses: { @@ -324,7 +316,7 @@ service Assets { get: "/archivist/v2/assets:caps" }; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { - tags: [ "Assets & Events", "Unsupported" ] // Unsupported is signal for future OAPI parsing + tags: [ "Assets and Events", "Unsupported" ] // Unsupported is signal for future OAPI parsing summary: "Get remaining capped resources for Assets" description: "Not stable or officially supported. Get remaining capped resources for Assets" responses: { @@ -341,4 +333,37 @@ service Assets { } }; } -} \ No newline at end of file +} + +service Events { + rpc Create(CreateEventRequest) returns (EventResponse) { + option (google.api.http) = { + post: "/archivist/v2/assets/{uuid}/events" + body: "*" + }; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { + tags: [ "Assets and Events" ] + summary: "Creates an Event" + description: "Creates an Event" + responses: { + key: "401"; + value: { + description: "Returned when the user is not authenticated to the system." + } + } + responses: { + key: "402"; + value: { + description: "Returned when the user's quota of Events has been reached." + } + } + responses: { + key: "429"; + value: { + description: "Returned when a user exceeds their subscription's rate limit for requests."; + } + } + }; + } +} + From 8c30a9edce16adbe4094c0c9cf380a128e25edd1 Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Thu, 23 Jan 2025 15:56:02 +0000 Subject: [PATCH 2/2] Use apim The assets proto file is now suitable for APIM Removed the requirement for avid-builder Added standard default and environment runes Removed workflow(docker-all) git hub action AB#10288 --- .env.tools | 2 +- .github/workflows/ci.yml | 3 +- .github/workflows/workflow.yml | 25 ---------------- Taskfile.yml | 53 ++++++---------------------------- taskfiles/Taskfile_apis.yml | 7 ++--- 5 files changed, 13 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/workflow.yml diff --git a/.env.tools b/.env.tools index 10565b5..cc1221b 100644 --- a/.env.tools +++ b/.env.tools @@ -1 +1 @@ -export TOOLS_BUILDNUMBER=20241007.3 +export TOOLS_BUILDNUMBER=20241108.3 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a7ddbc..f13d0c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Generate, Test and Export run: | - # Note: it is by design that we don't use the builder - task local-all + task all - name: Show exports working-directory: exported/ diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index 53eec60..0000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Developer Workflow Tests - -on: - # we ony run these on pull request as they significantly hinder developer feed - # back if they are triggered on push - pull_request: - branches: - - main -jobs: - docker-all: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup go-task - uses: arduino/setup-task@v1 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: docker all - run: | - - echo "TODO: permissions on accessing our acr prevent this from working" - exit 0 - # ensure the all target (which runs everything in docker) works - # task all diff --git a/Taskfile.yml b/Taskfile.yml index 9c33a8e..d8bf544 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -21,60 +21,25 @@ includes: # dev-notes/build-warts.md#always-set-dir-relative-to-current-task-file dir: ./taskfiles -vars: - ACR: rkvstdev.azurecr.io - # Default to the stage master-latest builder image - # Note: the CI tags builder images with the build number of the avid-builder pipeline run, - # i.e. 20220607.10 - # For local builder development build run `task builder` in the avid-builder git repo to - # build a new local builder image, then run these tasks like - # task generate BUILDER_ACR=rkvstdev.azurecr.io BUILDER_IMAGE_TAG=dev-$USER - BUILDER_ACR: rkvststage.azurecr.io - BUILDER_IMAGE_TAG: master-latest - - BUILD_CONTAINER: go-datatrails-common-api-builder - tasks: - ### ------------------------- - # Docker container tasks - ### ------------------------- - - builder-start: - summary: | - Creates the build container for go-datatrails-common-api - - Many tasks docker exec into this container so DO NOT ADD --rm. - - Use defer to cleanup the docker container in top level workflow tasks - which create the builder container - vars: - BUILDER_IMAGE: '{{.BUILDER_ACR}}/avid-builder:{{.BUILDER_IMAGE_TAG}}' + default: + desc: Default task lists all available tasks to run. cmds: + - which task - | - docker run \ - --env CONCURRENCY={{.SKAFFOLD_CONCURRENCY | default 2}} \ - --name {{.BUILD_CONTAINER}} \ - -dt -u $(id -u):$(id -g) \ - --mount type=bind,src=$(pwd),dst=/avid -w /avid \ - {{.BUILDER_IMAGE}} + task --list --sort=none - builder-cleanup: - summary: cleans up the forestrie build container + environment: + desc: Show environment and location of tools executables. cmds: - - docker rm -f {{.BUILD_CONTAINER}} + - env | sort + - which go ### ------------------------- # Primary workflow tasks ### ------------------------- all: - desc: "do everything necessary after clone (or rebase) in the builder" - cmds: - - task: builder-start - - defer: {task: builder-cleanup} - - docker exec -t {{.BUILD_CONTAINER}} task local-all - - local-all: desc: | do everything necessary after clone (or rebase) (faster as it RUNS ON NATIVE HOST) @@ -105,4 +70,4 @@ tasks: generate: desc: generates all the artifacts we need pre-build cmds: - - task: apis:generate \ No newline at end of file + - task: apis:generate diff --git a/taskfiles/Taskfile_apis.yml b/taskfiles/Taskfile_apis.yml index 3410b4c..99a20dd 100644 --- a/taskfiles/Taskfile_apis.yml +++ b/taskfiles/Taskfile_apis.yml @@ -80,7 +80,6 @@ tasks: rm -rf {{.GOOGLE_COMMON_INC_ARCHIVE_BASEDIR}}-{{.GOOGLE_COMMON_PROTOS_VERSION}} find proto-include -type d - bootstrap:grpc_health: desc: download and prepare the grpc health definitions we require under proto-include dir: '..' @@ -114,15 +113,12 @@ tasks: github.com/envoyproxy/protoc-gen-validate cmds: - | - go get github.com/datatrails/go-datatrails-common-api-gen/tools - go install \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ google.golang.org/protobuf/cmd/protoc-gen-go \ google.golang.org/grpc/cmd/protoc-gen-go-grpc \ github.com/envoyproxy/protoc-gen-validate \ - github.com/lyft/protoc-gen-star/v2 \ github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc - for: { var: API_PROTO_DEPS } cmd: | @@ -156,6 +152,7 @@ tasks: dir: '../datatrails-common-api' cmds: - go mod tidy + - go mod verify generate: desc: generate api grpc bindings @@ -278,4 +275,4 @@ tasks: # OUTPATH: datatrails-common-api OUTPATH: . GOBIN: - sh: echo ${GOBIN:-$HOME/go/bin} \ No newline at end of file + sh: echo ${GOBIN:-$HOME/go/bin}