Update registered exports

.github/workflows/main-ci.yml

@@ -42,6 +42,8 @@ jobs:
           make PortalDataset
           make PortalStudy
           make Superdataset
+          make PortalPublication
+          make Protocol
 
       # Set up supported python.
       - uses: actions/setup-python@v5

.github/workflows/propagate.yml

@@ -39,20 +39,4 @@ jobs:
       - name: Push to Synapse schema registry only JSON schemas that changed
         run: |
           CHANGED=$(git diff --name-only HEAD HEAD~1 registered-json-schemas)
-          for FILE in $CHANGED
-          do
-            REQUEST_BODY=$(jq '. + { concreteType: "org.sagebionetworks.repo.model.schema.CreateSchemaRequest", dryRun: false }' $FILE)
-            RESPONSE=$(curl -X POST https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/start \
-            -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d "$REQUEST_BODY")
-
-            # Will error if token not within response
-            TOKEN=$( echo $RESPONSE | jq -e -r '.token')
-            sleep 1
-            STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
-            -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
-
-            echo $STATUS | jq -e '.newVersionInfo'
-          done
-
+          .github/workflows/register-schema.sh $CHANGED

.github/workflows/register-schema.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+for FILE in $1
+do
+    # Synapse does not allow additionalProperties in schema, so remove that first
+    jq 'del(.additionalProperties)' $FILE > temp.json
+    REQUEST_BODY=$(jq '{schema : ., concreteType: "org.sagebionetworks.repo.model.schema.CreateSchemaRequest", dryRun: false }' temp.json)
+    RESPONSE=$(curl -X POST https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/start \
+    -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$REQUEST_BODY")
+    rm temp.json
+
+    # Will error if token not within response
+    echo $RESPONSE
+    TOKEN=$( echo $RESPONSE | jq -e -r '.token')
+    sleep 1
+    STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
+    -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
+    while true; do
+        if echo "$STATUS" | jq -e '.jobState == "PROCESSING"' > /dev/null; then
+            sleep 1
+            STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
+            -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
+        elif echo "$STATUS" | jq -e '.concreteType == "org.sagebionetworks.repo.model.ErrorResponse"' > /dev/null; then
+            echo "Error: $(echo $STATUS | jq -r '.reason')"
+            exit 1
+        else
+            echo "Success:"
+            echo "$STATUS" | jq -e '.newVersionInfo'
+            break
+        fi
+    done
+done

Makefile

@@ -13,19 +13,10 @@ PortalDataset:
 	yq '.slots |= with_entries(select(.value.in_subset[] == "portal"))' modules/props.yaml > relevant_props.yaml
 	yq ea '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/Assay/Assay.yaml modules/Sample/Species.yaml modules/DCC/Portal.yaml > relevant_enums.yaml
 	cat header.yaml relevant_props.yaml relevant_enums.yaml modules/Template/PortalDataset.yaml > temp.yaml
-	gen-json-schema --inline --no-metadata --not-closed temp.yaml > tmp.json
-	rm relevant_props.yaml relevant_enums.yaml temp.yaml
+	gen-json-schema --inline --no-metadata --title-from=title --not-closed temp.yaml > tmp.json
 	json-dereference -s tmp.json -o tmp.json
-	jq '{ schema: { "$$schema": "http://json-schema.org/draft-07/schema#", "$$id": "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portaldataset", properties: ."$$defs".PortalDataset.properties, required: ."$$defs".PortalDataset.required }}' tmp.json > registered-json-schemas/PortalDataset.json
-	rm tmp.json
-# There is a LinkML bug that needs some titles to be adjusted manually
-	yq -iP '.schema.properties.accessType.title="accessType"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.assay.title="assay"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.dataType.title="dataType"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.fundingAgency.title="fundingAgency"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.license.title="license"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.manifestation.title="manifestation"' registered-json-schemas/PortalDataset.json -o json
-	yq -iP '.schema.properties.species.title="species"' registered-json-schemas/PortalDataset.json -o json
+	jq '."$$defs".PortalDataset | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portaldataset"' tmp.json > registered-json-schemas/PortalDataset.json
+	rm -f relevant_props.yaml relevant_enums.yaml temp.yaml tmp.json
 	@echo "--- Saved registered-json-schemas/PortalDataset.json ---"
 
 
@@ -34,14 +25,33 @@ PortalStudy:
 	yq eval-all '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/DCC/Portal.yaml modules/Other/Organization.yaml > relevant_enums.yaml
 	cat header.yaml relevant_enums.yaml modules/Template/PortalStudy.yaml > temp.yaml
 	gen-json-schema --inline --no-metadata --not-closed temp.yaml > tmp.json
-	rm relevant_enums.yaml temp.yaml
 	json-dereference -s tmp.json -o tmp.json
-	jq '{ schema: { "$$schema": "http://json-schema.org/draft-07/schema#", "$$id": "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalstudy", properties: ."$$defs".PortalStudy.properties, required: ."$$defs".PortalStudy.required }}' tmp.json > registered-json-schemas/PortalStudy.json
-	rm tmp.json
+	jq '."$$defs".PortalStudy | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalstudy"' tmp.json > registered-json-schemas/PortalStudy.json
+	rm -f relevant_enums.yaml temp.yaml tmp.json
 	@echo "--- Saved registered-json-schemas/PortalStudy.json ---"
 
+Protocol:
+	yq '.slots |= with_entries(select(.value.in_subset[] == "portal" or .value.in_subset[] == "registered"))' modules/props.yaml > relevant_props.yaml
+	yq ea '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/DCC/Portal.yaml modules/Assay/Assay.yaml > relevant_enums.yaml
+	cat header.yaml relevant_props.yaml relevant_enums.yaml modules/Template/Protocol.yaml > temp.yaml
+	gen-json-schema --inline --no-metadata --not-closed --title-from=title temp.yaml > tmp.json
+	json-dereference -s tmp.json -o tmp.json
+	jq '."$$defs".Protocol | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-protocol"' tmp.json > registered-json-schemas/Protocol.json
+	rm -f relevant_props.yaml relevant_enums.yaml temp.yaml tmp.json 
+	@echo "--- Saved registered-json-schemas/Protocol.json ---"
+
 Superdataset:
-	jq '.schema += input | del('.schema.required') | .schema["$$id"] = "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-superdataset"' registered-json-schemas/PortalDataset.json registered-json-schemas/super_rules.json > registered-json-schemas/Superdataset.json
+	jq '. += input | del(.required) | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-superdataset"' registered-json-schemas/PortalDataset.json registered-json-schemas/super_rules.json > registered-json-schemas/Superdataset.json
+	@echo "--- Saved registered-json-schemas/Superdataset.json ---"
+
+PortalPublication:
+	yq ea '. as $$item ireduce ({}; . * $$item )' modules/DCC/Portal.yaml modules/Other/PublicationEnum.yaml > relevant_enums.yaml
+	cat header.yaml modules/Template/PortalPublication.yaml relevant_enums.yaml > temp.yaml
+	gen-json-schema --top-class=PortalPublication --no-metadata --not-closed --title-from=title temp.yaml > tmp.json
+	json-dereference -s tmp.json -o tmp.json
+	jq '."$$defs".PortalPublication | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalpublication"' tmp.json > registered-json-schemas/PortalPublication.json
+	rm -f tmp.json temp.yaml relevant_enums.yaml
+	@echo "--- Saved registered-json-schemas/PortalPublication.json ---"
 
 PortalPublication:
 	yq ea '. as $$item ireduce ({}; . * $$item )' modules/DCC/Portal.yaml modules/Other/PublicationEnum.yaml > relevant_enums.yaml