KMS-516: Implement CRUD for SKOS concepts (#10)

* KMS-516: Support for CRUDing SKOS concepts

---------

Co-authored-by: Christopher Gokey <[email protected]>

.gitignore

@@ -20,4 +20,5 @@ junit.xml
 
 cmr
 cdk.context.json
+cdk.out
 infrastructure/rdfdb/cdk/cdk.context.json

Dockerfile

@@ -0,0 +1,4 @@
+FROM node:18-bullseye
+COPY . /build
+WORKDIR /build
+RUN npm ci --omit=dev && npm run build

README.md

@@ -15,56 +15,114 @@ Keyword Management System (KMS) is a application for maintaining keywords (scien
 
 To install the necessary components, run:
 
-```bash
-    npm install
+```
+npm install
 ```
 
 ### Usage
 
-In order to run KMS locally, you first need to setup a RDF database.
-
 #### Running Serverless Offline (API Gateway/Lambdas)
 
 In order to run serverless-offline, which is used for mimicking API Gateway to call lambda functions, run:
 
-```bash
-    npm run offline
+```
+RDF4J_SERVICE_URL=http://localhost:8080 npm run offline
 ```
 
 ## Local Testing
 
 To run the test suite, run:
 
-```bash
-    npm run test
+```
+npm run test
+```
+## Setting up the RDF Database for local development
+In order to run KMS locally, you first need to setup a RDF database.
+### Prerequisites
+#### Set the RDFDB user name and password
+```
+export RDF4J_USER_NAME=[user name]
+export RDF4J_PASSWORD=[password]
+```
+### Building and Running the RDF Database
+#### Build the docker image
+```
+npm run rdf4j:build
+```
+#### Run the docker image
+```
+npm run rdf4j:start
+```
+#### Setup and load data into the RDF database
+```
+npm run rdf4j:setup
 ```
 
-## Deploying RDF Database
+# Deployments
+## Deploying RDF Database to AWS
 ### Prerequisites
 #### Copy your AWS credentials and set these up as env variables
-```export AWS_ACCESS_KEY_ID=[your access key id]
-export AWS_SECRET_ACCESS_KEY=[your access secret access key]
-export AWS_SESSION_TOKEN=[your session token]
+```
+export bamboo_STAGE_NAME=[sit|uat|prod]
+export bamboo_AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+export bamboo_AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+export bamboo_AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+export bamboo_LAMBDA_TIMEOUT=30
+export bamboo_SUBNET_ID_A={subnet #1}
+export bamboo_SUBNET_ID_B={subnet #2}
+export bamboo_SUBNET_ID_C={subnet #3}
+export bamboo_VPC_ID={your vpc id}
 ```
 #### Retrieve the VPC_ID from AWS
-```export VPC_ID=[your vpc id]
+```
+export VPC_ID=[your vpc id]
 ```
 #### Set the RDFDB user name and password
-```export RDFDB_USER_NAME=[your rdfdb user name]
-export RDFDB_PASSWORD=[your rdfdb password]
+```
+export RDF4J_USER_NAME=[your rdfdb user name]
+export RDF4J_PASSWORD=[your rdfdb password]
 ```
 
-### Deploy Docker Container to Registry
-```cd infrastructure/rdfdb/cdk/bin
-export 
+#### Deploy Docker Container to Registry
+```
+cd infrastructure/rdfdb/cdk/bin
 ./deploy_to_ecr.sh
 ```
 
-### Deploy ECS Service to AWS
-```cd infrastructure/rdfdb/cdk
+#### Deploy ECS Service to AWS
+#### Deploy IAM, EBS, LB, and ECS stacks
+```
+cd infrastructure/rdfdb/cdk
 cdk deploy rdf4jIamStack
-cdk deploy rdf4jEfsStack
+cdk deploy rdf4jEbsStack
+cdk deploy rdf4jLbStack
 cdk deploy rdf4jEcsStack
 ```
+#### Alternatively, you can deploy all stacks at once
+```
+cd infrastructure/rdfdb/cdk
+cdk deploy --all
+```
+One thing to note is if you destroy the rdf4jEbsStack and redeploy, this will create a new EBS file system.  You will need to copy the data from the old EBS file system to the new one.  This can be done by mounting the old EBS file system to an EC2 instance and copying the data to the new EBS file system.
+
+## Deploying KMS to AWS
+### Prerequisites
+#### Copy your AWS credentials and set these up as env variables
+```
+export bamboo_STAGE_NAME=[sit|uat|prod]
+export bamboo_AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+export bamboo_AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+export bamboo_AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+export bamboo_LAMBDA_TIMEOUT=30
+export bamboo_SUBNET_ID_A={subnet #1}
+export bamboo_SUBNET_ID_B={subnet #2}
+export bamboo_SUBNET_ID_C={subnet #3}
+export bamboo_VPC_ID={your vpc id}
+export bamboo_RDF4J_USER_NAME=[your rdfdb user name]
+export bamboo_RDF4J_PASSWORD=[your rdfdb password]
+```
+#### Deploy KMS Application
+```
+./bin/deploy_bamboo.sh
+```
 
-One thing to note is if you destroy the rdf4jEfsStack and redeploy, this will create a new EFS file system.  You will need to copy the data from the old EFS file system to the new one.  This can be done by mounting the old EFS file system to an EC2 instance and copying the data to the new EFS file system.

bin/deploy-bamboo.sh

@@ -3,6 +3,18 @@
 # Bail on unset variables, errors and trace execution
 set -eux
 
+# Deployment configuration/variables
+####################################
+
+# read in static.config.json
+config="`cat static.config.json`"
+
+# update keys for deployment
+config="`jq '.application.env = $newValue' --arg newValue $bamboo_STAGE_NAME <<< $config`"
+
+# overwrite static.config.json with new values
+echo $config > tmp.$$.json && mv tmp.$$.json static.config.json
+
 # Set up Docker image
 #####################
 
@@ -42,6 +54,8 @@ dockerRun() {
         --env "SUBNET_ID_B=$bamboo_SUBNET_ID_B" \
         --env "SUBNET_ID_C=$bamboo_SUBNET_ID_C" \
         --env "VPC_ID=$bamboo_VPC_ID" \
+        --env "RDF4J_USER_NAME=$bamboo_RDF4J_USER_NAME" \
+        --env "RDF4J_PASSWORD=$bamboo_RDF4J_PASSWORD" \
         $dockerTag "$@"
 }
 

infrastructure/rdfdb/cdk/bin/deploy_to_ecr.sh

@@ -3,7 +3,7 @@
 # Set variables
 REPO_NAME="rdf4j"
 REGION="us-east-1"
-DOCKER_FILE_PATH="../../docker"
+DOCKER_FILE_PATH="../docker"
 
 # Create ECR repository
 aws ecr create-repository --repository-name $REPO_NAME --region "us-east-1"

infrastructure/rdfdb/cdk/bin/rdf4j-aws-deployment.js

@@ -1,11 +1,16 @@
 #!/usr/bin/env node
 const cdk = require('aws-cdk-lib')
+const { EbsStack } = require('../lib/ebs-stack')
 const { EcsStack } = require('../lib/ecs-stack')
-const { EfsStack } = require('../lib/efs-stack')
 const { IamStack } = require('../lib/iam-stack')
+const { LoadBalancerStack } = require('../lib/lb-stack')
 
 async function main() {
-  const app = new cdk.App()
+  const app = new cdk.App({
+    context: {
+      '@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature': true
+    }
+  })
 
   const env = {
     account: process.env.CDK_DEFAULT_ACCOUNT,
@@ -18,25 +23,31 @@ async function main() {
     env,
     vpcId
   })
-  const efsStack = new EfsStack(app, 'rdf4jEfsStack', {
+
+  const lbStack = new LoadBalancerStack(app, 'rdf4jLoadBalancerStack', {
     env,
-    vpcId,
-    efsTaskSecurityGroup: iamStack.efsTaskSecurityGroup
+    vpcId
+  })
+
+  const ebsStack = new EbsStack(app, 'rdf4jEbsStack', {
+    env,
+    vpcId
   })
 
   const ecsStack = new EcsStack(app, 'rdf4jEcsStack', {
     env,
     vpcId,
-    role: iamStack.role,
-    ecsTasksSecurityGroup: efsStack.ecsTasksSecurityGroup,
-    fileSystem: efsStack.fileSystem,
-    accessPoint: efsStack.accessPoint
+    roleArn: iamStack.role.roleArn,
+    lbStack,
+    ebsStack
   })
 
   // Add dependencies
-  efsStack.addDependency(iamStack)
+  ebsStack.addDependency(iamStack)
+  lbStack.addDependency(iamStack)
   ecsStack.addDependency(iamStack)
-  ecsStack.addDependency(efsStack)
+  ecsStack.addDependency(ebsStack)
+  ecsStack.addDependency(lbStack)
 
   app.synth()
 }

infrastructure/rdfdb/cdk/docker/Dockerfile

@@ -0,0 +1,39 @@
+FROM eclipse/rdf4j-workbench:latest
+USER root
+
+# Create the directory for EFS mount and set permissions
+RUN mkdir -p /rdf4j-data
+RUN chown -R tomcat:nogroup /rdf4j-data
+RUN chmod -R 755 /rdf4j-data
+
+# Set the RDF4J data directory
+ENV RDF4J_DATA_DIR=/rdf4j-data
+
+# Install necessary tools
+RUN apt-get update && apt-get install -y unzip zip liblmdb-dev lmdb-utils
+
+# Copy configuration files
+COPY config/tomcat-users.xml /usr/local/tomcat/conf/tomcat-users.xml
+
+# Copy scripts
+COPY scripts/setup.sh /usr/local/tomcat/setup.sh
+COPY scripts/setup_web_auth.sh /usr/local/tomcat/setup_web_auth.sh
+COPY scripts/create_repository.sh /usr/local/tomcat/create_repository.sh
+
+RUN chown -R tomcat:nogroup /usr/local/tomcat
+# Uncomment the line below to include rdf4j workbench, which is is a web-based 
+# user interface that provides a graphical way to interact with and manage 
+# RDF4J repositories.   To access it, go here: http://localhost:8080/rdf4j-workbench
+# RUN rm -r /usr/local/tomcat/webapps/rdf4j-workbench*
+
+# We'll keep the container running as root to allow permission changes at runtime
+# The entrypoint script will switch to the tomcat user
+
+# Set correct permissions
+RUN chown -R tomcat:nogroup /rdf4j-data
+RUN chmod -R 755 /rdf4j-data
+
+USER tomcat
+
+# Use the new entrypoint script
+CMD ["/usr/local/tomcat/setup.sh"]

infrastructure/rdfdb/cdk/docker/data/server/conf/logback.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration debug="false" scan="true" scanPeriod="30 seconds">
+	<appender name="MainLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>${org.eclipse.rdf4j.common.logging.dir}/main.log</file>
+		<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+			<fileNamePattern>${org.eclipse.rdf4j.common.logging.dir}/main-%d{yyyy-MM-dd}.log</fileNamePattern>
+		</rollingPolicy>
+		<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
+			<layout class="org.eclipse.rdf4j.common.logging.file.logback.StackTracePatternLayout"/>
+			<charset>UTF-8</charset>
+		</encoder>
+		<logreader class="org.eclipse.rdf4j.common.logging.file.logback.FileLogReader" default="true"/>
+	</appender>
+	<root>
+		<level value="INFO"/>
+		<appender-ref ref="MainLog"/>
+	</root>
+</configuration>

infrastructure/rdfdb/cdk/docker/scripts/create_repository.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Function to create repository
+create_repository() {
+  # Wait for RDF4J server to be up
+  until curl --output /dev/null --silent --fail http://localhost:8080/rdf4j-server/protocol; do
+    echo "Waiting for RDF4J server to be ready..."
+    sleep 5
+  done
+
+  if [ ! -d "${RDF4J_DATA_DIR}/server/repositories/kms" ]; then
+    # Create the repository
+    echo "Repository 'kms' does not exist. Creating it..."
+    curl -u rdf4j:rdf4j -X PUT -H "Content-Type: application/x-turtle" --data-binary '
+#
+# RDF4J configuration template for a main-memory repository
+#
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix config: <tag:rdf4j.org,2023:config/>.
+
+[] a config:Repository ;
+   config:rep.id "kms" ;
+   rdfs:label "kms" ;
+   config:rep.impl [
+      config:rep.type "openrdf:SailRepository" ;
+      config:sail.impl [
+        config:sail.type "openrdf:NativeStore" ;
+        config:native.forceSync: true ;
+        config:sail.memory "false" ;
+        config:sail.reindex "true" ;
+        config:sail.writeThrough "true" ;
+    ]
+   ].
+    ' http://localhost:8080/rdf4j-server/repositories/kms
+    echo "Repository created successfully"
+  else
+    echo "Repository 'kms' already exists. Skipping creation."
+  fi
+}
+
+# Call the function
+create_repository

infrastructure/rdfdb/docker/scripts/setup.sh → infrastructure/rdfdb/cdk/docker/scripts/setup.sh

@@ -4,7 +4,7 @@ set -e
 echo 'org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource' >> /usr/local/tomcat/conf/catalina.properties
 echo "RDF4J_DATA_DIR is set to: ${RDF4J_DATA_DIR}"
 
-/usr/local/tomcat/modify_web_xml.sh
+/usr/local/tomcat/setup_web_auth.sh
 
 echo "showing rdf4j data directory"
 ls -l /rdf4j-data
@@ -14,4 +14,6 @@ export CATALINA_OPTS="-Dorg.eclipse.rdf4j.appdata.basedir=${RDF4J_DATA_DIR}"
 echo "JAVA_OPTS=\"\$JAVA_OPTS -Dorg.eclipse.rdf4j.appdata.basedir=${RDF4J_DATA_DIR}\"" >> /usr/local/tomcat/bin/setenv.sh
 
 # Start Web Application
+# Check the role and perform appropriate actions
+/usr/local/tomcat/create_repository.sh &
 catalina.sh run