diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9fdb796 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM tomcat:9.0.79-jdk17 + +# Pull the elasticsearch apm agent in for observability. +COPY --from=docker.elastic.co/observability/apm-agent-java:latest /usr/agent/elastic-apm-agent.jar elastic-apm-agent.jar + +COPY ./kbplus/build/libs/kbplus-8.0.0-SNAPSHOT.war /usr/local/tomcat/webapps/kbplus.war \ No newline at end of file diff --git a/README.md b/README.md index abe9e22..8b0f739 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,31 @@ # KBPlusG5 -Grails5 Port of KBPlus -prerequisites: +A Grails5 Port of [KBPlus](https://github.com/k-int/KBPlus) -# Deploy +## Documentation -## Build +See additional documentation from the original project wiki https://github.com/k-int/KBPlus/wiki + +## prerequisites: + +[opendk 17](https://openjdk.org/projects/jdk/17/) +or +[corretto 17](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/downloads-list.html) + +[docker and docker compose](https://www.docker.com/products/docker-desktop/) + +## Local Development + +See [`dev/README.md`](dev/README.md) + +## Deploy To apache tomcat + +### Build ./gradlew clean build -x integrationTest + +### Deploy + scp ./build/libs/kbplus-8.0.0-SNAPSHOT.war IP:/tmp ssh IP cd /srv/kbplus/apache-tomcat-9.0.73/webapps# @@ -17,14 +35,14 @@ prerequisites: ../bin/startup.sh -# Testing notes +## Testing notes -## Grails command to run for the test env +### Grails command to run for the test env grails -Dgrails.env=test run-app -## Ubuntu 2204+ +### Ubuntu 2204+ notes diff --git a/dev/README.md b/dev/README.md index 9404dc7..9de40ae 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,19 +1,54 @@ +# Development + +### Containerised Development environment + +First run gradle to build the app and create the war if you haven't already. + + cd ../kbplus/ + ./gradlew clean build -x integrationTest -x Test + +To run the full stack and bootstrap mysql and elasticsearch + + docker-compose -f docker-compose-dev-setup.yml -f docker-compose-dev-app.yml -f docker-compose-dev-bootstrap.yml up + +kbplus will be available at http://localhost:8080/kbplus/ + +To just run the datastores for local dev + + docker-compose -f docker-compose-dev-setup.yml + +To run the optional dev tools like Application performance monitoring and Kibana dashboards + + docker-compose -f docker-compose-dev-setup.yml -f docker-compose-dev-app.yml -f docker-compose-dev-tools.yml up + +kibana will be available at http://localhost:5601/ + +`docker-compose-dev-bootstrap.yml` only needs to be ran on first boot or when elasticsearch mappings change. + + +### Database Administration To use the mysql command line client, you will need to use -mysql -u root -h 127.0.0.1 -P 3306 -p + mysql -u root -h 127.0.0.1 -P 3306 -p -h localhost will likely fail as the client will try and use a socket instead of tcp. Set up your databases as described in the root readme.md -Alternatively, run mysql client in the container +Alternatively, run mysql client in the container or when using mariadb fire up a new client container. + +mysql + + docker exec -it jc_mysql mysql -u root -p + +mariadb -docker exec -it jc_mysql mysql -u root -p + docker run -it --network dev_default --rm mariadb:11 mariadb -hjc_mysql -uroot -p -Load a dump file storred on the container +Load a dump file stored on the container -cat backup.sql | docker exec -i jc_mysql mysql -u root --password=jc kbplus700prod + cat backup.sql | docker exec -i jc_mysql mysql -u root --password=jc kbplus700prod Or -cat backup.sql | docker exec -i jc_mysql mysql -u root --password=jc kbplus700dev + cat backup.sql | docker exec -i jc_mysql mysql -u root --password=jc kbplus700dev diff --git a/dev/docker-compose-dev-app.yml b/dev/docker-compose-dev-app.yml new file mode 100644 index 0000000..f807bec --- /dev/null +++ b/dev/docker-compose-dev-app.yml @@ -0,0 +1,29 @@ +version: "3.0" + +services: + kbplus: + build: + context: ../. + dockerfile: Dockerfile + image: kbplus:latest + environment: + CATALINA_OPTS: '-Dgrails.env=development' + DATASOURCE_URL: jdbc:mysql://jc_mysql/kbplus700dev?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 + DATASOURCE_USERNAME: k-int + DATASOURCE_PASSWORD: k-int + ELASTICSEARCH_SERVICE: jc_es6 + ports: + - 8080:8080 + depends_on: + jc_mysql: + condition: + service_healthy + jc_es6: + condition: + service_started + # healthcheck: + # test: curl --fail http://localhost:8080/kbplus/ || exit 1 + # interval: 10s + # retries: 5 + # start_period: 10s + # timeout: 3s \ No newline at end of file diff --git a/dev/docker-compose-dev-bootstrap.yml b/dev/docker-compose-dev-bootstrap.yml new file mode 100644 index 0000000..e7154d2 --- /dev/null +++ b/dev/docker-compose-dev-bootstrap.yml @@ -0,0 +1,40 @@ +version: "3.0" + +services: + + es_index: + image: curlimages/curl + command: "--connect-timeout 5 --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 20 -X PUT 'jc_es6:9200/kbplus'" + container_name: es_index + restart: 'no' + depends_on: + jc_es6: + condition: + service_started + jc_mysql: + condition: + service_started + kbplus: + condition: + service_started + + es_mappings: + image: curlimages/curl + command: "--connect-timeout 5 --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 20 -X PUT 'jc_es6:9200/kbplus/_mapping' --header 'Content-Type: application/json' -d @/mapping.json" + volumes: + - ./esinit/mapping.json:/mapping.json:ro + container_name: es_mappings + restart: on-failure:2 + depends_on: + jc_es6: + condition: + service_started + jc_mysql: + condition: + service_started + kbplus: + condition: + service_started + es_index: + condition: + service_completed_successfully \ No newline at end of file diff --git a/dev/docker-compose-dev-setup.yml b/dev/docker-compose-dev-setup.yml index 93cc654..1633b69 100644 --- a/dev/docker-compose-dev-setup.yml +++ b/dev/docker-compose-dev-setup.yml @@ -8,13 +8,11 @@ services: # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html # https://github.com/k-int/KBPlusG3/blob/dev-integration/SETUP-ES-DOCKER.txt - # see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html - # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html + # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html + # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html jc_es6: container_name: jc_es6 - # image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1 - # image: docker.elastic.co/elasticsearch/elasticsearch:6.6.1 - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.18 restart: 'no' environment: - bootstrap.memory_lock=true @@ -27,7 +25,6 @@ services: hard: -1 ports: - 9200:9200 - - 9300:9300 volumes: - jc_esdata:/usr/share/elasticsearch/data networks: @@ -39,12 +36,24 @@ services: # Connect with mysql --protocol=tcp -h localhost -u k-int -p kbplus700dev jc_mysql: container_name: jc_mysql - image: mariadb:10 + image: mariadb:11 restart: 'no' environment: MYSQL_ROOT_PASSWORD: jc + MARIADB_USER: admin volumes: - jc_mysqldata:/var/lib/mysql - ./mysqlinit:/docker-entrypoint-initdb.d + healthcheck: + interval: 30s + retries: 3 + test: + [ + "CMD", + "healthcheck.sh", + "--connect", + "--innodb_initialized" + ] + timeout: 30s ports: - 3306:3306 diff --git a/dev/docker-compose-dev-tools.yml b/dev/docker-compose-dev-tools.yml new file mode 100644 index 0000000..8025843 --- /dev/null +++ b/dev/docker-compose-dev-tools.yml @@ -0,0 +1,32 @@ +version: "3.0" + +services: + kbplus: + environment: + CATALINA_OPTS: '-javaagent:elastic-apm-agent.jar -Delastic.apm.server_urls=http://apm-server:8200 -Dgrails.env=development' + DATASOURCE_URL: jdbc:mysql://jc_mysql/kbplus700dev?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 + DATASOURCE_USERNAME: k-int + DATASOURCE_PASSWORD: k-int + ELASTICSEARCH_SERVICE: jc_es6 + kibana: + image: docker.elastic.co/kibana/kibana:7.17.18 + environment: + - ELASTICSEARCH_HOSTS=http://jc_es6:9200 + ports: + - 5601:5601 + depends_on: + - jc_es6 + apm-server: + image: docker.elastic.co/apm/apm-server:7.17.18 + cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"] + cap_drop: ["ALL"] + ports: + - 8200:8200 + command: > + apm-server -e + -E apm-server.rum.enabled=true + -E setup.kibana.host=kibana:5601 + -E setup.template.settings.index.number_of_replicas=0 + -E apm-server.kibana.enabled=true + -E apm-server.kibana.host=kibana:5601 + -E output.elasticsearch.hosts=["jc_es6:9200"] \ No newline at end of file diff --git a/dev/esinit/mapping.json b/dev/esinit/mapping.json new file mode 100644 index 0000000..deac187 --- /dev/null +++ b/dev/esinit/mapping.json @@ -0,0 +1,452 @@ +{ + "properties": { + "availableToOrgs": { + "type": "long" + }, + "consortiaId": { + "type": "long" + }, + "consortiaName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "contact": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "cp": { + "properties": { + "variantNames": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "cpid": { + "type": "long" + }, + "cpname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dbId": { + "type": "long" + }, + "endDate": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "endYear": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "endYearAndMonth": { + "type": "date" + }, + "identifier": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "identifiers": { + "properties": { + "type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "impId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "isPublic": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "jiscLicenseId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "keyTitle": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "lastModified": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "licenseStatus": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "licenseTerms": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "note": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "licenseType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "licenseUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "licenseeRef": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "licensorRef": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "normTitle": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "noticePeriod": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "packages": { + "properties": { + "cpid": { + "type": "long" + }, + "cpname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "pkgid": { + "type": "long" + }, + "pkgidstr": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "pkgname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "pkg_scope": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "products": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "pubVariantNames": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "publisher": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "rectype": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "sector": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "sortTitle": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "sortname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "startDate": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "startYear": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "startYearAndMonth": { + "type": "date" + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "subtype": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "title": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "titleCount": { + "type": "long" + }, + "tokname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "variantNames": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "visible": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } +} \ No newline at end of file diff --git a/kbplus/README.md b/kbplus/README.md deleted file mode 100644 index b50c453..0000000 --- a/kbplus/README.md +++ /dev/null @@ -1,16 +0,0 @@ - - -# Build and deploy -./gradlew -x integrationTest clean build - -scp ./build/libs/kbplus-8.0.0-SNAPSHOT.war kbplus_3_test: - - -SSH to server as root - -cd /srv/kbplus/apache-tomcat-9.0.73 -cd bin -./shutdown.sh -rm -Rf ../webapps/test2* -cp ~/wherever/kbplus-8.0.0-SNAPSHOT.war ../webapps -./startup.sh