Release of version 2024.09.0 (#65)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality.

* Testing 3.10

* Update to python 3.10.  No longer need 2 stage build

* Update python 3.10

* Add gridappsd branch (#61)

* Develop (#49)

* Update Dockerfile.gridappsd_base

Pin helics to v2.6.1 per Andy's request.

* Update main.yml

Fix tag parsing for github actions

* helics 2.7.1 (#45)

* Split build, add helics 2.7 back (#47)

* Update main.yml

* Update main.yml

Docker tag parsing

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#51)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#52)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#53)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#54)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#55)

* Gridlab-d is introducing breaking changes. Update gridlab-d branch to gridappsd-branch to maintain functionality. (#56)

* Update scripts (#64)

---------

Co-authored-by: Craig <[email protected]>

Author: tonya1

README.md

@@ -4,9 +4,12 @@ This repository is used to build the base container for gridappsd.
 
 It also includes a few scripts one to create the releases and another to create a blazegraph container.
 
-./scripts/release/create_release.sh - Requires github username and token.  
+
+
+## Create a gridappsd release 
 
 To create a release, the create_release.sh script is run multiple times, between each run there are manual testing and validation steps.
+Requires github username and token.  
 
 ### 1.  Clone the build repository ###
 ```
@@ -61,10 +64,30 @@ cd ../../../gridappsd-docker
 ./run.sh -t vVERSION
 ```
 
-./create_blazegraph.sh - From the lyrasis/blazegraph:2.1.4 container, 
- * add a custom configuration file 
- * from https://github.com/GRIDAPPSD/Powergrid-Models/
-   * import the XML files 
-   * extract the measurements 
-   * import measurements
-   * add the houses
+
+
+## Build the blazegraph container
+
+### 1.  Clone the build repository ###
+```
+git clone https://github.com/GRIDAPPSD/gridappsd-docker-build
+```
+
+### 2.  Create a virtual environment and install the python requirements ###
+```
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements_blazegraph.txt
+```
+
+### 3.  Run the create_release.sh script to create the local blazegraph container  ###
+```
+./create_blazegraph.sh
+```
+  - pulls the base lyrasis container
+  - clones the GitHub repositories for the CIMHub and Powergrid0Models
+  - imports the PowerGridModels/platform/ files, and inserts measurements and houses
+
+### 4.  Verify the import was successfull ###
+Review the bzbuild/build_timestamp/create.log file to verify all files, measurements, and houses were imported correctly.
+

create_blazegraph.sh

@@ -10,6 +10,12 @@
 # print instructions for committing the build container, tagging and pushing to dockerhub
 ####
 
+# set the tag container
+GRIDAPPSD_TAG=':develop'
+
+
+export PYTHONWARNINGS="ignore"
+
 usage () {
   /bin/echo "Usage:  $0 [-d]"
   /bin/echo "        -d      debug"
@@ -55,14 +61,42 @@ http_status_container() {
   fi
 }
 
+build_dir=bzbuild/build_$(date +"%Y%m%d%H%M%S")
+
+if [ -d $build_dir ]; then
+  echo "$build_dir exists"
+  echo "Exiting..."
+  exit 1
+fi
+
+echo "Build directory: $build_dir"
+mkdir -p $build_dir
+cp $0 $build_dir
+cp Dockerfile.gridappsd_blazegraph $build_dir
+mkdir ${build_dir}/conf
+cp -rp ./conf/rwstore.properties ${build_dir}/conf
+cd $build_dir
+
+echo "Logging to : ${build_dir}/create.log"
+
+# Close STDOUT file descriptor
+exec 1<&-
+# Close STDERR FD
+exec 2<&-
+
+# Open STDOUT as $LOG_FILE file for read and write.
+exec 1<>create.log
+
+# Redirect STDERR to STDOUT
+exec 2>&1
+
+date
+
 url_viz="http://localhost:8080/"
-blazegraph_models="ACEP_PSIL.xml EPRI_DPV_J1.xml IEEE123.xml IEEE123_PV.xml IEEE13.xml IEEE13_Assets.xml IEEE8500.xml IEEE8500_3subs.xml R2_12_47_2.xml"
-url_blazegraph="http://localhost:8889/bigdata/"
-data_dir="Powergrid-Models/blazegraph/test"
+url_blazegraph="http://localhost:8889/bigdata/namespace/kb/"
+data_dir="Powergrid-Models/platform/cimxml"
 debug=0
 exists=0
-# set the default tag for the gridappsd and viz containers
-GRIDAPPSD_TAG=':develop'
 
 # parse options
 while getopts dpt: option ; do
@@ -114,7 +148,8 @@ if [ -d Powergrid-Models ]; then
   git pull -v
   cd $cwd
 else
-  git clone http://github.com/GRIDAPPSD/Powergrid-Models -b develop
+  git clone http://github.com/GRIDAPPSD/Powergrid-Models  -b gridappsd 
+  git clone http://github.com/GRIDAPPSD/CIMHub  -b gridappsd 
 fi
 
 GITHASH=`git -C Powergrid-Models log -1 --pretty=format:"%h"`
@@ -123,118 +158,24 @@ http_status_container 'blazegraph'
 
 bz_load_status=0
 echo " "
-echo "Checking blazegraph data"
+echo "Importing blazegraph data"
 
-echo " "
-# Check if blazegraph data is already loaded
-rangeCount=`curl -s -G -H 'Accept: application/xml' "${url_blazegraph}sparql" --data-urlencode ESTCARD | sed 's/.*rangeCount=\"\([0-9]*\)\".*/\1/'`
-if [ x"$rangeCount" == x"0" ]; then
-  for blazegraph_file in $blazegraph_models; do
-    echo "Ingesting blazegraph data $data_dir/$blazegraph_file ${url_blazegraph}sparql ($rangeCount)"
-    debug_msg "curl -s -D- -H 'Content-Type: application/xml' --upload-file \"$data_dir/$blazegraph_file\" -X POST \"${url_blazegraph}sparql\""
-    curl_output=`curl -s -D- -H 'Content-Type: application/xml' --upload-file "$data_dir/$blazegraph_file" -X POST "${url_blazegraph}sparql"`
-    debug_msg "curl output: $curl_output"
-    bz_status=`echo $curl_output | grep -c 'data modified='`
-
-    if [ ${bz_status:-0} -ne 1 ]; then
-      echo "Error could not ingest blazegraph data file $data_dir/$blazegraph_file"
-      echo $curl_output
-      bz_load_status=1
-    fi
-    #echo "Verifying blazegraph data"
-    rangeCount=`curl -s -G -H 'Accept: application/xml' "${url_blazegraph}sparql" --data-urlencode ESTCARD | sed 's/.*rangeCount=\"\([0-9]*\)\".*/\1/'`
-  done
-
-  if [ ${rangeCount:-0} -gt 0  -a $bz_load_status == 0 ]; then
-    echo "Finished ingesting blazegraph data files ($rangeCount)"
-  else
-    echo "Error ingesting blazegraph data files ($rangeCount)"
-    echo "Exiting "
-    echo " "
-    #echo $curl_output
-    exit 1
-  fi
-else
-  echo "Error Blazegrpah already contains data ($rangeCount)"
-  exit 1
+cd Powergrid-Models/platform
+if [ ! -f envars.sh ]; then
+  cp envars_docker.sh envars.sh
 fi
-
-echo " "
-echo "docker commit $did gridappsd/blazegraph:${TIMESTAMP}_${GITHASH}_models "
-docker commit $did gridappsd/blazegraph:${TIMESTAMP}_${GITHASH}_models
-echo " "
-
-# load the measurements 
-echo " "
-echo "Loading the measurements"
-
-echo " "
-echo "Modifying the Powergrid-Models/Meas/constants.py file for loading data into local docker container"
-sed -i'.bak' -e 's/^blazegraph_url.*$/blazegraph_url = \"http:\/\/localhost:8889\/bigdata\/sparql\"/' Powergrid-Models/Meas/constants.py
-
-cd Powergrid-Models/Meas
-if [ ! -d tmp ]; then 
-  mkdir tmp
-else
-  rm tmp/*txt 2> /dev/null
+if [ ! -f cimhubconfig.json ]; then
+  cp cimhubdocker.json cimhubconfig.json
 fi
-cd tmp
-
-echo " "
-echo "Generating measurements files"
-python3 ../ListFeeders.py | grep -v 'binding keys' | while read line; do
-  echo "  Generating measurements files for $line"
-  python3 ../ListMeasureables.py $line
-done
-
-echo " "
-echo "Measurements found"
-#shasum * | shasum | cut -d' ' -f1
-wc -l *txt | grep total | awk '{print $1}'
 
-echo " "
-echo "Loading measurements files"
-ls -1 *txt | xargs -P 12 -n 1 -I, bash -c 'echo "  Loading measurments file ,";python3 ../InsertMeasurements.py ,'
-
-echo " "
-rangeCount=`curl -s -G -H 'Accept: application/xml' "${url_blazegraph}sparql" --data-urlencode ESTCARD | sed 's/.*rangeCount=\"\([0-9]*\)\".*/\1/'`
-echo "Finished loading blazegraph measurements ($rangeCount)"
-echo " "
-echo "docker commit $did gridappsd/blazegraph:${TIMESTAMP}_${GITHASH}_measurements "
-docker commit $did gridappsd/blazegraph:${TIMESTAMP}_${GITHASH}_measurements
-
-cd ../../houses
-# assign random climate zones (1-5) to lookup
-zone_acep_psil=1
-zone_ieee123=2
-zone_ieee123pv=3
-zone_ieee13nodeckt=4
-zone_ieee13nodecktassets=5
-zone_ieee8500=3
-zone_ieee8500new_335=4
-zone_j1=1
-zone_sourceckt=2
-
-echo " "
-echo "Loading houses"
-python3 ../Meas/ListFeeders.py | grep -v 'binding keys' | while read fdrname fdrid ; do
-  # use lookup
-  climate_zone="zone_$fdrname"
-  echo "  Loading houses for $fdrname : $fdrid : ${!climate_zone}"
-  python3 insertHouses.py $fdrid ${!climate_zone}
-  status=$?
-  if [ $status -gt 0 ]; then
-    echo " "
-    echo "  Failed loading houses for $fdrname: python3 insertHouses.py $fdrid ${!climate_zone}"
-    echo " "
-  fi
-done
+./import_all.sh
+status=$?
+echo "status: $status"
 
 echo " "
 rangeCount=`curl -s -G -H 'Accept: application/xml' "${url_blazegraph}sparql" --data-urlencode ESTCARD | sed 's/.*rangeCount=\"\([0-9]*\)\".*/\1/'`
 echo "Finished loading blazegraph houses ($rangeCount)"
 
-
 echo " "
 echo "----"
 echo "docker commit $did gridappsd/blazegraph:${TIMESTAMP}_${GITHASH} "
@@ -244,7 +185,7 @@ docker stop $did
 echo " "
 echo "Run these commands to commit the container and push the container to dockerhub"
 echo "----"
-echo "docker tag gridappsd/blazegraph:${TIMESTAMP}_${GITHASH} gridappsd/blazegraph:develop"
+echo "docker tag gridappsd/blazegraph:${TIMESTAMP}_${GITHASH} gridappsd/blazegraph${GRIDAPPSD_TAG}"
 echo "docker push gridappsd/blazegraph:${TIMESTAMP}_${GITHASH}"
 echo "docker push gridappsd/blazegraph:develop"
 

requirements_blazegraph.txt

@@ -0,0 +1,6 @@
+cimhub
+pandas<2.0
+pyparsing==3.0.8
+python-dateutil==2.8.2
+six==1.16.0
+SPARQLWrapper==2.0.0