Merge pull request #149 from ncbo/develop

Merge develop to master,  release v5.29.0

.github/workflows/ruby-unit-test.yml

@@ -16,7 +16,7 @@ jobs:
         triplestore: ['fs', 'ag']
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo apt-get -y install raptor2-utils
     - name: Set up Ruby
@@ -30,5 +30,8 @@ jobs:
     - name: Run tests
       run: bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
-
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: unittests
+        verbose: true

Gemfile

@@ -21,4 +21,4 @@ group :profiling do
   gem "thin"
 end
 
-gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master'
+gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop'

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/ncbo/sparql-client.git
-  revision: d418d56a6c9ff5692f925b45739a2a1c66bca851
-  branch: master
+  revision: 55e7dbf858eb571c767bc67868f9af61663859cb
+  branch: develop
   specs:
     sparql-client (1.0.1)
       json_pure (>= 1.4)
@@ -35,7 +35,7 @@ GEM
     base64 (0.2.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     cube-ruby (0.0.3)
     daemons (1.4.1)
@@ -58,7 +58,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.1205)
+    mime-types-data (3.2024.0305)
     minitest (4.7.5)
     multi_json (1.15.0)
     mustermann (3.0.0)
@@ -69,7 +69,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.4)
-    rack (2.2.8)
+    rack (2.2.8.1)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
@@ -80,9 +80,9 @@ GEM
     rake (13.1.0)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (5.0.8)
+    redis (5.1.0)
       redis-client (>= 0.17.0)
-    redis-client (0.19.1)
+    redis-client (0.21.0)
       connection_pool
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -140,4 +140,4 @@ DEPENDENCIES
   uuid
 
 BUNDLED WITH
-   2.3.22
+   2.4.22

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       retries: 30
 
   solr-ut:
-    image: ontoportal/solr-ut:0.0.2
+    image: ontoportal/solr-ut:0.1.0
     ports:
       - 8983:8983
     healthcheck:
@@ -21,7 +21,7 @@ services:
       retries: 5
 
   agraph-ut:
-    image: franzinc/agraph:v8.0.0
+    image: franzinc/agraph:v8.1.0
     platform: linux/amd64
     environment:
       - AGRAPH_SUPER_USER=test
@@ -38,12 +38,12 @@ services:
         ; agtool users add anonymous
         ; agtool users grant anonymous root:ontoportal_test:rw
         ; tail -f /agraph/data/agraph.log"
-    # healthcheck:
-    #   test: ["CMD-SHELL", "curl -m 1 -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"]
-    #   start_period: 60s
-    #   interval: 10s
-    #   timeout: 5s
-    #   retries: 5
+    healthcheck:
+      test: ["CMD-SHELL", "curl -m 1 -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"]
+      start_period: 60s
+      interval: 10s
+      timeout: 5s
+      retries: 5
     profiles:
       - ag
 

lib/goo/sparql/client.rb

@@ -83,41 +83,28 @@ def delete_data_graph(graph)
       def append_triples_no_bnodes(graph,file_path,mime_type_in)
         bnodes_filter = nil
         dir = nil
-
-        if file_path.end_with?("ttl")
+        response = nil
+        if file_path.end_with?('ttl')
           bnodes_filter = file_path
         else
-          bnodes_filter,dir = bnodes_filter_file(file_path,mime_type_in)
-        end
-        mime_type = "text/turtle"
-
-        if mime_type_in == "text/x-nquads"
-          mime_type = "text/x-nquads"
-          graph = "http://data.bogus.graph/uri"
+          bnodes_filter, dir = bnodes_filter_file(file_path, mime_type_in)
+        end
+        chunk_lines = 500_000 # number of line
+        file = File.foreach(bnodes_filter)
+        lines = []
+        file.each_entry do |line|
+          lines << line
+          if lines.size == chunk_lines
+            response = execute_append_request graph, lines.join, mime_type_in
+            lines.clear
+          end
         end
 
-        data_file = File.read(bnodes_filter)
-        params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
-        backend_name = Goo.sparql_backend_name
-
-        if backend_name == BACKEND_4STORE
-          params[:payload] = {
-            graph: graph.to_s,
-            data: data_file,
-            "mime-type" => mime_type
-          }
-          #for some reason \\\\ breaks parsing
-          params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
-        else
-          params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
-          params[:payload] = data_file
-        end
+        response = execute_append_request graph, lines.join, mime_type_in unless lines.empty?
 
-        response = RestClient::Request.execute(params)
 
         unless dir.nil?
           File.delete(bnodes_filter)
-
           begin
             FileUtils.rm_rf(dir)
           rescue => e
@@ -196,6 +183,35 @@ def status
         resp[:outstanding] = outstanding
         resp
       end
+
+      private
+
+      def execute_append_request(graph, data_file, mime_type_in)
+        mime_type = "text/turtle"
+
+        if mime_type_in == "text/x-nquads"
+          mime_type = "text/x-nquads"
+          graph = "http://data.bogus.graph/uri"
+        end
+
+        params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
+        backend_name = Goo.sparql_backend_name
+
+        if backend_name == BACKEND_4STORE
+          params[:payload] = {
+            graph: graph.to_s,
+            data: data_file,
+            'mime-type' => mime_type
+          }
+          #for some reason \\\\ breaks parsing
+          params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
+        else
+          params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
+          params[:payload] = data_file
+        end
+
+        RestClient::Request.execute(params)
+      end
     end
   end
 end

lib/goo/sparql/solutions_mapper.rb

@@ -35,10 +35,14 @@ def map_each_solutions(select)
         list_attributes = Set.new(klass.attributes(:list))
         all_attributes = Set.new(klass.attributes(:all))
 
-        # for using prefixes before queries
-        # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch)
-        # that enables implicit internal ordering. The patch requires the prefix below
-        select.prefix('franzOption_imposeImplicitBasicOrdering: <franz:yes>') if @options[:page]
+        if @options[:page]
+          # for using prefixes before queries
+          # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch)
+          # that enables implicit internal ordering. The patch requires the prefix below
+          select.prefix('franzOption_imposeImplicitBasicOrdering: <franz:yes>')
+          # mdorf, 1/24/2024, AllegroGraph 8 introduced a new feature that allows caching OFFSET/LIMIT queries
+          select.prefix('franzOption_allowCachingResults: <franz:yes>')
+        end
 
         # for troubleshooting specific queries (write 1 of 3)
         # ont_to_parse = 'OAE'