Synchronise forward if we complete backward synchronisation

Author: simkim

app/models/hosts/gitlab.rb

@@ -304,17 +304,46 @@ def crawl_repositories_async
     end
 
     def crawl_repositories
+      # legacy way to synchronize on ecosyste.ms
+      crawl_repositories_backwards
+    end
+
+    def crawl_repositories_two_ways
+      # new experimental way to synchronize
+      if ! crawl_repositories_backwards
+        crawl_repositories_forward
+      end
+    end
+
+    def crawl_repositories_backwards
       last_id = REDIS.get("gitlab_last_id:#{@host.id}")
       repos = api_client.projects(per_page: 100, archived: false, id_before: last_id, simple: true)
       if repos.present? && repos.any?
         repos.reject! { |repo| repo.dig("namespace", "kind") == "user" } if ENV["SKIP_USER_REPOS"]
         repos.each { |repo| @host.sync_repository(repo["path_with_namespace"], uuid: repo["id"]) }
         REDIS.set("gitlab_last_id:#{@host.id}", repos.last["id"])
+        return true
       end
     rescue *IGNORABLE_EXCEPTIONS
       nil
     end
 
+    def crawl_repositories_forward
+      recent_id = REDIS.get("gitlab_recent_id:#{@host.id}")
+      if recent_id.nil?
+        recent_id = @host.repositories.maximum(:id)
+      end
+      repos = api_client.projects(per_page: 100, archived: false, id_after: recent_id, simple: true)
+      repos.reject! { |repo| repo.dig("namespace", "kind") == "user" } if ENV["SKIP_USER_REPOS"]
+      if repos.present?
+        repos.each { |repo| @host.sync_repository(repo["path_with_namespace"], uuid: repo["id"]) }
+        REDIS.set("gitlab_recent_id:#{@host.id}", repos.last["id"])
+      end
+    rescue *IGNORABLE_EXCEPTIONS
+      nil
+    end
+
+
     def load_owner_repos_names(owner)
       if owner.user?
         api_client.user_projects(owner.login, per_page: 100, archived: false, simple: true).map { |repo| repo["path_with_namespace"] }

lib/tasks/dinum.rake

@@ -130,7 +130,7 @@ module Dinum
     host_reset(host) if host.repositories_count == 0
     loop do
       repo_count = host.repositories_count
-      host.crawl_repositories
+      host.crawl_repositories_two_ways
       sleep 1
       break if host.reload.repositories_count == repo_count
     end