Merge pull request #278 from jeafreezy/new-frontend

.github/workflows/backend_build.yml

@@ -35,14 +35,6 @@ jobs:
         with:
           python-version: 3.8
 
-      - name: Get my current working dir
-        run: pwd
-
-      - name: Test env vars for python
-        env:
-          TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }}
-        run: python -c "import os; print(os.environ['TESTING_TOKEN'])"
-
       - name: Clone Ramp
         run: git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code
 
@@ -92,7 +84,6 @@ jobs:
       - name: Create env
         run: |
           cd backend/
-          mv sample_env .env
           export DATABASE_URL=postgis://admin:password@localhost:5432/ai
           export RAMP_HOME="/home/runner/work/fAIr/fAIr"
           export TRAINING_WORKSPACE="/home/runner/work/fAIr/fAIr/backend/training"
@@ -120,6 +111,9 @@ jobs:
       - name: Run migrations
         env:
           TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }}
+          OSM_CLIENT_ID: ${{ secrets.OSM_CLIENT_ID }}
+          OSM_CLIENT_SECRET: ${{ secrets.OSM_CLIENT_SECRET }}
+          OSM_SECRET_KEY: ${{ secrets.OSM_SECRET_KEY }}
         run: |
           cd backend/
           python manage.py makemigrations
@@ -134,7 +128,7 @@ jobs:
           TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }}
           OSM_CLIENT_ID: ${{ secrets.OSM_CLIENT_ID }}
           OSM_CLIENT_SECRET: ${{ secrets.OSM_CLIENT_SECRET }}
-          OSM_SECRET_KEY: ""
+          OSM_SECRET_KEY: ${{ secrets.OSM_SECRET_KEY }}
 
         run : |
           cd backend/

.github/workflows/docker_build.yml

@@ -0,0 +1,81 @@
+name: Build Docker Images for backend
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "backend/**"
+      - ".github/workflows/docker_build.yml"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "backend/**"
+      - ".github/workflows/docker_build.yml"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-api-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for API Docker
+        id: meta_api
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_api
+
+      - name: Build and push API Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: backend/
+          file: backend/Dockerfile_CPU
+          push: false
+          tags: ${{ steps.meta_api.outputs.tags }}
+          labels: ${{ steps.meta_api.outputs.labels }}
+
+  build-and-push-worker-image:
+    needs: build-and-push-api-image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Worker Docker
+        id: meta_worker
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_worker
+
+      - name: Build and push Worker Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: backend/
+          file: backend/Dockerfile
+          push: false
+          tags: ${{ steps.meta_worker.outputs.tags }}
+          labels: ${{ steps.meta_worker.outputs.labels }}

Readme.md

@@ -32,6 +32,7 @@ To eliminate model biases, fAIr is built to work with the local communities and
 |✅| mapping predictions into OSM using JOSM |
 |⚙️| improvements to the prediction algorithm: adopting YOLOv8 model |
 |⚙️| UI/UX redesign to enhance the user experience |
+|⚙️| fAIr evaluation research with Masaryk University & Missing Maps Czechia and Slovakia, welcome to join the efforts, [here](https://hotosm.slack.com/archives/C0542U7DLA0/p1725445186654019) |
 | | remove overlapping predictions from final output |
 | | improvements to testing suite to improve code resilience going forward |
 | | multi-mask training for better building feature prediction |
@@ -71,4 +72,4 @@ Checkout Docker Installation [docs](./docs/Docker-installation.md)
 ## Get involved!
 
 * Start by reading our [Code of conduct](https://github.com/hotosm/fAIr/blob/master/docs/Code-of-Conduct.md)
-* Get familiar with our [contributor guidelines](https://github.com/hotosm/fAIr/blob/master/docs/Contribution.md) explaining the different ways in which you can support this project! We need your help!
+* Get familiar with our [contributor guidelines](CONTRIBUTING.md) explaining the different ways in which you can support this project! We need your help!

backend/Dockerfile

@@ -17,17 +17,17 @@ RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal"
 COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt
 RUN pip install -r /tmp/docker-requirements.txt
 
-
-RUN pip install --upgrade setuptools
+# RUN pip install --upgrade pip
+# RUN pip install --upgrade setuptools
 COPY requirements.txt /tmp/requirements.txt
 COPY api-requirements.txt /tmp/api-requirements.txt
 
-RUN pip install -r /tmp/requirements.txt
+RUN pip install --use-deprecated=legacy-resolver -r /tmp/requirements.txt 
 
 
 COPY docker/ramp/solaris /tmp/solaris
 RUN pip install /tmp/solaris --use-feature=in-tree-build && \
     pip install scikit-fmm --use-feature=in-tree-build
-    
+
 WORKDIR /app
 COPY . /app

backend/Dockerfile.API

@@ -0,0 +1,62 @@
+## docker build -t fair-api -f Dockerfile.API . 
+
+## For Development:
+
+## docker run --env-file .env --rm -p 8000:8000 -v $(pwd):/app/code --name fair-api-container fair-api
+
+## in order to access localhost port to your system from docker env you might wanna use : host.docker.internal or ip 
+
+FROM python:3.11-slim-bookworm AS build
+
+RUN apt-get update && apt-get --no-install-recommends -y install \
+    build-essential \
+    libpq-dev \
+    libgdal-dev \
+    python3-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+WORKDIR /app
+
+RUN python3 -m venv /app/venv
+
+RUN /app/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel
+
+COPY api-requirements.txt api-requirements.txt ./
+
+RUN /app/venv/bin/pip install --no-cache-dir -r api-requirements.txt
+
+WORKDIR /app/code
+COPY aiproject /app/code/aiproject
+COPY core /app/code/core
+COPY login /app/code/login
+COPY manage.py /app/code/manage.py
+COPY tests /app/code/tests
+
+FROM python:3.11-slim-bookworm
+
+RUN apt-get update && apt-get --no-install-recommends -y install \
+    libgdal-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+WORKDIR /app
+
+COPY --from=build /app/venv /app/venv
+
+COPY --from=build /app/code /app/code
+ENV PATH="/app/venv/bin:$PATH"
+COPY api-entrypoint.sh ./api-entrypoint-lock.sh
+RUN chmod +x ./api-entrypoint-lock.sh
+
+WORKDIR /app/code
+
+EXPOSE 8000
+
+ENTRYPOINT ["/app/api-entrypoint-lock.sh"]
+
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]

backend/Dockerfile_CPU

@@ -18,11 +18,12 @@ COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt
 RUN pip install -r /tmp/docker-requirements.txt
 
 
-RUN pip install --upgrade setuptools
+# RUN pip install --upgrade setuptools
 
 COPY requirements.txt /tmp/requirements.txt
 COPY api-requirements.txt /tmp/api-requirements.txt
-RUN pip install -r /tmp/requirements.txt
+# Don't use legacy resolver , TODO : fix this dependencies 
+RUN pip install --use-deprecated=legacy-resolver -r /tmp/requirements.txt 
 
 COPY docker/ramp/solaris /tmp/solaris
 RUN pip install /tmp/solaris --use-feature=in-tree-build && \

backend/aiproject/settings.py

@@ -11,7 +11,7 @@
 """
 
 import os
-
+import logging
 import dj_database_url
 import environ
 from corsheaders.defaults import default_headers
@@ -56,7 +56,9 @@
 # Limiter
 EPOCHS_LIMIT = env("EPOCHS_LIMIT", default=30)
 BATCH_SIZE_LIMIT = env("BATCH_SIZE_LIMIT", default=8)
-TRAINING_WORKSPACE_DOWNLOAD_LIMIT = env("TRAINING_WORKSPACE_DOWNLOAD_LIMIT", default=200)
+TRAINING_WORKSPACE_DOWNLOAD_LIMIT = env(
+    "TRAINING_WORKSPACE_DOWNLOAD_LIMIT", default=200
+)
 
 
 # Application definition
@@ -98,7 +100,7 @@
 
 CORS_ORIGIN_WHITELIST = ALLOWED_ORIGINS
 
-CORS_ORIGIN_ALLOW_ALL = env("CORS_ORIGIN_ALLOW_ALL", default= False)
+CORS_ORIGIN_ALLOW_ALL = env("CORS_ORIGIN_ALLOW_ALL", default=False)
 
 REST_FRAMEWORK = {
     "DEFAULT_SCHEMA_CLASS": "rest_framework.schemas.coreapi.AutoSchema",
@@ -185,6 +187,7 @@
 STATIC_ROOT = os.path.join(BASE_DIR, "api_static")
 
 if DEBUG:
+    logging.info("Enabling oauthlib insecure transport in debug mode")
     os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
 
 
@@ -204,10 +207,16 @@
     }
 }
 # get ramp home and set it to environ
-RAMP_HOME = env("RAMP_HOME")
-os.environ["RAMP_HOME"] = RAMP_HOME
+RAMP_HOME = env("RAMP_HOME",default=None)
+if RAMP_HOME:
+    os.environ["RAMP_HOME"] = RAMP_HOME
 
 # training workspace
 TRAINING_WORKSPACE = env(
     "TRAINING_WORKSPACE", default=os.path.join(os.getcwd(), "training")
 )
+
+ENABLE_PREDICTION_API = env("ENABLE_PREDICTION_API", default=False)
+
+
+TEST_RUNNER = 'tests.test_runners.NoDestroyTestRunner'

backend/api-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e 
+echo "Applying database migrations..."
+python manage.py makemigrations login core
+python manage.py migrate
+echo "Starting Django server..."
+exec "$@"

backend/api-requirements.txt

@@ -20,4 +20,8 @@ geojson2osm==0.0.1
 osmconflator==0.0.11
 orthogonalizer==0.0.4
 fairpredictor==0.0.26
-tflite-runtime==2.14.0
+
+rasterio==1.3.8
+numpy<2.0.0
+
+

backend/core/models.py

@@ -2,6 +2,7 @@
 from django.contrib.postgres.fields import ArrayField
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
+
 from login.models import OsmUser
 
 # Create your models here.
@@ -133,3 +134,17 @@ class FeedbackLabel(models.Model):
 
     geom = geomodels.PolygonField(srid=4326)
     created_at = models.DateTimeField(auto_now_add=True)
+
+
+class ApprovedPredictions(models.Model):
+    training = models.ForeignKey(Training, to_field="id", on_delete=models.DO_NOTHING)
+    config = models.JSONField(
+        null=True, blank=True
+    )  ### Config meant to be kept for vectorization config / zoom config , to know what user is using for the most of the time
+    geom = geomodels.GeometryField(
+        srid=4326
+    )  ## Making this geometry field to support point/line prediction later on
+    approved_at = models.DateTimeField(auto_now_add=True)
+    approved_by = models.ForeignKey(
+        OsmUser, to_field="osm_id", on_delete=models.CASCADE
+    )

backend/core/serializers.py

@@ -1,10 +1,11 @@
 from django.conf import settings
-from login.models import OsmUser
 from rest_framework import serializers
 from rest_framework_gis.serializers import (
     GeoFeatureModelSerializer,  # this will be used if we used to serialize as geojson
 )
 
+from login.models import OsmUser
+
 from .models import *
 
 # from .tasks import train_model
@@ -113,6 +114,13 @@ class Meta:
         # read_only_fields = ("created_at", "osm_id")
 
 
+class ApprovedPredictionsSerializer(GeoFeatureModelSerializer):
+    class Meta:
+        model = ApprovedPredictions
+        geo_field = "geom"
+        fields = "__all__"
+
+
 class FeedbackLabelSerializer(GeoFeatureModelSerializer):
     class Meta:
         model = FeedbackLabel