Merge branch 'master' into patch-3

CONTRIBUTING.md

@@ -1,11 +1,16 @@
 # Contributing to fAIr
 
-## Welcome
+## :hugs: Welcome
 
 :+1::tada: First off, I'm really glad you're reading this, because we need volunteer developers to help with the development of fAIr! :tada::+1:
 We welcome and encourage contributors of all skill levels and we are committed to making sure your participation in our tech collective is inclusive, enjoyable and rewarding. If you have never contributed to an open-source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask!
 
-## :handshake: Thank you
 
-Thank you very much in advance for your contributions!! Please ensure you refer to our [Code of Conduct](https://github.com/hotosm/fAIr/blob/master/docs/Code-of-Conduct.md).
-If you've read the guidelines, but you are still not sure how to contribute on Github, please reach out to us via our Slack #geospatial-tech-and-innovation.
+## Code contributions
+
+Fork repo, Maintain your local changes on branch and Create pull requests (PRs) for changes that you think are needed. We would really appreciate your help! 
+
+
+## Documentation contributions 
+
+Create pull requests (PRs) for changes that you think are needed to the documentation of fAIr.As of now you can find the documentation work at the [docs](./docs) directory.

backend/aiproject/settings.py

@@ -33,7 +33,7 @@
 HOSTNAME = env("HOSTNAME", default="127.0.0.1")
 EXPORT_TOOL_API_URL = env(
     "EXPORT_TOOL_API_URL",
-    default="	https://api-prod.raw-data.hotosm.org/v1",
+    default="https://api-prod.raw-data.hotosm.org/v1",
 )
 
 ALLOWED_HOSTS = ["localhost", "127.0.0.1", HOSTNAME]

backend/core/models.py

@@ -41,6 +41,7 @@ class Label(models.Model):
     aoi = models.ForeignKey(AOI, to_field="id", on_delete=models.CASCADE)
     geom = geomodels.GeometryField(srid=4326)
     osm_id = models.BigIntegerField(null=True, blank=True)
+    tags = models.JSONField(null=True, blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
 
 
@@ -101,7 +102,7 @@ class Feedback(models.Model):
         validators=[MinValueValidator(18), MaxValueValidator(23)]
     )
     feedback_type = models.CharField(choices=FEEDBACK_TYPE, max_length=10)
-    comments = models.TextField(max_length=100,null=True,blank=True)
+    comments = models.TextField(max_length=100, null=True, blank=True)
     user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE)
     source_imagery = models.URLField()
 
@@ -111,6 +112,7 @@ class DownloadStatus(models.IntegerChoices):
         DOWNLOADED = 1
         NOT_DOWNLOADED = -1
         RUNNING = 0
+
     training = models.ForeignKey(Training, to_field="id", on_delete=models.CASCADE)
     geom = geomodels.PolygonField(srid=4326)
     label_status = models.IntegerField(default=-1, choices=DownloadStatus.choices)
@@ -123,6 +125,10 @@ class DownloadStatus(models.IntegerChoices):
 
 class FeedbackLabel(models.Model):
     osm_id = models.BigIntegerField(null=True, blank=True)
-    feedback_aoi = models.ForeignKey(FeedbackAOI, to_field="id", on_delete=models.CASCADE)
+    feedback_aoi = models.ForeignKey(
+        FeedbackAOI, to_field="id", on_delete=models.CASCADE
+    )
+    tags = models.JSONField(null=True, blank=True)
+
     geom = geomodels.PolygonField(srid=4326)
     created_at = models.DateTimeField(auto_now_add=True)

backend/core/serializers.py

@@ -126,15 +126,15 @@ class Meta:
         model = Label
         geo_field = "geom"
         # auto_bbox = True
-        fields = ("osm_id",)
+        fields = ("osm_id", "tags")
 
 
 class FeedbackLabelFileSerializer(GeoFeatureModelSerializer):
     class Meta:
         model = FeedbackLabel
         geo_field = "geom"
         # auto_bbox = True
-        fields = ("osm_id",)
+        fields = ("osm_id", "tags")
 
 
 class FeedbackFileSerializer(GeoFeatureModelSerializer):

backend/core/tasks.py

@@ -10,24 +10,24 @@
 import ramp.utils
 import tensorflow as tf
 from celery import shared_task
-from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
-from core.serializers import (
-    FeedbackFileSerializer,
-    FeedbackLabelFileSerializer,
-    LabelFileSerializer,
-)
-from predictor import download_imagery,get_start_end_download_coords
-from core.utils import (
-    bbox,
-    is_dir_empty,
-)
 from django.conf import settings
 from django.contrib.gis.db.models.aggregates import Extent
 from django.contrib.gis.geos import GEOSGeometry
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
 from hot_fair_utilities import preprocess, train
 from hot_fair_utilities.training import run_feedback
+from predictor import download_imagery, get_start_end_download_coords
+
+from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
+from core.serializers import (
+    AOISerializer,
+    FeedbackAOISerializer,
+    FeedbackFileSerializer,
+    FeedbackLabelFileSerializer,
+    LabelFileSerializer,
+)
+from core.utils import bbox, is_dir_empty
 
 logger = logging.getLogger(__name__)
 
@@ -56,8 +56,8 @@ def train_model(
     try:
         ## -----------IMAGE DOWNLOADER---------
         os.makedirs(settings.LOG_PATH, exist_ok=True)
-        if training_instance.task_id is None or training_instance.task_id.strip() == '': 
-            training_instance.task_id=train_model.request.id
+        if training_instance.task_id is None or training_instance.task_id.strip() == "":
+            training_instance.task_id = train_model.request.id
             training_instance.save()
         log_file = os.path.join(
             settings.LOG_PATH, f"run_{train_model.request.id}_log.txt"
@@ -77,6 +77,8 @@ def train_model(
             if feedback:
                 try:
                     aois = FeedbackAOI.objects.filter(training=feedback)
+                    aoi_serializer = FeedbackAOISerializer(aois, many=True)
+
                 except FeedbackAOI.DoesNotExist:
                     raise ValueError(
                         f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first",
@@ -85,11 +87,12 @@ def train_model(
             else:
                 try:
                     aois = AOI.objects.filter(dataset=dataset_id)
+                    aoi_serializer = AOISerializer(aois, many=True)
+
                 except AOI.DoesNotExist:
                     raise ValueError(
                         f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
                     )
-
             for obj in aois:
                 bbox_coords = bbox(obj.geom.coords[0])
                 for z in zoom_level:
@@ -223,15 +226,31 @@ def train_model(
 
             logger.info(model.inputs)
             logger.info(model.outputs)
-            
+
             # Convert the model to tflite for android/ios.
             converter = tf.lite.TFLiteConverter.from_keras_model(model)
             tflite_model = converter.convert()
 
             # Save the model.
-            with open(os.path.join(output_path, "checkpoint.tflite"), 'wb') as f:
+            with open(os.path.join(output_path, "checkpoint.tflite"), "wb") as f:
                 f.write(tflite_model)
 
+            # dump labels to output folder as well
+            with open(
+                os.path.join(output_path, "labels.geojson"),
+                "w",
+                encoding="utf-8",
+            ) as f:
+                f.write(json.dumps(serialized_field.data))
+
+            # dump used aois as featurecollection in output
+            with open(
+                os.path.join(output_path, "aois.geojson"),
+                "w",
+                encoding="utf-8",
+            ) as f:
+                f.write(json.dumps(aoi_serializer.data))
+
             # now remove the ramp-data all our outputs are copied to our training workspace
             shutil.rmtree(base_path)
             training_instance.accuracy = float(final_accuracy)

backend/core/utils.py

@@ -189,6 +189,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
     """Multi thread process of features"""
     properties = feature["properties"]
     osm_id = properties["osm_id"]
+    tags = properties["tags"]
     geometry = feature["geometry"]
     if feedback:
         if FeedbackLabel.objects.filter(
@@ -199,7 +200,12 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
             ).delete()
 
         label = FeedbackLabelSerializer(
-            data={"osm_id": int(osm_id), "geom": geometry, "feedback_aoi": aoi_id}
+            data={
+                "osm_id": int(osm_id),
+                "tags": tags,
+                "geom": geometry,
+                "feedback_aoi": aoi_id,
+            }
         )
 
     else:
@@ -211,7 +217,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
             ).delete()
 
         label = LabelSerializer(
-            data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id}
+            data={"osm_id": int(osm_id), "tags": tags, "geom": geometry, "aoi": aoi_id}
         )
     if label.is_valid():
         label.save()
@@ -239,7 +245,10 @@ def process_geojson(geojson_file_path, aoi_id, feedback=False):
     max_workers = (
         (os.cpu_count() - 1) if os.cpu_count() != 1 else 1
     )  # leave one cpu free always
-
+    if feedback:
+        FeedbackLabel.objects.filter(feedback_aoi__id=aoi_id).delete()
+    else : 
+        Label.objects.filter(aoi__id=aoi_id).delete()
     # max_workers = os.cpu_count()  # get total cpu count available on the
 
     with open(geojson_file_path) as f:

backend/core/views.py

@@ -192,6 +192,7 @@ class FeedbackLabelViewset(viewsets.ModelViewSet):
     bbox_filter_field = "geom"
     filter_backends = (
         InBBoxFilter,  # it will take bbox like this api/v1/label/?in_bbox=-90,29,-89,35 ,
+        DjangoFilterBackend
     )
     bbox_filter_include_overlapping = True
     filterset_fields = ["feedback_aoi", "feedback_aoi__training"]

docs/Docker-installation.md

@@ -33,11 +33,11 @@ Docker Compose is created with redis , worker , postgis database ,  api and fron
         ```
         mkdir ramp
         ```
-    - Download BaseModel Checkpoint from [here](https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing) 
+    - Download BaseModel Checkpoint from [here](https://drive.google.com/file/d/1YQsY61S_rGfJ_f6kLQq4ouYE2l3iRe1k/view) 
         OR You can use basemodel from [Model Ramp Baseline](https://github.com/radiantearth/model_ramp_baseline/tree/main/data/input/checkpoint.tf)
         ```
         pip install gdown
-        gdown --fuzzy https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing
+        gdown --fuzzy https://drive.google.com/file/d/1YQsY61S_rGfJ_f6kLQq4ouYE2l3iRe1k/view
         ```
     - Clone Ramp Code 
 

frontend/src/components/Layout/Feedback/Feedback.js

@@ -101,9 +101,14 @@ const Feedback = (props) => {
       } else {
         const datasetId = res.data.dataset;
         setDatasetId(datasetId);
-        const resAOIs = await axios.get(`/aoi/?dataset=${datasetId}`, null, {
-          headers,
-        });
+        const resAOIs = await axios.get(
+          `/workspace/download/dataset_${datasetId}/output/training_${trainingId}/aois.geojson`,
+          null,
+          {
+            headers,
+          }
+        );
+        // console.log("resAOIs", resAOIs);
         setOriginalAOIs(resAOIs.data);
       }
     } catch (e) {
@@ -247,7 +252,7 @@ const Feedback = (props) => {
         "access-token": accessToken,
       };
       const res = await axios.get(
-        `/feedback-label/?in_bbox=${box._southWest.lng},${box._southWest.lat},${box._northEast.lng},${box._northEast.lat}`,
+        `/feedback-label/?in_bbox=${box._southWest.lng},${box._southWest.lat},${box._northEast.lng},${box._northEast.lat}&feedback_aoi__training=${trainingId}`,
         { headers }
       );
       console.log("res from getLabels ", res);