From 536fcd3a591633daf6c9d287f449a872608da94c Mon Sep 17 00:00:00 2001
From: NicholasTurner23 <mugtrade@gmail.com>
Date: Tue, 10 Dec 2024 14:31:35 +0300
Subject: [PATCH 01/35] Update dashboard chart query

---
 src/analytics/api/models/events.py         |  6 ++----
 src/analytics/api/utils/data_formatters.py | 25 ++++++++++++++++++++++
 src/analytics/api/views/dashboard.py       | 11 +++-------
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/src/analytics/api/models/events.py b/src/analytics/api/models/events.py
index 4ac9e2febb..c17c32fa21 100644
--- a/src/analytics/api/models/events.py
+++ b/src/analytics/api/models/events.py
@@ -413,6 +413,7 @@ def download_from_bigquery(
             frequency (str): Data frequency (e.g., 'raw', 'daily', 'hourly').
             pollutants (list): List of pollutants to include in the data.
             data_type (str): Type of data ('raw' or 'aggregated').
+            filter_columns(list)
             weather_fields (list): List of weather fields to retrieve.
 
         Returns:
@@ -1273,9 +1274,7 @@ def get_d3_chart_events(self, sites, start_date, end_date, pollutant, frequency)
         )
 
     @cache.memoize()
-    def get_d3_chart_events_v2(
-        self, sites, start_date, end_date, pollutant, frequency, tenant
-    ):
+    def get_d3_chart_events_v2(self, sites, start_date, end_date, pollutant, frequency):
         if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]:
             raise Exception("Invalid pollutant")
 
@@ -1293,7 +1292,6 @@ def get_d3_chart_events_v2(
           JOIN {self.BIGQUERY_SITES} ON {self.BIGQUERY_SITES}.id = {self.BIGQUERY_EVENTS}.site_id 
           WHERE  {self.BIGQUERY_EVENTS}.timestamp >= '{start_date}'
           AND {self.BIGQUERY_EVENTS}.timestamp <= '{end_date}'
-          AND {self.BIGQUERY_EVENTS}.tenant = '{tenant}'
           AND `airqo-250220.metadata.sites`.id in UNNEST({sites})
         """
 
diff --git a/src/analytics/api/utils/data_formatters.py b/src/analytics/api/utils/data_formatters.py
index bace9559de..0c6fb6b377 100644
--- a/src/analytics/api/utils/data_formatters.py
+++ b/src/analytics/api/utils/data_formatters.py
@@ -328,6 +328,31 @@ def filter_non_private_sites(filter_type: str, sites: List[str]) -> Dict[str, An
         logger.exception(f"Error while filtering non private devices {rex}")
 
 
+def validate_network(self, network_name: str) -> bool:
+    """
+    Validate if a given network name exists in the list of networks.
+
+    Args:
+        network_name (str): The name of the network to validate.
+
+    Returns:
+        bool: True if the network name exists, False otherwise.
+    """
+    if not network_name:
+        return False
+
+    endpoint: str = "/users/networks"
+    airqo_requests = AirQoRequests()
+    response = airqo_requests.request(endpoint=endpoint, method="get")
+
+    if response and "networks" in response:
+        networks = response["networks"]
+        # TODO Could add an active network filter
+        return any(network.get("net_name") == network_name for network in networks)
+
+    return False
+
+
 def filter_non_private_devices(filter_type: str, devices: List[str]) -> Dict[str, Any]:
     """
     FilterS out private device IDs from a provided array of device IDs.
diff --git a/src/analytics/api/views/dashboard.py b/src/analytics/api/views/dashboard.py
index 62d31bc01e..3e4429495e 100644
--- a/src/analytics/api/views/dashboard.py
+++ b/src/analytics/api/views/dashboard.py
@@ -206,7 +206,6 @@ def _get_validated_filter(self, json_data):
         return filter_type, validated_data, error_message
 
     def post(self):
-        tenant = request.args.get("tenant", "airqo")
 
         json_data = request.get_json()
 
@@ -219,20 +218,16 @@ def post(self):
         except Exception as e:
             logger.exception(f"An error has occured; {e}")
 
-        sites = filter_non_private_sites("sites", json_data.get("sites", {})).get(
-            "sites", []
-        )
-
         start_date = json_data["startDate"]
         end_date = json_data["endDate"]
         frequency = json_data["frequency"]
         pollutant = json_data["pollutant"]
         chart_type = json_data["chartType"]
 
-        events_model = EventsModel(tenant)
-        # data = events_model.get_d3_chart_events(sites, start_date, end_date, pollutant, frequency)
+        events_model = EventsModel("airqo")
+
         data = events_model.get_d3_chart_events_v2(
-            filter_value, start_date, end_date, pollutant, frequency, tenant
+            filter_value, start_date, end_date, pollutant, frequency
         )
 
         if chart_type.lower() == "pie":

From 370f4583d4297e2b2e395cc3bb5fc870dfb1e50a Mon Sep 17 00:00:00 2001
From: NicholasTurner23 <mugtrade@gmail.com>
Date: Tue, 10 Dec 2024 14:43:14 +0300
Subject: [PATCH 02/35] Drop timestamp column

---
 src/analytics/api/views/data.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/analytics/api/views/data.py b/src/analytics/api/views/data.py
index 6038e80497..07ef72c0a2 100644
--- a/src/analytics/api/views/data.py
+++ b/src/analytics/api/views/data.py
@@ -166,6 +166,7 @@ def post(self):
                 data_frame.drop(
                     columns=[
                         "site_id",
+                        "timestamp",
                     ],
                     inplace=True,
                 )

From 368c59d9af60fbf432b807b668c9d82d491f3bd8 Mon Sep 17 00:00:00 2001
From: NicholasTurner23 <mugtrade@gmail.com>
Date: Tue, 10 Dec 2024 17:22:49 +0300
Subject: [PATCH 03/35] Handle non calibrated values

---
 src/analytics/api/models/events.py | 55 +++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/src/analytics/api/models/events.py b/src/analytics/api/models/events.py
index c17c32fa21..a3883752f2 100644
--- a/src/analytics/api/models/events.py
+++ b/src/analytics/api/models/events.py
@@ -439,14 +439,11 @@ def download_from_bigquery(
         weather_columns = []
         for pollutant in pollutants:
 
-            if pollutant == "raw":
-                key = pollutant
-            else:
-                key = f"{pollutant}_{data_type}"
-
+            key = f"{pollutant}_{data_type}"
             pollutant_mapping = BIGQUERY_FREQUENCY_MAPPER.get(frequency, {}).get(
                 key, []
             )
+
             pollutant_columns.extend(
                 cls.get_columns(
                     cls,
@@ -460,6 +457,10 @@ def download_from_bigquery(
 
             # TODO Clean up by use using `get_columns` helper method
             if pollutant in {"pm2_5", "pm10", "no2"}:
+                if data_type == "raw":
+                    # Add dummy column to fix union column number missmatch.
+                    bam_pollutant_columns.append("-1 as pm2_5")
+
                 if frequency in ["weekly", "monthly", "yearly"]:
                     bam_pollutant_columns.extend(
                         [f"ROUND(AVG({pollutant}), {decimal_places}) AS {key}_value"]
@@ -468,6 +469,7 @@ def download_from_bigquery(
                     bam_pollutant_columns.extend(
                         [f"ROUND({pollutant}, {decimal_places}) AS {key}_value"]
                     )
+
         # TODO Fix query when weather data is included. Currently failing
         if weather_fields:
             for field in weather_fields:
@@ -537,12 +539,55 @@ def download_from_bigquery(
             drop_columns.append("datetime")
             sorting_cols.append("datetime")
 
+        if data_type == "raw":
+            cls.simple_data_cleaning(dataframe)
+
         dataframe.drop_duplicates(subset=drop_columns, inplace=True, keep="first")
         dataframe.sort_values(sorting_cols, ascending=True, inplace=True)
         dataframe["frequency"] = frequency
         dataframe = dataframe.replace(np.nan, None)
         return dataframe
 
+    @classmethod
+    def simple_data_cleaning(cls, data: pd.DataFrame) -> pd.DataFrame:
+        """
+         Perform data cleaning on a pandas DataFrame to handle specific conditions
+         related to "pm2_5" and "pm2_5_raw_value" columns.
+
+         The cleaning process includes:
+         1. Ensuring correct numeric data types for "pm2_5" and "pm2_5_raw_value".
+         2. Removing "pm2_5" values where "pm2_5_raw_value" has data.
+         3. Dropping the "pm2_5_raw_value" column if it has no data at all.
+         4. Retaining "pm2_5" values where "pm2_5_raw_value" has no data, and removing
+         "pm2_5" values where "pm2_5_raw_value" has data.
+         5. Dropping any column (including "pm2_5" and "pm2_5_raw_value") if it is
+        entirely empty.
+
+         Args:
+             cls: Class reference (used in classmethods).
+             data (pd.DataFrame): Input pandas DataFrame with "pm2_5" and
+                                 "pm2_5_raw_value" columns.
+
+         Returns:
+             pd.DataFrame: Cleaned DataFrame with updates applied in place.
+
+        """
+        data["pm2_5_raw_value"] = pd.to_numeric(
+            data["pm2_5_raw_value"], errors="coerce"
+        )
+        data["pm2_5"] = pd.to_numeric(data["pm2_5"], errors="coerce")
+
+        data.loc[~data["pm2_5_raw_value"].isna(), "pm2_5"] = np.nan
+
+        if data["pm2_5_raw_value"].isna().all():
+            data.drop(columns=["pm2_5_raw_value"], inplace=True)
+
+        data["pm2_5"] = data["pm2_5"].where(data["pm2_5_raw_value"].isna(), np.nan)
+
+        data.dropna(how="all", axis=1, inplace=True)
+
+        return data
+
     @classmethod
     def data_export_query(
         cls,

From c1cf65941d5aebd63f1f769305b52df8eb6f4e78 Mon Sep 17 00:00:00 2001
From: NicholasTurner23 <mugtrade@gmail.com>
Date: Tue, 10 Dec 2024 17:23:38 +0300
Subject: [PATCH 04/35] Clean up

---
 src/analytics/api/utils/pollutants/pm_25.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/analytics/api/utils/pollutants/pm_25.py b/src/analytics/api/utils/pollutants/pm_25.py
index 6e3f2a986b..9c16abf988 100644
--- a/src/analytics/api/utils/pollutants/pm_25.py
+++ b/src/analytics/api/utils/pollutants/pm_25.py
@@ -56,19 +56,14 @@
 
 COMMON_POLLUTANT_MAPPING = {
     "pm2_5_calibrated": ["pm2_5_calibrated_value"],
-    "pm2_5_raw": ["pm2_5_raw_value"],
+    "pm2_5_raw": ["pm2_5_raw_value", "pm2_5"],
     "pm10_calibrated": ["pm10_calibrated_value"],
-    "pm10_raw": ["pm10_raw_value"],
+    "pm10_raw": ["pm10_raw_value", "pm10"],
     "no2_calibrated": ["no2_calibrated_value"],
     "no2_raw": ["no2_raw_value"],
 }
 
 BIGQUERY_FREQUENCY_MAPPER = {
-    "raw": {
-        "pm2_5": ["pm2_5", "s1_pm2_5", "s2_pm2_5"],
-        "pm10": ["pm10", "s1_pm10", "s2_pm10"],
-        "no2": ["no2"],
-    },
     "daily": COMMON_POLLUTANT_MAPPING,
     "hourly": COMMON_POLLUTANT_MAPPING,
     "weekly": COMMON_POLLUTANT_MAPPING,

From 84577f29ddfb0d3144219c7886e3a1b5be8ae4fb Mon Sep 17 00:00:00 2001
From: Nicholas Bob <mr.nnturner@gmail.com>
Date: Tue, 10 Dec 2024 17:44:02 +0300
Subject: [PATCH 05/35] Update data_formatters.py

Clean up
---
 src/analytics/api/utils/data_formatters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/analytics/api/utils/data_formatters.py b/src/analytics/api/utils/data_formatters.py
index 0c6fb6b377..d1f5eda567 100644
--- a/src/analytics/api/utils/data_formatters.py
+++ b/src/analytics/api/utils/data_formatters.py
@@ -328,7 +328,7 @@ def filter_non_private_sites(filter_type: str, sites: List[str]) -> Dict[str, An
         logger.exception(f"Error while filtering non private devices {rex}")
 
 
-def validate_network(self, network_name: str) -> bool:
+def validate_network(network_name: str) -> bool:
     """
     Validate if a given network name exists in the list of networks.
 

From 99b38f93f39383d15406714825f6014474953c1d Mon Sep 17 00:00:00 2001
From: baalmart <bbaalmart@gmail.com>
Date: Tue, 10 Dec 2024 18:05:45 +0300
Subject: [PATCH 06/35] allow sites and devices to belong to more than one
 Group

---
 .../config/global/db-projections.js           | 12 +--
 src/device-registry/models/Activity.js        |  6 +-
 src/device-registry/models/Airqloud.js        |  6 +-
 src/device-registry/models/Cohort.js          | 18 ++---
 src/device-registry/models/Device.js          | 76 ++++++++++---------
 src/device-registry/models/Grid.js            |  8 +-
 src/device-registry/models/Location.js        |  8 ++
 src/device-registry/models/Photo.js           |  8 +-
 src/device-registry/models/Site.js            | 37 ++++++---
 src/device-registry/routes/v2/sites.js        | 18 +++++
 src/device-registry/utils/generate-filter.js  | 24 ++++--
 .../validators/device.validators.js           | 18 +++++
 12 files changed, 157 insertions(+), 82 deletions(-)

diff --git a/src/device-registry/config/global/db-projections.js b/src/device-registry/config/global/db-projections.js
index 8282998021..37f14abffc 100644
--- a/src/device-registry/config/global/db-projections.js
+++ b/src/device-registry/config/global/db-projections.js
@@ -135,7 +135,7 @@ const dbProjections = {
     lat_long: 1,
     country: 1,
     network: 1,
-    group: 1,
+    groups: 1,
     data_provider: 1,
     district: 1,
     sub_county: 1,
@@ -379,7 +379,7 @@ const dbProjections = {
     mobility: 1,
     status: 1,
     network: 1,
-    group: 1,
+    groups: 1,
     api_code: 1,
     serial_number: 1,
     authRequired: 1,
@@ -591,7 +591,7 @@ const dbProjections = {
     shape: 1,
     createdAt: 1,
     network: 1,
-    group: 1,
+    groups: 1,
     sites: "$sites",
     numberOfSites: {
       $cond: {
@@ -675,7 +675,7 @@ const dbProjections = {
     name: 1,
     description: 1,
     cohort_tags: 1,
-    group: 1,
+    groups: 1,
     createdAt: 1,
     visibility: 1,
     cohort_codes: 1,
@@ -817,7 +817,7 @@ const dbProjections = {
     airqloud_tags: 1,
     isCustom: 1,
     network: 1,
-    group: 1,
+    groups: 1,
     metadata: 1,
     center_point: 1,
     sites: "$sites",
@@ -1065,7 +1065,7 @@ const dbProjections = {
     date: 1,
     description: 1,
     network: 1,
-    group: 1,
+    groups: 1,
     activityType: 1,
     maintenanceType: 1,
     recallType: 1,
diff --git a/src/device-registry/models/Activity.js b/src/device-registry/models/Activity.js
index 0e5863949e..9cd5ada002 100644
--- a/src/device-registry/models/Activity.js
+++ b/src/device-registry/models/Activity.js
@@ -40,8 +40,8 @@ const activitySchema = new Schema(
       type: String,
       trim: true,
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     activityType: { type: String, trim: true },
@@ -70,7 +70,7 @@ activitySchema.methods = {
       _id: this._id,
       device: this.device,
       network: this.network,
-      group: this.group,
+      groups: this.groups,
       date: this.date,
       description: this.description,
       activityType: this.activityType,
diff --git a/src/device-registry/models/Airqloud.js b/src/device-registry/models/Airqloud.js
index beacd653db..1be15e8683 100644
--- a/src/device-registry/models/Airqloud.js
+++ b/src/device-registry/models/Airqloud.js
@@ -113,8 +113,8 @@ const airqloudSchema = new Schema(
       type: String,
       trim: true,
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     airqloud_tags: {
@@ -152,7 +152,7 @@ airqloudSchema.methods.toJSON = function() {
     name: this.name,
     long_name: this.long_name,
     network: this.network,
-    group: this.group,
+    groups: this.groups,
     description: this.description,
     airqloud_tags: this.airqloud_tags,
     admin_level: this.admin_level,
diff --git a/src/device-registry/models/Cohort.js b/src/device-registry/models/Cohort.js
index 788811e60a..e5aeb92d20 100644
--- a/src/device-registry/models/Cohort.js
+++ b/src/device-registry/models/Cohort.js
@@ -17,8 +17,8 @@ const cohortSchema = new Schema(
       trim: true,
       required: [true, "the network is required!"],
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     name: {
@@ -81,7 +81,7 @@ cohortSchema.methods.toJSON = function() {
     cohort_tags,
     cohort_codes,
     network,
-    group,
+    groups,
     visibility,
   } = this;
   return {
@@ -92,7 +92,7 @@ cohortSchema.methods.toJSON = function() {
     cohort_tags,
     cohort_codes,
     network,
-    group,
+    groups,
   };
 };
 
@@ -202,7 +202,7 @@ cohortSchema.statics.list = async function(
         name: 1,
         createdAt: 1,
         network: 1,
-        group: 1,
+        groups: 1,
         devices: {
           $cond: {
             if: { $eq: [{ $size: "$devices" }, 0] },
@@ -214,7 +214,7 @@ cohortSchema.statics.list = async function(
       .sort({ createdAt: -1 })
       .project(inclusionProjection)
       .project(exclusionProjection)
-      .group({
+      .groups({
         _id: "$_id",
         visibility: { $first: "$visibility" },
         cohort_tags: { $first: "$cohort_tags" },
@@ -222,7 +222,7 @@ cohortSchema.statics.list = async function(
         name: { $first: "$name" },
         createdAt: { $first: "$createdAt" },
         network: { $first: "$network" },
-        group: { $first: "$group" },
+        groups: { $first: "$groups" },
         devices: { $first: "$devices" },
       })
       .skip(skip ? parseInt(skip) : 0)
@@ -240,7 +240,7 @@ cohortSchema.statics.list = async function(
         name: cohort.name,
         network: cohort.network,
         createdAt: cohort.createdAt,
-        group: cohort.group,
+        groups: cohort.groups,
         numberOfDevices: cohort.devices ? cohort.devices.length : 0,
         devices: cohort.devices
           ? cohort.devices
@@ -250,7 +250,7 @@ cohortSchema.statics.list = async function(
                 status: device.status,
                 name: device.name,
                 network: device.network,
-                group: device.group,
+                groups: device.groups,
                 device_number: device.device_number,
                 description: device.description,
                 long_name: device.long_name,
diff --git a/src/device-registry/models/Device.js b/src/device-registry/models/Device.js
index a5b10c7500..0f5b08749d 100644
--- a/src/device-registry/models/Device.js
+++ b/src/device-registry/models/Device.js
@@ -74,8 +74,8 @@ const deviceSchema = new mongoose.Schema(
       trim: true,
       required: [true, "the network is required!"],
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     serial_number: {
@@ -237,6 +237,20 @@ deviceSchema.plugin(uniqueValidator, {
   message: `{VALUE} must be unique!`,
 });
 
+const checkDuplicates = (arr, fieldName) => {
+  const duplicateValues = arr.filter(
+    (value, index, self) => self.indexOf(value) !== index
+  );
+
+  if (duplicateValues.length > 0) {
+    return new HttpError(
+      `Duplicate values found in ${fieldName} array.`,
+      httpStatus.BAD_REQUEST
+    );
+  }
+  return null;
+};
+
 deviceSchema.pre(
   [
     "update",
@@ -326,18 +340,16 @@ deviceSchema.pre(
           this.device_codes.push(this.serial_number);
         }
 
-        // Check for duplicate values in cohorts array
-        const duplicateValues = this.cohorts.filter(
-          (value, index, self) => self.indexOf(value) !== index
-        );
+        // Check for duplicates in cohorts
+        const cohortsDuplicateError = checkDuplicates(this.cohorts, "cohorts");
+        if (cohortsDuplicateError) {
+          return next(cohortsDuplicateError);
+        }
 
-        if (duplicateValues.length > 0) {
-          return next(
-            new HttpError(
-              "Duplicate values found in cohorts array.",
-              httpStatus.BAD_REQUEST
-            )
-          );
+        // Check for duplicates in groups
+        const groupsDuplicateError = checkDuplicates(this.groups, "groups");
+        if (groupsDuplicateError) {
+          return next(groupsDuplicateError);
         }
       }
 
@@ -371,28 +383,20 @@ deviceSchema.pre(
         updateData.access_code = access_code.toUpperCase();
       }
 
-      // Handle $addToSet for device_codes, previous_sites, and pictures
-      const addToSetUpdates = {};
-
-      if (updateData.device_codes) {
-        addToSetUpdates.device_codes = { $each: updateData.device_codes };
-        delete updateData.device_codes; // Remove from main update object
-      }
-
-      if (updateData.previous_sites) {
-        addToSetUpdates.previous_sites = { $each: updateData.previous_sites };
-        delete updateData.previous_sites; // Remove from main update object
-      }
-
-      if (updateData.pictures) {
-        addToSetUpdates.pictures = { $each: updateData.pictures };
-        delete updateData.pictures; // Remove from main update object
-      }
-
-      // If there are any $addToSet updates, merge them into the main update object
-      if (Object.keys(addToSetUpdates).length > 0) {
-        updateData.$addToSet = addToSetUpdates;
-      }
+      // Handle array fields using $addToSet
+      const arrayFieldsToAddToSet = [
+        "device_codes",
+        "previous_sites",
+        "groups",
+        "pictures",
+      ];
+      arrayFieldsToAddToSet.forEach((field) => {
+        if (updateData[field]) {
+          updateData.$addToSet = updateData.$addToSet || {};
+          updateData.$addToSet[field] = { $each: updateData[field] };
+          delete updateData[field];
+        }
+      });
 
       next();
     } catch (error) {
@@ -415,7 +419,7 @@ deviceSchema.methods = {
       alias: this.alias,
       mobility: this.mobility,
       network: this.network,
-      group: this.group,
+      groups: this.groups,
       api_code: this.api_code,
       serial_number: this.serial_number,
       authRequired: this.authRequired,
diff --git a/src/device-registry/models/Grid.js b/src/device-registry/models/Grid.js
index 53e8e5c027..231704c5b2 100644
--- a/src/device-registry/models/Grid.js
+++ b/src/device-registry/models/Grid.js
@@ -43,8 +43,8 @@ const gridSchema = new Schema(
       trim: true,
       required: [true, "the network is required!"],
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     geoHash: {
@@ -121,7 +121,7 @@ gridSchema.methods.toJSON = function() {
     name,
     long_name,
     network,
-    group,
+    groups,
     visibility,
     description,
     grid_tags,
@@ -139,7 +139,7 @@ gridSchema.methods.toJSON = function() {
     description,
     grid_tags,
     network,
-    group,
+    groups,
     admin_level,
     grid_codes,
     centers,
diff --git a/src/device-registry/models/Location.js b/src/device-registry/models/Location.js
index b37bd7c349..2b894a812f 100644
--- a/src/device-registry/models/Location.js
+++ b/src/device-registry/models/Location.js
@@ -84,6 +84,10 @@ const locationSchema = new Schema(
       type: String,
       trim: true,
     },
+    groups: {
+      type: [String],
+      trim: true,
+    },
     location_tags: {
       type: Array,
       default: [],
@@ -124,6 +128,7 @@ locationSchema.methods = {
       isCustom: this.isCustom,
       location: this.location,
       network: this.network,
+      groups: this.groups,
       metadata: this.metadata,
     };
   },
@@ -190,6 +195,7 @@ locationSchema.statics = {
         isCustom: 1,
         metadata: 1,
         network: 1,
+        groups: 1,
         sites: "$sites",
       };
 
@@ -200,6 +206,7 @@ locationSchema.statics = {
         admin_level: 1,
         description: 1,
         network: 1,
+        group: 1,
         metadata: 1,
       };
 
@@ -306,6 +313,7 @@ locationSchema.statics = {
           description: 1,
           admin_level: 1,
           network: 1,
+          group: 1,
           isCustom: 1,
           metadata: 1,
         },
diff --git a/src/device-registry/models/Photo.js b/src/device-registry/models/Photo.js
index 78fb7c6679..385be95009 100644
--- a/src/device-registry/models/Photo.js
+++ b/src/device-registry/models/Photo.js
@@ -18,8 +18,8 @@ const photoSchema = new Schema(
       type: String,
       trim: true,
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     device_id: {
@@ -83,7 +83,7 @@ photoSchema.methods = {
       tags: this.tags,
       name: this.name,
       network: this.network,
-      group: this.group,
+      groups: this.groups,
       image_url: this.image_url,
       device_id: this.device_id,
       site_id: this.site_id,
@@ -158,7 +158,7 @@ photoSchema.statics = {
           description: 1,
           metadata: 1,
           network: 1,
-          group: 1,
+          groups: 1,
         })
         .skip(skip ? skip : 0)
         .limit(limit ? limit : 1000)
diff --git a/src/device-registry/models/Site.js b/src/device-registry/models/Site.js
index 101b5fa1a0..3e0946db40 100644
--- a/src/device-registry/models/Site.js
+++ b/src/device-registry/models/Site.js
@@ -75,8 +75,8 @@ const siteSchema = new Schema(
       trim: true,
       required: [true, "network is required!"],
     },
-    group: {
-      type: String,
+    groups: {
+      type: [String],
       trim: true,
     },
     data_provider: {
@@ -364,6 +364,20 @@ const siteSchema = new Schema(
   }
 );
 
+const checkDuplicates = (arr, fieldName) => {
+  const duplicateValues = arr.filter(
+    (value, index, self) => self.indexOf(value) !== index
+  );
+
+  if (duplicateValues.length > 0) {
+    return new HttpError(
+      `Duplicate values found in ${fieldName} array.`,
+      httpStatus.BAD_REQUEST
+    );
+  }
+  return null;
+};
+
 siteSchema.pre(
   ["updateOne", "findOneAndUpdate", "updateMany", "update", "save"],
   function(next) {
@@ -411,6 +425,7 @@ siteSchema.pre(
           "land_use",
           "site_codes",
           "airqlouds",
+          "groups",
           "grids",
         ];
         arrayFieldsToAddToSet.forEach((field) => {
@@ -442,12 +457,16 @@ siteSchema.pre(
         if (this[field]) this.site_codes.push(this[field]);
       });
 
-      // Check for duplicate grid values
-      const duplicateValues = this.grids.filter(
-        (value, index, self) => self.indexOf(value) !== index
-      );
-      if (duplicateValues.length > 0) {
-        return next(new Error("Duplicate values found in grids array."));
+      // Check for duplicates in grids
+      const gridsDuplicateError = checkDuplicates(this.grids, "grids");
+      if (gridsDuplicateError) {
+        return next(gridsDuplicateError);
+      }
+
+      // Check for duplicates in groups
+      const groupsDuplicateError = checkDuplicates(this.groups, "groups");
+      if (groupsDuplicateError) {
+        return next(groupsDuplicateError);
       }
     }
 
@@ -473,7 +492,7 @@ siteSchema.methods = {
       generated_name: this.generated_name,
       search_name: this.search_name,
       network: this.network,
-      group: this.group,
+      groups: this.groups,
       data_provider: this.data_provider,
       location_name: this.location_name,
       formatted_name: this.formatted_name,
diff --git a/src/device-registry/routes/v2/sites.js b/src/device-registry/routes/v2/sites.js
index 8d3198e89e..8117e500d0 100644
--- a/src/device-registry/routes/v2/sites.js
+++ b/src/device-registry/routes/v2/sites.js
@@ -486,6 +486,15 @@ router.post(
         .bail()
         .notEmpty()
         .withMessage("the site_tags should not be empty"),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("airqlouds")
         .optional()
         .custom((value) => {
@@ -860,6 +869,15 @@ router.put(
         .withMessage(
           "Invalid site_category format, crosscheck the types or content of all the provided nested fields. latitude, longitude & search_radius should be numbers. tags should be an array of strings. category, search_tags & search_radius are required fields"
         ),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
     ],
   ]),
   siteController.update
diff --git a/src/device-registry/utils/generate-filter.js b/src/device-registry/utils/generate-filter.js
index e6320d57b3..e8366e409f 100644
--- a/src/device-registry/utils/generate-filter.js
+++ b/src/device-registry/utils/generate-filter.js
@@ -1116,7 +1116,7 @@ const generateFilter = {
     // }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1238,7 +1238,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1370,7 +1370,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1437,7 +1437,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1500,7 +1500,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1597,7 +1597,7 @@ const generateFilter = {
     }
   },
   locations: (req, next) => {
-    let { id, name, admin_level, summary, network } = {
+    let { id, name, admin_level, summary, network, group } = {
       ...req.query,
       ...req.params,
     };
@@ -1623,6 +1623,14 @@ const generateFilter = {
       );
     }
 
+    if (group) {
+      filter.groups = handlePredefinedValueMatch(
+        group,
+        constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
+        { matchCombinations: true }
+      );
+    }
+
     if (admin_level) {
       filter["admin_level"] = admin_level;
     }
@@ -1671,7 +1679,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
@@ -1759,7 +1767,7 @@ const generateFilter = {
     }
 
     if (group) {
-      filter.group = handlePredefinedValueMatch(
+      filter.groups = handlePredefinedValueMatch(
         group,
         constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS,
         { matchCombinations: true }
diff --git a/src/device-registry/validators/device.validators.js b/src/device-registry/validators/device.validators.js
index e0b4916f55..b08baea97d 100644
--- a/src/device-registry/validators/device.validators.js
+++ b/src/device-registry/validators/device.validators.js
@@ -126,6 +126,15 @@ const validateCreateDevice = [
         .isInt()
         .withMessage("the generation should be an integer")
         .toInt(),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("mountType")
         .optional()
         .notEmpty()
@@ -384,6 +393,15 @@ const validateUpdateDevice = [
     .trim()
     .isBoolean()
     .withMessage("isActive must be Boolean"),
+  body("groups")
+    .optional()
+    .custom((value) => {
+      return Array.isArray(value);
+    })
+    .withMessage("the groups should be an array")
+    .bail()
+    .notEmpty()
+    .withMessage("the groups should not be empty"),
   body("isRetired")
     .optional()
     .notEmpty()

From 2594d9129d10b6aacb6ad3711beb52aa15441f29 Mon Sep 17 00:00:00 2001
From: baalmart <bbaalmart@gmail.com>
Date: Tue, 10 Dec 2024 18:14:54 +0300
Subject: [PATCH 07/35] input validators for groups and cohorts

---
 src/device-registry/routes/v2/cohorts.js | 18 ++++++++++++++++++
 src/device-registry/routes/v2/grids.js   | 18 ++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/src/device-registry/routes/v2/cohorts.js b/src/device-registry/routes/v2/cohorts.js
index e2d5a07a2b..8f047c0054 100644
--- a/src/device-registry/routes/v2/cohorts.js
+++ b/src/device-registry/routes/v2/cohorts.js
@@ -128,6 +128,15 @@ router.put(
         .trim()
         .isBoolean()
         .withMessage("visibility must be Boolean"),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("network")
         .optional()
         .notEmpty()
@@ -169,6 +178,15 @@ router.post(
         .trim()
         .optional()
         .notEmpty(),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("network")
         .trim()
         .exists()
diff --git a/src/device-registry/routes/v2/grids.js b/src/device-registry/routes/v2/grids.js
index f70426a4b6..a8678a7a8e 100644
--- a/src/device-registry/routes/v2/grids.js
+++ b/src/device-registry/routes/v2/grids.js
@@ -195,6 +195,15 @@ router.post(
         .withMessage(
           "admin_level values include but not limited to: province, state, village, county, etc. Update your GLOBAL configs"
         ),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("network")
         .trim()
         .optional()
@@ -371,6 +380,15 @@ router.put(
         .optional()
         .notEmpty()
         .withMessage("the description should not be empty if provided"),
+      body("groups")
+        .optional()
+        .custom((value) => {
+          return Array.isArray(value);
+        })
+        .withMessage("the groups should be an array")
+        .bail()
+        .notEmpty()
+        .withMessage("the groups should not be empty"),
       body("network")
         .optional()
         .notEmpty()

From 68a0b0f4f5f11f08347f430c1aaf612e3c42e010 Mon Sep 17 00:00:00 2001
From: baalmart <bbaalmart@gmail.com>
Date: Tue, 10 Dec 2024 18:16:07 +0300
Subject: [PATCH 08/35] Fix inconsistent field name in projectSummary

---
 src/device-registry/models/Location.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/device-registry/models/Location.js b/src/device-registry/models/Location.js
index 2b894a812f..b5867155eb 100644
--- a/src/device-registry/models/Location.js
+++ b/src/device-registry/models/Location.js
@@ -206,7 +206,7 @@ locationSchema.statics = {
         admin_level: 1,
         description: 1,
         network: 1,
-        group: 1,
+        groups: 1,
         metadata: 1,
       };
 
@@ -313,7 +313,7 @@ locationSchema.statics = {
           description: 1,
           admin_level: 1,
           network: 1,
-          group: 1,
+          groups: 1,
           isCustom: 1,
           metadata: 1,
         },

From ec8123a62ff7bcae81d700654dcc26c3d5395eb4 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:21:55 +0300
Subject: [PATCH 09/35] Update AirQo exceedance production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/exceedance/values-prod-airqo.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
index 8fd10c0c0c..8eaf5c0da7 100644
--- a/k8s/exceedance/values-prod-airqo.yaml
+++ b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''

From e78124c360b232c9575479730213618777982c52 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:22:02 +0300
Subject: [PATCH 10/35] Update KCCA exceedance production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/exceedance/values-prod-kcca.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
index 72030a5345..afea05a6f6 100644
--- a/k8s/exceedance/values-prod-kcca.yaml
+++ b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''

From 4280b7ed7b477ca1ddbe2a75f6a1b49816a2e1c7 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:22:31 +0300
Subject: [PATCH 11/35] Update device registry staging image tag to
 stage-c1e369be-1733844028

---
 k8s/device-registry/values-stage.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/device-registry/values-stage.yaml b/k8s/device-registry/values-stage.yaml
index c3d8dccf77..54b83d55b0 100644
--- a/k8s/device-registry/values-stage.yaml
+++ b/k8s/device-registry/values-stage.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 2
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-device-registry-api
-  tag: stage-dfe6eb16-1733832983
+  tag: stage-c1e369be-1733844028
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From a7b43476b587fa030f593b56d238af84daef2027 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:22:48 +0300
Subject: [PATCH 12/35] Update device registry production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/device-registry/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml
index b4c32c4d56..9f7570f9bc 100644
--- a/k8s/device-registry/values-prod.yaml
+++ b/k8s/device-registry/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From 3640ff8f6397d4fe1e93db09abc7cde11e6fa7ff Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:23:17 +0300
Subject: [PATCH 13/35] Update website production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/website/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml
index 6460bba388..342d8be4aa 100644
--- a/k8s/website/values-prod.yaml
+++ b/k8s/website/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-website-api
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From fedfa5e7428aa99b7ad59a9cf0149323527a202c Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:23:42 +0300
Subject: [PATCH 14/35] Update workflows prod image tag to
 prod-ae2e15a3-1733844050

---
 k8s/workflows/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml
index 723152603a..d0549ff120 100644
--- a/k8s/workflows/values-prod.yaml
+++ b/k8s/workflows/values-prod.yaml
@@ -10,7 +10,7 @@ images:
     initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
     redisContainer: eu.gcr.io/airqo-250220/airqo-redis
     containers: eu.gcr.io/airqo-250220/airqo-workflows
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From f6100bad0b46cc1c3571800b141c57c2e4891878 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:24:51 +0300
Subject: [PATCH 15/35] Update predict production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/predict/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml
index 70cf680436..18ec2275a1 100644
--- a/k8s/predict/values-prod.yaml
+++ b/k8s/predict/values-prod.yaml
@@ -7,7 +7,7 @@ images:
     predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
     trainJob: eu.gcr.io/airqo-250220/airqo-train-job
     predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 api:
   name: airqo-prediction-api
   label: prediction-api

From ec1ff31aff44e64d2a541b5ef3f63c5f29032c74 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:29:32 +0300
Subject: [PATCH 16/35] Update spatial production image tag to
 prod-ae2e15a3-1733844050

---
 k8s/spatial/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml
index 3e448286e8..94644d2a35 100644
--- a/k8s/spatial/values-prod.yaml
+++ b/k8s/spatial/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-spatial-api
-  tag: prod-ee15b958-1733833086
+  tag: prod-ae2e15a3-1733844050
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From f8bf5b7d8e54e5433ebb0865e877549444660509 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:52:41 +0300
Subject: [PATCH 17/35] Update AirQo exceedance production image tag to
 prod-5e45c956-1733849508

---
 k8s/exceedance/values-prod-airqo.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
index 8eaf5c0da7..d26ed0773e 100644
--- a/k8s/exceedance/values-prod-airqo.yaml
+++ b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''

From e375c0ff79fa2af280f7045e61b4964b67e0ca98 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:52:48 +0300
Subject: [PATCH 18/35] Update KCCA exceedance production image tag to
 prod-5e45c956-1733849508

---
 k8s/exceedance/values-prod-kcca.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
index afea05a6f6..da76b77e9d 100644
--- a/k8s/exceedance/values-prod-kcca.yaml
+++ b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''

From 826c43613ad7640487c1c275dc5ddcfe55c37ddd Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:53:32 +0300
Subject: [PATCH 19/35] Update device registry production image tag to
 prod-5e45c956-1733849508

---
 k8s/device-registry/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml
index 9f7570f9bc..b98df12889 100644
--- a/k8s/device-registry/values-prod.yaml
+++ b/k8s/device-registry/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From 0607721b8814b1793ee3ac2da34d876753bb61f8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:53:35 +0300
Subject: [PATCH 20/35] Update analytics staging images tag to
 stage-dd764c29-1733849460

---
 k8s/analytics/values-stage.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/analytics/values-stage.yaml b/k8s/analytics/values-stage.yaml
index 6c2109404c..114d458f1a 100644
--- a/k8s/analytics/values-stage.yaml
+++ b/k8s/analytics/values-stage.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-stage-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-devices-summary-job
-  tag: stage-014ccd0f-1733315863
+  tag: stage-dd764c29-1733849460
 api:
   name: airqo-stage-analytics-api
   label: sta-alytics-api

From ec8c7899ac429cbeca43968121e7a08cb59ec6df Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:53:51 +0300
Subject: [PATCH 21/35] Update website production image tag to
 prod-5e45c956-1733849508

---
 k8s/website/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml
index 342d8be4aa..4f049cd0a1 100644
--- a/k8s/website/values-prod.yaml
+++ b/k8s/website/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-website-api
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From f9ad01f5e9ed23cd420b22e21acbf2b1236654a2 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:54:32 +0300
Subject: [PATCH 22/35] Update workflows prod image tag to
 prod-5e45c956-1733849508

---
 k8s/workflows/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml
index d0549ff120..e80c45e124 100644
--- a/k8s/workflows/values-prod.yaml
+++ b/k8s/workflows/values-prod.yaml
@@ -10,7 +10,7 @@ images:
     initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
     redisContainer: eu.gcr.io/airqo-250220/airqo-redis
     containers: eu.gcr.io/airqo-250220/airqo-workflows
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From 2c2e01c516694dbec6bd90f61fe24b4c53c21767 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:54:42 +0300
Subject: [PATCH 23/35] Update analytics production image tag to
 prod-5e45c956-1733849508

---
 k8s/analytics/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml
index 98861f66e8..0a76851664 100644
--- a/k8s/analytics/values-prod.yaml
+++ b/k8s/analytics/values-prod.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-3d3f6c14-1733315928
+  tag: prod-5e45c956-1733849508
 api:
   name: airqo-analytics-api
   label: analytics-api

From 1641a392ab59b08ec6c83564b7974a7a887f6985 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:56:24 +0300
Subject: [PATCH 24/35] Update predict production image tag to
 prod-5e45c956-1733849508

---
 k8s/predict/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml
index 18ec2275a1..9c89c531f4 100644
--- a/k8s/predict/values-prod.yaml
+++ b/k8s/predict/values-prod.yaml
@@ -7,7 +7,7 @@ images:
     predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
     trainJob: eu.gcr.io/airqo-250220/airqo-train-job
     predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 api:
   name: airqo-prediction-api
   label: prediction-api

From 5fac056f746868d22e5abbd9d1051a22210d4269 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 20:01:48 +0300
Subject: [PATCH 25/35] Update spatial production image tag to
 prod-5e45c956-1733849508

---
 k8s/spatial/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml
index 94644d2a35..dd1053451b 100644
--- a/k8s/spatial/values-prod.yaml
+++ b/k8s/spatial/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-spatial-api
-  tag: prod-ae2e15a3-1733844050
+  tag: prod-5e45c956-1733849508
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From b41583a3428ffdc0f14811239bd9f7149ef8e23c Mon Sep 17 00:00:00 2001
From: baalmart <bbaalmart@gmail.com>
Date: Tue, 10 Dec 2024 23:25:04 +0300
Subject: [PATCH 26/35] addressing a runtime error for listing cohorts

---
 src/device-registry/models/Cohort.js          |  2 +-
 .../utils/scripts/new-bulk-script.js          | 71 +++++++++++++++++++
 2 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 src/device-registry/utils/scripts/new-bulk-script.js

diff --git a/src/device-registry/models/Cohort.js b/src/device-registry/models/Cohort.js
index e5aeb92d20..a00884dacd 100644
--- a/src/device-registry/models/Cohort.js
+++ b/src/device-registry/models/Cohort.js
@@ -214,7 +214,7 @@ cohortSchema.statics.list = async function(
       .sort({ createdAt: -1 })
       .project(inclusionProjection)
       .project(exclusionProjection)
-      .groups({
+      .group({
         _id: "$_id",
         visibility: { $first: "$visibility" },
         cohort_tags: { $first: "$cohort_tags" },
diff --git a/src/device-registry/utils/scripts/new-bulk-script.js b/src/device-registry/utils/scripts/new-bulk-script.js
new file mode 100644
index 0000000000..37b7f4c23b
--- /dev/null
+++ b/src/device-registry/utils/scripts/new-bulk-script.js
@@ -0,0 +1,71 @@
+const axios = require("axios");
+const isEmpty = require("is-empty");
+
+const url = "http://localhost:3000/api/v2/devices/sites";
+const config = {
+  headers: {
+    Authorization: "",
+  },
+};
+
+const NETWORK_MAPPINGS = {
+  //   iqair: "permian-health",
+  //   usembassy: "us-embassy",
+  //   urbanbetter: "urban-better",
+  //   kcca: "kcca",
+  //   airqo: "airqo",
+  // Add more mappings as needed
+};
+
+const DEFAULT_GROUP = "unknown";
+
+axios
+  .get(url, config)
+  .then(async (response) => {
+    const groups = response.data.sites
+      .map((site) => {
+        // Look up the group based on network, with a fallback to a default
+        const group = NETWORK_MAPPINGS[site.network] || DEFAULT_GROUP;
+
+        // Optionally log devices with unknown networks
+        if (group === DEFAULT_GROUP) {
+          console.log(
+            `Unrecognized network for device ${site.name}: ${site.network}`
+          );
+        }
+
+        return group;
+      })
+      // Remove any 'unknown' groups if you want only mapped networks
+      .filter((group) => group !== DEFAULT_GROUP);
+
+    console.log("the data:");
+    console.dir({ groups });
+
+    // Process devices in batches
+    for (let i = 0; i < response.data.sites.length; i += 10) {
+      const batch = response.data.sites.slice(i, i + 10);
+
+      for (const site of batch) {
+        const group = NETWORK_MAPPINGS[site.network] || DEFAULT_GROUP;
+
+        if (group !== DEFAULT_GROUP) {
+          const url = `http://localhost:3000/api/v2/devices/sites?id=${site._id}`;
+          const data = { groups: [group] };
+          //   console.log("the data:");
+          //   console.dir(data);
+
+          try {
+            // Uncomment if you want to make the PUT request
+            const putResponse = await axios.put(url, data, config);
+            console.log("PUT response:", putResponse.data);
+          } catch (error) {
+            console.error("PUT error:", error.message);
+          }
+        }
+      }
+    }
+  })
+  .catch((error) => {
+    console.error("GET error:", error);
+  });

From 10c64f136a9af958d83990a3a9a39d98fdbae935 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:31:31 +0300
Subject: [PATCH 27/35] Update device registry staging image tag to
 stage-9f584165-1733862604

---
 k8s/device-registry/values-stage.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/device-registry/values-stage.yaml b/k8s/device-registry/values-stage.yaml
index 54b83d55b0..a92ee18671 100644
--- a/k8s/device-registry/values-stage.yaml
+++ b/k8s/device-registry/values-stage.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 2
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-device-registry-api
-  tag: stage-c1e369be-1733844028
+  tag: stage-9f584165-1733862604
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From 817fd0225f972cf5fa3092694a00d96f7b4d0a9d Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:32:50 +0300
Subject: [PATCH 28/35] Update AirQo exceedance production image tag to
 prod-3c9423e2-1733862695

---
 k8s/exceedance/values-prod-airqo.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
index d26ed0773e..d941e8d9e8 100644
--- a/k8s/exceedance/values-prod-airqo.yaml
+++ b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''

From 4470535f88133ddb05c3d0d86966b3f4ccd54522 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:33:01 +0300
Subject: [PATCH 29/35] Update KCCA exceedance production image tag to
 prod-3c9423e2-1733862695

---
 k8s/exceedance/values-prod-kcca.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
index da76b77e9d..dfc47322a5 100644
--- a/k8s/exceedance/values-prod-kcca.yaml
+++ b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''

From c1213ea1798bdfc9904d0ff6e5612c25065adab6 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:33:40 +0300
Subject: [PATCH 30/35] Update device registry production image tag to
 prod-3c9423e2-1733862695

---
 k8s/device-registry/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml
index b98df12889..a359fb2e9b 100644
--- a/k8s/device-registry/values-prod.yaml
+++ b/k8s/device-registry/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From a4fcc3d312ae50e10591af86ab1e4ec815e6687c Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:33:53 +0300
Subject: [PATCH 31/35] Update website production image tag to
 prod-3c9423e2-1733862695

---
 k8s/website/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml
index 4f049cd0a1..759c1c62f3 100644
--- a/k8s/website/values-prod.yaml
+++ b/k8s/website/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-website-api
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From b21851dca999ff27d967b6c12ecc85d9414c41c8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:34:19 +0300
Subject: [PATCH 32/35] Update workflows prod image tag to
 prod-3c9423e2-1733862695

---
 k8s/workflows/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml
index e80c45e124..c736740f1d 100644
--- a/k8s/workflows/values-prod.yaml
+++ b/k8s/workflows/values-prod.yaml
@@ -10,7 +10,7 @@ images:
     initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
     redisContainer: eu.gcr.io/airqo-250220/airqo-redis
     containers: eu.gcr.io/airqo-250220/airqo-workflows
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

From 74a78393da6069d64ad7fb5c999d7c3a8ca83cef Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:34:58 +0300
Subject: [PATCH 33/35] Update analytics production image tag to
 prod-3c9423e2-1733862695

---
 k8s/analytics/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml
index 0a76851664..8d3b5292bc 100644
--- a/k8s/analytics/values-prod.yaml
+++ b/k8s/analytics/values-prod.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 api:
   name: airqo-analytics-api
   label: analytics-api

From 8441af6d68cea3057244dbfb5f474ddf0d6dd999 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:35:47 +0300
Subject: [PATCH 34/35] Update predict production image tag to
 prod-3c9423e2-1733862695

---
 k8s/predict/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml
index 9c89c531f4..ef7a86c858 100644
--- a/k8s/predict/values-prod.yaml
+++ b/k8s/predict/values-prod.yaml
@@ -7,7 +7,7 @@ images:
     predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
     trainJob: eu.gcr.io/airqo-250220/airqo-train-job
     predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 api:
   name: airqo-prediction-api
   label: prediction-api

From 741573b5022fd60bd16a72d59fcdf737fe368dab Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 23:40:30 +0300
Subject: [PATCH 35/35] Update spatial production image tag to
 prod-3c9423e2-1733862695

---
 k8s/spatial/values-prod.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml
index dd1053451b..149d92b402 100644
--- a/k8s/spatial/values-prod.yaml
+++ b/k8s/spatial/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-spatial-api
-  tag: prod-5e45c956-1733849508
+  tag: prod-3c9423e2-1733862695
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}