IPROD-545: Enable prometheus remote write and read (#275)

* IPROD-545: Enable remote write on client prometheus

* fix url address

* extract configs in params

* test disabling remote write

* refactor

* add remote read configs

* added default values for central monitoring configs

* remove a comment

terraform/gitops/generate-files/templates/monitoring/install/values-prom-operator.yaml.tpl

@@ -28,6 +28,26 @@ prometheus:
     value: ${prom_tsdb_min_block_duration}
   - name: storage.tsdb.max-block-duration
     value: ${prom_tsdb_max_block_duration}
+  externalLabels:
+    cluster: ${cluster_label}
+
+%{if enable_central_observability_write ~}
+  remoteWrite:
+  - name: central-monitoring
+    url: ${central_observability_endpoint}/api/v1/push
+    headers:
+      X-Scope-OrgID: ${central_observability_tenant_id}    
+%{endif ~} 
+
+%{if enable_central_observability_read ~}
+  remoteRead:
+  - name: central-monitoring
+    url: ${central_observability_endpoint}/api/v1/read
+    headers:
+      X-Scope-OrgID: ${central_observability_tenant_id}    
+%{endif ~} 
+
+
 operator:
   nodeAffinityPreset:
     type: hard

terraform/gitops/k8s-cluster-config/monitoring.tf

@@ -54,6 +54,13 @@ module "generate_monitoring_files" {
     grafana_fqdn                        = local.grafana_fqdn
     grafana_istio_gateway_namespace     = local.grafana_istio_gateway_namespace
     grafana_istio_wildcard_gateway_name = local.vault_istio_wildcard_gateway_name
+
+    # central observability configs
+    cluster_label                      = var.cluster_name # cluster identifier in central observability stack
+    enable_central_observability_write = try(var.common_var_map.enable_central_observability_write, local.enable_central_observability_write)
+    enable_central_observability_read  = try(var.common_var_map.enable_central_observability_read, local.enable_central_observability_read)
+    central_observability_endpoint     = try(var.common_var_map.central_observability_endpoint, local.central_observability_endpoint)
+    central_observability_tenant_id    = try(var.common_var_map.central_observability_tenant_id, local.central_observability_tenant_id)
   }
   file_list       = [for f in fileset(local.monitoring_template_path, "**/*.tpl") : trimsuffix(f, ".tpl") if !can(regex(local.monitoring_app_file, f))]
   template_path   = local.monitoring_template_path
@@ -136,4 +143,8 @@ locals {
   grafana_fqdn                        = local.grafana_wildcard_gateway == "external" ? "grafana.${var.public_subdomain}" : "grafana.${var.private_subdomain}"
   grafana_istio_gateway_namespace     = local.grafana_wildcard_gateway == "external" ? var.istio_external_gateway_namespace : var.istio_internal_gateway_namespace
   grafana_istio_wildcard_gateway_name = local.grafana_wildcard_gateway == "external" ? local.istio_external_wildcard_gateway_name : local.istio_internal_wildcard_gateway_name
+  enable_central_observability_write  = false
+  enable_central_observability_read   = false
+  central_observability_endpoint      = "http://to-be-updated"
+  central_observability_tenant_id     = "infitx"
 }