diff --git a/bootstrap/feature/main.tf b/bootstrap/feature/main.tf index d1d2980..01cb2d4 100644 --- a/bootstrap/feature/main.tf +++ b/bootstrap/feature/main.tf @@ -14,6 +14,74 @@ variable "topology_zone" { type = string } -variable "image_tag" { +variable "postgres_image_tag" { + type = string +} + +variable "postgres_resources" { + type = object({ + requests = map(string) + limits = map(string) + }) + + default = { + "limits" = { + memory = "2Gi" + cpu = "4000m" + } + "requests" = { + memory = "2Gi" + cpu = "100m" + } + } +} + +variable "postgres_settings" { + default = { + listen_addresses = "*" + max_connections = 1000 + shared_buffers = "8GB" + effective_cache_size = "24GB" + maintenance_work_mem = "2GB" + checkpoint_completion_target = 0.9 + wal_buffers = "16MB" + default_statistics_target = 500 + random_page_cost = 1.1 + effective_io_concurrency = 200 + work_mem = "1048kB" + huge_pages = "try" + min_wal_size = "4GB" + max_wal_size = "16GB" + max_worker_processes = 8 + max_parallel_workers_per_gather = 4 + max_parallel_workers = 8 + max_parallel_maintenance_workers = 4 + ssl = "off" + } +} + + +variable "operator_image_tag" { + type = string +} + +variable "metrics_delay" { + default = 30 +} + +variable "dcu_per_second" { + type = map(string) + default = { + "mainnet" = "10" + "preprod" = "5" + "preview" = "5" + } +} + +variable "postgres_host" { + type = string +} + +variable "postgres_secret_name" { type = string } \ No newline at end of file diff --git a/bootstrap/feature/monitor.tf b/bootstrap/feature/monitor.tf index e714abe..33f7322 100644 --- a/bootstrap/feature/monitor.tf +++ b/bootstrap/feature/monitor.tf @@ -25,3 +25,31 @@ resource "kubernetes_manifest" "postgres_podmonitor" { } } } + +resource "kubernetes_manifest" "operator_monitor" { + manifest = { + apiVersion = "monitoring.coreos.com/v1" + kind = "PodMonitor" + metadata = { + labels = { + "app.kubernetes.io/component" = "o11y" + "app.kubernetes.io/part-of" = "demeter" + } + name = "operator" + namespace = var.namespace + } + spec = { + selector = { + matchLabels = { + role = "operator" + } + } + podMetricsEndpoints = [ + { + port = "metrics", + path = "/metrics" + } + ] + } + } +} \ No newline at end of file diff --git a/bootstrap/feature/operator.tf b/bootstrap/feature/operator.tf new file mode 100644 index 0000000..25ecc5f --- /dev/null +++ b/bootstrap/feature/operator.tf @@ -0,0 +1,135 @@ +resource "kubernetes_deployment_v1" "operator" { + wait_for_rollout = false + + metadata { + namespace = var.namespace + name = "operator" + labels = { + role = "operator" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + role = "operator" + } + } + + template { + metadata { + labels = { + role = "operator" + } + } + + spec { + container { + image = "ghcr.io/demeter-run/ext-cardano-dbsync-serverless:${var.operator_image_tag}" + name = "main" + + env { + name = "K8S_IN_CLUSTER" + value = "true" + } + + env { + name = "METRICS_DELAY" + value = var.metrics_delay + } + + env { + name = "DCU_PER_SECOND_MAINNET" + value = var.dcu_per_second["mainnet"] + } + + env { + name = "DCU_PER_SECOND_PREPROD" + value = var.dcu_per_second["preprod"] + } + + env { + name = "DCU_PER_SECOND_PREVIEW" + value = var.dcu_per_second["preview"] + } + + env { + name = "ADDR" + value = "0.0.0.0:5000" + } + + env { + name = "POSTGRES_PASSWORD" + value_from { + secret_key_ref { + name = var.postgres_secret_name + key = "password" + } + } + } + + env { + name = "DB_URL_MAINNET" + value = "postgres://postgres:$(POSTGRES_PASSWORD)@${var.postgres_host}:5432/dbsync-mainnet" + } + + env { + name = "DB_URL_PREPROD" + value = "postgres://postgres:$(POSTGRES_PASSWORD)@${var.postgres_host}:5432/dbsync-preprod" + } + + env { + name = "DB_URL_PREVIEW" + value = "postgres://postgres:$(POSTGRES_PASSWORD)@${var.postgres_host}:5432/dbsync-preview" + } + + env { + name = "RUST_BACKTRACE" + value = "1" + } + + + resources { + limits = { + memory = "256Mi" + } + requests = { + cpu = "50m" + memory = "256Mi" + } + } + + port { + name = "metrics" + container_port = 5000 + protocol = "TCP" + } + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Equal" + value = "general-purpose" + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = "x86" + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = "consistent" + } + } + } + } +} + diff --git a/bootstrap/feature/pg-configs.tf b/bootstrap/feature/pg-configs.tf new file mode 100644 index 0000000..58e9a33 --- /dev/null +++ b/bootstrap/feature/pg-configs.tf @@ -0,0 +1,10 @@ +resource "kubernetes_config_map" "postgres_config" { + metadata { + namespace = var.namespace + name = "postgres-config" + } + + data = { + "postgresql.conf" = file("${path.module}/postgresql.conf") + } +} diff --git a/bootstrap/feature/postgres.tf b/bootstrap/feature/postgres.tf index 9c72bee..d176bdf 100644 --- a/bootstrap/feature/postgres.tf +++ b/bootstrap/feature/postgres.tf @@ -33,6 +33,7 @@ resource "kubernetes_stateful_set_v1" "postgres" { "demeter.run/instance" = var.instance_name } } + template { metadata { labels = { @@ -40,18 +41,32 @@ resource "kubernetes_stateful_set_v1" "postgres" { } } spec { + affinity { + node_affinity { + required_during_scheduling_ignored_during_execution { + node_selector_term { + match_expressions { + key = "topology.kubernetes.io/zone" + operator = "In" + values = [var.topology_zone] + } + } + } + } + } security_context { fs_group = 1000 } container { name = "main" - image = "postgres:${var.image_tag}" + image = "postgres:${var.postgres_image_tag}" + args = ["-c", "config_file=/etc/postgresql/postgresql.conf"] image_pull_policy = "Always" port { - container_port = 1442 - name = "http" + container_port = 5432 + name = "postgres" protocol = "TCP" } @@ -59,7 +74,7 @@ resource "kubernetes_stateful_set_v1" "postgres" { name = "POSTGRES_PASSWORD" value_from { secret_key_ref { - name = "postgres-${var.instance_name}" + name = "postgres.${var.instance_name}" key = "password" } } @@ -72,12 +87,12 @@ resource "kubernetes_stateful_set_v1" "postgres" { resources { limits = { - cpu = var.resources.limits.cpu - memory = var.resources.limits.memory + cpu = var.postgres_resources.limits.cpu + memory = var.postgres_resources.limits.memory } requests = { - cpu = var.resources.requests.cpu - memory = var.resources.requests.memory + cpu = var.postgres_resources.requests.cpu + memory = var.postgres_resources.requests.memory } } @@ -86,6 +101,11 @@ resource "kubernetes_stateful_set_v1" "postgres" { name = "data" } + volume_mount { + mount_path = "/etc/postgresql/postgresql.conf" + name = "config" + sub_path = "postgresql.conf" + } } container { @@ -93,15 +113,20 @@ resource "kubernetes_stateful_set_v1" "postgres" { image = "quay.io/prometheuscommunity/postgres-exporter:v0.12.0" env { name = "DATA_SOURCE_URI" - value = "localhost:5432/dbsync-mainnet?sslmode=disable,localhost:5432/dbsync-preview?sslmode=disable,localhost:5432/dbsync-preprod?sslmode=disable" + value = "localhost:5432/postgres?sslmode=disable" } env { name = "DATA_SOURCE_USER" - value = "$(POSTGRES_USER)" + value = "postgres" } env { - name = "DATA_SOURCE_PASS" - value = "$(POSTGRES_PASSWORD)" + name = "DATA_SOURCE_PASS" + value_from { + secret_key_ref { + name = "postgres.${var.instance_name}" + key = "password" + } + } } env { name = "PG_EXPORTER_CONSTANT_LABELS" @@ -120,6 +145,13 @@ resource "kubernetes_stateful_set_v1" "postgres" { } } + volume { + name = "config" + config_map { + name = "postgres-config" + } + } + toleration { effect = "NoSchedule" key = "demeter.run/compute-profile" @@ -145,119 +177,24 @@ resource "kubernetes_stateful_set_v1" "postgres" { } } +resource "kubernetes_service_v1" "postgres" { + metadata { + name = var.instance_name + namespace = var.namespace + labels = { + "demeter.run/kind" = "DbsyncPostgres" + } + } + spec { + selector = { + "demeter.run/instance" = var.instance_name + } + type = "ClusterIP" + port { + port = 5432 + target_port = 5432 + name = "postgres" + } + } +} - -# resource "kubernetes_manifest" "postgres" { -# field_manager { -# force_conflicts = true -# } -# manifest = { -# "apiVersion" = "acid.zalan.do/v1" -# "kind" = "postgresql" -# "metadata" = { -# "name" = local.postgres_host -# "namespace" = var.namespace -# "labels" = { -# "dbsync-status" = "ready" -# } -# } -# "spec" = { -# "env" : [ -# { -# "name" : "ALLOW_NOSSL" -# "value" : "true" -# } -# ] -# "numberOfInstances" = var.postgres_replicas -# "enableMasterLoadBalancer" = var.enable_master_load_balancer -# "enableReplicaLoadBalancer" = var.enable_replica_load_balancer -# "allowedSourceRanges" = [ -# "0.0.0.0/0" -# ] -# "dockerImage" : "ghcr.io/zalando/spilo-15:3.0-p1" -# "teamId" = "dmtr" -# "tolerations" = [ -# { -# "effect" = "NoSchedule" -# "key" = "demeter.run/compute-profile" -# "operator" = "Equal" -# "value" = "disk-intesive" -# }, -# { -# "effect" = "NoSchedule" -# "key" = "demeter.run/compute-arch" -# "operator" = "Equal" -# "value" = "x86" -# }, -# { -# "effect" = "NoSchedule" -# "key" = "demeter.run/availability-sla" -# "operator" = "Equal" -# "value" = "consistent" -# } -# ] -# "nodeAffinity" = var.topology_zone != null ? local.node_affinity : null -# "serviceAnnotations" : { -# "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" -# "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" -# "service.beta.kubernetes.io/aws-load-balancer-type" = "external" -# } -# "databases" = { -# "cardanodbsync" = "dmtrdb" -# } -# "postgresql" = { -# "version" = "14" -# "parameters" = var.postgres_params -# } -# "users" = { -# "dmtrdb" = [ -# "superuser", -# "createdb", -# "login" -# ], -# "dmtrro" = [ -# "login" -# ] -# } -# "resources" = { -# "limits" = var.postgres_resources.limits -# "requests" = var.postgres_resources.requests -# } -# "volume" = { -# "storageClass" = var.postgres_volume.storage_class -# "size" = var.postgres_volume.size -# } -# "" -# sidecars = [ -# { -# name : "exporter" -# image : "quay.io/prometheuscommunity/postgres-exporter:v0.12.0" -# env : [ -# { -# name : "DATA_SOURCE_URI" -# value : "localhost:5432/cardanodbsync?sslmode=disable" -# }, -# { -# name : "DATA_SOURCE_USER" -# value : "$(POSTGRES_USER)" -# }, -# { -# name : "DATA_SOURCE_PASS" -# value : "$(POSTGRES_PASSWORD)" -# }, -# { -# name : "PG_EXPORTER_CONSTANT_LABELS" -# value : "service=dbsync-${local.postgres_host}" -# } -# ] -# ports : [ -# { -# name : "metrics" -# containerPort : 9187 -# } -# ] -# } -# ] -# } -# } -# } diff --git a/bootstrap/feature/postgresql.conf b/bootstrap/feature/postgresql.conf new file mode 100644 index 0000000..28b4a9c --- /dev/null +++ b/bootstrap/feature/postgresql.conf @@ -0,0 +1,20 @@ +listen_addresses = '*' +max_connections = 1000 +shared_buffers = 8GB +effective_cache_size = 24GB +maintenance_work_mem = 2GB +checkpoint_completion_target = 0.9 +wal_buffers = 16MB +default_statistics_target = 500 +random_page_cost = 1.1 +effective_io_concurrency = 200 +work_mem = 1048kB +huge_pages = try +min_wal_size = 4GB +max_wal_size = 16GB +max_worker_processes = 8 +max_parallel_workers_per_gather = 4 +max_parallel_workers = 8 +max_parallel_maintenance_workers = 4 +ssl = off +shared_preload_libraries = 'pg_stat_statements' diff --git a/bootstrap/feature/rbac.tf b/bootstrap/feature/rbac.tf new file mode 100644 index 0000000..da2b2e6 --- /dev/null +++ b/bootstrap/feature/rbac.tf @@ -0,0 +1,27 @@ +resource "kubernetes_cluster_role" "cluster_role" { + metadata { + name = var.namespace + } + + rule { + api_groups = ["", "demeter.run", "networking.k8s.io", "gateway.networking.k8s.io", "configuration.konghq.com"] + resources = ["*"] + verbs = ["*"] + } +} + +resource "kubernetes_cluster_role_binding" "cluster_role_binding" { + metadata { + name = var.namespace + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = var.namespace + } + subject { + kind = "ServiceAccount" + name = "default" + namespace = var.namespace + } +} diff --git a/bootstrap/feature/secret.tf b/bootstrap/feature/secret.tf index c74beec..a8af495 100644 --- a/bootstrap/feature/secret.tf +++ b/bootstrap/feature/secret.tf @@ -12,7 +12,7 @@ resource "random_password" "dmtrro" { resource "kubernetes_secret" "postgres" { metadata { - name = "postgres-${var.instance_name}" + name = "postgres.${var.instance_name}" namespace = var.namespace } data = { @@ -23,7 +23,7 @@ resource "kubernetes_secret" "postgres" { resource "kubernetes_secret" "dmtrro" { metadata { - name = "dmtrro-${var.instance_name}" + name = "dmtrro.${var.instance_name}" namespace = var.namespace } data = { diff --git a/bootstrap/instance/dbsync.tf b/bootstrap/instance/dbsync.tf index bd09e40..44e3ffc 100644 --- a/bootstrap/instance/dbsync.tf +++ b/bootstrap/instance/dbsync.tf @@ -21,6 +21,7 @@ resource "kubernetes_persistent_volume_claim" "state" { } resource "kubernetes_deployment_v1" "db_sync" { + wait_for_rollout = false metadata { labels = { salt = var.salt @@ -128,7 +129,7 @@ resource "kubernetes_deployment_v1" "db_sync" { env { name = "POSTGRES_HOST" - value = local.postgres_host + value = var.postgres_instance_name } env { diff --git a/bootstrap/pgbouncer/deployment.tf b/bootstrap/pgbouncer/deployment.tf new file mode 100644 index 0000000..0bddac9 --- /dev/null +++ b/bootstrap/pgbouncer/deployment.tf @@ -0,0 +1,174 @@ +resource "kubernetes_deployment_v1" "dbsyncv2_pgcat" { + wait_for_rollout = false + metadata { + labels = { + role = "pgbouncer" + } + name = "pgbouncer" + namespace = var.namespace + } + + spec { + replicas = 2 + + strategy { + rolling_update { + max_surge = 1 + max_unavailable = 0 + } + } + + selector { + match_labels = { + role = "pgbouncer" + } + } + + template { + metadata { + labels = { + role = "pgbouncer" + } + } + + spec { + container { + name = "main" + image = "bitnami/pgbouncer:${var.image_tag}" + + resources { + limits = { + memory = "250Mi" + } + requests = { + cpu = "100m" + memory = "250Mi" + } + } + + port { + container_port = 9930 + name = "metrics" + protocol = "TCP" + } + + port { + container_port = 6432 + name = "pgbouncer" + protocol = "TCP" + } + + env { + name = "PGBOUNCER_DATABASE" + value = "*" + } + + env { + name = "POSTGRESQL_USERNAME" + value = "postgres" + } + + env { + name = "POSTGRESQL_PASSWORD" + value_from { + secret_key_ref { + name = "postgres.postgres-dbsync-v3" + key = "password" + } + } + } + + env { + name = "POSTGRESQL_HOST" + value = "postgres-dbsync-v3" + } + + env { + name = "POSTGRESQL_PORT" + value = "5432" + } + + env { + name = "PGBOUNCER_DSN_0" + value = "mainnet=host=postgres-dbsync-v3 port=5432 dbname=dbsync-mainnet auth_user=pgbouncer" + } + + env { + name = "PGBOUNCER_DSN_1" + value = "preview=host=postgres-dbsync-v3 port=5432 dbname=dbsync-preview auth_user=pgbouncer" + } + + env { + name = "PGBOUNCER_DSN_2" + value = "preprod=host=postgres-dbsync-v3 port=5432 dbname=dbsync-preprod auth_user=pgbouncer" + } + + env { + name = "PGBOUNCER_AUTH_USER" + value = "pgbouncer" + } + + env { + name = "PGBOUNCER_AUTH_QUERY" + value = "SELECT usename, passwd FROM user_search($1)" + } + + env { + name = "PGBOUNCER_IGNORE_STARTUP_PARAMETERS" + value = "ignore_startup_parameters = extra_float_digits" + } + + + env { + name = "PGBOUNCER_USERLIST_FILE" + value = "/etc/pgbouncer/users.txt" + } + + volume_mount { + name = "pgbouncer-config" + mount_path = "/etc/pgbouncer" + } + + } + + volume { + name = "pgbouncer-config" + config_map { + name = "pgbouncer-config" + } + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Exists" + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = "x86" + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = "best-effort" + } + } + } + } +} + +resource "kubernetes_config_map" "dbsync_pgbouncer_config" { + metadata { + namespace = var.namespace + name = "pgbouncer-config" + } + + data = { + "users.txt" = "${file("${path.module}/users.txt")}" + } +} diff --git a/bootstrap/pgbouncer/main.tf b/bootstrap/pgbouncer/main.tf new file mode 100644 index 0000000..3b5b337 --- /dev/null +++ b/bootstrap/pgbouncer/main.tf @@ -0,0 +1,7 @@ +variable namespace { + type = string +} + +variable image_tag { + default = "1.21.0" +} diff --git a/bootstrap/pgbouncer/service.tf b/bootstrap/pgbouncer/service.tf new file mode 100644 index 0000000..7e96e3f --- /dev/null +++ b/bootstrap/pgbouncer/service.tf @@ -0,0 +1,26 @@ +resource "kubernetes_service_v1" "dbsync_pgbouncer_elb" { + metadata { + namespace = var.namespace + name = "dbsync-v3-pgbouncer" + annotations = { + "beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-type" = "external" + } + } + + spec { + type = "LoadBalancer" + load_balancer_class = "service.k8s.aws/nlb" + + port { + protocol = "TCP" + port = 5432 + target_port = 6432 + } + + selector = { + "role" = "pgbouncer" + } + } +} diff --git a/bootstrap/pgbouncer/users.txt b/bootstrap/pgbouncer/users.txt new file mode 100644 index 0000000..85b2058 --- /dev/null +++ b/bootstrap/pgbouncer/users.txt @@ -0,0 +1 @@ +"pgbouncer" "pgbouncertest" \ No newline at end of file