From ceec08e395111c642a9097e843f7067710a85c5a Mon Sep 17 00:00:00 2001
From: CS Zhang <zhangcho@us.ibm.com>
Date: Thu, 1 Jul 2021 11:38:10 -0500
Subject: [PATCH 01/14] Enable external DNS/LB support

Signed-off-by: CS Zhang <zhangcho@us.ibm.com>
---
 docs/var.tfvars-doc.md                              |  6 ++++++
 modules/3_helpernode/helpernode.tf                  |  2 ++
 modules/3_helpernode/templates/helpernode_vars.yaml |  6 ++++++
 modules/3_helpernode/variables.tf                   |  2 ++
 ocp.tf                                              |  2 ++
 variables.tf                                        | 10 ++++++++++
 6 files changed, 28 insertions(+)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index 3ba787b7a3..e507fdd298 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -134,6 +134,12 @@ The total length of `cluster_id_prefix`.`cluster_id` should not exceed 14 charac
 
 These variables provides miscellaneous customizations. For common usage scenarios these are not required and should be left unchanged.
 
+The following variables are used to set the external DNS and load balancer, use IP address for these two variables. They can be used only when external DNS and load balancer are pre-configured for the OCP installation. 
+```
+lb_ipaddr                       = ""
+ext_dns                         = ""
+```
+
 The following variable is used to set the network adapter type for the VMs. By default the VMs will use SEA. If SRIOV is required then uncomment the variable
 ```
 network_type                = "SRIOV"
diff --git a/modules/3_helpernode/helpernode.tf b/modules/3_helpernode/helpernode.tf
index ebe46693de..b521f413fc 100644
--- a/modules/3_helpernode/helpernode.tf
+++ b/modules/3_helpernode/helpernode.tf
@@ -38,6 +38,8 @@ locals {
         bastion_master_ip   = var.bastion_ip[0]
         bastion_backup_ip   = length(var.bastion_ip) > 1 ? slice(var.bastion_ip, 1, length(var.bastion_ip)) : []
         forwarders      = var.dns_forwarders
+        lb_ipaddr       = var.lb_ipaddr
+        ext_dns         = var.ext_dns
         gateway_ip      = var.gateway_ip
         netmask         = cidrnetmask(var.cidr)
         broadcast       = cidrhost(var.cidr,-1)
diff --git a/modules/3_helpernode/templates/helpernode_vars.yaml b/modules/3_helpernode/templates/helpernode_vars.yaml
index 5f3edc414c..9cb8a6a7e0 100644
--- a/modules/3_helpernode/templates/helpernode_vars.yaml
+++ b/modules/3_helpernode/templates/helpernode_vars.yaml
@@ -21,10 +21,16 @@ dns:
   domain: "${cluster_domain}"
   clusterid: "${cluster_id}"
   forwarder1: "${forwarders}"
+%{ if lb_ipaddr != "" }
+  lb_ipaddr: "${lb_ipaddr}"
+%{ endif }
 dhcp:
   router: "${gateway_ip}"
   bcast: "${broadcast}"
   netmask: "${netmask}"
+%{ if ext_dns != "" }
+  dns: "${ext_dns}"
+%{ endif }
   ipid: "${ipid}"
   netmaskid: "${netmask}"
   poolstart: "${pool.start}"
diff --git a/modules/3_helpernode/variables.tf b/modules/3_helpernode/variables.tf
index e17d6f869a..30d9e747af 100644
--- a/modules/3_helpernode/variables.tf
+++ b/modules/3_helpernode/variables.tf
@@ -29,6 +29,8 @@ variable "dns_forwarders" {
     default   = "8.8.8.8; 9.9.9.9"
 }
 
+variable "lb_ipaddr" {}
+variable "ext_dns" {}
 variable "gateway_ip" {}
 variable "cidr" {}
 variable "allocation_pools" {}
diff --git a/ocp.tf b/ocp.tf
index fe4c3529dd..14cece9a18 100644
--- a/ocp.tf
+++ b/ocp.tf
@@ -87,6 +87,8 @@ module "helpernode" {
     cluster_domain                  = var.cluster_domain
     cluster_id                      = local.cluster_id
     dns_forwarders                  = var.dns_forwarders
+    lb_ipaddr                       = var.lb_ipaddr
+    ext_dns                         = var.ext_dns
     gateway_ip                      = module.network.gateway_ip
     cidr                            = module.network.cidr
     allocation_pools                = module.network.allocation_pools
diff --git a/variables.tf b/variables.tf
index 7c0a31e975..1d3afa0a14 100644
--- a/variables.tf
+++ b/variables.tf
@@ -310,6 +310,16 @@ variable "dns_forwarders" {
     default   = "8.8.8.8; 8.8.4.4"
 }
 
+variable "lb_ipaddr" {
+    description = "Define the preconfigured external Load Balancer"
+    default = ""    
+}
+
+variable "ext_dns" {
+    description = "Define the preconfigured external DNS and Load Balancer"
+    default = ""
+}
+
 variable "mount_etcd_ramdisk" {
     description = "Whether mount etcd directory in the ramdisk (Only for dev/test) on low performance disk"
     default     = false

From cab69442598ed00c63ba141cb6ba107f6600f927 Mon Sep 17 00:00:00 2001
From: CS Zhang <zhangcho@us.ibm.com>
Date: Thu, 1 Jul 2021 12:46:51 -0500
Subject: [PATCH 02/14] Update the doc

Signed-off-by: CS Zhang <zhangcho@us.ibm.com>
---
 docs/var.tfvars-doc.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index e507fdd298..a6be7e53b0 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -134,7 +134,7 @@ The total length of `cluster_id_prefix`.`cluster_id` should not exceed 14 charac
 
 These variables provides miscellaneous customizations. For common usage scenarios these are not required and should be left unchanged.
 
-The following variables are used to set the external DNS and load balancer, use IP address for these two variables. They can be used only when external DNS and load balancer are pre-configured for the OCP installation. 
+The following variables are used to define the IP address for the preconfigured external DNS and the Load-balancer 
 ```
 lb_ipaddr                       = ""
 ext_dns                         = ""

From 12e61a0226365dbcbd9328dca3b45a3e483a4ff5 Mon Sep 17 00:00:00 2001
From: CS Zhang <zhangcho@us.ibm.com>
Date: Thu, 2 Sep 2021 15:30:05 -0500
Subject: [PATCH 03/14] Update the helpernode_tag to latest level

Signed-off-by: CS Zhang <zhangcho@us.ibm.com>
---
 variables.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/variables.tf b/variables.tf
index 1d3afa0a14..cb6259024f 100644
--- a/variables.tf
+++ b/variables.tf
@@ -240,7 +240,7 @@ variable "helpernode_repo" {
 variable "helpernode_tag" {
     description = "Set the branch/tag name or commit# for using ocp4-helpernode repo"
     # Checkout level for https://github.com/RedHatOfficial/ocp4-helpernode which is used for setting up services required on bastion node
-    default = "1ac7f276b537cd734240eda9ed554a254ba80629"
+    default = "324e09e3d303101874f540730c993cd986ddbc04"
 }
 
 variable "install_playbook_repo" {

From 80eb83c142848b3c586d608596c7326394e4e1c3 Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Mon, 2 Aug 2021 21:35:04 +0530
Subject: [PATCH 04/14] Add RHCOS kernel options before installation

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 docs/var.tfvars-doc.md                        | 10 +++++
 modules/5_install/install.tf                  | 45 ++++++++++---------
 modules/5_install/templates/install_vars.yaml |  2 +
 modules/5_install/variables.tf                |  1 +
 ocp.tf                                        |  1 +
 var.tfvars                                    |  1 +
 variables.tf                                  |  8 +++-
 7 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index 0c96877827..10b3af0c4b 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -204,6 +204,16 @@ This variable specifies the external DNS servers to forward DNS queries that can
 dns_forwarders              = "1.1.1.1; 9.9.9.9"
 ```
 
+List of [day-1 kernel arguments](https://docs.openshift.com/container-platform/4.8/installing/install_config/installing-customizing.html#installation-special-config-kargs_installing-customizing) for the cluster nodes.
+To add kernel arguments to master or worker nodes, using MachineConfig object and inject that object into the set of manifest files used by Ignition during cluster setup.
+```
+rhcos_pre_kernel_options        = []
+```
+- Example 1
+  ```
+  rhcos_pre_kernel_options   = ["rd.multipath=default","root=/dev/disk/by-label/dm-mpath-root"]
+  ```
+
 List of [kernel arguments](https://docs.openshift.com/container-platform/4.4/nodes/nodes/nodes-nodes-working.html#nodes-nodes-kernel-arguments_nodes-nodes-working) for the cluster nodes.
 Note that this will be applied after the cluster is installed and all the nodes are in `Ready` status.
 ```
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index fce5343704..a1fcc0b073 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -40,28 +40,29 @@ locals {
     local_registry_ocp_image = "registry.${var.cluster_id}.${local.cluster_domain}:5000/${local.ocp_release_repo}:${var.ocp_release_tag}"
 
     install_vars = {
-        bastion_vip             = var.bastion_vip
-        cluster_id              = var.cluster_id
-        cluster_domain          = local.cluster_domain
-        pull_secret             = var.pull_secret
-        public_ssh_key          = var.public_key
-        storage_type            = var.storage_type
-        log_level               = var.log_level
-        release_image_override  = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
-        enable_local_registry   = var.enable_local_registry
-        node_connection_timeout = 60 * var.connection_timeout
-        rhcos_kernel_options    = var.rhcos_kernel_options
-        sysctl_tuned_options    = var.sysctl_tuned_options
-        sysctl_options          = var.sysctl_options
-        match_array             = indent(2,var.match_array)
-        setup_squid_proxy       = var.setup_squid_proxy
-        squid_source_range      = var.cidr
-        proxy_url               = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
-        no_proxy                = var.cidr
-        chrony_config           = var.chrony_config
-        chrony_config_servers   = var.chrony_config_servers
-        chrony_allow_range      = var.cidr
-        cni_network_provider    = var.cni_network_provider
+        bastion_vip              = var.bastion_vip
+        cluster_id               = var.cluster_id
+        cluster_domain           = local.cluster_domain
+        pull_secret              = var.pull_secret
+        public_ssh_key           = var.public_key
+        storage_type             = var.storage_type
+        log_level                = var.log_level
+        release_image_override   = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
+        enable_local_registry    = var.enable_local_registry
+        node_connection_timeout  = 60 * var.connection_timeout
+        rhcos_pre_kernel_options = var.rhcos_pre_kernel_options
+        rhcos_kernel_options     = var.rhcos_kernel_options
+        sysctl_tuned_options     = var.sysctl_tuned_options
+        sysctl_options           = var.sysctl_options
+        match_array              = indent(2,var.match_array)
+        setup_squid_proxy        = var.setup_squid_proxy
+        squid_source_range       = var.cidr
+        proxy_url                = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
+        no_proxy                 = var.cidr
+        chrony_config            = var.chrony_config
+        chrony_config_servers    = var.chrony_config_servers
+        chrony_allow_range       = var.cidr
+        cni_network_provider     = var.cni_network_provider
     }
 
     upgrade_vars = {
diff --git a/modules/5_install/templates/install_vars.yaml b/modules/5_install/templates/install_vars.yaml
index f83262ab94..e33d0e9334 100644
--- a/modules/5_install/templates/install_vars.yaml
+++ b/modules/5_install/templates/install_vars.yaml
@@ -13,6 +13,8 @@ enable_local_registry: ${enable_local_registry}
 
 node_connection_timeout: ${node_connection_timeout}
 
+rhcos_pre_kernel_options: [%{ for opt in rhcos_pre_kernel_options ~}"${opt}",%{ endfor ~}]
+
 rhcos_kernel_options: [%{ for opt in rhcos_kernel_options ~}"${opt}",%{ endfor ~}]
 
 sysctl_tuned_options: ${sysctl_tuned_options}
diff --git a/modules/5_install/variables.tf b/modules/5_install/variables.tf
index 076dd66a81..34058b9c84 100644
--- a/modules/5_install/variables.tf
+++ b/modules/5_install/variables.tf
@@ -54,6 +54,7 @@ variable "storage_type" {}
 variable "log_level" {}
 
 variable "ansible_extra_options" {}
+variable "rhcos_pre_kernel_options" {}
 variable "rhcos_kernel_options" {}
 
 variable "sysctl_tuned_options" {}
diff --git a/ocp.tf b/ocp.tf
index 26a3de64ca..0df142e914 100644
--- a/ocp.tf
+++ b/ocp.tf
@@ -168,6 +168,7 @@ module "install" {
     install_playbook_tag            = var.install_playbook_tag
     log_level                       = var.installer_log_level
     ansible_extra_options           = var.ansible_extra_options
+    rhcos_pre_kernel_options        = var.rhcos_pre_kernel_options
     rhcos_kernel_options            = var.rhcos_kernel_options
     sysctl_tuned_options            = var.sysctl_tuned_options
     sysctl_options                  = var.sysctl_options
diff --git a/var.tfvars b/var.tfvars
index ccdc5bddf7..3cc9347217 100644
--- a/var.tfvars
+++ b/var.tfvars
@@ -66,6 +66,7 @@ cluster_id                  = ""         # It will use random generated id with
 #ansible_extra_options      = "-v"
 #ansible_repo_name          = "ansible-2.9-for-rhel-8-ppc64le-rpms"
 #dns_forwarders             = "1.1.1.1; 9.9.9.9"
+#rhcos_pre_kernel_options   = []
 #rhcos_kernel_options       = []
 #chrony_config              = true
 #chrony_config_servers      = [ {server = "0.centos.pool.ntp.org", options = "iburst"}, {server = "1.centos.pool.ntp.org", options = "iburst"} ]
diff --git a/variables.tf b/variables.tf
index 8dd5c45c38..8b1ba22d54 100644
--- a/variables.tf
+++ b/variables.tf
@@ -199,6 +199,12 @@ variable "rhel_subscription_org" {
 variable "rhel_subscription_activationkey" {
     default = ""
 }
+
+variable "rhcos_pre_kernel_options" {
+    description = "List of kernel arguments for the cluster nodes for pre-installation"
+    default     = []
+}
+
 variable "rhcos_kernel_options" {
     description = "List of kernel arguments for the cluster nodes"
     default     = []
@@ -275,7 +281,7 @@ variable "install_playbook_repo" {
 variable "install_playbook_tag" {
     description = "Set the branch/tag name or commit# for using ocp4-playbooks repo"
     # Checkout level for https://github.com/ocp-power-automation/ocp4-playbooks which is used for running ocp4 installations steps
-    default = "10fec74c9e987b39f7af1127abe304a9e41f8e65"
+    default = "7c5c0158fb96df7816b79da2274ff21b2fd61c1c"
 }
 
 variable "ansible_extra_options" {

From e2e63e7aaec65957126b8388eb603c0c04e42243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Chabrolles?= <s.chabrolles@fr.ibm.com>
Date: Fri, 24 Dec 2021 17:58:32 +0100
Subject: [PATCH 05/14] Allow OCP network customization before installation.
 (#224)

add clusterNetwork_CIDR, serviceNetwork, hostprefix vars
---
 docs/var.tfvars-doc.md                        |  3 ++
 modules/5_install/install.tf                  | 49 ++++++++++---------
 modules/5_install/templates/install_vars.yaml |  4 ++
 modules/5_install/variables.tf                |  3 ++
 ocp.tf                                        |  3 ++
 var.tfvars                                    |  3 ++
 variables.tf                                  | 17 ++++++-
 7 files changed, 58 insertions(+), 24 deletions(-)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index 10b3af0c4b..08e6758904 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -271,4 +271,7 @@ This variable is used to set the default Container Network Interface (CNI) netwo
 
 ```
 cni_network_provider       = "OpenshiftSDN"
+cluster_network_cidr        = "10.128.0.0/14"
+cluster_network_hostprefix  = "23"
+service_network             = "172.30.0.0/16"
 ```
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index a1fcc0b073..b581fdbc26 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -40,29 +40,32 @@ locals {
     local_registry_ocp_image = "registry.${var.cluster_id}.${local.cluster_domain}:5000/${local.ocp_release_repo}:${var.ocp_release_tag}"
 
     install_vars = {
-        bastion_vip              = var.bastion_vip
-        cluster_id               = var.cluster_id
-        cluster_domain           = local.cluster_domain
-        pull_secret              = var.pull_secret
-        public_ssh_key           = var.public_key
-        storage_type             = var.storage_type
-        log_level                = var.log_level
-        release_image_override   = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
-        enable_local_registry    = var.enable_local_registry
-        node_connection_timeout  = 60 * var.connection_timeout
-        rhcos_pre_kernel_options = var.rhcos_pre_kernel_options
-        rhcos_kernel_options     = var.rhcos_kernel_options
-        sysctl_tuned_options     = var.sysctl_tuned_options
-        sysctl_options           = var.sysctl_options
-        match_array              = indent(2,var.match_array)
-        setup_squid_proxy        = var.setup_squid_proxy
-        squid_source_range       = var.cidr
-        proxy_url                = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
-        no_proxy                 = var.cidr
-        chrony_config            = var.chrony_config
-        chrony_config_servers    = var.chrony_config_servers
-        chrony_allow_range       = var.cidr
-        cni_network_provider     = var.cni_network_provider
+        bastion_vip                = var.bastion_vip
+        cluster_id                 = var.cluster_id
+        cluster_domain             = local.cluster_domain
+        pull_secret                = var.pull_secret
+        public_ssh_key             = var.public_key
+        storage_type               = var.storage_type
+        log_level                  = var.log_level
+        release_image_override     = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
+        enable_local_registry      = var.enable_local_registry
+        node_connection_timeout    = 60 * var.connection_timeout
+        rhcos_pre_kernel_options   = var.rhcos_pre_kernel_options
+        rhcos_kernel_options       = var.rhcos_kernel_options
+        sysctl_tuned_options       = var.sysctl_tuned_options
+        sysctl_options             = var.sysctl_options
+        match_array                = indent(2,var.match_array)
+        setup_squid_proxy          = var.setup_squid_proxy
+        squid_source_range         = var.cidr
+        proxy_url                  = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}"
+        no_proxy                   = var.cidr
+        chrony_config              = var.chrony_config
+        chrony_config_servers      = var.chrony_config_servers
+        chrony_allow_range         = var.cidr
+        cni_network_provider       = var.cni_network_provider
+        cluster_network_cidr       = var.cluster_network_cidr
+        cluster_network_hostprefix = var.cluster_network_hostprefix
+        service_network            = var.service_network
     }
 
     upgrade_vars = {
diff --git a/modules/5_install/templates/install_vars.yaml b/modules/5_install/templates/install_vars.yaml
index e33d0e9334..c4292245ac 100644
--- a/modules/5_install/templates/install_vars.yaml
+++ b/modules/5_install/templates/install_vars.yaml
@@ -56,3 +56,7 @@ bastion_vip: "${bastion_vip}"
 %{ endif ~}
 
 cni_network_provider: "${cni_network_provider}"
+
+cluster_network_cidr: "${cluster_network_cidr}"
+cluster_network_hostprefix: "${cluster_network_hostprefix}"
+service_network: "${service_network}"
diff --git a/modules/5_install/variables.tf b/modules/5_install/variables.tf
index 34058b9c84..d1469ca62a 100644
--- a/modules/5_install/variables.tf
+++ b/modules/5_install/variables.tf
@@ -73,3 +73,6 @@ variable "upgrade_pause_time" {}
 variable "upgrade_delay_time" {}
 
 variable "cni_network_provider" {}
+variable "cluster_network_cidr" {}
+variable "cluster_network_hostprefix" {}
+variable "service_network" {}
diff --git a/ocp.tf b/ocp.tf
index 0df142e914..a94ef01291 100644
--- a/ocp.tf
+++ b/ocp.tf
@@ -183,4 +183,7 @@ module "install" {
     chrony_config                   = var.chrony_config
     chrony_config_servers           = var.chrony_config_servers
     cni_network_provider            = var.cni_network_provider
+    cluster_network_cidr            = var.cluster_network_cidr
+    cluster_network_hostprefix      = var.cluster_network_hostprefix
+    service_network                 = var.service_network
 }
diff --git a/var.tfvars b/var.tfvars
index 3cc9347217..4fcfa1ca62 100644
--- a/var.tfvars
+++ b/var.tfvars
@@ -91,3 +91,6 @@ cluster_id                  = ""         # It will use random generated id with
 #upgrade_delay_time         = "600"
 
 #cni_network_provider       = "OpenshiftSDN"
+#cluster_network_cidr        = "10.128.0.0/14"
+#cluster_network_hostprefix  = "23"
+#service_network             = "172.30.0.0/16"
diff --git a/variables.tf b/variables.tf
index 8b1ba22d54..40b62f55ec 100644
--- a/variables.tf
+++ b/variables.tf
@@ -281,7 +281,7 @@ variable "install_playbook_repo" {
 variable "install_playbook_tag" {
     description = "Set the branch/tag name or commit# for using ocp4-playbooks repo"
     # Checkout level for https://github.com/ocp-power-automation/ocp4-playbooks which is used for running ocp4 installations steps
-    default = "7c5c0158fb96df7816b79da2274ff21b2fd61c1c"
+    default = "86e8f06fa3e008fbdd6188659cb45a3cbe716e26"
 }
 
 variable "ansible_extra_options" {
@@ -417,6 +417,21 @@ variable "cni_network_provider" {
     default = "OpenshiftSDN"
 }
 
+variable "cluster_network_cidr" {
+    description = "blocks of IP addresses from which pod IP addresses are allocated."
+    default = "10.128.0.0/14"
+}
+
+variable "cluster_network_hostprefix" {
+    description = "The subnet prefix length to assign to each individual node."
+    default = "23"
+}
+
+variable "service_network" {
+    description = "blocks of IP addresses from which service addresses are allocated."
+    default = "172.30.0.0/16"
+}
+
 ################################################################
 # Local registry variables ( used only in disconnected install )
 ################################################################

From 3e4715715bbdc888db855cc896dddd74217c2410 Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Thu, 2 Dec 2021 22:28:33 +0530
Subject: [PATCH 06/14] To set mtu on private network

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 docs/var.tfvars-doc.md                        |  5 +++
 modules/5_install/install.tf                  | 33 +++++++++++++++++++
 modules/5_install/templates/install_vars.yaml |  1 +
 modules/5_install/variables.tf                |  3 ++
 ocp.tf                                        |  2 ++
 var.tfvars                                    |  1 +
 variables.tf                                  |  6 ++++
 7 files changed, 51 insertions(+)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index 08e6758904..74d001fbd6 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -193,6 +193,11 @@ install_playbook_repo      = "https://github.com/ocp-power-automation/ocp4-playb
 install_playbook_tag       = "02a598faa332aa2c3d53e8edd0e840440ff74bd5"
 ```
 
+This variable specify the MTU value for the private network interface on RHEL and RHCOS nodes. The CNI network will have <private_network_mtu> - 50 for OpenshiftSDN and <private_network_mtu> - 100 for OVNKubernetes network provider.
+```
+private_network_mtu         = 1450
+```
+
 These variables can be used when debugging ansible playbooks
 ```
 installer_log_level         = "info"
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index b581fdbc26..6a26e56ac5 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -24,6 +24,8 @@ locals {
 
     ocp_release_repo    = "ocp4/openshift4"
 
+    bastion_count = lookup(var.bastion, "count", 1)
+
     install_inventory = {
         bastion_hosts   = [for ix in range(length(var.bastion_ip)) : "${var.cluster_id}-bastion-${ix}"]
         bootstrap_host  = var.bootstrap_ip == "" ? "" : "bootstrap"
@@ -66,6 +68,9 @@ locals {
         cluster_network_cidr       = var.cluster_network_cidr
         cluster_network_hostprefix = var.cluster_network_hostprefix
         service_network            = var.service_network
+        # Set CNI network MTU to MTU - 100 for OVNKubernetes and MTU - 50 for OpenShiftSDN(default).
+        # Add new conditions here when we have more network providers
+        cni_network_mtu = var.cni_network_provider == "OVNKubernetes" ? var.private_network_mtu - 100 : var.private_network_mtu - 50
     }
 
     upgrade_vars = {
@@ -77,7 +82,35 @@ locals {
     }
 }
 
+resource "null_resource" "pre_install" {
+  count      = local.bastion_count
+
+  connection {
+    type        = "ssh"
+    user        = var.rhel_username
+    host        = var.bastion_ip[count.index]
+    private_key = var.private_key
+    agent       = var.ssh_agent
+    timeout     = "${var.connection_timeout}m"
+    bastion_host = var.jump_host
+  }
+
+  # DHCP config for setting MTU; Since helpernode DHCP template does not support MTU setting
+  provisioner "remote-exec" {
+    inline = [
+      # Set specified mtu for private interface.
+      "sudo ip link set dev $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}') mtu ${var.private_network_mtu}",
+      "echo MTU=${var.private_network_mtu} | sudo tee -a /etc/sysconfig/network-scripts/ifcfg-$(ip r | grep ${var.cidr} | awk '{print $3}')",
+      # DHCP config for setting MTU;
+      "sed -i.mtubak '/option routers/i option interface-mtu ${var.private_network_mtu};' /etc/dhcp/dhcpd.conf",
+      "sudo systemctl restart dhcpd.service"
+    ]
+  }
+}
+
 resource "null_resource" "install" {
+    depends_on = [null_resource.pre_install]
+
     triggers = {
         worker_count    = length(var.worker_ips)
     }
diff --git a/modules/5_install/templates/install_vars.yaml b/modules/5_install/templates/install_vars.yaml
index c4292245ac..1e4145699a 100644
--- a/modules/5_install/templates/install_vars.yaml
+++ b/modules/5_install/templates/install_vars.yaml
@@ -60,3 +60,4 @@ cni_network_provider: "${cni_network_provider}"
 cluster_network_cidr: "${cluster_network_cidr}"
 cluster_network_hostprefix: "${cluster_network_hostprefix}"
 service_network: "${service_network}"
+cni_network_mtu: "${cni_network_mtu}"
diff --git a/modules/5_install/variables.tf b/modules/5_install/variables.tf
index d1469ca62a..844d765792 100644
--- a/modules/5_install/variables.tf
+++ b/modules/5_install/variables.tf
@@ -35,6 +35,7 @@ variable "ssh_agent" {}
 variable "connection_timeout" {}
 variable "jump_host" {}
 
+variable "bastion" {}
 variable "bootstrap_ip" {}
 variable "master_ips" {}
 variable "worker_ips" {}
@@ -43,6 +44,8 @@ variable "public_key" {}
 variable "pull_secret" {}
 variable "release_image_override" {}
 
+variable "private_network_mtu" {}
+
 variable "enable_local_registry" {}
 variable "local_registry_image" {}
 variable "ocp_release_tag" {}
diff --git a/ocp.tf b/ocp.tf
index a94ef01291..acb7d2951a 100644
--- a/ocp.tf
+++ b/ocp.tf
@@ -147,6 +147,7 @@ module "install" {
     cluster_domain                  = var.cluster_domain
     cluster_id                      = local.cluster_id
     cidr                            = module.network.cidr
+    bastion                         = var.bastion
     bastion_vip                     = module.network.bastion_vip
     bastion_ip                      = module.bastion.bastion_ip
     rhel_username                   = var.rhel_username
@@ -161,6 +162,7 @@ module "install" {
     pull_secret                     = file(coalesce(var.pull_secret_file, "/dev/null"))
     storage_type                    = local.storage_type
     release_image_override          = var.release_image_override
+    private_network_mtu             = var.private_network_mtu
     enable_local_registry           = var.enable_local_registry
     local_registry_image            = var.local_registry_image
     ocp_release_tag                 = var.ocp_release_tag
diff --git a/var.tfvars b/var.tfvars
index 4fcfa1ca62..adea66e9a5 100644
--- a/var.tfvars
+++ b/var.tfvars
@@ -94,3 +94,4 @@ cluster_id                  = ""         # It will use random generated id with
 #cluster_network_cidr        = "10.128.0.0/14"
 #cluster_network_hostprefix  = "23"
 #service_network             = "172.30.0.0/16"
+#private_network_mtu         = "1450"
diff --git a/variables.tf b/variables.tf
index 40b62f55ec..5ecf7527c9 100644
--- a/variables.tf
+++ b/variables.tf
@@ -255,6 +255,12 @@ variable "jump_host" {
     default     = ""
 }
 
+variable "private_network_mtu" {
+  type        = number
+  description = "MTU value for the private network interface on RHEL and RHCOS nodes"
+  default     = 1450
+}
+
 variable "installer_log_level" {
     description = "Set the log level required for openshift-install commands"
     default = "info"

From 5917dfe35e3b44ff3c4a9e23280ad28f4d21d5cb Mon Sep 17 00:00:00 2001
From: Yussuf Shaikh <yussuf.shaikh@ibm.com>
Date: Wed, 5 Jan 2022 21:49:59 +0530
Subject: [PATCH 07/14] Added cs-zhang as approver

---
 OWNERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/OWNERS b/OWNERS
index eaf34e8020..5a0748a84b 100644
--- a/OWNERS
+++ b/OWNERS
@@ -7,4 +7,5 @@ reviewers:
   - cs-zhang
 approvers:
   - bpradipt
+  - cs-zhang
   - yussufsh

From 7e33f0fa16ee0515477e011e201655cba4bb9483 Mon Sep 17 00:00:00 2001
From: Manjunath Kumatagi <mkumatag@in.ibm.com>
Date: Thu, 3 Mar 2022 14:50:25 +0530
Subject: [PATCH 08/14] remove mkumatag from reviewer list

Not actively involved, hence removing my entry from the reviewers to avoid getting assigned automatically for the review
---
 OWNERS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/OWNERS b/OWNERS
index 5a0748a84b..cd719e6a79 100644
--- a/OWNERS
+++ b/OWNERS
@@ -1,5 +1,4 @@
 reviewers:
-  - mkumatag
   - Prajyot-Parab
   - sudeeshjohn
   - yussufsh

From de0d39cde0dc5568eac33212c8b84c2955ab2db6 Mon Sep 17 00:00:00 2001
From: Sebastien Chabrolles <s.chabrolles@fr.ibm.com>
Date: Thu, 24 Feb 2022 17:47:19 +0100
Subject: [PATCH 09/14] force centos stream to use ansible 2.9 like rhel8

---
 modules/1_bastion/bastion.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/1_bastion/bastion.tf b/modules/1_bastion/bastion.tf
index 9d0a3a28d5..10cc99e8ba 100644
--- a/modules/1_bastion/bastion.tf
+++ b/modules/1_bastion/bastion.tf
@@ -270,7 +270,7 @@ resource "null_resource" "bastion_packages" {
     }
     provisioner "remote-exec" {
         inline = [
-           "sudo yum install -y ansible"
+           "sudo yum install -y ansible-2.9.*"
         ]
     }
     provisioner "remote-exec" {

From c2b5c2a7517dfe8d1578f9d6860d48dfb749798c Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Fri, 21 Jan 2022 15:02:23 +0530
Subject: [PATCH 10/14] Accessing cluster using non-root user

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 docs/var.tfvars-doc.md                              | 3 ++-
 modules/1_bastion/bastion.tf                        | 6 +++---
 modules/3_helpernode/helpernode.tf                  | 5 +++--
 modules/3_helpernode/templates/helpernode_inventory | 2 +-
 modules/5_install/install.tf                        | 1 +
 modules/5_install/templates/install_inventory       | 2 +-
 var.tfvars                                          | 2 +-
 variables.tf                                        | 2 +-
 8 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index 74d001fbd6..b2d1c510ad 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -83,10 +83,11 @@ worker                      = {instance_type    = "<worker-compute-template>", i
 ```
 These set of variables specify the username and the SSH key to be used for accessing the bastion node.
 ```
-rhel_username               = "root"
+rhel_username               = "root"  #Set it to an appropriate username for non-root user access
 public_key_file             = "data/id_rsa.pub"
 private_key_file            = "data/id_rsa"
 ```
+rhel_username is set to root. rhel_username can be set to an appropriate username having superuser privileges with no password prompt.
 Please note that only OpenSSH formatted keys are supported. Refer to the following links for instructions on creating SSH key based on your platform.
 - Windows 10 - https://phoenixnap.com/kb/generate-ssh-key-windows-10
 - Mac OSX - https://www.techrepublic.com/article/how-to-generate-ssh-keys-on-macos-mojave/
diff --git a/modules/1_bastion/bastion.tf b/modules/1_bastion/bastion.tf
index 9d0a3a28d5..9c2cde8c56 100644
--- a/modules/1_bastion/bastion.tf
+++ b/modules/1_bastion/bastion.tf
@@ -327,11 +327,11 @@ resource "null_resource" "setup_nfs_disk" {
     }
     provisioner "remote-exec" {
         inline = [
-            "rm -rf mkdir ${local.storage_path}; mkdir -p ${local.storage_path}; chmod -R 755 ${local.storage_path}",
+            "sudo rm -rf mkdir ${local.storage_path}; sudo mkdir -p ${local.storage_path}; sudo chmod -R 755 ${local.storage_path}",
             "sudo chmod +x /tmp/create_disk_link.sh",
             # Fix for copying file from Windows OS having CR
-            "sed -i 's/\r//g' /tmp/create_disk_link.sh",
-            "/tmp/create_disk_link.sh",
+            "sudo sed -i 's/\r//g' /tmp/create_disk_link.sh",
+            "sudo /tmp/create_disk_link.sh",
             "sudo mkfs.ext4 -F /dev/${local.disk_config.disk_name}",
             "echo '/dev/${local.disk_config.disk_name} ${local.storage_path} ext4 defaults 0 0' | sudo tee -a /etc/fstab > /dev/null",
             "sudo mount ${local.storage_path}",
diff --git a/modules/3_helpernode/helpernode.tf b/modules/3_helpernode/helpernode.tf
index b521f413fc..44036c774f 100644
--- a/modules/3_helpernode/helpernode.tf
+++ b/modules/3_helpernode/helpernode.tf
@@ -73,7 +73,8 @@ locals {
         install_tarball          = var.openshift_install_tarball
     }
     helpernode_inventory = {
-        bastion_ip  = var.bastion_ip
+        rhel_username = var.rhel_username
+        bastion_ip    = var.bastion_ip
     }
 }
 
@@ -119,7 +120,7 @@ resource "null_resource" "config" {
         inline = [
             "sed -i \"/^helper:.*/a \\ \\ networkifacename: $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}')\" ocp4-helpernode/helpernode_vars.yaml",
             "echo 'Running ocp4-helpernode playbook...'",
-            "cd ocp4-helpernode && ansible-playbook  -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options}"
+            "cd ocp4-helpernode && ansible-playbook  -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options} --become"
         ]
     }
 }
diff --git a/modules/3_helpernode/templates/helpernode_inventory b/modules/3_helpernode/templates/helpernode_inventory
index ac1eaddde8..d6b2ad0591 100644
--- a/modules/3_helpernode/templates/helpernode_inventory
+++ b/modules/3_helpernode/templates/helpernode_inventory
@@ -1,4 +1,4 @@
 [vmhost]
 %{ for ip in bastion_ip ~}
-${ip} ansible_connection=ssh ansible_user=root
+${ip} ansible_connection=ssh ansible_user=${rhel_username}
 %{ endfor ~}
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index 6a26e56ac5..94cc54f370 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -27,6 +27,7 @@ locals {
     bastion_count = lookup(var.bastion, "count", 1)
 
     install_inventory = {
+        rhel_username   = var.rhel_username
         bastion_hosts   = [for ix in range(length(var.bastion_ip)) : "${var.cluster_id}-bastion-${ix}"]
         bootstrap_host  = var.bootstrap_ip == "" ? "" : "bootstrap"
         master_hosts    = [for ix in range(length(var.master_ips)) : "master-${ix}"]
diff --git a/modules/5_install/templates/install_inventory b/modules/5_install/templates/install_inventory
index aec3cd88c8..5afd4aff3e 100644
--- a/modules/5_install/templates/install_inventory
+++ b/modules/5_install/templates/install_inventory
@@ -1,6 +1,6 @@
 [bastion]
 %{ for bastion in bastion_hosts ~}
-${bastion} ansible_connection=ssh ansible_user=root
+${bastion} ansible_connection=ssh ansible_user=${rhel_username}
 %{ endfor ~}
 
 %{ if bootstrap_host != "" ~}
diff --git a/var.tfvars b/var.tfvars
index adea66e9a5..baf9f0c21c 100644
--- a/var.tfvars
+++ b/var.tfvars
@@ -21,7 +21,7 @@ worker                      = {instance_type    = "<worker-compute-template>",
 # worker                      = {instance_type    = "<worker-compute-template>",    image_id    = "<image-uuid-rhcos>",  availability_zone = "<availability zone>",  "count"   = 2, data_volume_count  = 0, data_volume_size  = 100}
 
 
-rhel_username               = "root"
+rhel_username               = "root"  #Set it to an appropriate username for non-root user access
 public_key_file             = "data/id_rsa.pub"
 private_key_file            = "data/id_rsa"
 rhel_subscription_username  = "<subscription-id>"          #Leave this as-is if using CentOS as bastion image
diff --git a/variables.tf b/variables.tf
index 5ecf7527c9..f3cdfd8ca2 100644
--- a/variables.tf
+++ b/variables.tf
@@ -287,7 +287,7 @@ variable "install_playbook_repo" {
 variable "install_playbook_tag" {
     description = "Set the branch/tag name or commit# for using ocp4-playbooks repo"
     # Checkout level for https://github.com/ocp-power-automation/ocp4-playbooks which is used for running ocp4 installations steps
-    default = "86e8f06fa3e008fbdd6188659cb45a3cbe716e26"
+    default = "a328a8d03c043d4f7c38300f35bba471bc81bd37"
 }
 
 variable "ansible_extra_options" {

From 04eed592656450ab00ada1c2289d14660bf5b73d Mon Sep 17 00:00:00 2001
From: Sebastien Chabrolles <s.chabrolles@fr.ibm.com>
Date: Tue, 15 Mar 2022 12:46:34 +0100
Subject: [PATCH 11/14] bastion fqdn with clusterID as subdmain

---
 modules/1_bastion/bastion.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/1_bastion/bastion.tf b/modules/1_bastion/bastion.tf
index 9d0a3a28d5..f0983f65dc 100644
--- a/modules/1_bastion/bastion.tf
+++ b/modules/1_bastion/bastion.tf
@@ -103,8 +103,8 @@ resource "null_resource" "bastion_init" {
         inline = [
             "sudo chmod 600 $HOME/.ssh/id_rsa*",
             "sudo sed -i.bak -e 's/^ - set_hostname/# - set_hostname/' -e 's/^ - update_hostname/# - update_hostname/' /etc/cloud/cloud.cfg",
-            "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}",
-            "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null",
+            "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}",
+            "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null",
             "sudo hostname -F /etc/hostname",
             "echo 'vm.max_map_count = 262144' | sudo tee --append /etc/sysctl.conf > /dev/null",
         ]

From e52052c09e2b731fe279c900f7a36299050e2cd2 Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Tue, 15 Mar 2022 18:03:48 +0530
Subject: [PATCH 12/14] To remove the scp error with Terraform v1.1.x

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 modules/1_bastion/bastion.tf       | 6 +++---
 modules/3_helpernode/helpernode.tf | 6 +++---
 modules/5_install/install.tf       | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/modules/1_bastion/bastion.tf b/modules/1_bastion/bastion.tf
index f510655964..7559bc2b77 100644
--- a/modules/1_bastion/bastion.tf
+++ b/modules/1_bastion/bastion.tf
@@ -93,15 +93,15 @@ resource "null_resource" "bastion_init" {
     }
     provisioner "file" {
         content = var.private_key
-        destination = "$HOME/.ssh/id_rsa"
+        destination = ".ssh/id_rsa"
     }
     provisioner "file" {
         content = var.public_key
-        destination = "$HOME/.ssh/id_rsa.pub"
+        destination = ".ssh/id_rsa.pub"
     }
     provisioner "remote-exec" {
         inline = [
-            "sudo chmod 600 $HOME/.ssh/id_rsa*",
+            "sudo chmod 600 .ssh/id_rsa*",
             "sudo sed -i.bak -e 's/^ - set_hostname/# - set_hostname/' -e 's/^ - update_hostname/# - update_hostname/' /etc/cloud/cloud.cfg",
             "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}",
             "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null",
diff --git a/modules/3_helpernode/helpernode.tf b/modules/3_helpernode/helpernode.tf
index 44036c774f..c5bbd51ce3 100644
--- a/modules/3_helpernode/helpernode.tf
+++ b/modules/3_helpernode/helpernode.tf
@@ -106,15 +106,15 @@ resource "null_resource" "config" {
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/helpernode_inventory", local.helpernode_inventory)
-        destination = "$HOME/ocp4-helpernode/inventory"
+        destination = "ocp4-helpernode/inventory"
     }
     provisioner "file" {
         content     = var.pull_secret
-        destination = "$HOME/.openshift/pull-secret"
+        destination = ".openshift/pull-secret"
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/helpernode_vars.yaml", local.helpernode_vars)
-        destination = "$HOME/ocp4-helpernode/helpernode_vars.yaml"
+        destination = "ocp4-helpernode/helpernode_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index 94cc54f370..59e5a8f17a 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -136,11 +136,11 @@ resource "null_resource" "install" {
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/install_inventory", local.install_inventory)
-        destination = "$HOME/ocp4-playbooks/inventory"
+        destination = "ocp4-playbooks/inventory"
     }
     provisioner "file" {
         content     = templatefile("${path.module}/templates/install_vars.yaml", local.install_vars)
-        destination = "$HOME/ocp4-playbooks/install_vars.yaml"
+        destination = "ocp4-playbooks/install_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [
@@ -166,7 +166,7 @@ resource "null_resource" "upgrade" {
 
     provisioner "file" {
         content     = templatefile("${path.module}/templates/upgrade_vars.yaml", local.upgrade_vars)
-        destination = "$HOME/ocp4-playbooks/upgrade_vars.yaml"
+        destination = "ocp4-playbooks/upgrade_vars.yaml"
     }
     provisioner "remote-exec" {
         inline = [

From 41603aa43d1456ba10cdc85898118930762255cc Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Tue, 15 Mar 2022 18:09:35 +0530
Subject: [PATCH 13/14] To Update the Terraform Version

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 docs/automation_host_prereqs.md  | 2 +-
 modules/1_bastion/versions.tf    | 2 +-
 modules/2_network/versions.tf    | 2 +-
 modules/3_helpernode/versions.tf | 2 +-
 modules/4_nodes/versions.tf      | 2 +-
 modules/5_install/versions.tf    | 2 +-
 versions.tf                      | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/automation_host_prereqs.md b/docs/automation_host_prereqs.md
index 00fa56e9b8..901c09710b 100644
--- a/docs/automation_host_prereqs.md
+++ b/docs/automation_host_prereqs.md
@@ -22,7 +22,7 @@ Install the following packages on the automation host. Select the appropriate in
 **Terraform >= 0.13.0**: Please refer to the [link](https://learn.hashicorp.com/terraform/getting-started/install.html) for instructions on installing Terraform. For validating the version run `terraform version` command after install.
 
 Install Terraform and providers for Power environment:
-1. Download the Terraform binary version 0.13.5 from https://www.power-devops.com/terraform and install it to /usr/local/bin.
+1. Download and install the Terraform binary (>= 0.13.0) for Linux/ppc64le from https://www.power-devops.com/terraform.
 2. Download the required Terraform providers for Power into your TF project directory:
 ```
 $ cd <path_to_TF_project>
diff --git a/modules/1_bastion/versions.tf b/modules/1_bastion/versions.tf
index 052785e228..afb32cda1e 100644
--- a/modules/1_bastion/versions.tf
+++ b/modules/1_bastion/versions.tf
@@ -33,5 +33,5 @@ terraform {
       version = "~> 2.3"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }
diff --git a/modules/2_network/versions.tf b/modules/2_network/versions.tf
index efa46b12eb..fcb9ad20f6 100644
--- a/modules/2_network/versions.tf
+++ b/modules/2_network/versions.tf
@@ -25,5 +25,5 @@ terraform {
       version = "~> 1.32"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }
diff --git a/modules/3_helpernode/versions.tf b/modules/3_helpernode/versions.tf
index 103cb6a141..f280c68c9a 100644
--- a/modules/3_helpernode/versions.tf
+++ b/modules/3_helpernode/versions.tf
@@ -25,5 +25,5 @@ terraform {
       version = "~> 2.1"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }
diff --git a/modules/4_nodes/versions.tf b/modules/4_nodes/versions.tf
index 2b54931e99..dd23445829 100644
--- a/modules/4_nodes/versions.tf
+++ b/modules/4_nodes/versions.tf
@@ -33,5 +33,5 @@ terraform {
       version = "~> 2.3"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }
diff --git a/modules/5_install/versions.tf b/modules/5_install/versions.tf
index 103cb6a141..f280c68c9a 100644
--- a/modules/5_install/versions.tf
+++ b/modules/5_install/versions.tf
@@ -25,5 +25,5 @@ terraform {
       version = "~> 2.1"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }
diff --git a/versions.tf b/versions.tf
index 7016f11abf..47460246b1 100644
--- a/versions.tf
+++ b/versions.tf
@@ -29,5 +29,5 @@ terraform {
       version = "~> 2.3"
     }
   }
-  required_version = "~> 0.13.0"
+  required_version = ">= 0.13.0"
 }

From 84c4f4a624293c18bc7b9b5524ef5c40df9448ca Mon Sep 17 00:00:00 2001
From: Aishwarya Kamat <aishwarya.kamat@ibm.com>
Date: Thu, 31 Mar 2022 10:29:44 +0530
Subject: [PATCH 14/14] FIPS enablement

Signed-off-by: Aishwarya Kamat <aishwarya.kamat@ibm.com>
---
 docs/var.tfvars-doc.md                        | 8 ++++++++
 modules/5_install/install.tf                  | 1 +
 modules/5_install/templates/install_vars.yaml | 1 +
 modules/5_install/variables.tf                | 1 +
 ocp.tf                                        | 1 +
 var.tfvars                                    | 2 +-
 variables.tf                                  | 8 +++++++-
 7 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md
index b2d1c510ad..fe16fbca88 100644
--- a/docs/var.tfvars-doc.md
+++ b/docs/var.tfvars-doc.md
@@ -140,6 +140,14 @@ If `cluster_if_prefix` is not set, the `cluster_id` will be used only without pr
 A random value will be used for `cluster_id` if not set.
 The total length of `cluster_id_prefix`.`cluster_id` should not exceed 14 characters.
 
+### FIPS Variable for OpenShift deployment
+
+These variables will be used for deploying OCP in FIPS mode.
+Change the values as per your requirement.
+```
+fips_compliant      = false
+```
+
 ### Misc Customizations
 
 These variables provides miscellaneous customizations. For common usage scenarios these are not required and should be left unchanged.
diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf
index 59e5a8f17a..b0d7ab984e 100644
--- a/modules/5_install/install.tf
+++ b/modules/5_install/install.tf
@@ -52,6 +52,7 @@ locals {
         log_level                  = var.log_level
         release_image_override     = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override
         enable_local_registry      = var.enable_local_registry
+        fips_compliant             = var.fips_compliant
         node_connection_timeout    = 60 * var.connection_timeout
         rhcos_pre_kernel_options   = var.rhcos_pre_kernel_options
         rhcos_kernel_options       = var.rhcos_kernel_options
diff --git a/modules/5_install/templates/install_vars.yaml b/modules/5_install/templates/install_vars.yaml
index 1e4145699a..244ec6b9a9 100644
--- a/modules/5_install/templates/install_vars.yaml
+++ b/modules/5_install/templates/install_vars.yaml
@@ -10,6 +10,7 @@ storage_type: ${storage_type}
 log_level: ${log_level}
 release_image_override: '${release_image_override}'
 enable_local_registry: ${enable_local_registry}
+fips_compliant: "${fips_compliant}"
 
 node_connection_timeout: ${node_connection_timeout}
 
diff --git a/modules/5_install/variables.tf b/modules/5_install/variables.tf
index 844d765792..97408ff459 100644
--- a/modules/5_install/variables.tf
+++ b/modules/5_install/variables.tf
@@ -43,6 +43,7 @@ variable "worker_ips" {}
 variable "public_key" {}
 variable "pull_secret" {}
 variable "release_image_override" {}
+variable "fips_compliant" {}
 
 variable "private_network_mtu" {}
 
diff --git a/ocp.tf b/ocp.tf
index acb7d2951a..92955e0eb3 100644
--- a/ocp.tf
+++ b/ocp.tf
@@ -164,6 +164,7 @@ module "install" {
     release_image_override          = var.release_image_override
     private_network_mtu             = var.private_network_mtu
     enable_local_registry           = var.enable_local_registry
+    fips_compliant                  = var.fips_compliant
     local_registry_image            = var.local_registry_image
     ocp_release_tag                 = var.ocp_release_tag
     install_playbook_repo           = var.install_playbook_repo
diff --git a/var.tfvars b/var.tfvars
index baf9f0c21c..fe449bf8ba 100644
--- a/var.tfvars
+++ b/var.tfvars
@@ -41,7 +41,7 @@ pull_secret_file            = "data/pull-secret.txt"
 cluster_domain              = "ibm.com"  # Set domain to nip.io or xip.io if you prefer using online wildcard domain and avoid modifying /etc/hosts
 cluster_id_prefix           = "test-ocp" # Set it to empty if just want to use cluster_id without prefix
 cluster_id                  = ""         # It will use random generated id with cluster_id_prefix if this is not set
-
+#fips_compliant             = false   # Set it true if you prefer to use FIPS enable in ocp deployment
 
 ### Misc Customizations
 
diff --git a/variables.tf b/variables.tf
index f3cdfd8ca2..4e8564bb2f 100644
--- a/variables.tf
+++ b/variables.tf
@@ -287,7 +287,7 @@ variable "install_playbook_repo" {
 variable "install_playbook_tag" {
     description = "Set the branch/tag name or commit# for using ocp4-playbooks repo"
     # Checkout level for https://github.com/ocp-power-automation/ocp4-playbooks which is used for running ocp4 installations steps
-    default = "a328a8d03c043d4f7c38300f35bba471bc81bd37"
+    default = "284b597b3e88c635e3069b82926aa16812238492"
 }
 
 variable "ansible_extra_options" {
@@ -341,6 +341,12 @@ variable "cluster_id" {
     default   = ""
 }
 
+variable "fips_compliant" {
+  type        = bool
+  description = "Set to true to enable usage of FIPS for OCP deployment."
+  default     = false
+}
+
 variable "dns_forwarders" {
     default   = "8.8.8.8; 8.8.4.4"
 }