Fix: Force SCP mode for file transfer with bastion

askb · askb · commit d02b15305d8e · 2025-11-13T23:42:20.000+10:00
Force SCP mode for Ansible file transfers with bastion.
Add ANSIBLE_SCP_IF_SSH=True environment variable when
local_build=true to force Ansible to use SCP instead of SFTP
for file transfers.

Root cause: Ansible defaults to SFTP for file transfers, which causes
failures when connecting through bastion/jump hosts that don't have
sftp-server installed. The existing --scp-extra-args '-O' flag was
being ignored because Ansible was using SFTP, not SCP.

Error without this fix:
  failed to transfer file to /home/ubuntu/.ansible/tmp/...
  bash: line 1: /usr/lib/sftp-server: No such file or directory
  scp: Connection closed

Solution: Set ANSIBLE_SCP_IF_SSH=True to force Ansible to use SCP mode
when local_build=true (builds through bastion). Once SCP is being used,
the --scp-extra-args '-O' flag applies, forcing legacy SCP protocol
which works correctly through the bastion.

Changes:
- Added local.ansible_env_vars conditional in all templates
- When local_build=true: includes ANSIBLE_SCP_IF_SSH=True
- When local_build=false: standard Ansible environment (Jenkins builds)
- Updated provisioner blocks to use local.ansible_env_vars

Templates updated:
- builder.pkr.hcl
- docker.pkr.hcl
- devstack.pkr.hcl
- devstack-pre-pip-yoga.pkr.hcl
- windows-builder.pkr.hcl

Backward compatibility: Existing Jenkins builds continue to work
unchanged since local_build defaults to false. Only affects builds
with local_build=true (packer-build-action through bastion).

Change-Id: Iae0d6f2284fb2cdefa827c951bd11b51a5a56e19
Signed-off-by: Anil Belur &lt;abelur@linuxfoundation.org&gt;
diff --git a/releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml b/releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml
@@ -0,0 +1,50 @@
+---
+fixes:
+  - |
+    Fix Ansible file transfer failures when building through bastion/jump hosts.
+
+    **Problem**: Packer builds with ``local_build=true`` (bastion/jump host mode)
+    were failing during Ansible provisioning with SFTP errors:
+
+    .. code-block:: text
+
+       failed to transfer file to /home/ubuntu/.ansible/tmp/...
+       bash: line 1: /usr/lib/sftp-server: No such file or directory
+       scp: Connection closed
+
+    **Root Cause**: Ansible defaults to SFTP for file transfers, not SCP.
+    The existing ``--scp-extra-args '-O'`` flag was being ignored because
+    Ansible was using SFTP, not SCP. Many cloud instances and bastion hosts
+    don't have ``sftp-server`` installed, causing transfer failures.
+
+    **Resolution**: Added ``ANSIBLE_SCP_IF_SSH=True`` environment variable
+    when ``local_build=true`` to force Ansible to use SCP mode instead of
+    SFTP. This makes the ``--scp-extra-args '-O'`` flag take effect, which
+    forces the legacy SCP protocol that works through bastion hosts.
+
+    **Implementation**: Created conditional ``ansible_env_vars`` in the
+    locals block of all templates:
+
+    - When ``local_build=true``: Sets ``ANSIBLE_SCP_IF_SSH=True``
+    - When ``local_build=false``: Standard Ansible environment (no change)
+
+    **Backward Compatibility**: This change only affects builds with
+    ``local_build=true`` (packer-build-action through bastion). Existing
+    Jenkins builds continue to work unchanged since ``local_build``
+    defaults to false.
+
+    Templates updated:
+
+    - templates/builder.pkr.hcl
+    - templates/docker.pkr.hcl
+    - templates/devstack.pkr.hcl
+    - templates/devstack-pre-pip-yoga.pkr.hcl
+    - templates/windows-builder.pkr.hcl
+
+    **Usage**: When building through a bastion/jump host, set:
+
+    .. code-block:: bash
+
+       packer build -var local_build=true ...
+
+    Reference: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/ssh_connection.html
diff --git a/templates/builder.pkr.hcl b/templates/builder.pkr.hcl
@@ -158,14 +158,32 @@ variable "vm_volume_size" {
 }
 
 locals {
-  # SSH arguments for local builds only
+  # SSH arguments - SCP compatibility for local builds
   ssh_extra_args = var.local_build ? [
     "--scp-extra-args", "'-O'",
     "--ssh-extra-args",
     "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"
   ] : [
     "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa"
   ]
+
+  # Ansible environment variables - force SCP for local builds to work with bastion
+  ansible_env_vars = var.local_build ? [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_SCP_IF_SSH=True",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ] : [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ]
 }
 
 source "docker" "builder" {
@@ -216,14 +234,7 @@ build {
   }
 
   provisioner "ansible" {
-    ansible_env_vars   = [
-        "ANSIBLE_NOCOWS=1",
-        "ANSIBLE_PIPELINING=False",
-        "ANSIBLE_HOST_KEY_CHECKING=False",
-        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
-        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
-        "ANSIBLE_STDOUT_CALLBACK=debug"
-    ]
+    ansible_env_vars   = local.ansible_env_vars
     command            = "./common-packer/ansible-playbook.sh"
     extra_arguments    = local.ssh_extra_args
     playbook_file      = "provision/local-builder.yaml"
diff --git a/templates/devstack-pre-pip-yoga.pkr.hcl b/templates/devstack-pre-pip-yoga.pkr.hcl
@@ -161,7 +161,7 @@ variable "vm_volume_size" {
 }
 
 locals {
-  # SSH arguments for local builds only
+  # SSH arguments - SCP compatibility for local builds
   ssh_extra_args = var.local_build ? [
     "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga",
     "--scp-extra-args", "'-O'",
@@ -171,6 +171,24 @@ locals {
     "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga",
     "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa"
   ]
+
+  # Ansible environment variables - force SCP for local builds to work with bastion
+  ansible_env_vars = var.local_build ? [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_SCP_IF_SSH=True",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ] : [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ]
 }
 
 source "docker" "devstack-pre-pip-yoga" {
@@ -221,14 +239,7 @@ build {
   }
 
   provisioner "ansible" {
-    ansible_env_vars   = [
-        "ANSIBLE_NOCOWS=1",
-        "ANSIBLE_PIPELINING=False",
-        "ANSIBLE_HOST_KEY_CHECKING=False",
-        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
-        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
-        "ANSIBLE_STDOUT_CALLBACK=debug"
-    ]
+    ansible_env_vars   = local.ansible_env_vars
     command            = "./common-packer/ansible-playbook.sh"
     extra_arguments    = local.ssh_extra_args
     playbook_file      = "provision/devstack-pre-pip-centos.yaml"
diff --git a/templates/devstack.pkr.hcl b/templates/devstack.pkr.hcl
@@ -163,7 +163,7 @@ variable "vm_volume_size" {
 }
 
 locals {
-  # SSH arguments for local builds only
+  # SSH arguments - SCP compatibility for local builds
   ssh_extra_args = var.local_build ? [
     "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga",
     "--scp-extra-args", "'-O'",
@@ -173,6 +173,24 @@ locals {
     "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga",
     "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa"
   ]
+
+  # Ansible environment variables - force SCP for local builds to work with bastion
+  ansible_env_vars = var.local_build ? [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_SCP_IF_SSH=True",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ] : [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ]
 }
 
 source "docker" "devstack" {
@@ -223,14 +241,7 @@ build {
   }
 
   provisioner "ansible" {
-    ansible_env_vars   = [
-        "ANSIBLE_NOCOWS=1",
-        "ANSIBLE_PIPELINING=False",
-        "ANSIBLE_HOST_KEY_CHECKING=False",
-        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
-        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
-        "ANSIBLE_STDOUT_CALLBACK=debug"
-    ]
+    ansible_env_vars   = local.ansible_env_vars
     command            = "./common-packer/ansible-playbook.sh"
     extra_arguments    = local.ssh_extra_args
     playbook_file      = "provision/devstack-centos.yaml"
diff --git a/templates/docker.pkr.hcl b/templates/docker.pkr.hcl
@@ -164,14 +164,32 @@ variable "vm_volume_size" {
 }
 
 locals {
-  # SSH arguments for local builds only
+  # SSH arguments - SCP compatibility for local builds
   ssh_extra_args = var.local_build ? [
     "--scp-extra-args", "'-O'",
     "--ssh-extra-args",
     "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"
   ] : [
     "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa"
   ]
+
+  # Ansible environment variables - force SCP for local builds to work with bastion
+  ansible_env_vars = var.local_build ? [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_SCP_IF_SSH=True",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ] : [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ]
 }
 
 source "docker" "docker" {
@@ -222,14 +240,7 @@ build {
   }
 
   provisioner "ansible" {
-    ansible_env_vars   = [
-        "ANSIBLE_NOCOWS=1",
-        "ANSIBLE_PIPELINING=False",
-        "ANSIBLE_HOST_KEY_CHECKING=False",
-        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
-        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
-        "ANSIBLE_STDOUT_CALLBACK=debug"
-    ]
+    ansible_env_vars   = local.ansible_env_vars
     command            = "./common-packer/ansible-playbook.sh"
     extra_arguments    = local.ssh_extra_args
     playbook_file      = "provision/local-docker.yaml"
diff --git a/templates/windows-builder.pkr.hcl b/templates/windows-builder.pkr.hcl
@@ -160,7 +160,7 @@ variable "vm_volume_size" {
 }
 
 locals {
-  # SSH arguments for local builds only
+  # SSH arguments - SCP compatibility for local builds
   ssh_extra_args = var.local_build ? [
     "--extra-vars", "ansible_shell_type=powershell",
     "--extra-vars", "ansible_shell_executable=None",
@@ -172,6 +172,24 @@ locals {
     "--extra-vars", "ansible_shell_executable=None",
     "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa"
   ]
+
+  # Ansible environment variables - force SCP for local builds to work with bastion
+  ansible_env_vars = var.local_build ? [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_SCP_IF_SSH=True",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ] : [
+    "ANSIBLE_NOCOWS=1",
+    "ANSIBLE_PIPELINING=False",
+    "ANSIBLE_HOST_KEY_CHECKING=False",
+    "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+    "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+    "ANSIBLE_STDOUT_CALLBACK=debug"
+  ]
 }
 
 source "openstack" "windows-builder" {
@@ -209,14 +227,7 @@ build {
   }
 
   provisioner "ansible" {
-    ansible_env_vars   = [
-        "ANSIBLE_NOCOWS=1",
-        "ANSIBLE_PIPELINING=False",
-        "ANSIBLE_HOST_KEY_CHECKING=False",
-        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
-        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
-        "ANSIBLE_STDOUT_CALLBACK=debug"
-    ]
+    ansible_env_vars   = local.ansible_env_vars
     command            = "./common-packer/ansible-playbook.sh"
     extra_arguments    = local.ssh_extra_args
     playbook_file   = "provision/local-windows-builder.yaml"
