k8s: stream config data in chunks to aoid pipe buffer limits

src/backends/kubernetes/kubernetes_backend.py

@@ -778,7 +778,15 @@ def write_service_config(self, service_name: str, tar_buffer: IO[bytes], destina
             tty=False,
             _preload_content=False
         )
-        resp.write_stdin(tar_buffer.getvalue())
+        # Stream data in chunks, otherwise this breaks at 196608 bytes
+        # https://unix.stackexchange.com/questions/11946/how-big-is-the-pipe-buffer
+        # https://stackoverflow.com/a/53904789/1653320
+        tar_data = tar_buffer.getvalue()
+        chunk_size = 1024 * 32  # 32 kb
+        for i in range(0, len(tar_data), chunk_size):
+            chunk = tar_data[i:i+chunk_size]
+            resp.write_stdin(chunk)
+            self.log.info(f"Wrote {len(chunk)} bytes")
         resp.close()
         self.log.info(f"Finished writing tarball for {service_name}, unpacking...")
         # Unpack the archive