Open binary files in binary mode and do not encode them (#829)

* open binary files in binary mode and do not encode them

* reveal base64 string if binary data, otherwise decode

* fix linting, use raw data if binary for all cases

* handle None MIME type as non-binary

* provide --binary command line parameter

Co-authored-by: Ricardo Amaro <[email protected]>

Author: Tymolc

kapitan/cli.py

@@ -401,6 +401,12 @@ def main():
         action="store_true",
         default=from_dot_kapitan("refs", "base64", False),
     )
+    refs_parser.add_argument(
+        "--binary",
+        help="file content should be handled as binary data",
+        action="store_true",
+        default=from_dot_kapitan("refs", "binary", False),
+    )
     refs_parser.add_argument(
         "--reveal",
         "-r",

kapitan/refs/base64.py

@@ -40,7 +40,11 @@ def __init__(self, data, from_base64=False, **kwargs):
 
     def reveal(self):
         # TODO data should be bytes only
-        return base64.b64decode(self.data).decode()
+        decoded = base64.b64decode(self.data)
+        try:
+            return decoded.decode()
+        except:
+            return self.data
 
     def compile_embedded(self):
         dump = self.dump()

kapitan/refs/cmd_parser.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import sys
+import mimetypes
 
 from kapitan.errors import KapitanError, RefHashMismatchError
 from kapitan.refs.base import PlainRef, RefController, Revealer
@@ -37,6 +38,7 @@ def ref_write(args, ref_controller):
     "Write ref to ref_controller based on cli args"
     token_name = args.write
     file_name = args.file
+    is_binary = args.binary
     data = None
 
     if file_name is None:
@@ -46,8 +48,13 @@ def ref_write(args, ref_controller):
         for line in sys.stdin:
             data += line
     else:
-        with open(file_name) as fp:
-            data = fp.read()
+        mime_type = mimetypes.guess_type(file_name)
+        modifier = "rb" if is_binary else "r"
+        with open(file_name, modifier) as fp:
+            try:
+                data = fp.read()
+            except UnicodeDecodeError as e:
+                raise KapitanError("Could not read file. Please add '--binary' if the file contains binary data. ({})".format(e))
 
     if token_name.startswith("gpg:"):
         type_name, token_path = token_name.split(":")
@@ -167,7 +174,7 @@ def ref_write(args, ref_controller):
 
     elif token_name.startswith("base64:"):
         type_name, token_path = token_name.split(":")
-        _data = data.encode()
+        _data = data if is_binary else data.encode()
         encoding = "original"
         if args.base64:
             _data = base64.b64encode(_data).decode()
@@ -179,7 +186,7 @@ def ref_write(args, ref_controller):
 
     elif token_name.startswith("vaultkv:"):
         type_name, token_path = token_name.split(":")
-        _data = data.encode()
+        _data = data if is_binary else data.encode()
         vault_params = {}
         encoding = "original"
         if args.target_name:
@@ -213,7 +220,7 @@ def ref_write(args, ref_controller):
 
     elif token_name.startswith("plain:"):
         type_name, token_path = token_name.split(":")
-        _data = data.encode()
+        _data = data if is_binary else data.encode()
         encoding = "original"
         if args.base64:
             _data = base64.b64encode(_data).decode()
@@ -225,7 +232,7 @@ def ref_write(args, ref_controller):
 
     elif token_name.startswith("env:"):
         type_name, token_path = token_name.split(":")
-        _data = data.encode()
+        _data = data if is_binary else data.encode()
         encoding = "original"
         if args.base64:
             _data = base64.b64encode(_data).decode()