ng client: support for prefix_targets_with_hash

Add support for prefixing targets with their hashes when downloading or
using HASH.FILENAME.EXT as target names.
The introduction of prefix_targets_with_hash was necessary, because
there are use cases like Warehouse where you could use
consistent_snapshot, but without adding a hash prefix to your targets.

When prefix_targets_with_hash is set to True, target files conforming
the format HASH.FILENAME.EXT will be downloaded from the server, but
they will be saved on the client side without their hash prefixes or
FILENAME.EXT.
This makes sure the client won't understand the usage of
prefix_targets_with_hash.

Still, if you want to use HASH.FILENAME.EXT as target names when
downloading, then additionally you need to provide consistent_snapshot
set to True in your root.json. The reason is that the specification uses
consistent_snapshot for the same purpose:
"If consistent snapshots are not used (see § 6.2 Consistent snapshots),
then the filename used to download the target file is of the fixed form
FILENAME.EXT (e.g., foobar.tar.gz). Otherwise, the filename is of the
form HASH.FILENAME.EXT
(e.g., c14aeb4ac9f4a8fc0d83d12482b9197452f6adf3eb710e3b1e2b79e8d14cb681.foobar.tar.gz),
where HASH is one of the hashes of the targets file listed in the
targets metadata file found earlier in step § 5.6 Update the targets role.
In either case, the client MUST write the file to non-volatile
storage as FILENAME.EXT."

The same behavior of using two flags is used in the legacy code when
calling tuf.client.updater.download_target() in a repository using
prefix_targets_with_hash and consistent_snapshot.

See chapter 5.7.3:
https://theupdateframework.github.io/specification/latest/index.html#fetch-target
By default, prefix_targets_with_hash is set to true to make it easier
to the user to provide uniquely identifiable targets file names by
using consistent_snapshot set to True.

Signed-off-by: Martin Vrachev <[email protected]>

Author: MVrachev

tests/test_updater_ng.py

@@ -15,7 +15,10 @@
 import tuf.unittest_toolbox as unittest_toolbox
 
 from tests import utils
+from tuf.api.metadata import Metadata
 from tuf import ngclient
+from securesystemslib.signer import SSlibSigner
+from securesystemslib.interface import import_rsa_privatekey_from_file
 
 logger = logging.getLogger(__name__)
 
@@ -94,13 +97,13 @@ def setUp(self):
     url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + repository_basepath
 
-    metadata_url = f"{url_prefix}/metadata/"
-    targets_url = f"{url_prefix}/targets/"
+    self.metadata_url = f"{url_prefix}/metadata/"
+    self.targets_url = f"{url_prefix}/targets/"
     # Creating a repository instance.  The test cases will use this client
     # updater to refresh metadata, fetch target files, etc.
     self.repository_updater = ngclient.Updater(self.client_directory,
-                                              metadata_url,
-                                              targets_url)
+                                              self.metadata_url,
+                                              self.targets_url)
 
   def tearDown(self):
     # We are inheriting from custom class.
@@ -109,14 +112,91 @@ def tearDown(self):
     # Logs stdout and stderr from the sever subprocess.
     self.server_process_handler.flush_log()
 
+  def _create_consistent_target(self, targetname: str, target_hash:str) -> None:
+    """Create consistent targets copies of their non-consistent counterparts
+    inside the repository directory.
+
+    Args:
+      targetname: A string denoting the name of the target file.
+      target_hash: A string denoting the hash of the target.
+
+    """
+    consistent_target_name = f"{target_hash}.{targetname}"
+    source_path = os.path.join(self.repository_directory, "targets", targetname)
+    destination_path = os.path.join(
+      self.repository_directory, "targets", consistent_target_name
+    )
+    shutil.copy(source_path, destination_path)
+
+
+  def _make_root_file_with_consistent_snapshot_true(self) -> None:
+    """Swap the existing root file inside the client directory with a new root
+    file where the consistent_snapshot is set to true."""
+    root_path = os.path.join(self.client_directory, "root.json")
+    root = Metadata.from_file(root_path)
+    root.signed.consistent_snapshot = True
+    root_key_path = os.path.join(self.keystore_directory, "root_key")
+    root_key_dict = import_rsa_privatekey_from_file(
+            root_key_path, password="password"
+    )
+    root_signer = SSlibSigner(root_key_dict)
+    root.sign(root_signer)
+    # Remove the old root file and replace it with the newer root file.
+    os.remove(root_path)
+    root.to_file(root_path)
+
+
+  def test_refresh_on_consistent_targets(self):
+    # Generate a new root file where consistent_snapshot is set to true and
+    # replace the old root metadata file with it.
+    self._make_root_file_with_consistent_snapshot_true()
+    self.repository_updater = ngclient.Updater(self.client_directory,
+                                              self.metadata_url,
+                                              self.targets_url)
+    # All metadata is in local directory already
+    self.repository_updater.refresh()
+
+    # Get targetinfo for "file1.txt" listed in targets
+    targetinfo1 = self.repository_updater.get_one_valid_targetinfo("file1.txt")
+    # Get targetinfo for "file3.txt" listed in the delegated role1
+    targetinfo3 = self.repository_updater.get_one_valid_targetinfo("file3.txt")
+
+    # Create consistent targets with file path HASH.FILENAME.EXT
+    target1_hash = list(targetinfo1["fileinfo"].hashes.values())[0]
+    target3_hash = list(targetinfo3["fileinfo"].hashes.values())[0]
+    self._create_consistent_target("file1.txt", target1_hash)
+    self._create_consistent_target("file3.txt", target3_hash)
+
+    destination_directory = self.make_temp_directory()
+    updated_targets = self.repository_updater.updated_targets(
+      [targetinfo1, targetinfo3], destination_directory
+    )
+
+    self.assertListEqual(updated_targets, [targetinfo1, targetinfo3])
+    self.repository_updater.download_target(targetinfo1, destination_directory)
+    updated_targets = self.repository_updater.updated_targets(
+      updated_targets, destination_directory
+    )
+
+    self.assertListEqual(updated_targets, [targetinfo3])
+
+    self.repository_updater.download_target(targetinfo3, destination_directory)
+    updated_targets = self.repository_updater.updated_targets(
+      updated_targets, destination_directory
+    )
+
+    self.assertListEqual(updated_targets, [])
+
   def test_refresh(self):
+    # Test refresh without consistent targets - targets without hash prefixes.
+
     # All metadata is in local directory already
     self.repository_updater.refresh()
 
     # Get targetinfo for 'file1.txt' listed in targets
-    targetinfo1 = self.repository_updater.get_one_valid_targetinfo('file1.txt')
+    targetinfo1 = self.repository_updater.get_one_valid_targetinfo("file1.txt")
     # Get targetinfo for 'file3.txt' listed in the delegated role1
-    targetinfo3= self.repository_updater.get_one_valid_targetinfo('file3.txt')
+    targetinfo3 = self.repository_updater.get_one_valid_targetinfo("file3.txt")
 
     destination_directory = self.make_temp_directory()
     updated_targets = self.repository_updater.updated_targets([targetinfo1, targetinfo3],
@@ -146,7 +226,7 @@ def test_refresh_with_only_local_root(self):
     self.repository_updater.refresh()
 
     # Get targetinfo for 'file3.txt' listed in the delegated role1
-    targetinfo3= self.repository_updater.get_one_valid_targetinfo('file3.txt')
+    targetinfo3 = self.repository_updater.get_one_valid_targetinfo('file3.txt')
 
 if __name__ == '__main__':
   utils.configure_test_logging(sys.argv)

tuf/ngclient/config.py

@@ -15,3 +15,8 @@ class UpdaterConfig:
     timestamp_max_length: int = 16384  # bytes
     snapshot_max_length: int = 2000000  # bytes
     targets_max_length: int = 5000000  # bytes
+    # We need this variable because there are use cases like Warehouse where
+    # you could use consistent_snapshot, but without adding a hash prefix.
+    # By default, prefix_targets_with_hash is set to true to use uniquely
+    # identifiable targets file names for repositories.
+    prefix_targets_with_hash: bool = True

tuf/ngclient/updater.py

@@ -252,8 +252,12 @@ def download_target(
         else:
             target_base_url = _ensure_trailing_slash(target_base_url)
 
-        target_filepath = targetinfo["filepath"]
         target_fileinfo: "TargetFile" = targetinfo["fileinfo"]
+        target_filepath = targetinfo["filepath"]
+        consistent_snapshot = self._trusted_set.root.signed.consistent_snapshot
+        if consistent_snapshot and self.config.prefix_targets_with_hash:
+            hashes = list(target_fileinfo.hashes.values())
+            target_filepath = f"{hashes[0]}.{target_filepath}"
         full_url = parse.urljoin(target_base_url, target_filepath)
 
         with self._fetcher.download_file(
@@ -266,8 +270,11 @@ def download_target(
                     f"{target_filepath} length or hashes do not match"
                 ) from e
 
-            filepath = os.path.join(destination_directory, target_filepath)
-            sslib_util.persist_temp_file(target_file, filepath)
+            # Store the target file name without the HASH prefix.
+            local_filepath = os.path.join(
+                destination_directory, targetinfo["filepath"]
+            )
+            sslib_util.persist_temp_file(target_file, local_filepath)
 
     def _download_metadata(
         self, rolename: str, length: int, version: Optional[int] = None