disable LRU cache if we have to get statistics

This slows down the algorithm by 20% or more, but at least shows
correct stats.

This issue will go away once we can stream packs.

Author: Byron

README.md

@@ -68,6 +68,8 @@ The CLI uses various crates, please see _'Development Status'_ for details.
     * [ ] encode
       * [ ] create new packs
     * [x] verify pack with statistics
+    * [ ] pack streaming (i.e. indexing + resolution)
+      * [ ] use pack streaming for verification for performance and correctness
   * [ ] API documentation with examples
 * **git-transport**
   * [ ] via ssh

git-odb/src/pack/index/verify.rs

@@ -1,4 +1,4 @@
-use crate::pack::DecodeEntryResult;
+use crate::pack::{cache, DecodeEntryResult};
 use crate::{pack, pack::index};
 use git_features::progress::{self, Progress};
 use git_object::SHA1_SIZE;
@@ -88,16 +88,18 @@ impl index::File {
     /// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.
     /// It will be used to validate internal integrity of the pack before checking each objects integrity
     /// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.
-    pub fn verify_checksum_of_index<P>(
+    pub fn verify_checksum_of_index<P, C>(
         &self,
         pack: Option<&pack::File>,
         progress: Option<P>,
+        make_cache: impl Fn() -> C + Send + Sync,
     ) -> Result<(git_object::Id, Option<PackFileChecksumResult>), ChecksumError>
     where
         P: Progress,
         <P as Progress>::SubProgress: Send,
+        C: cache::DecodeEntry,
     {
-        use crate::pack::{cache, ResolvedBase};
+        use crate::pack::ResolvedBase;
         use git_features::parallel::{self, in_parallel_if};
 
         let mut root = progress::DoOrDiscard::from(progress);
@@ -224,7 +226,7 @@ impl index::File {
                     .init(Some(self.num_objects()), Some("objects"));
                 let state_per_thread = |index| {
                     (
-                        cache::DecodeEntryLRU::default(),
+                        make_cache(),
                         Vec::with_capacity(2048),
                         reduce_progress
                             .lock()

git-odb/tests/pack/index.rs

@@ -103,6 +103,7 @@ mod method {
 
 use common_macros::b_tree_map;
 use git_features::progress::Discard;
+use git_odb::pack::cache::DecodeEntryNoop;
 
 #[test]
 fn pack_lookup() {
@@ -113,14 +114,19 @@ fn pack_lookup() {
             index::PackFileChecksumResult {
                 average: DecodeEntryResult {
                     kind: git_object::Kind::Tree,
-                    num_deltas: 0,
+                    num_deltas: 1,
                     decompressed_size: 3456,
                     compressed_size: 1725,
                     object_size: 9621,
                 },
                 objects_per_chain_length: b_tree_map! {
                     0 => 18,
-                    1 => 12
+                    1 => 4,
+                    2 => 3,
+                    3 => 1,
+                    4 => 2,
+                    5 => 1,
+                    6 => 1,
                 },
                 total_compressed_entries_size: 51753,
                 total_decompressed_entries_size: 103701,
@@ -162,7 +168,8 @@ fn pack_lookup() {
                 },
                 objects_per_chain_length: b_tree_map! {
                     0 => 30,
-                    1 => 12
+                    1 => 6,
+                    2 => 6,
                 },
                 total_compressed_entries_size: 3604,
                 total_decompressed_entries_size: 4997,
@@ -177,7 +184,7 @@ fn pack_lookup() {
         assert_eq!(pack.kind(), pack::Kind::V2);
         assert_eq!(pack.num_objects(), idx.num_objects());
         assert_eq!(
-            idx.verify_checksum_of_index(Some(&pack), Discard.into())
+            idx.verify_checksum_of_index(Some(&pack), Discard.into(), || DecodeEntryNoop)
                 .unwrap(),
             (idx.checksum_of_index(), Some(stats.to_owned()))
         );
@@ -221,7 +228,8 @@ fn iter() {
         assert_eq!(idx.version(), *version);
         assert_eq!(idx.num_objects(), *num_objects);
         assert_eq!(
-            idx.verify_checksum_of_index(None, Discard.into()).unwrap(),
+            idx.verify_checksum_of_index(None, Discard.into(), || DecodeEntryNoop)
+                .unwrap(),
             (idx.checksum_of_index(), None)
         );
         assert_eq!(idx.checksum_of_index(), hex_to_id(index_checksum));

gitoxide-core/src/lib.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Context, Result};
 use bytesize::ByteSize;
 use git_features::progress::Progress;
+use git_object::Kind;
 use git_odb::pack::{self, index};
 use std::{io, path::Path};
 
@@ -38,7 +39,33 @@ where
                     Err(e)
                 })
                 .ok();
-            idx.verify_checksum_of_index(pack.as_ref(), progress)?
+            enum EitherCache {
+                Left(pack::cache::DecodeEntryNoop),
+                Right(pack::cache::DecodeEntryLRU),
+            };
+            impl pack::cache::DecodeEntry for EitherCache {
+                fn put(&mut self, offset: u64, data: &[u8], kind: Kind, compressed_size: usize) {
+                    match self {
+                        EitherCache::Left(v) => v.put(offset, data, kind, compressed_size),
+                        EitherCache::Right(v) => v.put(offset, data, kind, compressed_size),
+                    }
+                }
+
+                fn get(&mut self, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> {
+                    match self {
+                        EitherCache::Left(v) => v.get(offset, out),
+                        EitherCache::Right(v) => v.get(offset, out),
+                    }
+                }
+            }
+            idx.verify_checksum_of_index(pack.as_ref(), progress, || -> EitherCache {
+                if output_statistics {
+                    // turn off acceleration as we need to see entire chains all the time
+                    EitherCache::Left(pack::cache::DecodeEntryNoop)
+                } else {
+                    EitherCache::Right(pack::cache::DecodeEntryLRU::default())
+                }
+            })?
         }
         ext => {
             return Err(anyhow!(