apt-file: Add --stream-results parameter

Signed-off-by: Niels Thykier <[email protected]>

Author: nthykier

apt-file

@@ -296,6 +296,8 @@ sub do_grep($$) {
     my $matches = 0;
     my $quick_regexp = escape_parens($regexp);
     my $fd = open_data_pipeline($data);
+    my $stream_results = $Conf->{streaming};
+    tty_human_status('') if $stream_results;
     debug(1, "Pipeline open, waiting for input");
     while (<$fd>) {
 
@@ -305,13 +307,18 @@ sub do_grep($$) {
         next if !( ( $fname, $pkgs ) = /$regexp/o );
 
         debug_line ".";
-        # Put leading slash on file name
-        push(@{ $ret->{"/$fname"} },
-             # ... and extract the package name
-             map { basename($_) } split(m/,/, $pkgs)
-        );
-        if (++$matches % 10 == 0) {
-            tty_human_status("Searching, found $matches results so far ...");
+
+        if ($stream_results) {
+            stream_results("/$fname", map { basename($_) } split(m/,/, $pkgs));
+        } else {
+            # Put leading slash on file name
+            push(@{ $ret->{"/$fname"} },
+                 # ... and extract the package name
+                 map { basename($_) } split(m/,/, $pkgs)
+            );
+            if (++$matches % 10 == 0) {
+                tty_human_status("Searching, found $matches results so far ...");
+            }
         }
     }
     close($fd);
@@ -321,9 +328,22 @@ sub do_grep($$) {
         error("A subprocess exited uncleanly (raw: $?)");
     }
     debug(1, 'Read all input');
+    return if $stream_results;
     return reverse_hash($ret);
 }
 
+sub stream_results {
+    my ($filename, @pkgs) = @_;
+    if ($Conf->{package_only}) {
+        print map { "$_\n" } @pkgs;
+    } else {
+        for my $pkg (@pkgs) {
+            print "$pkg: $filename\n";
+        }
+    }
+    return;
+}
+
 sub escape_parens {
     my $pattern = shift;
 
@@ -577,6 +597,7 @@ Other options:
     --config           -c <file>        Parse the given APT config file [R]
     --option           -o <A::B>=<V>    Set the APT config option A::B to "V" [R]
     --package-only     -l               Only display packages name
+    --stream-results                    Emit results immediately (without deduplication)
     --verbose          -v               run in verbose mode [R]
     --help             -h               Show this help.
                        --               End of options (necessary if pattern
@@ -606,6 +627,7 @@ sub get_options() {
         "fixed-string|F"    => \$Conf->{fixed_strings},
         "from-file|f"       => \$Conf->{from_file},
         "from-deb|D"        => \$Conf->{from_deb},
+        'stream-results'    => \$Conf->{streaming},
         'filter-suites=s'   => sub { _set_option('in-suites', CONFIG_SEARCH_FILTER_SUITES, $_[1]); },
         'filter-origins=s'  => sub { _set_option('from-origin', CONFIG_SEARCH_FILTER_ORIGINS, $_[1]); },
         'config-file|c=s@'  => \&_parse_apt_config_file,

apt-file.pod

@@ -179,6 +179,21 @@ Ignore case when searching for pattern.
 
 Only display package name; do not display file names.
 
+=item --stream-results
+
+I<This is a special-usage option useful for dealing with searches that
+produce a high number of matches (10 000+) or/and automated processing
+of results.>
+
+Disable deduplication logic and immediately emit a result when a match
+is found.
+
+This can greatly reduce the memory requirements for apt-file when
+processing searches with many matches.  It will also reduce the time
+until the first match is emitted, which can be useful if matches can
+be processed as they are discovered and the consumer can handle
+duplicated matches.
+
 =item -o, --option APT::Option=Value
 
 Set a Configuration Option; This will set an arbitrary configuration

debian/changelog

@@ -1,8 +1,15 @@
-apt-file (3.2.3) UNRELEASED; urgency=medium
+apt-file (3.3) UNRELEASED; urgency=medium
 
+  [ Debian Janitor ]
   * Drop no longer supported add-log-mailing-address setting from
     debian/changelog.
 
+  [ Niels Thykier ]
+  * Add new --stream-results argument to apt-file.  This option
+    will make apt-file emit results as soon as they are found
+    but also emit duplicate results in many cases.  It can greatly
+    reduce memory usage and runtime spent up to first result.
+
  -- Debian Janitor <[email protected]>  Fri, 27 Dec 2019 01:58:57 +0000
 
 apt-file (3.2.2) unstable; urgency=medium