Skip to content

Commit

Permalink
parser/command: Parse and display sync events.
Browse files Browse the repository at this point in the history 
Just the basic functionality and 2 tests. Output includes the duration, but not (yet) any other info
like sync method or source or target. Will at least tweak the CLI and add outputs in more commands.
  • Loading branch information
@vincentdephily
vincentdephily committed Sep 5, 2018
1 parent e1241de commit 8ca594a
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 33 deletions.
10 changes: 7 additions & 3 deletions COMPARISON.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,12 @@ outputs more compact. Golop has a fairly spartan look: compact, machine-like, no
Emlop has a specific mode deticated to stats whereas {q,pq,go}lop include that at the end of other
outputs.

Pqlop doesn't handle newer log format for sync events.

| | genlop | qlop | emlop | pqlop | golop |
| :----------------------------------------------------------------- | :----: | :--: | :---: | :---: | :---: |
| Display sync and unmerges | yes | yes | no | yes | no |
| Display sync | yes | yes | yes | buggy | no |
| Display unmerges | yes | yes | no | yes | no |
| Display interrupted merges | no | no | no | yes | no |
| Display info about currently installed package like USE, CFLAGS... | yes | no | no | no | no |
| Display extra merge stats like total time/count, average... | no | yes | yes | yes | yes |
Expand Down Expand Up @@ -114,10 +117,11 @@ would estimate the resulting speedup factor.
| Show current merge | yes | yes | yes | yes | yes |
| Show current merge ETA | yes | no | yes | no | yes |
| Show current merge stage | no | no | no | yes | no |
| Show `emerge -p` merges global ETA | yes | no | yes | no | no |
| Show `emerge -p` merges | yes | no | yes | no | no |
| Show `emerge -p` merges global ETA | yes | n/a | yes | n/a | n/a |
| Show `emerge -p` merges individual ETAs | no | n/a | yes | n/a | n/a |
| Accuracy of time estimation | ok | n/a | good | n/a | ok |
| Query gentoo.linuxhowtos.org for unknown packages | yes | no | no | n/a | no |
| Query gentoo.linuxhowtos.org for unknown packages | yes | n/a | no | n/a | no |

## misc

Expand Down
26 changes: 24 additions & 2 deletions src/commands.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ use std::io::stdin;
pub fn cmd_list(args: &ArgMatches, subargs: &ArgMatches, st: Styles) -> Result<bool, Error> {
let hist = parser::new_hist(myopen(args.value_of("logfile").unwrap())?, args.value_of("logfile").unwrap(),
value_opt(args, "from", parse_date), value_opt(args, "to", parse_date),
true, subargs.is_present("sync"),
subargs.value_of("package"), subargs.is_present("exact"))?;
let mut started: HashMap<(String, String, String), i64> = HashMap::new();
let mut found_one = false;
let mut syncstart: i64 = 0;
for p in hist {
match p {
ParsedHist::Start{ts, ebuild, version, iter, ..} => {
Expand All @@ -27,6 +29,12 @@ pub fn cmd_list(args: &ArgMatches, subargs: &ArgMatches, st: Styles) -> Result<b
fmt_time(ts), st.dur_p, started.map_or(String::from("?"), |pt| fmt_duration(ts-pt)),
st.pkg_p, ebuild, version, st.pkg_s).unwrap_or(());
},
ParsedHist::SyncStart{ts} => {
syncstart = ts;
},
ParsedHist::SyncStop{ts} => {
writeln!(io::stdout(), "{} {}{:>9}{} Sync", fmt_time(ts), st.dur_p, fmt_duration(ts-syncstart), st.dur_s).unwrap_or(());
},
}
}
Ok(found_one)
Expand All @@ -39,6 +47,7 @@ pub fn cmd_list(args: &ArgMatches, subargs: &ArgMatches, st: Styles) -> Result<b
pub fn cmd_stats(tw: &mut TabWriter<io::Stdout>, args: &ArgMatches, subargs: &ArgMatches, st: Styles) -> Result<bool, Error> {
let hist = parser::new_hist(myopen(args.value_of("logfile").unwrap())?, args.value_of("logfile").unwrap(),
value_opt(args, "from", parse_date), value_opt(args, "to", parse_date),
true, false,
subargs.value_of("package"), subargs.is_present("exact"))?;
let lim = value(subargs, "limit", parse_limit);
let mut started: HashMap<(String, String, String), i64> = HashMap::new();
Expand All @@ -54,6 +63,8 @@ pub fn cmd_stats(tw: &mut TabWriter<io::Stdout>, args: &ArgMatches, subargs: &Ar
timevec.insert(0, ts-start_ts);
}
},
ParsedHist::SyncStart{..} => (),
ParsedHist::SyncStop{..} => (),
}
};
let found_one = !times.is_empty();
Expand Down Expand Up @@ -93,6 +104,7 @@ pub fn cmd_predict(tw: &mut TabWriter<io::Stdout>, args: &ArgMatches, subargs: &
// Parse emerge log.
let hist = parser::new_hist(myopen(args.value_of("logfile").unwrap())?, args.value_of("logfile").unwrap(),
value_opt(args, "from", parse_date), value_opt(args, "to", parse_date),
true, false,
None, false)?;
let mut started: HashMap<(String, String), i64> = HashMap::new();
let mut times: HashMap<String, Vec<i64>> = HashMap::new();
Expand All @@ -101,13 +113,15 @@ pub fn cmd_predict(tw: &mut TabWriter<io::Stdout>, args: &ArgMatches, subargs: &
// We're ignoring iter here (reducing the start->stop matching accuracy) because there's no iter in the pretend output.
ParsedHist::Start{ts, ebuild, version, ..} => {
started.insert((ebuild.clone(), version.clone()), ts);
}
},
ParsedHist::Stop{ts, ebuild, version, ..} => {
if let Some(start_ts) = started.remove(&(ebuild.clone(), version.clone())) {
let timevec = times.entry(ebuild.clone()).or_insert_with(|| vec![]);
timevec.insert(0, ts-start_ts);
}
}
},
ParsedHist::SyncStart{..} => (),
ParsedHist::SyncStop{..} => (),
}
}

Expand Down Expand Up @@ -201,6 +215,14 @@ mod tests {
2018-02-27 15:10:05 +00:00 43 media-libs/mlt-6.4.1-r6\n\
2018-02-27 16:48:40 +00:00 39 media-libs/mlt-6.4.1-r6\n"),
0),
// Check output of sync events
(&["-f","test/emerge.10000.log","l","--sync","--from","2018-03-07 10:42:00","--to","2018-03-07 14:00:00"],
indoc!("2018-03-07 10:43:10 +00:00 14 sys-apps/the_silver_searcher-2.0.0\n\
2018-03-07 11:37:05 +00:00 38 Sync\n\
2018-03-07 12:49:13 +00:00 1:01 sys-apps/util-linux-2.30.2-r1\n\
2018-03-07 13:56:09 +00:00 40 Sync\n\
2018-03-07 13:59:41 +00:00 24 dev-libs/nspr-4.18\n"),
0),
];
for (a,o,e) in t {
match e {
Expand Down
6 changes: 6 additions & 0 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ fn main() {
.long_help("Match package with a string instead of a regex. \
Regex is case-insensitive and matches on category/name (see https://docs.rs/regex/0.2.10/regex/index.html#syntax). \
String is case-sentitive and matches on whole name, or whole category/name if it contains a /.");//FIXME auto crate version
let arg_sync = Arg::with_name("sync")
.short("s")
.long("sync")
.help("Display sync history.")
.long_help("Display portage tree sync history in addition to package merges.");
let args = App::new("emlop")
.version(crate_version!())
.global_setting(AppSettings::ColoredHelp)
Expand Down Expand Up @@ -102,6 +107,7 @@ Accepts string like '2018-03-04', '2018-03-04 12:34:56', 'march', '1 month ago',
.long_about("Show list of completed merges.\n\
Merge date, merge time, package name-version.")
.help_message("Prints help information. Use --help for more details.")
.arg(&arg_sync)
.arg(&arg_exact)
.arg(&arg_pkg))
.subcommand(SubCommand::with_name("predict")
Expand Down
99 changes: 71 additions & 28 deletions src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,29 @@ use regex::{Regex, RegexBuilder};
use std;
use std::{io::{BufRead, BufReader, Read}, thread};

/// Items returned by `new_hist()`.
/// Items sent on the channel returned by `new_hist()`.
#[derive(Debug)]
pub enum ParsedHist {
/// Emerge started (might never complete)
/// Merge started (might never complete).
Start{ts: i64, ebuild: String, version: String, iter: String},
/// Emerge completed
/// Merge completed.
Stop{ts: i64, ebuild: String, version: String, iter: String},
/// Sync started (might never complete).
SyncStart{ts: i64},
/// Sync completed.
SyncStop{ts: i64},
}

/// Items returned by `new_pretend()`.
/// Items sent on the channel returned by `new_pretend()`.
#[derive(Debug)]
pub struct ParsedPretend{pub ebuild: String, pub version: String}

/// Parse emerge log into a channel of `Parsed` enums.
pub fn new_hist<R: Read>(reader: R, filename: &str, min_ts: Option<i64>, max_ts: Option<i64>, search_str: Option<&str>, search_exact: bool) -> Result<Receiver<ParsedHist>, Error> where R: std::marker::Send+'static {
pub fn new_hist<R: Read>(reader: R, filename: &str,
min_ts: Option<i64>, max_ts: Option<i64>,
parse_merge: bool, parse_sync: bool,
search_str: Option<&str>, search_exact: bool)
-> Result<Receiver<ParsedHist>, Error> where R: std::marker::Send+'static {
debug!("new_hist input={} min={:?} max={:?} str={:?} exact={}", filename, min_ts, max_ts, search_str, search_exact);
let (tx, rx): (Sender<ParsedHist>, Receiver<ParsedHist>) = unbounded();
let filter_ts = filter_ts_fn(min_ts, max_ts);
Expand All @@ -35,8 +43,10 @@ pub fn new_hist<R: Read>(reader: R, filename: &str, min_ts: Option<i64>, max_ts:
match l {
Ok(ref line) => { // Got a line, see if one of the funs match it
if let Some((t,s)) = parse_ts(line, &filter_ts) {
if let Some(found) = parse_start(t, s, &filter_pkg) {tx.send(found)}
else if let Some(found) = parse_stop(t, s, &filter_pkg) {tx.send(found)}
if let Some(found) = parse_start(parse_merge, t, s, &filter_pkg) {tx.send(found)}
else if let Some(found) = parse_stop(parse_merge, t, s, &filter_pkg) {tx.send(found)}
else if let Some(found) = parse_syncstart(parse_sync, t, s) {tx.send(found)}
else if let Some(found) = parse_syncstop(parse_sync, t, s) {tx.send(found)}
}
},
Err(e) => // Could be invalid UTF8, system read error...
Expand Down Expand Up @@ -134,8 +144,8 @@ fn parse_ts(line: &str, filter_ts: impl Fn(i64) -> bool) -> Option<(i64,&str)> {
if !(filter_ts)(ts) {return None}
Some((ts,&rest[1..]))
}
fn parse_start(ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option<ParsedHist> {
if !line.starts_with(" >>> emer") {return None}
fn parse_start(enabled: bool, ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option<ParsedHist> {
if !enabled || !line.starts_with(" >>> emer") {return None}
let mut tokens = line.split_whitespace(); //https://github.com/rust-lang/rust/issues/48656
let (t3,t5,t6) = (tokens.nth(2)?, tokens.nth(1)?, tokens.nth(0)?);
let (ebuild,version) = split_atom(t6)?;
Expand All @@ -145,8 +155,8 @@ fn parse_start(ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option
iter: format!("{} {}", t3, t5),
version: version.to_string()})
}
fn parse_stop(ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option<ParsedHist> {
if !line.starts_with(" ::: comp") {return None}
fn parse_stop(enabled: bool, ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option<ParsedHist> {
if !enabled || !line.starts_with(" ::: comp") {return None}
let mut tokens = line.split_whitespace();
let (t4,t6,t7) = (tokens.nth(3)?, tokens.nth(1)?, tokens.nth(0)?);
let (ebuild,version) = split_atom(t7)?;
Expand All @@ -156,6 +166,15 @@ fn parse_stop(ts: i64, line: &str, filter_pkg: impl Fn(&str) -> bool) -> Option<
iter: format!("{} {}", t4, t6),
version: version.to_string()})
}
fn parse_syncstart(enabled: bool, ts: i64, line: &str) -> Option<ParsedHist> {
if !enabled || line != " === sync" {return None}
Some(ParsedHist::SyncStart{ts})
}
fn parse_syncstop(enabled: bool, ts: i64, line: &str) -> Option<ParsedHist> {
// Old portage logs 'completed with <source>', new portage logs 'completed for <destination>'
if !enabled || !line.starts_with(" === Sync completed") {return None}
Some(ParsedHist::SyncStop{ts})
}
fn parse_pretend(line: &str, re: &Regex) -> Option<ParsedPretend> {
let c = re.captures(line)?;
Some(ParsedPretend{ebuild: c.get(1).unwrap().as_str().to_string(),
Expand All @@ -172,10 +191,11 @@ mod tests {
/// This checks parsing the given emerge.log.
fn parse_hist(filename: &str, mints: i64, maxts: i64,
filter_mints: Option<i64>, filter_maxts: Option<i64>,
parse_merge: bool, parse_sync: bool,
filter_pkg: Option<&str>, exact: bool,
expect_counts: Vec<(&str, usize)>) {
// Setup
let hist = new_hist(File::open(filename).unwrap(), filename.into(), filter_mints, filter_maxts, filter_pkg, exact).unwrap();
let hist = new_hist(File::open(filename).unwrap(), filename.into(), filter_mints, filter_maxts, parse_merge, parse_sync, filter_pkg, exact).unwrap();
let re_atom = Regex::new("^[a-z0-9-]+/[a-zA-Z0-9_+-]+$").unwrap(); //FIXME use catname.txt
let re_version = Regex::new("^[0-9][0-9a-z._-]*$").unwrap(); //Should match pattern used in *Parser
let re_iter = Regex::new("^\\([1-9][0-9]* [1-9][0-9]*\\)$").unwrap(); //Should match pattern used in *Parser
Expand All @@ -185,6 +205,8 @@ mod tests {
let (kind, ts, ebuild, version, iter) = match p {
ParsedHist::Start{ts, ebuild, version, iter} => ("start", ts, ebuild, version, iter),
ParsedHist::Stop{ts, ebuild, version, iter} => ("stop", ts, ebuild, version, iter),
ParsedHist::SyncStart{ts} => ("syncstart", ts, "c/e".into(), "1".into(), "(1 1)".into()),
ParsedHist::SyncStop{ts} => ("syncstop", ts, "c/e".into(), "1".into(), "(1 1)".into()),
};
*counts.entry(kind.to_string()).or_insert(0) += 1;
*counts.entry(ebuild.clone()).or_insert(0) += 1;
Expand All @@ -204,41 +226,46 @@ mod tests {
/// Simplified emerge log containing all the ebuilds in all the versions of the current portage tree (see test/generate.sh)
fn parse_hist_all() {
parse_hist("test/emerge.all.log", 1483228800, 1483747200,
None, None, None, false, vec![("start",37415),("stop",37415)]);
None, None, true, false, None, false,
vec![("start",37415),("stop",37415)]);
}

#[test]
/// Emerge log with various invalid data
fn parse_hist_nullbytes() {
parse_hist("test/emerge.nullbytes.log", 1327867709, 1327871057,
None, None, None, false, vec![("start",14),("stop",14)]);
None, None, true, false, None, false,
vec![("start",14),("stop",14)]);
}
#[test]
/// Emerge log with various invalid data
fn parse_hist_badtimestamp() {
parse_hist("test/emerge.badtimestamp.log", 1327867709, 1327871057,
None, None, None, false, vec![("start",2),("stop",3),
("media-libs/jpeg",1), //letter in timestamp
("dev-libs/libical",2),
("media-libs/libpng",2)]);
None, None, true, false, None, false,
vec![("start",2),("stop",3),
("media-libs/jpeg",1), //letter in timestamp
("dev-libs/libical",2),
("media-libs/libpng",2)]);
}
#[test]
/// Emerge log with various invalid data
fn parse_hist_badversion() {
parse_hist("test/emerge.badversion.log", 1327867709, 1327871057,
None, None, None, false, vec![("start",3),("stop",2),
("media-libs/jpeg",2),
("dev-libs/libical",2),
("media-libs/libpng",1)]); //missing version
None, None, true, false, None, false,
vec![("start",3),("stop",2),
("media-libs/jpeg",2),
("dev-libs/libical",2),
("media-libs/libpng",1)]); //missing version
}
#[test]
/// Emerge log with various invalid data
fn parse_hist_shortline() {
parse_hist("test/emerge.shortline.log", 1327867709, 1327871057,
None, None, None, false, vec![("start",3),("stop",2),
("media-libs/jpeg",2),
("dev-libs/libical",1), //missing end of line and spaces in iter
("media-libs/libpng",2)]);
None, None, true, false, None, false,
vec![("start",3),("stop",2),
("media-libs/jpeg",2),
("dev-libs/libical",1), //missing end of line and spaces in iter
("media-libs/libpng",2)]);
}

#[test]
Expand All @@ -256,7 +283,8 @@ mod tests {
(Some("File-Next"), true, 1, 1), // case-sensitive
] {
parse_hist("test/emerge.10000.log", 1517609348, 1520891098,
None, None, f, e, vec![("start",c1),("stop",c2)]);
None, None, true, false, f, e,
vec![("start",c1),("stop",c2)]);
}
}

Expand All @@ -278,7 +306,22 @@ mod tests {
(Some(1517959010), Some(1518176159), 24, 21),
] {
parse_hist("test/emerge.10000.log", 1517609348, 1520891098,
min, max, None, true, vec![("start",c1),("stop",c2)]);
min, max, true, false, None, true,
vec![("start",c1),("stop",c2)]);
}
}

#[test]
/// Enabling and disabling sync
fn parse_hist_sync_merge() {
for (m,s,c1,c2,c3,c4) in vec![(true, true, 889, 832, 163, 150),
(false, true, 0, 0, 163, 150),
(true, false, 889, 832, 0, 0),
(false, false, 0, 0, 0, 0),
] {
parse_hist("test/emerge.10000.log", 1517609348, 1520891098,
None, None, m, s, None, false,
vec![("start",c1),("stop",c2),("syncstart",c3),("syncstop",c4)]);
}
}

Expand Down

0 comments on commit 8ca594a

Please sign in to comment.