Merge pull request #1700 from cachix/devenv-generate

`devenv generate`

Author: domenkozar

Cargo.lock

@@ -25,10 +25,11 @@ xtask = { path = "xtask" }
 
 ansiterm = "0.12.2"
 blake3 = "1.5.4"
-clap = { version = "4.5.1", features = ["derive", "cargo"] }
+clap = { version = "4.5.1", features = ["derive", "cargo", "env"] }
 cli-table = "0.4.7"
 console = "0.15.8"
 dotlock = "0.5.0"
+dialoguer = "0.11.0"
 futures = "0.3.30"
 hex = "0.4.3"
 include_dir = "0.7.3"
@@ -40,7 +41,7 @@ nix = { version = "0.28.0", features = ["signal"] }
 petgraph = "0.6.5"
 pretty_assertions = { version = "1.4.0", features = ["unstable"] }
 regex = "1.10.3"
-reqwest = "0.11.26"
+reqwest = { version = "0.11.26", features = ["json", "stream"] }
 schemars = "0.8.16"
 schematic = { version = "0.14.3", features = [
     "schema",
@@ -72,6 +73,7 @@ tokio = { version = "1.39.3", features = [
 which = "6.0.0"
 whoami = "1.5.1"
 xdg = "2.5.2"
+tokio-tar = "0.3.1"
 
 # Always build optimized sqlx-macro to speed up query checks
 [profile.dev.package.sqlx-macros]

Cargo.toml

@@ -13,6 +13,26 @@
       },
       "parent": []
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1731738660,
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "e10ba121773f754a30d31b6163919a3e404a434f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -248,13 +268,30 @@
     "root": {
       "inputs": {
         "devenv": "devenv",
+        "fenix": "fenix",
         "git-hooks": "git-hooks",
         "nix": "nix",
         "nixpkgs": "nixpkgs_2",
         "pre-commit-hooks": [
           "git-hooks"
         ]
       }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1731827189,
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "ba56d9b9b5f7ae7311b4bd1cc47159d87eb033d0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
     }
   },
   "root": "root",

devenv.lock

@@ -20,6 +20,7 @@
   languages.nix.enable = true;
   # for cli
   languages.rust.enable = true;
+  languages.rust.channel = "stable";
   # for docs
   languages.python.enable = true;
   # it breaks glibc

devenv.nix

@@ -1,6 +1,11 @@
 # yaml-language-server: $schema=./docs/devenv.schema.json
 inputs:
-  devenv:
-    url: path:.?dir=src/modules
+  fenix:
+    url: github:nix-community/fenix
+    inputs:
+      nixpkgs:
+        follows: nixpkgs
   nix:
     url: github:domenkozar/nix/devenv-2.24
+  devenv:
+    url: path:.?dir=src/modules

devenv.yaml

@@ -17,6 +17,7 @@ nix-conf-parser.workspace = true
 clap.workspace = true
 cli-table.workspace = true
 console.workspace = true
+dialoguer.workspace = true
 dotlock.workspace = true
 hex.workspace = true
 include_dir.workspace = true
@@ -43,4 +44,8 @@ tracing-subscriber.workspace = true
 which.workspace = true
 whoami.workspace = true
 xdg.workspace = true
+tokio-tar.workspace = true
+tokio-util = { version = "0.7.12", features = ["io"] }
+similar = "2.6.0"
+binaryornot = "1.0.0"
 once_cell = "1.20.2"

devenv/Cargo.toml

@@ -184,6 +184,26 @@ pub enum Commands {
     #[command(about = "Scaffold devenv.yaml, devenv.nix, .gitignore and .envrc.")]
     Init { target: Option<PathBuf> },
 
+    #[command(about = "Generate devenv.yaml and devenv.nix using AI")]
+    Generate {
+        #[arg(num_args=0.., trailing_var_arg = true)]
+        description: Vec<String>,
+
+        #[clap(long, default_value = "https://devenv.new")]
+        host: String,
+
+        #[arg(
+            long,
+            help = "Paths to exclude during generation.",
+            value_name = "PATH"
+        )]
+        exclude: Vec<PathBuf>,
+
+        // https://consoledonottrack.com/
+        #[clap(long, env = "DO_NOT_TRACK", action = clap::ArgAction::SetTrue)]
+        disable_telemetry: bool,
+    },
+
     #[command(about = "Activate the developer environment. https://devenv.sh/basics/")]
     Shell {
         cmd: Option<String>,

devenv/src/cli.rs

@@ -3,12 +3,13 @@ use clap::crate_version;
 use cli_table::Table;
 use cli_table::{print_stderr, WithTitle};
 use include_dir::{include_dir, Dir};
-use miette::{bail, Result};
+use miette::{bail, IntoDiagnostic, Result};
 use nix::sys::signal;
 use nix::unistd::Pid;
 use once_cell::sync::Lazy;
 use serde::Deserialize;
 use sha2::Digest;
+use similar::{ChangeTag, TextDiff};
 use std::collections::HashMap;
 use std::io::Write;
 use std::os::unix::{fs::PermissionsExt, process::CommandExt};
@@ -154,14 +155,6 @@ impl Devenv {
             std::fs::create_dir_all(&target).expect("Failed to create target directory");
         }
 
-        // fails if any of the required files already exists
-        for filename in REQUIRED_FILES {
-            let file_path = target.join(filename);
-            if file_path.exists() && !EXISTING_REQUIRED_FILES.contains(&filename) {
-                bail!("File already exists {}", file_path.display());
-            }
-        }
-
         for filename in REQUIRED_FILES {
             info!("Creating {}", filename);
 
@@ -183,7 +176,15 @@ impl Devenv {
                     })
                     .expect("Failed to append to existing file");
             } else {
-                std::fs::write(&target_path, path.contents()).expect("Failed to write file");
+                if target_path.exists() && !EXISTING_REQUIRED_FILES.contains(&filename) {
+                    if let Some(utf8_contents) = path.contents_utf8() {
+                        confirm_overwrite(&target_path, utf8_contents.to_string())?;
+                    } else {
+                        bail!("Failed to read file contents as UTF-8");
+                    }
+                } else {
+                    std::fs::write(&target_path, path.contents()).expect("Failed to write file");
+                }
             }
         }
 
@@ -200,6 +201,120 @@ impl Devenv {
         Ok(())
     }
 
+    pub async fn generate(
+        &mut self,
+        description: Option<String>,
+        host: &str,
+        exclude: Vec<PathBuf>,
+        disable_telemetry: bool,
+    ) -> Result<()> {
+        let client = reqwest::Client::new();
+        let mut request = client
+            .post(host)
+            .query(&[("disable_telemetry", disable_telemetry)])
+            .header(reqwest::header::USER_AGENT, crate_version!());
+
+        let (asyncwriter, asyncreader) = tokio::io::duplex(256 * 1024);
+        let streamreader = tokio_util::io::ReaderStream::new(asyncreader);
+
+        let (body_sender, body) = match description {
+            Some(desc) => {
+                request = request.query(&[("q", desc)]);
+                (None, None)
+            }
+            None => {
+                let git_output = std::process::Command::new("git")
+                    .args(["ls-files", "-z"])
+                    .output()
+                    .map_err(|_| {
+                        miette::miette!("Failed to get list of files from git ls-files")
+                    })?;
+
+                let files = String::from_utf8_lossy(&git_output.stdout)
+                    .split('\0')
+                    .filter(|s| !s.is_empty())
+                    .filter(|s| !binaryornot::is_binary(s).unwrap_or(false))
+                    .map(PathBuf::from)
+                    .collect::<Vec<_>>();
+
+                if files.is_empty() {
+                    warn!("No files found. Are you in a git repository?");
+                    return Ok(());
+                }
+
+                if let Some(stderr) = String::from_utf8(git_output.stderr).ok() {
+                    if !stderr.is_empty() {
+                        warn!("{}", &stderr);
+                    }
+                }
+
+                let body = reqwest::Body::wrap_stream(streamreader);
+
+                request = request
+                    .body(body)
+                    .header(reqwest::header::CONTENT_TYPE, "application/x-tar");
+
+                (Some(tokio_tar::Builder::new(asyncwriter)), Some(files))
+            }
+        };
+
+        info!("Generating devenv.nix and devenv.yaml, this should take about a minute ...");
+
+        let response_future = request.send();
+
+        let tar_task = async {
+            if let (Some(mut builder), Some(files)) = (body_sender, body) {
+                for path in files {
+                    if path.is_file() && !exclude.iter().any(|exclude| path.starts_with(exclude)) {
+                        builder.append_path(&path).await?;
+                    }
+                }
+                builder.finish().await?;
+            }
+            Ok::<(), std::io::Error>(())
+        };
+
+        let (response, _) = tokio::join!(response_future, tar_task);
+
+        let response = response.into_diagnostic()?;
+        let status = response.status();
+        if !status.is_success() {
+            let error_text = &response
+                .text()
+                .await
+                .unwrap_or_else(|_| "No error details available".to_string());
+            bail!(
+                "Failed to generate (HTTP {}): {}",
+                &status.as_u16(),
+                match serde_json::from_str::<serde_json::Value>(error_text) {
+                    Ok(json) => json["message"]
+                        .as_str()
+                        .map(String::from)
+                        .unwrap_or_else(|| error_text.clone()),
+                    Err(_) => error_text.clone(),
+                }
+            );
+        }
+
+        let response_json: GenerateResponse = response.json().await.expect("Failed to parse JSON.");
+
+        confirm_overwrite(Path::new("devenv.nix"), response_json.devenv_nix)?;
+        confirm_overwrite(Path::new("devenv.yaml"), response_json.devenv_yaml)?;
+
+        info!(
+            "{}",
+            indoc::formatdoc!("
+              Generated devenv.nix and devenv.yaml 🎉
+
+              Treat these as templates and open an issue at https://github.com/cachix/devenv/issues if you think we can do better!
+
+              Start by running:
+
+                $ devenv shell
+            "));
+        Ok(())
+    }
+
     pub fn inputs_add(&mut self, name: &str, url: &str, follows: &[String]) -> Result<()> {
         self.config.add_input(name, url, follows);
         self.config.write();
@@ -893,6 +1008,46 @@ impl Devenv {
     }
 }
 
+fn confirm_overwrite(file: &Path, contents: String) -> Result<()> {
+    if std::fs::metadata(file).is_ok() {
+        // first output the old version and propose new changes
+        let before = std::fs::read_to_string(file).expect("Failed to read file");
+
+        let diff = TextDiff::from_lines(&before, &contents);
+
+        println!("\nChanges that will be made to {}:", file.to_string_lossy());
+        for change in diff.iter_all_changes() {
+            let sign = match change.tag() {
+                ChangeTag::Delete => "\x1b[31m-\x1b[0m",
+                ChangeTag::Insert => "\x1b[32m+\x1b[0m",
+                ChangeTag::Equal => " ",
+            };
+            print!("{}{}", sign, change);
+        }
+
+        let confirm = dialoguer::Confirm::new()
+            .with_prompt(format!(
+                "{} already exists. Do you want to overwrite it?",
+                file.to_string_lossy()
+            ))
+            .interact()
+            .into_diagnostic()?;
+
+        if confirm {
+            std::fs::write(file, contents).into_diagnostic()?;
+        }
+    } else {
+        std::fs::write(file, contents).into_diagnostic()?;
+    }
+    Ok(())
+}
+
+#[derive(Deserialize)]
+struct GenerateResponse {
+    devenv_nix: String,
+    devenv_yaml: String,
+}
+
 pub struct DevEnv {
     output: Vec<u8>,
     gc_root: PathBuf,

devenv/src/devenv.rs

@@ -132,6 +132,25 @@ async fn main() -> Result<()> {
             Ok(())
         }
         Commands::Init { target } => devenv.init(&target),
+        Commands::Generate {
+            description,
+            host,
+            exclude,
+            disable_telemetry,
+        } => {
+            devenv
+                .generate(
+                    if description.is_empty() {
+                        None
+                    } else {
+                        Some(description.join(" "))
+                    },
+                    &host,
+                    exclude,
+                    disable_telemetry,
+                )
+                .await
+        }
         Commands::Search { name } => devenv.search(&name).await,
         Commands::Gc {} => devenv.gc(),
         Commands::Info {} => devenv.info().await,