[ARCH-46] custom editor types (#33)

* initial swing at meta fields

* make meta values required

* test deep fields

* add custom editor types to manifest parsing

* better error messages in manifest parsing, nice display for editor types

* improve error message for invalid nested validator

* implement field validation

* fix recursive type def for ObjectDefinition

* add support for new path/field events, don't validate when caching

* fix missing editor_url in parsed manifest

* remove FieldValue::None

* make field value optional, remove fields when set to nothing

* make from_mime accept a string, move guess

* add clone to events

* add type override for file DisplayType for better type checking

* add get_children to value path, add method for updating URL to file type, more helpers

* add --no-deps to clippy

* move no-deps to clippy

* attempt to match rust-toolchain in ci

* clean up CI scripts

* update time dependency

.github/workflows/rust-ci.yml

@@ -18,51 +18,51 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          targets: wasm32-unknown-unknown
+          components: rustc-dev, clippy, rustfmt
       - uses: Swatinem/rust-cache@v2
 
-      # make sure all code has been formatted with rustfmt
-      - run: rustup component add rustfmt
       - name: check rustfmt
         run: cargo fmt -- --check --color always
 
-      # run clippy to verify we have no warnings
-      - run: rustup component add clippy
       - run: cargo fetch
       - name: cargo clippy
-        run: cargo clippy --all-features --all-targets -- -D warnings
+        run: cargo clippy --all-features --all-targets -- --no-deps -D warnings
 
   test:
     name: Tests
     strategy:
       matrix:
-        os: [
-            macOS-12,
-            # windows-2022,
-            ubuntu-22.04,
-          ]
+        os: [macOS-12, windows-2022, ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          targets: wasm32-unknown-unknown
+          components: rustc-dev, clippy, rustfmt
       - uses: Swatinem/rust-cache@v2
       - run: cargo fetch
       - name: cargo test build
         run: cargo build --tests --release
       - name: run tests
         run: ./test.sh
 
-  # msrv-check:
-  #   name: Minimum Stable Rust Version Check
-  #   runs-on: ubuntu-22.04
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - uses: dtolnay/rust-toolchain@master
-  #       with:
-  #         toolchain: "1.65.0"
-  #     - uses: Swatinem/rust-cache@v2
-  #     - run: cargo fetch
-  #     - name: cargo check
-  #       run: cargo check --all-targets
+  msrv-check:
+    name: Minimum Stable Rust Version Check
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "1.65.0"
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo fetch
+      - name: cargo check
+        run: cargo check --all-targets
 
   build-aarch64-apple-darwin:
     name: Build aarch64-apple-darwin
@@ -97,22 +97,6 @@ jobs:
   #     - uses: Swatinem/rust-cache@v2
   #     - run: cargo run --release --target x86_64-unknown-linux-musl -- generate --fail about.hbs
 
-  # Build `mdBook` documentation and upload it as a temporary build artifact
-  # doc-book:
-  #   name: Build the book
-  #   runs-on: ubuntu-22.04
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - run: |
-  #         set -e
-  #         curl -L https://github.com/rust-lang-nursery/mdBook/releases/download/v0.4.21/mdbook-v0.4.21-x86_64-unknown-linux-gnu.tar.gz | tar xzf -
-  #         echo `pwd` >> $GITHUB_PATH
-  #     - run: (cd docs && mdbook build)
-  #     - uses: actions/upload-artifact@v1
-  #       with:
-  #         name: doc-book
-  #         path: docs/book
-
   publish-check:
     name: Publish Check
     runs-on: ubuntu-22.04
@@ -178,28 +162,3 @@ jobs:
           files: "archival*"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  # publish:
-  #   name: Publish Docs
-  #   needs: [doc-book]
-  #   runs-on: ubuntu-22.04
-  #   if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-  #   steps:
-  #     - name: Download book
-  #       uses: actions/download-artifact@v1
-  #       with:
-  #         name: doc-book
-  #     - name: Assemble gh-pages
-  #       run: |
-  #         mv doc-book gh-pages
-  #     # If this is a push to the main branch push to the `gh-pages` using a
-  #     # deploy key. Note that a deploy key is necessary for now because otherwise
-  #     # using the default token for github actions doesn't actually trigger a page
-  #     # rebuild.
-  #     - name: Push to gh-pages
-  #       # Uses a rust script to setup and push to the gh-pages branch
-  #       run: curl -LsSf https://git.io/fhJ8n | rustc - && (cd gh-pages && ../rust_out)
-  #       env:
-  #         GITHUB_DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
-  #         BUILD_REPOSITORY_ID: ${{ github.repository }}
-  #         BUILD_SOURCEVERSION: ${{ github.sha }}

Cargo.toml

@@ -32,7 +32,6 @@ binary = [
     "dep:rand",
     "dep:base64",
     "dep:urlencoding",
-    "dep:serde_json",
 ]
 import-csv = ["dep:csv"]
 stdlib-fs = ["dep:notify", "dep:fs_extra", "dep:walkdir"]
@@ -66,7 +65,7 @@ reqwest = { version = "0.11.24", features = [
     "rustls-tls",
     "json",
 ], default-features = false, optional = true }
-serde_json = { version = "1.0.113", optional = true }
+serde_json = { version = "1.0.113" }
 home = { version = "0.5.9", optional = true }
 rsa = { version = "0.9.6", optional = true, features = ["sha2"] }
 rand = { version = "0.8.5", optional = true }
@@ -93,7 +92,7 @@ thiserror = "1.0.56"
 zip = { version = "0.6.6", default-features = false, features = ["deflate"] }
 toml_datetime = "0.6.5"
 tracing = "0.1.37"
-time = { version = "0.3.31", features = ["local-offset"] }
+time = { version = "0.3.36", features = ["local-offset"] }
 semver = "1.0.22"
 once_cell = "1.19.0"
 data-encoding = "2.5.0"

events.d.ts

@@ -19,9 +19,23 @@ export type File = {
     "name": (string | null);
     "filename": string;
     "mime": string;
-    "display_type": string;
+    "display_type": "image"|"audio"|"video"|"upload";
     "url": string;
 };
+export type MetaValue = ({
+    "String": string;
+} | {
+    "Number": F64;
+} | {
+    "Boolean": boolean;
+} | {
+    "DateTime": DateTime;
+} | {
+    "Array": MetaValue[];
+} | {
+    "Map": Record<string, MetaValue>;
+});
+export type Meta = Record<string, MetaValue>;
 export type FieldValue = ({
     "String": string;
 } | {
@@ -30,10 +44,14 @@ export type FieldValue = ({
     "Number": F64;
 } | {
     "Date": DateTime;
+} | {
+    "Objects": Record<string, FieldValue>[];
 } | {
     "Boolean": boolean;
 } | {
     "File": File;
+} | {
+    "Meta": Meta;
 });
 export type AddObjectValue = {
     "path": ValuePath;
@@ -53,7 +71,8 @@ export type EditFieldEvent = {
     "object": string;
     "filename": string;
     "path": ValuePath;
-    "value": FieldValue;
+    "field": string;
+    "value": (FieldValue | null);
 };
 export type EditOrderEvent = {
     "object": string;

pre-commit.sh

@@ -4,5 +4,5 @@ set -e
 
 cd $(dirname "$0")
 
-cargo clippy --all-features --all-targets -- -D warnings
+cargo clippy --all-features --all-targets -- --no-deps -D warnings
 ./test.sh

src/binary/command/import.rs

@@ -234,7 +234,7 @@ impl BinaryCommand for Command {
             // Find the specified child, if present
             Some(
                 child_path
-                    .get_child_definition(obj_def)
+                    .get_definition(obj_def)
                     .map_err(|_| ImportError::InvalidField(child_path.to_string()))?,
             )
         } else {
@@ -367,7 +367,7 @@ impl Command {
                         }))
                         .map_err(|e| ImportError::WriteError(e.to_string()))?;
                     if let ArchivalEventResponse::Index(i) = r {
-                        current_path = current_path.join(ValuePathComponent::Index(i));
+                        current_path = current_path.append(ValuePathComponent::Index(i));
                     } else {
                         panic!("archival did not return an index for an inserted child");
                     }
@@ -396,18 +396,16 @@ impl Command {
                     name
                 };
                 if let Some(value) = row.get(from_name) {
-                    let field_path = current_path
-                        .clone()
-                        .join(ValuePathComponent::Key(name.to_string()));
                     // Validate type
                     let value = FieldValue::from_string(name, field_type, value.to_string())
                         .map_err(|e| ImportError::ParseError(e.to_string()))?;
                     archival
                         .send_event_no_rebuild(ArchivalEvent::EditField(EditFieldEvent {
                             object: object.to_string(),
                             filename: filename.to_string(),
-                            path: field_path,
-                            value,
+                            path: current_path.clone(),
+                            value: Some(value),
+                            field: name.to_string(),
                         }))
                         .map_err(|e| ImportError::WriteError(e.to_string()))?;
                 } else {

src/binary/command/upload.rs

@@ -138,7 +138,7 @@ impl BinaryCommand for Command {
         let sha = archival.sha_for_file(file_path)?;
         let upload_url = format!("{}/upload/{}/{}", API_URL, archival_site, sha);
         let mime = mime_guess::from_path(file_path);
-        let mut file = File::from_mime(mime);
+        let mut file = File::from_mime_guess(mime);
         file.sha = sha;
         file.filename = file_path
             .file_name()
@@ -196,8 +196,9 @@ impl BinaryCommand for Command {
         archival.send_event_no_rebuild(ArchivalEvent::EditField(EditFieldEvent {
             object: object_type.clone(),
             filename: object_name.clone(),
-            path: field_path.clone(),
-            value: field_data.clone(),
+            path: ValuePath::empty(),
+            value: Some(field_data.clone()),
+            field: field.to_string(),
         }))?;
         if let FieldValue::File(fd) = field_data {
             println!(

src/constants.rs

@@ -6,6 +6,7 @@ pub const OBJECTS_DIR_NAME: &str = "objects";
 pub const BUILD_DIR_NAME: &str = "dist";
 pub const STATIC_DIR_NAME: &str = "public";
 pub const LAYOUT_DIR_NAME: &str = "layout";
+pub const NESTED_TYPES: [&str; 5] = ["meta", "upload", "video", "audio", "image"];
 #[cfg(debug_assertions)]
 pub const UPLOADS_URL: &str = "http://localhost:7777";
 #[cfg(not(debug_assertions))]

src/events.rs

@@ -4,7 +4,7 @@ use typescript_type_def::TypeDef;
 
 use crate::{value_path::ValuePath, FieldValue};
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub enum ArchivalEvent {
     AddObject(AddObjectEvent),
@@ -14,44 +14,45 @@ pub enum ArchivalEvent {
     AddChild(ChildEvent),
     RemoveChild(ChildEvent),
 }
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub enum ArchivalEventResponse {
     None,
     Index(usize),
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct EditFieldEvent {
     pub object: String,
     pub filename: String,
     pub path: ValuePath,
-    pub value: FieldValue,
+    pub field: String,
+    pub value: Option<FieldValue>,
 }
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct EditOrderEvent {
     pub object: String,
     pub filename: String,
     pub order: i32,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct DeleteObjectEvent {
     pub object: String,
     pub filename: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct AddObjectValue {
     pub path: ValuePath,
     pub value: FieldValue,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct AddObjectEvent {
     pub object: String,
@@ -60,7 +61,7 @@ pub struct AddObjectEvent {
     pub values: Vec<AddObjectValue>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[cfg_attr(feature = "typescript", derive(TypeDef))]
 pub struct ChildEvent {
     pub object: String,