Add --format argument to nickel query command (#2015)

* add structured output formats for `nickel query`.

WIP on interface

WIP

WIP

WIP.

Fix CI; improve impl Serialize for MergePriority

remove `Default` impl for `ExportFormatCommon`

change naming.

fix clippy warning

improvements on handling markdown format

- make markdown the default export format
- print error message when user want to export markdown format to a file.

* prune null/empty output fields

* fix clippy warnings

* fixes

* make value field a pretty printed string

---------

Co-authored-by: Ben Yang <[email protected]>

cli/Cargo.toml

@@ -28,6 +28,7 @@ nickel-lang-core = { workspace = true, features = [ "markdown" ], default-featur
 
 clap = { workspace = true, features = ["derive", "string"] }
 serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
 directories.workspace = true
 
 tempfile = { workspace = true, optional = true }

cli/src/query.rs

@@ -1,4 +1,14 @@
-use nickel_lang_core::repl::query_print;
+use std::{fs, io::Write, path::PathBuf};
+
+use nickel_lang_core::{
+    error::{Error, IOError},
+    identifier::{Ident, LocIdent},
+    pretty::PrettyPrintCap,
+    repl::query_print,
+    serialize::{self, MetadataExportFormat},
+    term::{record::Field, LabeledType, MergePriority, Term},
+};
+use serde::Serialize;
 
 use crate::{
     cli::GlobalOptions,
@@ -7,6 +17,8 @@ use crate::{
     input::{InputOptions, Prepare},
 };
 
+const VALUE_EXPORT_MAX_WIDTH: usize = 80;
+
 #[derive(clap::Parser, Debug)]
 pub struct QueryCommand {
     #[arg(long)]
@@ -24,10 +36,77 @@ pub struct QueryCommand {
     #[arg(long)]
     pub value: bool,
 
+    /// Export the value and all metadata of selected field in the specified format.
+    /// Value is exported in its string representation, capped at 80 characters.
+    ///
+    /// This flag cannot be used along with the following flags: --doc, --contract, --type, --default, --value
+    #[arg(long, short, value_enum, default_value_t, conflicts_with_all(["doc", "contract", "typ", "default", "value"]))]
+    pub format: MetadataExportFormat,
+
+    /// Output file. Standard output by default
+    #[arg(short, long, conflicts_with_all(["doc", "contract", "typ", "default", "value"]))]
+    pub output: Option<PathBuf>,
+
     #[command(flatten)]
     pub inputs: InputOptions<ExtractFieldOnly>,
 }
 
+#[derive(Clone, Debug, Serialize)]
+struct QueryResult {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub doc: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub typ: Option<LabeledType>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub contracts: Vec<LabeledType>,
+    pub optional: bool,
+    pub not_exported: bool,
+    pub priority: MergePriority,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sub_fields: Option<Vec<Ident>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub value: Option<String>,
+}
+
+impl From<Field> for QueryResult {
+    fn from(field: Field) -> Self {
+        let sub_fields = match field.value {
+            Some(ref val) => match val.as_ref() {
+                Term::Record(record) if !record.fields.is_empty() => {
+                    let mut fields: Vec<_> = record.fields.keys().collect();
+                    fields.sort();
+                    Some(fields.into_iter().map(LocIdent::ident).collect())
+                }
+                Term::RecRecord(record, dyn_fields, ..) if !record.fields.is_empty() => {
+                    let mut fields: Vec<_> = record.fields.keys().map(LocIdent::ident).collect();
+                    fields.sort();
+                    let dynamic = Ident::from("<dynamic>");
+                    fields.extend(dyn_fields.iter().map(|_| dynamic));
+                    Some(fields)
+                }
+                // Empty record has empty sub_fields
+                Term::Record(..) | Term::RecRecord(..) => Some(Vec::new()),
+                // Non-record has no concept of sub-field
+                _ => None,
+            },
+            None => None,
+        };
+
+        QueryResult {
+            doc: field.metadata.doc,
+            typ: field.metadata.annotation.typ,
+            contracts: field.metadata.annotation.contracts,
+            optional: field.metadata.opt,
+            not_exported: field.metadata.not_exported,
+            priority: field.metadata.priority,
+            sub_fields,
+            value: field
+                .value
+                .map(|v| v.pretty_print_cap(VALUE_EXPORT_MAX_WIDTH)),
+        }
+    }
+}
+
 impl QueryCommand {
     fn attributes_specified(&self) -> bool {
         self.doc || self.contract || self.typ || self.default || self.value
@@ -48,29 +127,72 @@ impl QueryCommand {
         }
     }
 
+    fn export<T>(self, res: T, format: MetadataExportFormat) -> Result<(), Error>
+    where
+        T: Serialize,
+    {
+        // This is a near-verbatim copy of ExportCommand::export
+
+        // We only add a trailing newline for JSON exports. Both YAML and TOML
+        // exporters already append a trailing newline by default.
+        let trailing_newline = format == MetadataExportFormat::Json;
+
+        if let Some(file) = self.output {
+            let mut file = fs::File::create(file).map_err(IOError::from)?;
+            serialize::to_writer_metadata(&mut file, format, &res)?;
+
+            if trailing_newline {
+                writeln!(file).map_err(IOError::from)?;
+            }
+        } else {
+            serialize::to_writer_metadata(std::io::stdout(), format, &res)?;
+
+            if trailing_newline {
+                println!();
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn run(self, global: GlobalOptions) -> CliResult<()> {
         let mut program = self.inputs.prepare(&global)?;
 
         if self.inputs.customize_mode.field().is_none() {
             program.report(Warning::EmptyQueryPath, global.error_format);
         }
 
-        let found = program
-            .query()
-            .map(|field| {
-                query_print::write_query_result(
-                    &mut std::io::stdout(),
-                    &field,
-                    self.query_attributes(),
-                )
-                .unwrap()
-            })
-            .report_with_program(program)?;
-
-        if !found {
-            eprintln!("No metadata found for this field.")
-        }
+        use MetadataExportFormat::*;
+        match self.format {
+            Markdown => {
+                if self.output.is_some() {
+                    eprintln!("Output query result in markdown format to a file is currently not supported.")
+                } else {
+                    let found = program
+                        .query()
+                        .map(|field| {
+                            query_print::write_query_result(
+                                &mut std::io::stdout(),
+                                &field,
+                                self.query_attributes(),
+                            )
+                            .unwrap()
+                        })
+                        .report_with_program(program)?;
 
+                    if !found {
+                        eprintln!("No metadata found for this field.")
+                    }
+                }
+            }
+            format @ (Json | Toml | Yaml) => {
+                let _ = &program
+                    .query()
+                    .map(QueryResult::from)
+                    .map(|res| self.export(res, format))
+                    .report_with_program(program)?;
+            }
+        }
         Ok(())
     }
 }

core/src/repl/query_print.rs

@@ -1,4 +1,6 @@
 //! Rendering of the results of a metadata query.
+use serde::Serialize;
+
 use crate::{
     identifier::{Ident, LocIdent},
     pretty::PrettyPrintCap,
@@ -153,7 +155,7 @@ impl QueryPrinter for MarkdownRenderer {
 }
 
 /// Represent which metadata attributes are requested by a query.
-#[derive(Clone, Copy, Eq, PartialEq)]
+#[derive(Clone, Copy, Eq, PartialEq, Serialize)]
 pub struct Attributes {
     pub doc: bool,
     pub contract: bool,

core/src/serialize.rs

@@ -26,7 +26,6 @@ use once_cell::sync::Lazy;
 use std::{fmt, io, rc::Rc};
 
 /// Available export formats.
-// If you add or remove variants, remember to update the CLI docs in `src/bin/nickel.rs'
 #[derive(Copy, Clone, Eq, PartialEq, Debug, Default, clap::ValueEnum)]
 pub enum ExportFormat {
     Raw,
@@ -47,8 +46,27 @@ impl fmt::Display for ExportFormat {
     }
 }
 
-// TODO: This type is publicly exposed, but never constructed.
-#[allow(dead_code)]
+/// Available metadata export formats.
+#[derive(Copy, Clone, Eq, PartialEq, Debug, Default, clap::ValueEnum)]
+pub enum MetadataExportFormat {
+    #[default]
+    Markdown,
+    Json,
+    Yaml,
+    Toml,
+}
+
+impl fmt::Display for MetadataExportFormat {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Markdown => write!(f, "markdown"),
+            Self::Json => write!(f, "json"),
+            Self::Yaml => write!(f, "yaml"),
+            Self::Toml => write!(f, "toml"),
+        }
+    }
+}
+
 #[derive(Clone, Eq, PartialEq, Debug)]
 pub struct ParseFormatError(String);
 
@@ -370,6 +388,34 @@ pub fn validate(format: ExportFormat, t: &RichTerm) -> Result<(), ExportError> {
     }
 }
 
+pub fn to_writer_metadata<W, T>(
+    mut writer: W,
+    format: MetadataExportFormat,
+    item: &T,
+) -> Result<(), ExportError>
+where
+    W: io::Write,
+    T: ?Sized + Serialize,
+{
+    // This is a near-verbatim copy of `to_writer`
+    match format {
+        MetadataExportFormat::Markdown => unimplemented!(),
+        MetadataExportFormat::Json => serde_json::to_writer_pretty(writer, &item)
+            .map_err(|err| ExportErrorData::Other(err.to_string())),
+        MetadataExportFormat::Yaml => serde_yaml::to_writer(writer, &item)
+            .map_err(|err| ExportErrorData::Other(err.to_string())),
+        MetadataExportFormat::Toml => toml::to_string_pretty(item)
+            .map_err(|err| ExportErrorData::Other(err.to_string()))
+            .and_then(|s| {
+                writer
+                    .write_all(s.as_bytes())
+                    .map_err(|err| ExportErrorData::Other(err.to_string()))
+            }),
+    }?;
+
+    Ok(())
+}
+
 pub fn to_writer<W>(mut writer: W, format: ExportFormat, rt: &RichTerm) -> Result<(), ExportError>
 where
     W: io::Write,

core/src/term/mod.rs

@@ -46,7 +46,7 @@ pub use malachite::{
     Integer, Rational,
 };
 
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize, Serializer};
 
 // Because we use `IndexMap` for recors, consumer of Nickel (as a library) might have to
 // manipulate values of this type, so we re-export this type.
@@ -680,6 +680,15 @@ impl fmt::Display for MergePriority {
     }
 }
 
+impl Serialize for MergePriority {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
 /// A branch of a match expression.
 #[derive(Debug, PartialEq, Clone)]
 pub struct MatchBranch {
@@ -730,6 +739,15 @@ impl LabeledType {
     }
 }
 
+impl Serialize for LabeledType {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.label.typ.to_string())
+    }
+}
+
 impl Traverse<RichTerm> for LabeledType {
     // Note that this function doesn't traverse the label, which is most often what you want. The
     // terms that may hide in a label are mostly types used for error reporting, but are never