pauljd/vectorcraft
1
0
Fork 0
mirror of https://github.com/storytold/vectorcraft.git synced 2026-11-12 07:45:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Variables: Import Data Sets reads every column as a variable and replaces the data sets

Browse Source 
Squashed from #5151 onto main.
This commit is contained in:
dotCooCoo
2026-11-03 16:38:29 +07:00
committed by bflatastic
parent fbab0bdd43
commit 86deb546ad
8 changed files with 335 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/engine/src/cmd/variablecmds/csv.rs
+54 -73
View File
@@ -102,7 +102,8 @@ fn tab_separated(text: &str) -> bool {
/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside),
/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted
/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and
/// drops white space after the closing quote at the end of the cell. Blank lines are skipped.
/// drops white space after the closing quote at the end of the cell. A line whose cells are all
/// blank and unquoted is skipped; a quoted empty cell keeps its line as a row.
fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {
let text = text.strip_prefix('\u{feff}').unwrap_or(text);
let sep = if tab_separated(text) { '\t' } else { ',' };
@@ -132,7 +133,10 @@ fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {
Some('\n') | None if !in_quotes => {
quoted.push(closed.is_some());
row.push(end_csv_cell(&mut cur, &mut closed));
if row.iter().any(|c| !c.trim().is_empty()) {
if row.len() > MAX_CSV_COLS {
return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns")));
}
if row.iter().any(|c| !c.trim().is_empty()) || quoted.contains(&true) {
rows.push((start, std::mem::take(&mut row), std::mem::take(&mut quoted)));
// +1 for the header.
if rows.len() > MAX_CSV_ROWS + 1 {
@@ -174,8 +178,10 @@ fn end_csv_cell(cur: &mut String, closed: &mut Option<usize>) -> String {
}
fn csv_escape(s: &str) -> String {
// An empty string is quoted so that it reads back as an empty value, not as a missing one.
if s.is_empty() || s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
// An empty string is quoted so that it reads back as an empty value, not as a missing one. A
// tab is quoted so that a header of one variable whose name holds a tab isn't read as
// tab-separated.
if s.is_empty() || s.contains([',', '"', '\n', '\r', '\t']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
}
/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive). An error names the
@@ -240,8 +246,7 @@ fn kinds_a_new_document_loses(vars: &Variables) -> (Vec<&str>, Vec<&str>) {
for (i, v) in vars.variables.iter().enumerate() {
let header = export_header(v);
let cell = header.trim();
// Column 1 holds the dataset names.
let read = if cell.is_empty() { None } else { header_variable(cell, i + 2, &none, &mut defined, C).ok() };
let read = if cell.is_empty() { None } else { header_variable(cell, i + 1, &none, &mut defined, C).ok() };
// The import refuses a header that names a variable an earlier column named.
let Some((name, kind)) = read.filter(|(name, _)| seen.insert(name.clone())) else {
refused.push(v.name.as_str());
@@ -260,30 +265,36 @@ fn kinds_a_new_document_loses(vars: &Variables) -> (Vec<&str>, Vec<&str>) {
(refused, other)
}
/// Build the CSV of every dataset (header `name` then each variable's [`export_header`] in
/// definition order). Refused when a variable's header is another variable's name, which the
/// import reads as that variable (a visibility variable `Show` next to a text variable `#Show`).
/// Build the CSV of every dataset: a header of each variable's [`export_header`] in definition
/// order, then one row per dataset. The dataset names are not written. Refused when a variable's
/// header is another variable's name, which the import reads as that variable (a visibility
/// variable `Show` next to a text variable `#Show`). A dataset with no value for any variable would
/// be a blank line, which the import skips, so its row gets a quoted empty cell: in the first column
/// whose variable isn't a text variable, where it reads back as no value, or else in the first
/// column, where it reads back as an empty text.
fn datasets_to_csv(vars: &Variables, cmd: &str) -> Result<String> {
let names: BTreeSet<&str> = vars.variables.iter().map(|v| v.name.as_str()).collect();
let mut out = String::from("name");
let mut header = Vec::with_capacity(vars.variables.len());
for v in &vars.variables {
let header = export_header(v);
if header != v.name && names.contains(header.as_str()) {
let (name, other) = (&v.name, &header);
return Err(bad(cmd, format!("the header `{header}` for `{name}` would read back as the variable `{other}`: rename one of them")));
let cell = export_header(v);
if cell != v.name && names.contains(cell.as_str()) {
let (name, other) = (&v.name, &cell);
return Err(bad(cmd, format!("the header `{cell}` for `{name}` would read back as the variable `{other}`: rename one of them")));
}
out.push(',');
out.push_str(&csv_escape(&header));
header.push(csv_escape(&cell));
}
let mut out = header.join(",");
out.push('\n');
let keep = vars.variables.iter().position(|v| v.kind != VariableKind::Text).unwrap_or(0);
for ds in &vars.datasets {
out.push_str(&csv_escape(&ds.name));
for v in &vars.variables {
out.push(',');
if let Some(val) = ds.values.get(&v.name) {
out.push_str(&csv_escape(&value_to_cell(val)));
}
let mut row: Vec<String> =
vars.variables.iter().map(|v| ds.values.get(&v.name).map(|val| csv_escape(&value_to_cell(val))).unwrap_or_default()).collect();
if row.iter().all(String::is_empty)
&& let Some(cell) = row.get_mut(keep)
{
*cell = csv_escape("");
}
out.push_str(&row.join(","));
out.push('\n');
}
Ok(out)
@@ -329,12 +340,11 @@ fn header_variable(
}
}
/// Import datasets from CSV. A first header cell `name` makes the first column the dataset
/// names; the other headers name variables, as [`header_variable`] reads them, and the
/// variables the document doesn't define yet are defined, unbound. Without the `name` column,
/// every header names a variable and the rows take the first free names "Data Set 1", "Data
/// Set 2" and so on. Two headers that name one variable are refused. Existing rows of the same
/// name are replaced; new names are appended. One undo step.
/// Import datasets from CSV. Every header cell names a variable, as [`header_variable`] reads it,
/// and the variables the document doesn't define yet are defined, unbound. Two headers that name
/// one variable are refused. The rows replace every dataset of the document and are named "Data
/// Set 1", "Data Set 2" and so on, in file order. The document's variables and their bindings
/// stay. One undo step, and none when the variables and datasets stay as they were.
pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "dataset.import";
let (bytes, given_as_text) = datasets_csv_bytes(p, C)?;
@@ -344,16 +354,10 @@ pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
let Some((_, header, _)) = rows.first() else {
return Err(bad(C, "CSV is empty"));
};
let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name"));
let (kinds, existing_names) = {
let st = s.doc()?;
let kinds: BTreeMap<String, VariableKind> = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();
let existing: BTreeSet<String> = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();
(kinds, existing)
};
let kinds: BTreeMap<String, VariableKind> = s.doc()?.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();
let mut columns: Vec<(usize, String, VariableKind)> = Vec::new();
let mut defined: Vec<Variable> = Vec::new();
for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) {
for (i, h) in header.iter().enumerate() {
let cell = h.trim();
if cell.is_empty() {
return Err(bad(C, format!("empty header in column {}", i + 1)));
@@ -373,27 +377,8 @@ pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
return Err(bad(C, "CSV has a header but no data rows"));
}
let mut parsed: Vec<DataSet> = Vec::new();
// The names of the rows read so far, and the last number given to a row without a name.
let (mut seen, mut number) = (BTreeSet::new(), 0usize);
for &(line, ref row, ref quoted) in rows.iter().skip(1) {
let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" };
let name = if raw_name.is_empty() {
// Same numbering as Capture Data Set when a row leaves the name blank or the file has
// no name column. The search starts after the last number given.
let (n, name) = (number + 1..)
.map(|n| (n, format!("Data Set {n}")))
.find(|(_, n)| !seen.contains(n) && !existing_names.contains(n))
.unwrap_or_else(|| (line, format!("Data Set {line}")));
number = n;
name
} else if raw_name.chars().count() > MAX_NAME {
return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters")));
} else {
raw_name.to_string()
};
if !seen.insert(name.clone()) {
return Err(bad(C, format!("line {line}: dataset `{name}` appears twice")));
}
for (n, &(line, ref row, ref quoted)) in rows.iter().skip(1).enumerate() {
let name = format!("Data Set {}", n + 1);
let mut values = BTreeMap::new();
for &(col, ref var, kind) in &columns {
let cell = row.get(col).map(String::as_str).unwrap_or("");
@@ -406,26 +391,23 @@ pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
}
let names: Vec<String> = parsed.iter().map(|d| d.name.clone()).collect();
let new_variables: Vec<String> = defined.iter().map(|v| v.name.clone()).collect();
let deleted: Vec<String> = s.doc()?.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();
// The import is one undo step, and an import that leaves the variables and data sets as they
// were adds none.
let (imported, updated) = super::edit_variables(s, "Import Data Sets", |vars| {
vars.variables.extend(defined);
let (mut imported, mut updated) = (0usize, 0usize);
for row in parsed {
if let Some(existing) = vars.datasets.iter_mut().find(|x| x.name == row.name) {
*existing = row;
updated += 1;
} else {
vars.datasets.push(row);
imported += 1;
}
// were adds none, so the active data set stays when nothing changes.
super::edit_variables(s, "Import Data Sets", |vars| {
if defined.is_empty() && vars.datasets == parsed {
return Ok(());
}
Ok((imported, updated))
vars.variables.extend(defined);
vars.datasets = parsed;
vars.active_dataset = None;
Ok(())
})?;
Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables}))
Ok(json!({"imported": names.len(), "names": names, "defined": new_variables, "deletedDataSets": deleted}))
}
/// Export every dataset as CSV ([`datasets_to_csv`]). With `path`, write the file; else return
/// Export every dataset as CSV ([`datasets_to_csv`]: one column per variable, no dataset names).
/// With `path`, write the file; else return
/// `{csv}`. The reply's `warning` names the variables whose column makes a document without them
/// refuse the file, or else the variables it doesn't read back with their kind
/// ([`kinds_a_new_document_loses`]); the app shows it in its status bar.
@@ -439,7 +421,6 @@ pub(super) fn dataset_export(s: &mut Session, p: &Value) -> Result<Value> {
if st.doc.variables.datasets.len() > MAX_CSV_ROWS {
return Err(bad(C, format!("more than {MAX_CSV_ROWS} data sets: Import Data Sets reads at most {MAX_CSV_ROWS}")));
}
// The name column comes first, so the import's column cap leaves room for this many variables.
if st.doc.variables.variables.len() > MAX_CSV_COLS {
return Err(bad(C, format!("more than {MAX_CSV_COLS} variables: Import Data Sets reads at most {MAX_CSV_COLS}")));
}