//! The Python part ([MS-XLSX] 2.1.23, 2.3.10): where a workbook keeps the code of its Python cells. //! //! A Python cell's formula is `_xlfn._xlws.PY(index, return_type, …)` ([MS-XLSX] 2.3.10.2): the //! first argument is the zero-based index of a `pythonScript` element in this part, whose `code` //! is the cell's Python code, and the second is 0 (an Excel value) or 1 (a Python object). Further //! arguments are references the code names as `%Px%`, `x` being the argument's zero-based index //! ([MS-XLSX] 2.3.10.4). The part's root is `python` ([MS-XLSX] 2.4.102, CT_Python 2.6.249), with //! `pythonScripts` (2.6.253) holding the `pythonScript` elements (2.6.252). //! //! GridCraft writes one script per Python cell, in the order the cells are written, with the code //! as it is (its `xl()` references stay text in the code, so no further arguments). Reading //! resolves the index and puts each `%Px%` back as the text of argument `x`. A formula that names //! no script the part has keeps only its cached value. Hostile input is bounded: the part's size, //! the number of scripts, each script's length, and the code all cells take from the part //! together. use gridcraft_formula::Expr; use gridcraft_model::PyCode; use gridcraft_model::python::{FILE_PREFIX, MAX_FORMULA_CHARS}; use crate::IoError; use crate::read::Ctx; use crate::xml::{self, El}; /// Namespace of the part. pub const NS: &str = "http://schemas.microsoft.com/office/spreadsheetml/2023/python"; /// Relationship type from the workbook part. pub const REL: &str = "http://schemas.microsoft.com/office/2023/09/relationships/Python"; /// Last segment of [`REL`], as [`crate::package::Rel::kind`] holds it. const REL_KIND: &str = "Python"; /// Content type of the part. pub const CONTENT_TYPE: &str = "application/vnd.ms-excel.python+xml"; /// Part name we write (the format names none; the relationship finds it). pub const PART: &str = "xl/python.xml"; /// Largest Python part read. const MAX_PART_BYTES: u64 = 64 * 1024 * 1024; /// Most scripts read from the part. const MAX_SCRIPTS: usize = 1_000_000; /// Most characters kept of one script: one more than a cell's code may have, so a longer script /// still fails [`PyCode::fits`] and its cell keeps only its value, with the usual warning. const MAX_SCRIPT_CHARS: usize = MAX_FORMULA_CHARS + 1; /// Most bytes of code all the cells of a workbook take from the part together. Several formulas /// may name one script, so without it a small part could fill memory. pub const MAX_CODE_BYTES: usize = 64 * 1024 * 1024; /// The scripts read from a workbook's Python part, and what the cells took from them. #[derive(Default)] pub struct ScriptsIn { scripts: Vec, /// Bytes of code the cells may still take ([`MAX_CODE_BYTES`] in all). budget: usize, /// Python cells that kept only their value: no such script, or over the budget. missing: usize, } impl ScriptsIn { /// The warning for the Python cells that kept only their value, if any. pub fn warning(&self) -> Option { (self.missing > 0).then(|| { format!( "{} Python cell(s) name code the file's Python part doesn't hold (or holds too much of); they kept only their values", self.missing ) }) } } /// Reads the workbook's Python part, if it has one. A damaged part is a warning: its cells keep /// their values. pub fn read(cx: &mut Ctx<'_>, rels: &[crate::package::Rel]) -> Result { let mut out = ScriptsIn { budget: MAX_CODE_BYTES, ..ScriptsIn::default() }; let Some(rel) = rels.iter().find(|r| !r.external && r.kind.eq_ignore_ascii_case(REL_KIND)) else { return Ok(out) }; let bytes = match cx.pkg.read_optional_limited(&rel.target, MAX_PART_BYTES) { Ok(Some(b)) => b, Ok(None) => { cx.warn(format!("missing part {}", rel.target)); return Ok(out); } Err(IoError::TooLarge(m)) => { cx.warn(format!("the Python part is too large ({m}); Python cells kept only their values")); return Ok(out); } // A damaged entry (a bad checksum or compressed stream) is a damaged part: its cells keep // their values, as when the part is missing. Err(e) => { cx.warn(format!("the Python part is damaged ({e}); Python cells kept only their values")); return Ok(out); } }; let mut over = 0usize; let parsed = xml::parse_streaming(&bytes, &["pythonScript"], &mut |el: El| { if out.scripts.len() < MAX_SCRIPTS { out.scripts.push(script_code(&el)); } else { over += 1; } Ok(()) }); match parsed { Ok(root) if root.name != "python" => { cx.warn(format!("the Python part has the root <{}>; Python cells kept only their values", root.name)); out.scripts.clear(); } Ok(_) => {} Err(e) => cx.warn(format!("the Python part is damaged ({e}); kept {} scripts", out.scripts.len())), } if over > 0 { cx.warn(format!("the Python part holds more than {MAX_SCRIPTS} scripts; {over} were not read")); } Ok(out) } /// Encoded characters of a script that are decoded: enough for its first [`MAX_SCRIPT_CHARS`] /// characters of code, whatever its escapes. One character takes at most 14 encoded characters /// (a UTF-16 pair of `_xHHHH_` escapes) and a `\r\n` line break becomes one character, so /// `2 × (MAX_SCRIPT_CHARS + 1)` decoded characters cover the code kept, and two more escapes' /// worth keep the last of them, and the line break after it, from being cut. const MAX_ENCODED_CHARS: usize = 14 * (2 * (MAX_SCRIPT_CHARS + 1) + 2); /// A `pythonScript` element's code (ST_Xstring), with `\n` line breaks, cut after /// [`MAX_SCRIPT_CHARS`] characters. The limit is measured on the decoded code, so escapes in the /// file (`_x000D_`, `_x0009_`) don't cut a script that fits; decoding is linear and reads at most /// [`MAX_ENCODED_CHARS`] characters. fn script_code(el: &El) -> String { let text = el.child("code").map(|c| c.text.as_str()).unwrap_or_default(); let encoded = match text.char_indices().nth(MAX_ENCODED_CHARS) { Some((at, _)) => text.get(..at).unwrap_or_default(), None => text, }; let code = gridcraft_model::python::normalize_code(&xml::decode_xstring(encoded)); match code.char_indices().nth(MAX_SCRIPT_CHARS) { Some((at, _)) => code.get(..at).unwrap_or_default().to_string(), None => code, } } /// `PY(index, 0|1, params…)`, as a file's formula holds it: the index, whether the return type is /// a Python object, and the parameters. `None` for anything else (the code in a string literal /// is [`PyCode::from_formula`]'s form). fn py_call(expr: &Expr) -> Option<(usize, bool, &[Expr])> { let Expr::Call(name, args) = expr else { return None }; let base = name.trim_start_matches("_XLFN.").trim_start_matches("_XLWS."); if !base.eq_ignore_ascii_case("PY") { return None; } let index = match args.first()? { Expr::Number(n) if *n >= 0.0 && n.fract() == 0.0 && *n < MAX_SCRIPTS as f64 => *n as usize, _ => return None, }; let object = match args.get(1)? { Expr::Number(n) if *n == 0.0 => false, Expr::Number(n) if *n == 1.0 => true, _ => return None, }; Some((index, object, args.get(2..).unwrap_or_default())) } /// What a file formula is to Python. #[derive(Debug, PartialEq)] pub enum PyFormula { /// Not a `PY(…)` call: an ordinary formula. Not, /// A Python cell with this code. Code(PyCode), /// A `PY(index, …)` call naming a script the part doesn't hold (or over the budget): the /// cell keeps only its cached value, since no calculation can evaluate the formula. Missing, } /// What a file formula makes: a Python cell from the Python part when it names a script, or from /// string literals ([`PyCode::from_formula`]). `formula` is the parsed formula, `raw` the /// formula's own text, which keeps line breaks in literal code. pub fn cell_code(scripts: &mut ScriptsIn, formula: &gridcraft_model::Formula, raw: &str) -> PyFormula { // The cheap test first: most formulas are not `PY(` calls. if !gridcraft_model::python::is_py_formula(&formula.text) { return PyFormula::Not; } let expr = formula.expr_arc(); let Some((index, object, params)) = expr.as_deref().and_then(py_call) else { return PyCode::from_formula(raw).map_or(PyFormula::Not, PyFormula::Code); }; let code = scripts.scripts.get(index).map(|code| with_params(code, params)); match code.filter(|c| c.len() <= scripts.budget) { Some(code) => { scripts.budget -= code.len(); PyFormula::Code(PyCode { object, ..PyCode::new(&code) }) } None => { scripts.missing += 1; PyFormula::Missing } } } /// `code` with every `%Px%` (`x` at least 2, naming one of `params`) replaced by the parameter's /// text. Stops growing past [`MAX_SCRIPT_CHARS`] characters: such code can't be a cell's. fn with_params(code: &str, params: &[Expr]) -> String { if params.is_empty() || !code.contains("%P") { return code.to_string(); } let texts: Vec = params.iter().map(gridcraft_formula::print).collect(); let mut out = String::with_capacity(code.len()); let mut chars = 0usize; let mut push = |out: &mut String, piece: &str| { out.push_str(piece); chars = chars.saturating_add(piece.chars().count()); chars > MAX_SCRIPT_CHARS }; let mut rest = code; while let Some(at) = rest.find("%P") { if push(&mut out, rest.get(..at).unwrap_or_default()) { return out; } let after = rest.get(at + 2..).unwrap_or_default(); let digits = after.bytes().take_while(u8::is_ascii_digit).count(); let param = (1..=6) .contains(&digits) .then(|| after.get(..digits).and_then(|d| d.parse::().ok())) .flatten() .filter(|_| after.as_bytes().get(digits) == Some(&b'%')) .and_then(|x| x.checked_sub(2)) .and_then(|x| texts.get(x)); let full = match param { Some(text) => { rest = after.get(digits + 1..).unwrap_or_default(); push(&mut out, text) } None => { rest = after; push(&mut out, "%P") } }; if full { return out; } } out.push_str(rest); out } /// The formula a Python cell is saved as: `_xlfn._xlws.PY(index,0|1)`, its code being script /// `index` of the part ([`write`]). pub fn file_formula(index: usize, object: bool) -> String { format!("{FILE_PREFIX}PY({index},{})", u8::from(object)) } /// The Python part holding `scripts`, in order. Only written when there is at least one: the /// format requires one `pythonScript` in `pythonScripts`. pub fn write(scripts: &[String]) -> String { let mut xml = format!("{}", crate::write::XML_DECL); for code in scripts { xml.push_str(""); xml.push_str(&xml::esc(&xml::encode_xstring(code))); xml.push_str(""); } xml.push_str(""); xml } #[cfg(test)] mod tests { use super::*; fn formula(text: &str) -> gridcraft_model::Formula { gridcraft_model::Formula::new(&crate::fmla::from_file(text)) } fn scripts(list: &[&str]) -> ScriptsIn { ScriptsIn { scripts: list.iter().map(|s| s.to_string()).collect(), budget: MAX_CODE_BYTES, missing: 0 } } #[test] fn reads_the_index_form_and_parameters() { let mut s = scripts(&["1 + 1", "df = xl(\"%P2%\", headers=True)\nxl('%P3%') + %P9% + %P1%"]); let code = |s: &mut ScriptsIn, text: &str| match cell_code(s, &formula(text), text) { PyFormula::Code(py) => Some(py), _ => None, }; let a = code(&mut s, "_xlfn._xlws.PY(0,1)").unwrap(); assert_eq!((a.code.as_str(), a.object), ("1 + 1", true)); let b = code(&mut s, "_xlfn._xlws.PY(1,0,A1:B3,Sheet2!$C$4)").unwrap(); assert_eq!(b.code, "df = xl(\"A1:B3\", headers=True)\nxl('Sheet2!$C$4') + %P9% + %P1%"); assert!(!b.object); // The literal form still reads (other applications, older files). assert_eq!(code(&mut s, "_xlfn._xlws.PY(\"x = 1\",0)").unwrap().code, "x = 1"); // A script the part doesn't hold: the cell keeps its value, and is counted for the warning. let gone = "_xlfn._xlws.PY(7,0)"; assert_eq!(cell_code(&mut s, &formula(gone), gone), PyFormula::Missing); assert!(s.warning().is_some()); for not in ["_xlfn._xlws.PY(-1,0)", "_xlfn._xlws.PY(0.5,0)", "_xlfn._xlws.PY(0,2)", "PY(A1,0)", "SUM(0,1)", "PYTHON_TYPE(A1)"] { assert_eq!(cell_code(&mut s, &formula(not), not), PyFormula::Not, "{not}"); } } #[test] fn hostile_scripts_are_bounded() { // Many formulas naming one script take from one budget. let code = "x".repeat(1_000); let mut s = ScriptsIn { scripts: vec![code], budget: 2_500, missing: 0 }; let f = formula("_xlfn._xlws.PY(0,0)"); let taken = (0..5).filter(|_| matches!(cell_code(&mut s, &f, ""), PyFormula::Code(_))).count(); assert_eq!(taken, 2); assert_eq!(s.missing, 3); // A parameter repeated many times stops growing the code just past a cell's limit. let many = "%P2%".repeat(5_000); let out = with_params(&many, &[Expr::Name("A_VERY_LONG_NAME_FOR_A_RANGE".into())]); assert!((MAX_SCRIPT_CHARS..MAX_SCRIPT_CHARS + 40).contains(&out.chars().count()), "{}", out.len()); assert!(!PyCode::new(&out).fits()); // Odd tokens stay as they are. assert_eq!(with_params("%P%P2%%P99999999%%P", &[Expr::Number(1.0)]), "%P1%P99999999%%P"); // A long script is cut just past the limit, so it fails the cell's check. let el = El { name: "pythonScript".into(), children: vec![El { name: "code".into(), text: "y".repeat(20_000), ..El::default() }], ..El::default() }; assert_eq!(script_code(&el).chars().count(), MAX_SCRIPT_CHARS); } #[test] fn written_scripts_read_back() { let codes = vec!["a = 1\nb = \"two\" & 'three' ".to_string(), "_x0041_ \u{1}".to_string(), String::new()]; let xml = write(&codes); assert!(xml.contains(NS) && xml.contains("a = 1\nb = \"two\" & 'three' <tag>"), "{xml}"); let mut read = Vec::new(); let root = xml::parse_streaming(xml.as_bytes(), &["pythonScript"], &mut |el| { read.push(script_code(&el)); Ok(()) }) .unwrap_or_default(); assert_eq!(root.name, "python"); assert_eq!(read, codes); assert_eq!(file_formula(3, true), "_xlfn._xlws.PY(3,1)"); } }