diff --git a/.github/ci/build.sh b/.github/ci/build.sh new file mode 100755 index 00000000..82f1f045 --- /dev/null +++ b/.github/ci/build.sh @@ -0,0 +1,64 @@ +#!/bin/bash +## on pull_request + +set -euxo pipefail + +export RUSTUP_HOME=/ci/cache/rustup +export CARGO_HOME=/ci/cache/cargo +export CARGO_TARGET_DIR=/ci/cache/target +export RUST_BACKTRACE=1 + +hashtime restore /ci/cache/filetime.json || true +hashtime save /ci/cache/filetime.json + +# Fetch enough history to compute the merge-base with main. +git remote add upstream https://github.com/embassy-rs/chiptool +git fetch --depth 50 upstream main +BASE=$(git merge-base HEAD upstream/main) +BASE=0c94b6c4c43712a455d687d34c7200909203f2d9 +HEAD_SHA=$(git rev-parse HEAD) + +# Reuse cached SVDs clone across CI runs. +mv /ci/cache/chiptool-test-svds tests/svds || true + +run_test() { + local out=$1 + rm -rf tests/output + cargo test --test svd2ir + rm -rf "$out" + mv tests/output "$out" +} + +# Generate YAMLs for HEAD. +run_test /tmp/head + +# Generate YAMLs for the merge base. +git -c advice.detachedHead=false checkout "$BASE" +run_test /tmp/base +git checkout - + +# Move cached SVDs back. +mv tests/svds /ci/cache/chiptool-test-svds + +# Produce a colored HTML diff. +mkdir -p /ci/artifacts +( + cd /tmp + # `--no-index` lets us diff two trees outside any repo. + # exit code 1 just means "differences found"; that's fine. + git --no-pager diff --no-index --color=always base head > /tmp/diff.ansi || true +) +aha --black < /tmp/diff.ansi > /ci/artifacts/diff.html + +# Also publish the raw YAML trees so reviewers can browse / download them. +cp -r /tmp/head /ci/artifacts/head +cp -r /tmp/base /ci/artifacts/base + +JOB_ID=$(jq -r .id < /ci/job.json) +cat > /ci/comment.md < String { + str.replace("[%s]", "").replace("%s", "") +} + +fn names(array: &MaybeArray, f: F) -> ExpandedMaybeArray +where + F: Fn(&T) -> &str, +{ + fn replace_placeholder(str: &str, replacement: &str) -> String { + str.replace("%s", replacement) + } + + fn as_array_name<'a>(r: &str) -> Option<&str> { + if let Some(array) = r.strip_suffix("[%s]") { + Some(array) + } else { + None + } + } + + match array { + MaybeArray::Single(r) => ExpandedMaybeArray::Single(f(r).to_string()), + MaybeArray::Array(r, dim_element) => { + if let Some(array_name) = as_array_name(f(r)) { + ExpandedMaybeArray::Array { + name: array_name.to_string(), + array: Array::Regular(RegularArray { + len: dim_element.dim, + stride: dim_element.dim_increment, + }), + } + } else { + // `dim_increment` may be 0 for `dim == 1` (degenerate "array" of + // one element); `step_by(0)` would panic, so clamp to 1. For + // `dim > 1`, a zero stride is nonsensical and we fail loudly. + let stride = if dim_element.dim_increment > 0 { + dim_element.dim_increment + } else { + assert!( + dim_element.dim == 1, + "dimIncrement=0 with dim={} > 1 in {}", + dim_element.dim, + f(r), + ); + 1 + }; + let offsets = (0..).step_by(stride as _); + + let values = offsets + .zip( + dim_element + .dim_index + .iter() + .flat_map(|v| v.iter()) + .map(|dim| replace_placeholder(f(r), dim)), + ) + .collect(); + + ExpandedMaybeArray::Many(values) + } + } + } +} + +#[derive(Clone)] +enum ExpandedMaybeArray { + Single(String), + Array { name: String, array: Array }, + Many(Vec<(u32, String)>), +} + +impl ExpandedMaybeArray { + pub fn array(&self) -> Option { + match self { + ExpandedMaybeArray::Array { array, .. } => Some(array.clone()), + _ => None, + } + } +} + +impl IntoIterator for ExpandedMaybeArray { + type Item = (u32, String); + + type IntoIter = std::vec::IntoIter<(u32, String)>; + + fn into_iter(self) -> Self::IntoIter { + match self { + ExpandedMaybeArray::Single(s) => vec![(0, s)].into_iter(), + ExpandedMaybeArray::Array { name, .. } => vec![(0, name)].into_iter(), + ExpandedMaybeArray::Many(items) => items.into_iter(), + } + } +} + +fn fieldset_name(mut block_name: Vec, reg_name: String) -> Vec { + block_name.push(reg_name); + block_name +} + pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<()> { let mut blocks = Vec::new(); let pname = p.header_struct_name.clone().unwrap_or(p.name.clone()); @@ -48,145 +148,184 @@ pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<() ); let enum_from_name = enum_map(&blocks); + let mut fieldsets: Vec = Vec::new(); let mut enums: Vec = Vec::new(); + // A map mapping fully expanded fieldset names to their unique + // equivalent. E.g. `blocka::REG%s` with dimensions `A`, `B`, `C` would + // add `blocka::REGA -> blocka::REG`, `blocka::REGB -> blocka::REG` and + // `blocka::REGC -> REG` to this map. + let mut fieldset_mapping = BTreeMap::new(); + let mut used_fieldset_names = BTreeSet::new(); + + let usable_register_clusters = blocks + .iter() + .flat_map(|b| std::iter::repeat(&b.name).zip(b.registers.iter())) + .filter_map(|(b, r)| { + let svd::RegisterCluster::Register(r) = r else { + return None; + }; - for block in &blocks { - for r in &block.registers { - if let svd::RegisterCluster::Register(r) = r { - if r.derived_from.is_some() { + if r.derived_from.is_some() { + return None; + } + + let Some(fields) = r.fields.as_ref() else { + return None; + }; + + let without_placeholder = remove_placeholder(&r.name); + let mut unique_fs_name = fieldset_name(b.clone(), without_placeholder.clone()); + + let mut counter = 0; + loop { + if counter != 0 { + let last = unique_fs_name.last_mut().unwrap(); + if last.ends_with('_') { + *last = format!("{}{}", last, counter); + } else { + *last = format!("{}_{}", without_placeholder, counter); + } + } + + counter += 1; + + if !used_fieldset_names.contains(&unique_fs_name) { + break; + } + } + + used_fieldset_names.insert(unique_fs_name.clone()); + + for (_, full_name) in names(r, |r| &r.name) { + let full_name: Vec<_> = fieldset_name(b.clone(), full_name); + fieldset_mapping.insert(full_name, unique_fs_name.clone()); + } + + Some((unique_fs_name, r.deref(), fields)) + }); + + for (fieldset_name, r, fields) in usable_register_clusters { + let mut field_name_counts: BTreeMap = BTreeMap::new(); + let mut out_fields = Vec::with_capacity(fields.len()); + + for f in fields { + if f.derived_from.is_some() { + continue; + } + + let mut enum_read = None; + let mut enum_write = None; + let mut enum_readwrite = None; + + let mut field_name = remove_placeholder(&f.name); + let field_name_count = field_name_counts.entry(field_name.clone()).or_insert(0); + *field_name_count += 1; + if *field_name_count > 1 { + field_name = format!("{}{}", field_name, field_name_count); + } + + for e in &f.enumerated_values { + let e = if let Some(derived_from) = &e.derived_from { + let Some(e) = enum_from_name.get(derived_from.as_str()) else { + warn!( + "unknown enum to derive from ({} -> {})", + field_name, derived_from + ); + continue; + }; + e + } else { + e + }; + + let usage = e.usage.unwrap_or(svd::Usage::ReadWrite); + let target = match usage { + svd::Usage::Read => &mut enum_read, + svd::Usage::Write => &mut enum_write, + svd::Usage::ReadWrite => &mut enum_readwrite, + }; + + if target.is_some() { + warn!("ignoring enum with dup usage {:?}", usage); continue; } - if let Some(fields) = &r.fields { - let mut fieldset_name = block.name.clone(); - let mut field_name_counts: BTreeMap = BTreeMap::new(); - fieldset_name.push(replace_suffix(&r.name, "")); - - let mut out_fields = Vec::with_capacity(fields.len()); - - for f in fields { - if f.derived_from.is_some() { - continue; - } - - let mut enum_read = None; - let mut enum_write = None; - let mut enum_readwrite = None; - - let mut field_name = replace_suffix(&f.name, ""); - - let field_name_count = - field_name_counts.entry(field_name.clone()).or_insert(0); - *field_name_count += 1; - if *field_name_count > 1 { - field_name = format!("{}{}", field_name, field_name_count); - } - - for e in &f.enumerated_values { - let e = if let Some(derived_from) = &e.derived_from { - let Some(e) = enum_from_name.get(derived_from.as_str()) else { - warn!( - "unknown enum to derive from ({} -> {})", - field_name, derived_from - ); - continue; - }; - e - } else { - e - }; - - let usage = e.usage.unwrap_or(svd::Usage::ReadWrite); - let target = match usage { - svd::Usage::Read => &mut enum_read, - svd::Usage::Write => &mut enum_write, - svd::Usage::ReadWrite => &mut enum_readwrite, - }; - - if target.is_some() { - warn!("ignoring enum with dup usage {:?}", usage); - continue; - } - - *target = Some(e) - } - - enum EnumSet<'a> { - Single(&'a svd::EnumeratedValues), - ReadWrite(&'a svd::EnumeratedValues, &'a svd::EnumeratedValues), - } - - let set = match (enum_read, enum_write, enum_readwrite) { - (None, None, None) => None, - (Some(e), None, None) => Some(EnumSet::Single(e)), - (None, Some(e), None) => Some(EnumSet::Single(e)), - (None, None, Some(e)) => Some(EnumSet::Single(e)), - (Some(r), Some(w), None) => Some(EnumSet::ReadWrite(r, w)), - (Some(r), None, Some(w)) => Some(EnumSet::ReadWrite(r, w)), - (None, Some(w), Some(r)) => Some(EnumSet::ReadWrite(r, w)), - (Some(_), Some(_), Some(_)) => { - bail!("cannot have enumeratedvalues for read, write and readwrite!") - } - }; - - let enumm = if let Some(set) = set { - let variants = match set { - EnumSet::Single(e) => e.values.clone(), - EnumSet::ReadWrite(r, w) => { - let r_values = r.values.iter().map(|v| v.value.unwrap()); - let w_values = w.values.iter().map(|v| v.value.unwrap()); - let values: BTreeSet<_> = r_values.chain(w_values).collect(); - let mut values: Vec<_> = values.iter().collect(); - values.sort(); - - let r_values: BTreeMap<_, _> = - r.values.iter().map(|v| (v.value.unwrap(), v)).collect(); - let w_values: BTreeMap<_, _> = - w.values.iter().map(|v| (v.value.unwrap(), v)).collect(); - - values - .into_iter() - .map(|&v| match (r_values.get(&v), w_values.get(&v)) { - (None, None) => unreachable!(), - (Some(&r), None) => r.clone(), - (None, Some(&w)) => w.clone(), - (Some(&r), Some(&w)) => { - let mut m = r.clone(); - if r.name != w.name { - m.name = format!("R_{}_W_{}", r.name, w.name); - } - m - } - }) - .collect() + *target = Some(e) + } + + enum EnumSet<'a> { + Single(&'a svd::EnumeratedValues), + ReadWrite(&'a svd::EnumeratedValues, &'a svd::EnumeratedValues), + } + + let set = match (enum_read, enum_write, enum_readwrite) { + (None, None, None) => None, + (Some(e), None, None) => Some(EnumSet::Single(e)), + (None, Some(e), None) => Some(EnumSet::Single(e)), + (None, None, Some(e)) => Some(EnumSet::Single(e)), + (Some(r), Some(w), None) => Some(EnumSet::ReadWrite(r, w)), + (Some(r), None, Some(w)) => Some(EnumSet::ReadWrite(r, w)), + (None, Some(w), Some(r)) => Some(EnumSet::ReadWrite(r, w)), + (Some(_), Some(_), Some(_)) => { + bail!("cannot have enumeratedvalues for read, write and readwrite!") + } + }; + + let enumm = if let Some(set) = set { + let variants = match set { + EnumSet::Single(e) => e.values.clone(), + EnumSet::ReadWrite(r, w) => { + let r_values = r.values.iter().map(|v| v.value.unwrap()); + let w_values = w.values.iter().map(|v| v.value.unwrap()); + let values: BTreeSet<_> = r_values.chain(w_values).collect(); + let mut values: Vec<_> = values.iter().collect(); + values.sort(); + + let r_values: BTreeMap<_, _> = + r.values.iter().map(|v| (v.value.unwrap(), v)).collect(); + let w_values: BTreeMap<_, _> = + w.values.iter().map(|v| (v.value.unwrap(), v)).collect(); + + values + .into_iter() + .map(|&v| match (r_values.get(&v), w_values.get(&v)) { + (None, None) => unreachable!(), + (Some(&r), None) => r.clone(), + (None, Some(&w)) => w.clone(), + (Some(&r), Some(&w)) => { + let mut m = r.clone(); + if r.name != w.name { + m.name = format!("R_{}_W_{}", r.name, w.name); + } + m } - }; - - let mut name = fieldset_name.clone(); - name.push(field_name); - enums.push(ProtoEnum { - name: name.clone(), - bit_size: f.bit_range.width, - variants, - }); - Some(name) - } else { - None - }; - - out_fields.push((f.clone(), enumm)); + }) + .collect() } - - fieldsets.push(ProtoFieldset { - name: fieldset_name.clone(), - description: r.description.clone(), - bit_size: r.properties.size.unwrap_or(32), - fields: out_fields, - }); }; - } + + let mut name = fieldset_name.clone(); + name.push(field_name); + enums.push(ProtoEnum { + name: name.clone(), + bit_size: f.bit_range.width, + variants, + }); + Some(name) + } else { + None + }; + + out_fields.push((f.clone(), enumm)); } + + fieldsets.push(ProtoFieldset { + name: fieldset_name.clone(), + description: r.description.clone(), + bit_size: r.properties.size.unwrap_or(32), + fields: out_fields, + }); } // Make all collected names unique by prefixing with parents' names if needed. @@ -210,45 +349,21 @@ pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<() continue; } - let fieldset_name = if r.fields.is_some() { - let mut fieldset_name = proto.name.clone(); - fieldset_name.push(replace_suffix(&r.name, "")); - Some(fieldset_names.get(&fieldset_name).unwrap().clone()) - } else { - None - }; - - let array = if let svd::Register::Array(_, dim) = r { - Some(Array::Regular(RegularArray { - len: dim.dim, - stride: dim.dim_increment, - })) - } else { - None - }; - - let access = match r.properties.access { - None => Access::ReadWrite, - Some(svd::Access::ReadOnly) => Access::Read, - Some(svd::Access::WriteOnly) => Access::Write, - Some(svd::Access::WriteOnce) => Access::Write, - Some(svd::Access::ReadWrite) => Access::ReadWrite, - Some(svd::Access::ReadWriteOnce) => Access::ReadWrite, - }; - - let block_item = BlockItem { - name: replace_suffix(&r.name, ""), - description: r.description.clone(), - array, - byte_offset: r.address_offset, - inner: BlockItemInner::Register(Register { - access, // todo - bit_size: r.properties.size.unwrap_or(32), - fieldset: fieldset_name.clone(), - }), - }; - - block.items.push(block_item) + let names = crate::svd2ir::names(&r, |r| &r.name); + let array = names.array(); + + for (offset, name) in names { + let block_item = create_block_item( + name, + offset, + proto, + &fieldset_mapping, + &fieldset_names, + array.as_ref(), + r, + ); + block.items.push(block_item) + } } svd::RegisterCluster::Cluster(c) => { if c.derived_from.is_some() { @@ -256,28 +371,22 @@ pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<() continue; } - let cname = replace_suffix(&c.name, ""); - - let array = if let svd::Cluster::Array(_, dim) = c { - Some(Array::Regular(RegularArray { - len: dim.dim, - stride: dim.dim_increment, - })) - } else { - None - }; - - let mut block_name = proto.name.clone(); - block_name.push(replace_suffix(&c.name, "")); - let block_name = block_names.get(&block_name).unwrap().clone(); - - block.items.push(BlockItem { - name: cname.clone(), - description: c.description.clone(), - array, - byte_offset: c.address_offset, - inner: BlockItemInner::Block(BlockItemBlock { block: block_name }), - }); + let names = names(c, |c| &c.name); + let array = names.array(); + + for (offset, cname) in names { + let mut block_name = proto.name.clone(); + block_name.push(cname.clone()); + let block_name = block_names.get(&block_name).unwrap().clone(); + + block.items.push(BlockItem { + name: cname.clone(), + description: c.description.clone(), + array: array.clone(), + byte_offset: c.address_offset + offset, + inner: BlockItemInner::Block(BlockItemBlock { block: block_name }), + }); + } } } } @@ -300,32 +409,26 @@ pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<() warn!("unsupported derived_from in fieldset"); } - let array = if let svd::Field::Array(_, dim) = f { - Some(Array::Regular(RegularArray { - len: dim.dim, - stride: dim.dim_increment, - })) - } else { - None - }; - - let field_name = replace_suffix(&f.name, ""); - - let field = Field { - name: field_name.clone(), - description: f.description.clone(), - bit_offset: BitOffset::Regular(f.bit_range.offset), - bit_size: f.bit_range.width, - array, - enumm: enumm.as_ref().map(|v| { - enum_names - .get(v) - .cloned() - .expect("All enums have a unique-name mapping") - }), - }; + let names = names(f, |f| &f.name); + let array = names.array(); + + for (offset, field_name) in names { + let field = Field { + name: field_name.clone(), + description: f.description.clone(), + bit_offset: BitOffset::Regular(f.bit_range.offset + offset), + bit_size: f.bit_range.width, + array: array.clone(), + enumm: enumm.as_ref().map(|v| { + enum_names + .get(v) + .cloned() + .expect("All enums have a unique-name mapping") + }), + }; - fieldset.fields.push(field) + fieldset.fields.push(field) + } } let fieldset_name = fieldset_names.get(&proto.name).unwrap().clone(); @@ -363,6 +466,45 @@ pub fn convert_peripheral(ir: &mut IR, p: &svd::Peripheral) -> anyhow::Result<() Ok(()) } +fn create_block_item( + name: String, + offset: u32, + proto: &ProtoBlock, + inner_block_to_fieldset: &BTreeMap, Vec>, + fieldset_names: &BTreeMap, String>, + array: Option<&Array>, + r: &MaybeArray, +) -> BlockItem { + let fieldset_name = if r.fields.is_some() { + let fieldset_full_name = fieldset_name(proto.name.clone(), name.clone()); + let fieldset_name = inner_block_to_fieldset.get(&fieldset_full_name).unwrap(); + Some(fieldset_names.get(fieldset_name).unwrap().clone()) + } else { + None + }; + + let access = match r.properties.access { + None => Access::ReadWrite, + Some(svd::Access::ReadOnly) => Access::Read, + Some(svd::Access::WriteOnly) => Access::Write, + Some(svd::Access::WriteOnce) => Access::Write, + Some(svd::Access::ReadWrite) => Access::ReadWrite, + Some(svd::Access::ReadWriteOnce) => Access::ReadWrite, + }; + + BlockItem { + name: name, + description: r.description.clone(), + array: array.cloned(), + byte_offset: r.address_offset + offset, + inner: BlockItemInner::Register(Register { + access, // todo + bit_size: r.properties.size.unwrap_or(32), + fieldset: fieldset_name.clone(), + }), + } +} + /// Convert an entire SVD to IR. pub fn convert_svd(svd: &svd::Device, namespace: NamespaceMode) -> anyhow::Result { let mut ir = IR::new(); @@ -488,9 +630,11 @@ fn collect_blocks( continue; } - let mut block_name = block_name.clone(); - block_name.push(replace_suffix(&c.name, "")); - collect_blocks(out, block_name, c.description.clone(), &c.children); + for (_, block) in names(c, |c| &c.name) { + let mut block_name = block_name.clone(); + block_name.push(block); + collect_blocks(out, block_name, c.description.clone(), &c.children); + } } } } @@ -584,11 +728,3 @@ pub fn namespace_names(peripheral: &PeripheralInfo, ir: &mut IR, namespace: Name } }); } - -pub fn replace_suffix(name: &str, suffix: &str) -> String { - if name.contains("[%s]") { - name.replace("[%s]", suffix) - } else { - name.replace("%s", suffix) - } -} diff --git a/tests/svd2ir.rs b/tests/svd2ir.rs new file mode 100644 index 00000000..fda5f335 --- /dev/null +++ b/tests/svd2ir.rs @@ -0,0 +1,99 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use chiptool::svd2ir::NamespaceMode; + +const SVDS_REPO_URL: &str = "https://github.com/embassy-rs/chiptool-test-svds.git"; +const SVDS_REPO_COMMIT: &str = "0e8e1c19b98b5e09d757a48f44c9ed1cfd1684db"; + +fn ensure_svds_repo() -> PathBuf { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("svds"); + + if dir.join(".git").is_dir() { + let head = Command::new("git") + .args(["-C"]) + .arg(&dir) + .args(["rev-parse", "HEAD"]) + .output() + .expect("git rev-parse failed"); + if head.status.success() + && std::str::from_utf8(&head.stdout).unwrap().trim() == SVDS_REPO_COMMIT + { + return dir; + } + } + + if dir.exists() { + fs::remove_dir_all(&dir).expect("failed to remove stale svds dir"); + } + + let status = Command::new("git") + .args(["clone", "--no-checkout", SVDS_REPO_URL]) + .arg(&dir) + .status() + .expect("git clone failed to spawn"); + assert!(status.success(), "git clone of {SVDS_REPO_URL} failed"); + + let status = Command::new("git") + .args(["-C"]) + .arg(&dir) + .args(["checkout", "--detach", SVDS_REPO_COMMIT]) + .status() + .expect("git checkout failed to spawn"); + assert!(status.success(), "git checkout {SVDS_REPO_COMMIT} failed"); + + dir +} + +fn process_svd(svd_path: &Path, out_dir: &Path) { + let chip = svd_path.file_stem().unwrap().to_str().unwrap(); + let chip_out = out_dir.join(chip); + if chip_out.exists() { + fs::remove_dir_all(&chip_out).unwrap(); + } + fs::create_dir_all(&chip_out).unwrap(); + + let xml = fs::read_to_string(svd_path).unwrap(); + let config = svd_parser::Config::default() + .expand_properties(true) + .validate_level(svd_parser::ValidateLevel::Disabled); + let device = svd_parser::parse_with_config(&xml, &config) + .unwrap_or_else(|e| panic!("failed to parse {}: {e}", svd_path.display())); + + for p in &device.peripherals { + if p.derived_from.is_some() { + continue; + } + let ir = chiptool::commands::extract_peripheral(p, NamespaceMode::None) + .unwrap_or_else(|e| panic!("extract_peripheral failed for {}/{}: {e}", chip, p.name)); + + let yaml = serde_yaml::to_string(&ir).unwrap(); + fs::write(chip_out.join(format!("{}.yaml", p.name)), yaml).unwrap(); + } +} + +#[test] +fn svd2ir() { + let svds_dir = ensure_svds_repo(); + let out_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("output"); + fs::create_dir_all(&out_dir).unwrap(); + + let mut svds: Vec<_> = fs::read_dir(&svds_dir) + .unwrap() + .filter_map(|e| { + let p = e.unwrap().path(); + (p.extension().and_then(|s| s.to_str()) == Some("svd")).then_some(p) + }) + .collect(); + svds.sort(); + assert!(!svds.is_empty(), "no SVDs found in {}", svds_dir.display()); + + for svd in &svds { + process_svd(svd, &out_dir); + } +}