mirror of
https://github.com/ruvnet/RuView
synced 2026-07-31 18:51:42 +00:00
--convert-model rejected the published model.safetensors because its JSON header is padded to an 8-byte boundary with trailing NUL bytes, which a strict serde_json::from_slice parse treats as trailing characters. Trim the padding before parsing, with a regression test that reproduces the exact boundary-padded shape of the real HF file. docs/huggingface/MODEL_CARD.md had drifted from the card actually published on the Hub: every file in its "Files in this repo" table (pretrained-encoder.onnx, pretrained-heads.onnx, pretrained.rvf, room-profiles.json) does not exist in ruvnet/wifi-densepose-pretrained. Replaced it with the content live on the Hub and added a section documenting the --convert-model / --model RVF conversion path, which neither card mentioned.
This commit is contained in:
@@ -209,7 +209,18 @@ pub fn safetensors_to_rvf(data: &[u8], model_id: &str) -> Result<Vec<u8>, ModelL
|
||||
.filter(|&e| e <= data.len())
|
||||
.ok_or_else(|| fail(format!("declared header length {header_len} exceeds file size")))?;
|
||||
|
||||
let header: serde_json::Value = serde_json::from_slice(&data[header_start..header_end])
|
||||
// The reference safetensors format pads the header to an 8-byte boundary
|
||||
// with NUL bytes after the closing brace; a strict single-shot parse over
|
||||
// the full declared-length slice rejects that padding. Trim trailing NUL
|
||||
// (and whitespace) before parsing — every published HF safetensors file
|
||||
// exercises this padding.
|
||||
let raw_header = &data[header_start..header_end];
|
||||
let trimmed_end = raw_header
|
||||
.iter()
|
||||
.rposition(|&b| b != 0 && !b.is_ascii_whitespace())
|
||||
.map(|i| i + 1)
|
||||
.unwrap_or(0);
|
||||
let header: serde_json::Value = serde_json::from_slice(&raw_header[..trimmed_end])
|
||||
.map_err(|e| fail(format!("safetensors header is not valid JSON: {e}")))?;
|
||||
let obj = header
|
||||
.as_object()
|
||||
@@ -427,6 +438,43 @@ mod tests {
|
||||
assert_eq!(detect_format(&rvfs, "model.rvf"), ModelFormat::Rvf);
|
||||
}
|
||||
|
||||
/// Build a safetensors buffer with the header padded to an 8-byte boundary
|
||||
/// with NUL bytes, matching the reference format and the published
|
||||
/// `model.safetensors` (issue #1480).
|
||||
fn make_safetensors_nul_padded(weights: &[f32]) -> Vec<u8> {
|
||||
let n = weights.len();
|
||||
let header = serde_json::json!({
|
||||
"weight": {
|
||||
"dtype": "F32",
|
||||
"shape": [n],
|
||||
"data_offsets": [0, n * 4],
|
||||
}
|
||||
});
|
||||
let mut header_bytes = serde_json::to_vec(&header).unwrap();
|
||||
let padded_len = (header_bytes.len() + 7) & !7;
|
||||
header_bytes.resize(padded_len, 0);
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&(header_bytes.len() as u64).to_le_bytes());
|
||||
out.extend_from_slice(&header_bytes);
|
||||
for &w in weights {
|
||||
out.extend_from_slice(&w.to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// REGRESSION #1480: a NUL-padded header (the reference safetensors format,
|
||||
/// and what the published `model.safetensors` actually ships) must convert,
|
||||
/// not fail with "safetensors header is not valid JSON: trailing characters".
|
||||
#[test]
|
||||
fn safetensors_nul_padded_header_converts() {
|
||||
let st = make_safetensors_nul_padded(&[1.0, 2.0, 3.0, 4.0, 5.0]);
|
||||
let rvf = safetensors_to_rvf(&st, "wifi-densepose-pretrained")
|
||||
.expect("NUL-padded header must convert");
|
||||
let mut loader = ProgressiveLoader::new(&rvf).expect("converted RVF must load");
|
||||
let lc = loader.load_layer_c().expect("Layer C");
|
||||
assert_eq!(lc.all_weights, vec![1.0, 2.0, 3.0, 4.0, 5.0]);
|
||||
}
|
||||
|
||||
/// CORE #894 PROOF: the published safetensors converts to a container the
|
||||
/// ProgressiveLoader loads (Layer A succeeds, weights present) — the old
|
||||
/// path returned the opaque "invalid magic … 0x77455735" and gave up.
|
||||
|
||||
Reference in New Issue
Block a user