diff --git a/docs/development/OpenNexus生产化实施进度-2026-09-08.md b/docs/development/OpenNexus生产化实施进度-2026-09-08.md index ec3a8d3..12e34d8 100644 --- a/docs/development/OpenNexus生产化实施进度-2026-09-08.md +++ b/docs/development/OpenNexus生产化实施进度-2026-09-08.md @@ -625,3 +625,13 @@ Core 的独立数据目录目前不等于已授权 Vault。Python 旧笔记写 - 首轮全目标编译发现删除决策 fingerprint 仍需要生产 hash 导入,已恢复其非 test 范围,同时移除 sync_inbox 中不再使用的 fs 导入;该问题在全目标回归期间发现,未将仅 lib test 编译通过当作全部目标通过。 - 外部变更扫描、初始绑定对账和部分逻辑记录/API 仍有整块读取;未将这些功能测试当作 RSS 基准或完整强杀/断电矩阵。大附件端到端内存与完整生产化目标继续未完成。 - 最终 desktop 全目标累计 135 通过、12 ignored(库 120、Host 8、其余集成 7),包括实际 Sync 服务与故障恢复集成;日志 `.build/sync-conflict-stream-full.log`。desktop 全目标 Clippy -D warnings 通过,日志 `.build/sync-conflict-stream-clippy.log`。本轮没有重新执行 ignored 沙箱长时验收,全部生产化条件仍未满足。 + + +## 增量:扫描、首次对账与重新绑定的流式读取 + +- 共用 hash_file_info 返回实际读取的摘要与长度;sync_file_info 对普通文件有界流式读取,对逻辑记录仍最多读取 1 MiB 加 1 字节并执行现有 schema 校验。sync_discover 不再读入整块文件,发现变化后用 store_payload_file 保存经过复核的快照再提交观察记录与 outbox。 +- 首次合并预览使用流式摘要/实际长度,确认后的本地准备使用流式 payload 快照,内容变化仍拒绝旧预览。Workspace.scan 的 Markdown 摘要也改为有界读取,避免 UI 扫描先分配整个大文件。 +- 空远端绑定原先只调用 Markdown scan;重新绑定时已观察过的附件没有新 outbox,后续 discover 也可能因摘要未变而跳过。本轮改为枚举同步白名单路径(含允许的附件/记录),归档旧绑定队列后为当前文件准备流式快照与本地写入意图,再建立新绑定队列;不再仅以 Markdown 集合作为绑定数据集。 +- 新增 100 MiB Markdown 与 100 MiB 附件测试,分别走首次合并与空远端绑定:仅改末字节即可使旧预览失效,绑定后重复发现不增加事件,UI scan 后的外部修改仍被 discover 识别一次,重开不重复;切换远端 Vault/账号重新绑定时两类文件均进入新队列且原 file_id 保持。测试通过,耗时 10.36 秒,日志 `.build/sync-discovery-stream-large.log`。先前同步回归 17 项通过,日志 `.build/sync-discovery-stream-tests.log`。 +- 测试补充阶段修正了 Binding 不实现 Debug 时不能使用 unwrap_err 的测试写法,没有为调试输出扩大生产类型的可见字段。上述测试不是真实进程峰值内存测量、10000 文件基准或完整首次绑定故障矩阵;这些验收仍需继续。 +- 最终 desktop 全目标累计 136 通过、12 ignored(库 121、Host 8、其余集成 7),包含真实 Sync 服务与故障恢复集成,日志 `.build/sync-discovery-stream-full.log`;desktop 全目标 Clippy -D warnings 通过,日志 `.build/sync-discovery-stream-clippy.log`。仍需独立进程峰值内存和完整基准/部署/发布验收,完整生产化目标未完成。 diff --git a/frontend/src-tauri/src/payloads.rs b/frontend/src-tauri/src/payloads.rs index c6cc029..84bf036 100644 --- a/frontend/src-tauri/src/payloads.rs +++ b/frontend/src-tauri/src/payloads.rs @@ -165,6 +165,21 @@ pub(crate) fn validate_record(file: &mut fs::File, path: &str, size: u64) -> Res Ok(()) } pub(crate) fn hash_file(path: &Path) -> Result { + hash_file_info(path).map(|(digest, _)| digest) +} +pub(crate) fn sync_file_info(source: &Path, path: &str) -> Result<(String, u64)> { + if crate::records::is_record(path) { + let mut bytes = Vec::new(); + fs::File::open(source)? + .take(1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + crate::records::validate(path, &bytes)?; + Ok((hash(&bytes), bytes.len() as u64)) + } else { + hash_file_info(source) + } +} +fn hash_file_info(path: &Path) -> Result<(String, u64)> { let mut file = fs::File::open(path)?; let mut buffer = vec![0u8; VERIFY_BUFFER_BYTES]; let mut hasher = Sha256::new(); @@ -180,7 +195,7 @@ pub(crate) fn hash_file(path: &Path) -> Result { } hasher.update(&buffer[..count]); } - Ok(format!("{:x}", hasher.finalize())) + Ok((format!("{:x}", hasher.finalize()), total)) } pub(crate) fn verify(path: &Path, digest: &str, size: u64) -> Result<()> { open_verified(path, digest, size).map(drop) diff --git a/frontend/src-tauri/src/sync_discovery.rs b/frontend/src-tauri/src/sync_discovery.rs index 82ac11d..b4575b3 100644 --- a/frontend/src-tauri/src/sync_discovery.rs +++ b/frontend/src-tauri/src/sync_discovery.rs @@ -1,7 +1,7 @@ //! Reconcile externally edited files against committed snapshots, never against UI read caches. -use crate::workspace::{hash, HostError, Result, Workspace}; +use crate::workspace::{HostError, Result, Workspace}; use rusqlite::{params, OptionalExtension}; -use std::{collections::HashSet, fs, io::Read, path::Path}; +use std::{collections::HashSet, fs, path::Path}; use uuid::Uuid; /// File transport only. Logical records receive their own versioned whitelist separately. pub fn allowed(path: &str) -> bool { @@ -93,20 +93,7 @@ impl Workspace { let mut changes = 0; for path in paths { let target = self.resolve(&path)?; - if fs::metadata(&target)?.len() > 104857600 { - return Err(HostError::new("FILE_TOO_LARGE")); - } - let mut bytes = Vec::new(); - fs::File::open(&target)? - .take(104857601) - .read_to_end(&mut bytes)?; - if bytes.len() > 104857600 { - return Err(HostError::new("FILE_TOO_LARGE")); - } - if crate::records::is_record(&path) { - crate::records::validate(&path, &bytes)?; - } - let digest = hash(&bytes); + let (digest, _) = crate::payloads::sync_file_info(&target, &path)?; let previous = self.entry(&path)?; let observed: Option<(String, String, bool)> = if let Some(entry) = &previous { self.db @@ -129,9 +116,8 @@ impl Workspace { continue; } // Confirm the snapshot without writing back over an external editor. - crate::payloads::verify(&target, &digest, bytes.len() as u64)?; let operation = Uuid::new_v4().to_string(); - self.store_payload(&operation, &bytes)?; + self.store_payload_file(&operation, &target, &digest)?; let file_id = previous.map_or_else(|| Uuid::new_v4().to_string(), |e| e.file_id); let tx = self.db.transaction()?; tx.execute("INSERT INTO files VALUES (?1,?2,?3,1,0) ON CONFLICT(path) DO UPDATE SET hash=excluded.hash,revision=files.revision+1,deleted=0",params![file_id,path,digest])?; diff --git a/frontend/src-tauri/src/sync_initial.rs b/frontend/src-tauri/src/sync_initial.rs index 4e61eb2..6a5a85e 100644 --- a/frontend/src-tauri/src/sync_initial.rs +++ b/frontend/src-tauri/src/sync_initial.rs @@ -6,7 +6,9 @@ use crate::{ }; use rusqlite::{params, OptionalExtension}; use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, fs}; +use std::collections::BTreeMap; +#[cfg(test)] +use std::fs; use uuid::Uuid; #[derive(Clone, Serialize, Deserialize)] pub struct Snapshot { @@ -36,17 +38,11 @@ impl Workspace { .into_iter() .map(|path| { let source = self.resolve(&path)?; - if fs::metadata(&source)?.len() > 104857600 { - return Err(HostError::new("FILE_TOO_LARGE")); - } - let bytes = fs::read(source)?; - if crate::records::is_record(&path) { - crate::records::validate(&path, &bytes)?; - } + let (hash, size) = crate::payloads::sync_file_info(&source, &path)?; Ok(Local { path, - hash: hash(&bytes), - size: bytes.len(), + hash, + size: size as usize, }) }) .collect() @@ -131,11 +127,14 @@ impl Workspace { let mut prepared = Vec::new(); for item in local { let operation = Uuid::new_v4().to_string(); - let bytes = fs::read(self.resolve(&item.path)?)?; - if hash(&bytes) != item.hash { - return Err(HostError::new("SYNC_PREVIEW_CHANGED")); - } - self.store_payload(&operation, &bytes)?; + self.store_payload_file(&operation, &self.resolve(&item.path)?, &item.hash) + .map_err(|error| { + if error.code == "REVISION_CONFLICT" { + HostError::new("SYNC_PREVIEW_CHANGED") + } else { + error + } + })?; let old = self.entry(&item.path)?; let remote = snapshot .items @@ -271,6 +270,98 @@ mod tests { } } #[test] + fn hundred_mib_discovery_preview_and_rebinding_preserve_all_current_files() { + use std::io::{Seek, SeekFrom, Write}; + for initial in [false, true] { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("attachments")).unwrap(); + let source = root.path().join("large.md"); + let mut file = fs::File::create(&source).unwrap(); + let block = vec![b'a'; 64 * 1024]; + for _ in 0..1600 { + file.write_all(&block).unwrap(); + } + file.sync_all().unwrap(); + drop(file); + fs::copy(&source, root.path().join("attachments/large.bin")).unwrap(); + let mut ws = Workspace::open(root.path()).unwrap(); + let snapshot = Snapshot { + boundary: 0, + items: Vec::new(), + }; + let binding = if initial { + let preview = ws + .sync_preview("https://sync.example", "remote", "account", &snapshot) + .unwrap(); + let mut file = fs::OpenOptions::new().write(true).open(&source).unwrap(); + file.seek(SeekFrom::End(-1)).unwrap(); + file.write_all(b"b").unwrap(); + file.sync_all().unwrap(); + drop(file); + assert_eq!( + ws.sync_bind_initial( + "https://sync.example", + "remote", + "account", + &snapshot, + &preview.fingerprint + ) + .err() + .unwrap() + .code, + "SYNC_PREVIEW_CHANGED" + ); + let preview = ws + .sync_preview("https://sync.example", "remote", "account", &snapshot) + .unwrap(); + ws.sync_bind_initial( + "https://sync.example", + "remote", + "account", + &snapshot, + &preview.fingerprint, + ) + .unwrap() + } else { + ws.sync_bind_empty("https://sync.example", "remote", "account") + .unwrap() + }; + assert_eq!(ws.sync_discover(&binding.id).unwrap(), 0); + ws.sync_capture(&binding.id).unwrap(); + assert_eq!(ws.pending_count().unwrap(), 2); + let identity = ws.entry("large.md").unwrap().unwrap().file_id; + let mut file = fs::OpenOptions::new().write(true).open(&source).unwrap(); + file.seek(SeekFrom::End(-1)).unwrap(); + file.write_all(b"c").unwrap(); + file.sync_all().unwrap(); + drop(file); + ws.scan().unwrap(); + assert_eq!(ws.sync_discover(&binding.id).unwrap(), 1); + assert_eq!(ws.entry("large.md").unwrap().unwrap().file_id, identity); + drop(ws); + ws = Workspace::open(root.path()).unwrap(); + assert_eq!(ws.sync_discover(&binding.id).unwrap(), 0); + assert_eq!(ws.pending_count().unwrap(), 3); + ws.sync_unbind(&binding.id).unwrap(); + let rebound = ws + .sync_bind_empty("https://sync.example", "another-vault", "another-account") + .unwrap(); + assert_eq!(ws.pending_count().unwrap(), 2); + assert_eq!( + ws.db + .query_row( + "SELECT COUNT(*) FROM sync_jobs WHERE binding=?1", + [&rebound.id], + |r| r.get::<_, i64>(0) + ) + .unwrap(), + 2 + ); + assert_eq!(ws.sync_discover(&rebound.id).unwrap(), 0); + assert_eq!(ws.entry("large.md").unwrap().unwrap().file_id, identity); + } + } + #[test] fn initial_snapshot_cursor_waits_for_all_files_and_recovers_twenty_rounds() { for _ in 0..20 { for committed in [false, true] { diff --git a/frontend/src-tauri/src/sync_state.rs b/frontend/src-tauri/src/sync_state.rs index 685a050..8637564 100644 --- a/frontend/src-tauri/src/sync_state.rs +++ b/frontend/src-tauri/src/sync_state.rs @@ -71,7 +71,7 @@ impl Workspace { .query_row("SELECT EXISTS(SELECT 1 FROM sync_bindings)", [], |r| { r.get(0) })?; - let entries = self.scan()?; + let paths = self.sync_paths()?; let id = Uuid::new_v4().to_string(); // Rebinding explicitly starts from the current snapshot, never an old account's queue. if had_binding { @@ -80,15 +80,30 @@ impl Workspace { [], )?; } - for entry in entries.into_iter().filter(|e| !e.is_folder && !e.deleted) { - let queued: bool = self.db.query_row( - "SELECT EXISTS(SELECT 1 FROM outbox WHERE file_id=?1 AND state='pending')", - [&entry.file_id], - |r| r.get(0), - )?; + for path in paths { + let entry = self.entry(&path)?; + let queued = if let Some(entry) = &entry { + self.db.query_row( + "SELECT EXISTS(SELECT 1 FROM outbox WHERE file_id=?1 AND state='pending')", + [&entry.file_id], + |r| r.get::<_, bool>(0), + )? + } else { + false + }; if !queued { - let content = fs::read(self.resolve(&entry.path)?)?; - self.write(&entry.path, &entry.hash, &content, "local")?; + let source = self.resolve(&path)?; + let (digest, size) = crate::payloads::sync_file_info(&source, &path)?; + let operation = Uuid::new_v4().to_string(); + self.store_payload_file(&operation, &source, &digest)?; + self.write_spooled_with_identity( + &path, + &digest, + (&digest, size), + "local", + &operation, + None, + )?; } } self.db.execute( diff --git a/frontend/src-tauri/src/workspace.rs b/frontend/src-tauri/src/workspace.rs index 13971ab..16af59c 100644 --- a/frontend/src-tauri/src/workspace.rs +++ b/frontend/src-tauri/src/workspace.rs @@ -298,8 +298,7 @@ impl Workspace { }); continue; } - let content = fs::read(self.resolve(&path)?)?; - let digest = hash(&content); + let digest = crate::payloads::hash_file(&self.resolve(&path)?)?; let previous = self.entry(&path)?; if previous .as_ref()