ZDDC
1f8b4e4aaa
Previously every .zip was auto-expanded into a folder of members (and was even double-represented as both a file and a folder node). Now a .zip is one classifiable file by default; right-click it → "Expand as folder" to pull its members into the fileset, and right-click an expanded archive (or a member) → "Collapse to single file" to go back. The toggle sits with Exclude in the context menu. - scanner: stop creating zip-root nodes during the scan; expandZipAsFolder / collapseZipToFile mutate the tree in place (re-reading members from the live handle or, for a restored workspace, lazily from the root) and recompute subtree totals. Mode is encoded by the tree shape, so it persists in the snapshot as-is. - classify.dropAssignments clears the assignments that cease to exist when a zip flips mode (the single-file key on expand; the member keys on collapse). - copy already handles both: a zip-as-file copies whole; members extract from the archive. Also: a folder whose entire subtree is excluded now renders its name struck through, mirroring the excluded-file style. Tests: collapse restores the single .zip + drops member assignments; a fully-excluded folder gets the struck-through class (48 green). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1007 lines
44 KiB
JavaScript
1007 lines
44 KiB
JavaScript
/**
|
||
* Directory Scanner Module
|
||
* Scans directories and collects files
|
||
*/
|
||
(function() {
|
||
'use strict';
|
||
|
||
// Store ZIP data for later access
|
||
const zipCache = new Map(); // path -> { zip: JSZip, fileHandle: FileSystemFileHandle }
|
||
|
||
// ── Incremental-scan state ───────────────────────────────────────────────
|
||
// The scan no longer reads the whole tree before rendering. It walks
|
||
// breadth-first behind a small worker pool, renders progressively (top
|
||
// levels appear first), and shows live status — so a huge network drive
|
||
// never looks stalled. Each folder tracks its own scan state + counts.
|
||
let scanGen = 0; // bumped per scan; stale workers bail
|
||
let scanStats = null; // { folders, files, current, done, startedAt }
|
||
let renderTimer = null; // throttle for progressive re-render
|
||
// How many directories to read in flight at once. The scan is I/O-bound —
|
||
// each readdir is a round-trip to the backing store (cloud-sync / network
|
||
// mounts like OneDrive or Samba have high per-op latency), so the lever is
|
||
// parallel in-flight reads, not CPU threads. This only helps the
|
||
// many-folders case; a single fat folder is enumerated one entry at a time
|
||
// by the File System Access API and can't be parallelized. Raise it if the
|
||
// store tolerates more concurrency; too high risks cloud-provider
|
||
// throttling.
|
||
var SCAN_CONCURRENCY = 32;
|
||
|
||
function scheduleRender() {
|
||
if (renderTimer) return;
|
||
renderTimer = setTimeout(function () {
|
||
renderTimer = null;
|
||
try { window.app.modules.tree.render(); } catch (_) { /* ignore */ }
|
||
updateScanStatus();
|
||
}, 180);
|
||
}
|
||
|
||
function flushRender() {
|
||
if (renderTimer) { clearTimeout(renderTimer); renderTimer = null; }
|
||
try { window.app.modules.tree.render(); } catch (_) { /* ignore */ }
|
||
updateScanStatus();
|
||
}
|
||
|
||
// elapsed since the scan started, e.g. "3.2s" or "1m 04s".
|
||
function elapsedStr() {
|
||
if (!scanStats) return '0s';
|
||
const ms = Date.now() - scanStats.startedAt;
|
||
if (ms < 60000) return (ms / 1000).toFixed(1) + 's';
|
||
const m = Math.floor(ms / 60000);
|
||
const s = Math.round((ms % 60000) / 1000);
|
||
return m + 'm ' + (s < 10 ? '0' : '') + s + 's';
|
||
}
|
||
|
||
// Render the running scan status (with live elapsed time) into the footer.
|
||
function updateScanStatus() {
|
||
const el = document.getElementById('scanStatus');
|
||
if (!el || !scanStats) return;
|
||
if (scanStats.done) {
|
||
el.textContent = 'Scanned ' + scanStats.folders + ' folders · '
|
||
+ scanStats.files + ' files in ' + elapsedStr();
|
||
el.classList.remove('scanning');
|
||
} else {
|
||
el.textContent = 'Scanning… ' + scanStats.folders + ' folders · '
|
||
+ scanStats.files + ' files · ' + elapsedStr()
|
||
+ (scanStats.current ? ' — ' + scanStats.current : '');
|
||
el.classList.add('scanning');
|
||
}
|
||
}
|
||
|
||
// Make a tree node. scanState: 'pending' (children not read) →
|
||
// 'scanning' → 'children' (immediate children read, subtree still going) →
|
||
// 'done' (entire subtree enumerated). The UI greys a node until 'done'.
|
||
function makeNode(handle, path, parent) {
|
||
const node = {
|
||
name: handle.name,
|
||
path: path,
|
||
handle: handle,
|
||
parent: parent || null,
|
||
files: [],
|
||
fileCount: 0, // direct files in this folder
|
||
subdirCount: 0, // direct subfolders
|
||
runFiles: 0, // files in the whole subtree (grows as scanned; final on 'done')
|
||
runDirs: 0, // subfolders in the whole subtree
|
||
children: [],
|
||
expanded: false,
|
||
scanState: 'pending',
|
||
pending: 0, // child dirs not yet 'done'
|
||
};
|
||
if (handle.isZipRoot) { node.isZipRoot = true; node.zipPath = handle.zipPath; }
|
||
if (handle.isVirtualDir) { node.isVirtualDir = true; node.zipPath = handle.zipPath; }
|
||
return node;
|
||
}
|
||
|
||
// Mark a node's subtree fully scanned: roll up recursive totals and
|
||
// propagate completion to the parent (which flips to 'done' once all its
|
||
// children are done). This is what turns a folder from grey to solid.
|
||
function markDone(node) {
|
||
if (node.scanState === 'done') return;
|
||
// runFiles/runDirs were accumulated into this node (and its ancestors)
|
||
// as each descendant was scanned, so by the time the subtree is
|
||
// complete they already hold the final totals — nothing to compute.
|
||
node.scanState = 'done';
|
||
const p = node.parent;
|
||
if (p && p.scanState !== 'done') {
|
||
p.pending -= 1;
|
||
if (p.pending <= 0 && (p.scanState === 'children' || p.scanState === 'scanning')) {
|
||
markDone(p);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Translate a File System Access API error into accurate, actionable text.
|
||
// The browser's raw DOMException messages are cryptic and often read like a
|
||
// permission problem when they aren't — we key off err.name (reliable)
|
||
// rather than the message. Returns a plain-language explanation; the raw
|
||
// name + message are still appended by the caller for troubleshooting.
|
||
function describeFsError(err) {
|
||
var name = err && err.name ? err.name : '';
|
||
switch (name) {
|
||
case 'NotAllowedError':
|
||
return 'Permission to read this folder was denied or revoked. '
|
||
+ 'Re-pick the root folder to re-grant access.';
|
||
case 'InvalidStateError':
|
||
// The handle was read once, then the directory changed underneath
|
||
// it (common on a network/SMB share that's being written to, or
|
||
// after a disconnect/reconnect). NOT a permissions problem.
|
||
return 'The folder changed on disk since it was first read '
|
||
+ '(common on a busy or reconnecting network share). '
|
||
+ 'Rescan to pick up the current contents.';
|
||
case 'NotFoundError':
|
||
return 'The folder no longer exists — it may have been moved, '
|
||
+ 'renamed, or deleted since the scan started.';
|
||
case 'NotReadableError':
|
||
return 'The folder could not be read — the share may have '
|
||
+ 'disconnected, or the OS denied access.';
|
||
case 'SecurityError':
|
||
return 'The browser blocked access to this folder for security '
|
||
+ 'reasons.';
|
||
case 'TypeMismatchError':
|
||
return 'Expected a folder here but found a file (or vice-versa).';
|
||
case 'AbortError':
|
||
return 'Reading this folder was aborted.';
|
||
default:
|
||
return 'Could not read this folder.';
|
||
}
|
||
}
|
||
|
||
// One-shot toast for scan errors (permission denied, stale handles, network
|
||
// hiccups on a share). De-duped per path so a flaky folder doesn't spam.
|
||
const scanErrorsSeen = new Set();
|
||
function reportScanError(path, err) {
|
||
console.error('Scan error:', path, err);
|
||
if (scanErrorsSeen.has(path)) return;
|
||
scanErrorsSeen.add(path);
|
||
// Plain-language explanation, then the raw error in parentheses so the
|
||
// user can copy it (toasts are selectable) for deeper troubleshooting.
|
||
var raw = err && err.name
|
||
? err.name + (err.message ? ': ' + err.message : '')
|
||
: (err && err.message ? err.message : String(err));
|
||
var msg = 'Couldn’t scan ' + path + ' — ' + describeFsError(err)
|
||
+ '\n\n(' + raw + ')';
|
||
if (window.zddc && typeof window.zddc.toast === 'function') {
|
||
window.zddc.toast(msg, 'error');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Scan directory and build folder tree with files
|
||
*/
|
||
async function scanDirectory(dirHandle, preserveState = false) {
|
||
// Preserve which folders were expanded across a rescan (e.g. after a
|
||
// ZIP extract) so the user doesn't lose their place.
|
||
let savedExpanded = new Set();
|
||
let savedSelected = new Set();
|
||
if (preserveState) {
|
||
savedExpanded = getExpandedPaths(window.app.folderTree);
|
||
savedSelected = new Set(window.app.selectedFolders);
|
||
}
|
||
|
||
const myGen = ++scanGen;
|
||
zipCache.clear();
|
||
scanStats = { folders: 0, files: 0, current: dirHandle.name, done: false, startedAt: Date.now() };
|
||
|
||
// Root node — render immediately so the pane never sits blank.
|
||
const root = makeNode(dirHandle, dirHandle.name, null);
|
||
root.expanded = true;
|
||
window.app.folderTree = [root];
|
||
window.app.modules.store.setFolderTree(window.app.folderTree);
|
||
if (!preserveState) {
|
||
// Select the root so the grid shows its immediate files at once,
|
||
// instead of auto-loading the ENTIRE drive (the old behaviour,
|
||
// which is exactly what stalled on a large share).
|
||
window.app.selectedFolders = new Set([root.path]);
|
||
window.app.lastSelectedFolderPath = root.path;
|
||
window.app.modules.store.setSelectedFolders(window.app.selectedFolders);
|
||
} else {
|
||
window.app.selectedFolders = savedSelected;
|
||
window.app.modules.store.setSelectedFolders(savedSelected);
|
||
}
|
||
flushRender();
|
||
|
||
// Tick the footer's elapsed time once a second even if no new folder
|
||
// landed (so a slow directory doesn't make the timer look frozen).
|
||
const ticker = setInterval(function () {
|
||
if (myGen !== scanGen || (scanStats && scanStats.done)) { clearInterval(ticker); return; }
|
||
updateScanStatus();
|
||
}, 1000);
|
||
|
||
// Continuous breadth-first walk: up to SCAN_CONCURRENCY directory reads
|
||
// in flight at once, pulling newly-discovered child dirs as they land
|
||
// (no per-level barrier, so the pool stays saturated). Top levels still
|
||
// appear first (FIFO). The cap is the lever — see SCAN_CONCURRENCY.
|
||
await drainQueue([root], myGen, SCAN_CONCURRENCY);
|
||
if (preserveState && savedExpanded.size) {
|
||
restoreExpandedPaths(window.app.folderTree, savedExpanded);
|
||
}
|
||
clearInterval(ticker);
|
||
if (myGen !== scanGen) return; // superseded by a newer scan
|
||
|
||
scanStats.done = true;
|
||
scanStats.current = '';
|
||
flushRender();
|
||
|
||
// Completion toast with the totals + elapsed time.
|
||
if (window.zddc && typeof window.zddc.toast === 'function') {
|
||
window.zddc.toast(
|
||
'Scan complete — ' + scanStats.folders + ' folders, '
|
||
+ scanStats.files + ' files in ' + elapsedStr() + '.',
|
||
'success');
|
||
}
|
||
}
|
||
|
||
// Continuous worker pool over a shared queue: keep up to `conc` directory
|
||
// reads in flight at once, pulling newly-discovered child dirs as they land
|
||
// — no per-level barrier, so workers never idle waiting on the slowest dir
|
||
// in a level. Roughly breadth-first (FIFO; a node's children are enqueued
|
||
// after it), so top levels still surface first. Resolves when the queue is
|
||
// drained and no read is in flight (clean termination, no empty-queue race).
|
||
function drainQueue(seed, myGen, conc) {
|
||
const queue = seed.slice();
|
||
let active = 0;
|
||
return new Promise(function (resolve) {
|
||
function finishIfIdle() {
|
||
if (queue.length === 0 && active === 0) resolve();
|
||
}
|
||
function pump() {
|
||
while (myGen === scanGen && active < conc && queue.length) {
|
||
const node = queue.shift();
|
||
active++;
|
||
Promise.resolve(scanNodeChildren(node, myGen)).then(function () {
|
||
active--;
|
||
if (myGen === scanGen) {
|
||
const kids = node.children;
|
||
for (let i = 0; i < kids.length; i++) {
|
||
if (kids[i].scanState === 'pending') queue.push(kids[i]);
|
||
}
|
||
}
|
||
pump();
|
||
finishIfIdle();
|
||
}, function () { active--; pump(); finishIfIdle(); });
|
||
}
|
||
finishIfIdle();
|
||
}
|
||
pump();
|
||
});
|
||
}
|
||
|
||
// Force a folder's subtree to scan NOW (jumped ahead of the background
|
||
// walk). Called when the user opens a folder, so an opened folder always
|
||
// shows complete contents. Idempotent + shares the live scan generation.
|
||
async function ensureScanned(node) {
|
||
if (!node || !node.handle || node.scanState === 'done') return;
|
||
await drainQueue([node], scanGen, SCAN_CONCURRENCY);
|
||
flushRender();
|
||
}
|
||
|
||
/**
|
||
* Get all expanded folder paths from tree
|
||
*/
|
||
function getExpandedPaths(folders, paths = new Set()) {
|
||
for (const folder of folders) {
|
||
if (folder.expanded) {
|
||
paths.add(folder.path);
|
||
}
|
||
if (folder.children) {
|
||
getExpandedPaths(folder.children, paths);
|
||
}
|
||
}
|
||
return paths;
|
||
}
|
||
|
||
/**
|
||
* Restore expanded state to tree
|
||
*/
|
||
function restoreExpandedPaths(folders, expandedPaths) {
|
||
for (const folder of folders) {
|
||
folder.expanded = expandedPaths.has(folder.path);
|
||
if (folder.children) {
|
||
restoreExpandedPaths(folder.children, expandedPaths);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Read ONE directory's immediate entries: files into node.files, child
|
||
// directories into node.children (left 'pending' for the BFS to descend).
|
||
// A .zip becomes an expandable zip-root child, scanned inline (its
|
||
// contents are already in memory once the entry is read). Idempotent:
|
||
// only a 'pending' node is scanned, so concurrent callers (background +
|
||
// open-prioritised) don't double-scan.
|
||
async function scanNodeChildren(node, myGen) {
|
||
// A .zip is a lazy node — read its contents only when opened.
|
||
if (node.scanState === 'zip-pending') { await scanZipNode(node); return; }
|
||
if (node.scanState !== 'pending') return;
|
||
node.scanState = 'scanning';
|
||
if (scanStats) scanStats.current = node.path;
|
||
const files = [];
|
||
const childDirs = [];
|
||
try {
|
||
for await (const entry of node.handle.values()) {
|
||
if (myGen !== scanGen) { node.scanState = 'pending'; return; } // cancelled
|
||
if (entry.kind === 'file') {
|
||
const fo = createFileObject(entry, node.handle);
|
||
fo.folderPath = node.path;
|
||
files.push(fo);
|
||
if (scanStats) scanStats.files++;
|
||
// A .zip is a single file by default (one classifiable unit).
|
||
// The user can later "Expand as folder" (expandZipAsFolder) to
|
||
// pull its members into the fileset.
|
||
} else if (entry.kind === 'directory') {
|
||
const childPath = node.path + '/' + entry.name;
|
||
childDirs.push(makeNode(entry, childPath, node));
|
||
if (scanStats) scanStats.folders++;
|
||
}
|
||
}
|
||
} catch (err) {
|
||
node.scanError = true;
|
||
reportScanError(node.path, err);
|
||
}
|
||
node.files = files;
|
||
node.fileCount = files.length;
|
||
node.children = childDirs;
|
||
node.subdirCount = childDirs.length;
|
||
// Roll this folder's own files/dirs into the running subtree totals of
|
||
// this node + every ancestor. Real child dirs add their share when they
|
||
// get scanned; lazy zip nodes add theirs when opened (scanZipNode).
|
||
const addF = files.length;
|
||
const addD = childDirs.length;
|
||
for (let a = node; a; a = a.parent) { a.runFiles += addF; a.runDirs += addD; }
|
||
// Only real unscanned dirs hold the parent open; zip-pending children
|
||
// are lazy, so they don't.
|
||
node.pending = childDirs.filter(function (c) { return c.scanState === 'pending'; }).length;
|
||
if (node.pending === 0) {
|
||
markDone(node);
|
||
} else {
|
||
node.scanState = 'children';
|
||
}
|
||
scheduleRender();
|
||
}
|
||
|
||
// Read a lazy zip node's contents on demand (when opened), building its
|
||
// child nodes and folding its internal totals into ancestors.
|
||
async function scanZipNode(node) {
|
||
if (node.scanState !== 'zip-pending') return;
|
||
var fileObj = node._zipFileObj;
|
||
if (!fileObj) {
|
||
// Restored from a snapshot — no live file object. Resolve the .zip
|
||
// from the workspace root by its path so it can be opened on demand.
|
||
if (!window.app.rootHandle || !node.zipPath) return;
|
||
try {
|
||
var dir = await resolveDirHandle(window.app.rootHandle, relFromRoot(parentPath(node.zipPath)));
|
||
fileObj = { handle: await dir.getFileHandle(baseName(node.zipPath)), folderHandle: dir };
|
||
} catch (e) {
|
||
reportScanError(node.path, e); node.scanState = 'done'; node.runFiles = 0; node.runDirs = 0; return;
|
||
}
|
||
}
|
||
node.scanState = 'scanning';
|
||
scheduleRender();
|
||
try {
|
||
await scanZipIntoNode(node, fileObj); // builds children, runFiles/runDirs, sets 'done'
|
||
} catch (e) {
|
||
reportScanError(node.path, e);
|
||
node.scanState = 'done';
|
||
node.runFiles = 0;
|
||
node.runDirs = 0;
|
||
}
|
||
node._zipFileObj = null;
|
||
// The zip counted as 1 dir in its parent already; now fold in its
|
||
// internal files/dirs to every ancestor's running totals.
|
||
for (let a = node.parent; a; a = a.parent) {
|
||
a.runFiles += node.runFiles;
|
||
a.runDirs += node.runDirs;
|
||
}
|
||
scheduleRender();
|
||
}
|
||
|
||
// Build a zip-root node's children from its archive contents (in memory),
|
||
// marking the whole zip subtree 'done' immediately. Mirrors the on-disk
|
||
// node shape so the rest of the app treats zip folders like real ones.
|
||
async function scanZipIntoNode(zipNode, fileObj) {
|
||
const f = await fileObj.handle.getFile();
|
||
const zip = await JSZip.loadAsync(await f.arrayBuffer());
|
||
const zipPath = zipNode.path;
|
||
zipCache.set(zipPath, { zip: zip, fileHandle: fileObj.handle, folderHandle: fileObj.folderHandle });
|
||
const dirNodes = new Map();
|
||
dirNodes.set(zipPath, zipNode);
|
||
function ensureDir(dirPath) {
|
||
if (dirNodes.has(dirPath)) return dirNodes.get(dirPath);
|
||
const parentPath = dirPath.substring(0, dirPath.lastIndexOf('/'));
|
||
const parent = ensureDir(parentPath);
|
||
const name = dirPath.substring(dirPath.lastIndexOf('/') + 1);
|
||
const vh = { name: name, kind: 'directory', isVirtualDir: true, zipPath: zipPath, virtualPath: dirPath };
|
||
const child = makeNode(vh, dirPath, parent);
|
||
parent.children.push(child);
|
||
dirNodes.set(dirPath, child);
|
||
return child;
|
||
}
|
||
zip.forEach(function (relativePath, entry) {
|
||
if (entry.dir) {
|
||
ensureDir(zipPath + '/' + relativePath.replace(/\/$/, ''));
|
||
} else {
|
||
const fileName = relativePath.split('/').pop();
|
||
const fileDir = relativePath.includes('/')
|
||
? zipPath + '/' + relativePath.substring(0, relativePath.lastIndexOf('/'))
|
||
: zipPath;
|
||
const dirNode = ensureDir(fileDir);
|
||
const split = zddc.splitExtension(fileName);
|
||
dirNode.files.push({
|
||
originalFilename: split.name,
|
||
extension: split.extension,
|
||
size: entry._data ? entry._data.uncompressedSize : 0,
|
||
lastModified: entry.date ? entry.date.getTime() : Date.now(),
|
||
isVirtual: true,
|
||
zipPath: zipPath,
|
||
zipEntryPath: relativePath,
|
||
folderPath: dirNode.path,
|
||
trackingNumber: '', revision: '', status: '', title: '',
|
||
isDirty: false, error: false, errorMessage: '', validation: null, sha256: null
|
||
});
|
||
}
|
||
});
|
||
finalizeZipNode(zipNode);
|
||
}
|
||
|
||
// Roll up a zip node's counts + mark its whole subtree 'done'.
|
||
function finalizeZipNode(node) {
|
||
node.fileCount = node.files.length;
|
||
node.subdirCount = node.children.length;
|
||
let rf = node.files.length, rd = node.children.length;
|
||
for (const c of node.children) { finalizeZipNode(c); rf += c.runFiles; rd += c.runDirs; }
|
||
node.runFiles = rf;
|
||
node.runDirs = rd;
|
||
node.scanState = 'done';
|
||
node.pending = 0;
|
||
}
|
||
|
||
/**
|
||
* Scan a ZIP file and add its contents as virtual folders
|
||
*/
|
||
async function scanZipFile(zipFileObj, foldersMap, parentPath, parentItems) {
|
||
try {
|
||
const fileObj = await zipFileObj.handle.getFile();
|
||
const arrayBuffer = await fileObj.arrayBuffer();
|
||
const zip = await JSZip.loadAsync(arrayBuffer);
|
||
|
||
const zipPath = parentPath + '/' + zddc.joinExtension(zipFileObj.originalFilename, zipFileObj.extension);
|
||
|
||
// Cache the ZIP for later extraction
|
||
zipCache.set(zipPath, {
|
||
zip: zip,
|
||
fileHandle: zipFileObj.handle,
|
||
folderHandle: zipFileObj.folderHandle
|
||
});
|
||
|
||
// Mark the file as a ZIP container
|
||
zipFileObj.isZipContainer = true;
|
||
zipFileObj.zipPath = zipPath;
|
||
|
||
// Build virtual folder structure from ZIP contents
|
||
const virtualFolders = new Map(); // path -> { files: [], subdirs: Set }
|
||
virtualFolders.set(zipPath, { files: [], subdirs: new Set() });
|
||
|
||
zip.forEach((relativePath, zipEntry) => {
|
||
if (zipEntry.dir) {
|
||
// It's a directory
|
||
const dirPath = zipPath + '/' + relativePath.replace(/\/$/, '');
|
||
if (!virtualFolders.has(dirPath)) {
|
||
virtualFolders.set(dirPath, { files: [], subdirs: new Set() });
|
||
}
|
||
// Add to parent's subdirs
|
||
const parentDir = dirPath.substring(0, dirPath.lastIndexOf('/'));
|
||
if (virtualFolders.has(parentDir)) {
|
||
virtualFolders.get(parentDir).subdirs.add(dirPath);
|
||
}
|
||
} else {
|
||
// It's a file
|
||
const fileName = relativePath.split('/').pop();
|
||
const fileDir = relativePath.includes('/')
|
||
? zipPath + '/' + relativePath.substring(0, relativePath.lastIndexOf('/'))
|
||
: zipPath;
|
||
|
||
// Ensure parent directories exist
|
||
ensureVirtualPath(virtualFolders, zipPath, fileDir);
|
||
|
||
// Create virtual file object
|
||
const split = zddc.splitExtension(fileName);
|
||
|
||
const virtualFile = {
|
||
originalFilename: split.name,
|
||
extension: split.extension,
|
||
size: zipEntry._data ? zipEntry._data.uncompressedSize : 0,
|
||
lastModified: zipEntry.date ? zipEntry.date.getTime() : Date.now(),
|
||
|
||
// Virtual file markers
|
||
isVirtual: true,
|
||
zipPath: zipPath,
|
||
zipEntryPath: relativePath,
|
||
|
||
// Editable fields
|
||
trackingNumber: '',
|
||
revision: '',
|
||
status: '',
|
||
title: '',
|
||
|
||
// State
|
||
isDirty: false,
|
||
error: false,
|
||
errorMessage: '',
|
||
validation: null,
|
||
sha256: null
|
||
};
|
||
|
||
virtualFolders.get(fileDir).files.push(virtualFile);
|
||
}
|
||
});
|
||
|
||
// Convert virtual folders to format compatible with tree builder
|
||
// Create a virtual handle for the ZIP root
|
||
const zipVirtualHandle = {
|
||
name: zddc.joinExtension(zipFileObj.originalFilename, zipFileObj.extension),
|
||
kind: 'directory',
|
||
isZipRoot: true,
|
||
zipPath: zipPath
|
||
};
|
||
|
||
// Store virtual folder data
|
||
buildVirtualFolderMap(virtualFolders, zipPath, foldersMap, zipVirtualHandle);
|
||
|
||
// Add ZIP as a virtual directory in parent
|
||
parentItems.push({
|
||
handle: zipVirtualHandle,
|
||
isDirectory: true,
|
||
isZipRoot: true
|
||
});
|
||
|
||
} catch (err) {
|
||
console.error('Error scanning ZIP file:', zipFileObj.originalFilename, err);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Ensure all parent directories exist in virtual folder map
|
||
*/
|
||
function ensureVirtualPath(virtualFolders, zipPath, targetPath) {
|
||
if (virtualFolders.has(targetPath)) return;
|
||
|
||
const parts = targetPath.substring(zipPath.length + 1).split('/').filter(p => p);
|
||
let currentPath = zipPath;
|
||
|
||
for (const part of parts) {
|
||
const parentPath = currentPath;
|
||
currentPath = currentPath + '/' + part;
|
||
|
||
if (!virtualFolders.has(currentPath)) {
|
||
virtualFolders.set(currentPath, { files: [], subdirs: new Set() });
|
||
}
|
||
|
||
if (virtualFolders.has(parentPath)) {
|
||
virtualFolders.get(parentPath).subdirs.add(currentPath);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Build virtual folder entries for the foldersMap
|
||
* Uses path strings as keys for virtual folders to avoid object reference issues
|
||
*/
|
||
function buildVirtualFolderMap(virtualFolders, zipPath, foldersMap, zipVirtualHandle) {
|
||
const rootData = virtualFolders.get(zipPath);
|
||
if (!rootData) return;
|
||
|
||
// Create items array for ZIP root
|
||
const rootItems = [...rootData.files];
|
||
|
||
// Add subdirectories
|
||
for (const subdirPath of rootData.subdirs) {
|
||
const subdirName = subdirPath.split('/').pop();
|
||
const subdirHandle = {
|
||
name: subdirName,
|
||
kind: 'directory',
|
||
isVirtualDir: true,
|
||
virtualPath: subdirPath,
|
||
zipPath: zipPath
|
||
};
|
||
rootItems.push({
|
||
handle: subdirHandle,
|
||
isDirectory: true,
|
||
isVirtualDir: true
|
||
});
|
||
|
||
// Recursively add subdir contents
|
||
buildVirtualSubfolder(virtualFolders, subdirPath, foldersMap, zipPath);
|
||
}
|
||
|
||
// Store with both the handle object AND the path string as keys
|
||
// This ensures lookup works regardless of which reference is used
|
||
foldersMap.set(zipVirtualHandle, rootItems);
|
||
foldersMap.set(zipPath, rootItems); // Path-based key for tree building
|
||
}
|
||
|
||
/**
|
||
* Recursively build virtual subfolder entries
|
||
*/
|
||
function buildVirtualSubfolder(virtualFolders, folderPath, foldersMap, zipPath) {
|
||
const folderData = virtualFolders.get(folderPath);
|
||
if (!folderData) return;
|
||
|
||
const folderName = folderPath.split('/').pop();
|
||
const folderHandle = {
|
||
name: folderName,
|
||
kind: 'directory',
|
||
isVirtualDir: true,
|
||
virtualPath: folderPath,
|
||
zipPath: zipPath
|
||
};
|
||
|
||
const items = [...folderData.files];
|
||
|
||
// Store with path string key for tree building lookup
|
||
foldersMap.set(folderPath, items);
|
||
|
||
// Add subdirectories
|
||
for (const subdirPath of folderData.subdirs) {
|
||
const subdirName = subdirPath.split('/').pop();
|
||
const subdirHandle = {
|
||
name: subdirName,
|
||
kind: 'directory',
|
||
isVirtualDir: true,
|
||
virtualPath: subdirPath,
|
||
zipPath: zipPath
|
||
};
|
||
items.push({
|
||
handle: subdirHandle,
|
||
isDirectory: true,
|
||
isVirtualDir: true
|
||
});
|
||
|
||
// Recursively add subdir contents
|
||
buildVirtualSubfolder(virtualFolders, subdirPath, foldersMap, zipPath);
|
||
}
|
||
|
||
foldersMap.set(folderHandle, items);
|
||
}
|
||
|
||
/**
|
||
* Get cached ZIP data
|
||
*/
|
||
function getZipCache(zipPath) {
|
||
return zipCache.get(zipPath);
|
||
}
|
||
|
||
/**
|
||
* Extract a ZIP file to its parent directory
|
||
*/
|
||
async function extractZip(zipPath) {
|
||
const cached = zipCache.get(zipPath);
|
||
if (!cached) {
|
||
throw new Error('ZIP not found in cache');
|
||
}
|
||
|
||
const { zip, folderHandle } = cached;
|
||
|
||
// Get the ZIP filename without extension for the extract folder name
|
||
const zipName = zipPath.split('/').pop();
|
||
const extractFolderName = zipName.replace(/\.zip$/i, '');
|
||
|
||
// Create extraction folder
|
||
const extractFolder = await folderHandle.getDirectoryHandle(extractFolderName, { create: true });
|
||
|
||
// Extract all files
|
||
const entries = [];
|
||
zip.forEach((relativePath, zipEntry) => {
|
||
if (!zipEntry.dir) {
|
||
entries.push({ path: relativePath, entry: zipEntry });
|
||
}
|
||
});
|
||
|
||
for (const { path, entry } of entries) {
|
||
try {
|
||
// Create subdirectories if needed
|
||
const parts = path.split('/');
|
||
const fileName = parts.pop();
|
||
|
||
let currentDir = extractFolder;
|
||
for (const part of parts) {
|
||
if (part) {
|
||
currentDir = await currentDir.getDirectoryHandle(part, { create: true });
|
||
}
|
||
}
|
||
|
||
// Write file
|
||
const content = await entry.async('arraybuffer');
|
||
const fileHandle = await currentDir.getFileHandle(fileName, { create: true });
|
||
const writable = await fileHandle.createWritable();
|
||
await writable.write(content);
|
||
await writable.close();
|
||
} catch (err) {
|
||
console.error('Error extracting file:', path, err);
|
||
}
|
||
}
|
||
|
||
return extractFolderName;
|
||
}
|
||
|
||
/**
|
||
* Create file object with metadata
|
||
*/
|
||
// Build a file row from JUST the directory entry — no getFile(). Listing a
|
||
// network share is already slow; the old code opened EVERY file to read
|
||
// size/lastModified (which the grid doesn't even display), turning a
|
||
// listing into one network round-trip per file. size/lastModified are now
|
||
// loaded on demand by preview / SHA / rename, which call getFile()
|
||
// themselves. The scan is now a pure directory listing.
|
||
function createFileObject(fileHandle, folderHandle) {
|
||
const split = zddc.splitExtension(fileHandle.name);
|
||
return {
|
||
handle: fileHandle,
|
||
folderHandle: folderHandle,
|
||
originalFilename: split.name,
|
||
extension: split.extension,
|
||
size: null,
|
||
lastModified: null,
|
||
// Editable fields
|
||
trackingNumber: '',
|
||
revision: '',
|
||
status: '',
|
||
title: '',
|
||
// State
|
||
isDirty: false,
|
||
error: false,
|
||
errorMessage: '',
|
||
validation: null,
|
||
sha256: null
|
||
// folderPath added by the caller.
|
||
};
|
||
}
|
||
|
||
// ── Workspace snapshot (scan once, resume without re-walking the FS) ────
|
||
|
||
// Serialize the completed scan to compact JSON (short keys: large trees).
|
||
// Zip subtrees ARE preserved: a scanned archive keeps its virtual folders +
|
||
// members so classifications inside it survive reopen; copy/preview re-load
|
||
// the archive lazily from the root (ensureZipLoaded). An archive that was
|
||
// never opened persists as a lazy 'zip' node that reopens on demand.
|
||
function snapshotTree() {
|
||
function serFile(f) {
|
||
var o = { o: f.originalFilename, e: f.extension, p: f.folderPath };
|
||
if (f.isVirtual) { o.z = f.zipPath; o.ze = f.zipEntryPath; } // zip member
|
||
return o;
|
||
}
|
||
function serNode(n) {
|
||
var o = { n: n.name, p: n.path };
|
||
if (n.isZipRoot) o.zr = 1; // archive root (zipPath === n.path)
|
||
else if (n.isVirtualDir) o.vd = n.zipPath; // folder inside an archive
|
||
if (n.files && n.files.length) o.f = n.files.map(serFile);
|
||
if (n.children && n.children.length) o.c = n.children.map(serNode);
|
||
// Record scan progress so an interrupted scan can resume: 'children'
|
||
// = direct entries fully read (kids may still be pending); anything
|
||
// unfinished → 'pending' to re-read. An unopened archive persists as
|
||
// 'zip' (reopen lazily, never a real dir re-walk). 'done' is the
|
||
// default and omitted.
|
||
var st = n.scanState;
|
||
if (n.isZipRoot && st !== 'done') o.s = 'zip';
|
||
else if (st && st !== 'done') o.s = (st === 'children') ? 'children' : 'pending';
|
||
return o;
|
||
}
|
||
return (window.app.folderTree || []).map(serNode);
|
||
}
|
||
|
||
// Rebuild window.app.folderTree from a snapshot — handle-less nodes, marked
|
||
// 'done', subtree totals recomputed. Handles are resolved lazily from the
|
||
// workspace root handle at copy/preview time.
|
||
function loadSnapshot(snap) {
|
||
function deFile(sf) {
|
||
var fo = {
|
||
handle: null, folderHandle: null,
|
||
originalFilename: sf.o, extension: sf.e,
|
||
size: null, lastModified: null,
|
||
trackingNumber: '', revision: '', status: '', title: '',
|
||
isDirty: false, error: false, errorMessage: '', validation: null, sha256: null,
|
||
folderPath: sf.p,
|
||
};
|
||
if (sf.z) { fo.isVirtual = true; fo.zipPath = sf.z; fo.zipEntryPath = sf.ze; }
|
||
return fo;
|
||
}
|
||
function deNode(sn, parent) {
|
||
var desc = { name: sn.n, kind: 'directory' };
|
||
if (sn.zr) { desc.isZipRoot = true; desc.zipPath = sn.p; }
|
||
else if (sn.vd) { desc.isVirtualDir = true; desc.zipPath = sn.vd; }
|
||
var node = makeNode(desc, sn.p, parent);
|
||
node.handle = null;
|
||
if (sn.zr || sn.vd) node.virtualPath = sn.p;
|
||
// 'zip' restores an unopened archive (reopen lazily); else resume marker.
|
||
node.scanState = sn.s === 'zip' ? 'zip-pending' : (sn.s || 'done');
|
||
node.expanded = false;
|
||
node.files = (sn.f || []).map(deFile);
|
||
node.children = (sn.c || []).map(function (c) { return deNode(c, node); });
|
||
node.fileCount = node.files.length;
|
||
node.subdirCount = node.children.length;
|
||
return node;
|
||
}
|
||
var roots = (snap || []).map(function (sn) { return deNode(sn, null); });
|
||
if (roots[0]) roots[0].expanded = true;
|
||
(function totals(nodes) {
|
||
nodes.forEach(function (n) {
|
||
totals(n.children);
|
||
var rf = n.files.length, rd = n.children.length;
|
||
n.children.forEach(function (c) { rf += c.runFiles; rd += c.runDirs; });
|
||
n.runFiles = rf; n.runDirs = rd;
|
||
});
|
||
})(roots);
|
||
window.app.folderTree = roots;
|
||
if (window.app.modules.store && window.app.modules.store.setFolderTree) {
|
||
window.app.modules.store.setFolderTree(roots);
|
||
}
|
||
return roots;
|
||
}
|
||
|
||
// ── Lazy handle resolution (snapshot files carry paths, not handles) ────
|
||
function relFromRoot(p) { var i = (p || '').indexOf('/'); return i < 0 ? '' : p.slice(i + 1); }
|
||
function parentPath(p) { var i = (p || '').lastIndexOf('/'); return i < 0 ? '' : p.slice(0, i); }
|
||
function baseName(p) { var i = (p || '').lastIndexOf('/'); return i < 0 ? p : p.slice(i + 1); }
|
||
// Load (and cache) a zip archive by its tree path. After a snapshot restore
|
||
// the in-memory cache is empty, so resolve the .zip from the workspace root
|
||
// and parse it on demand. Returns the cache record { zip, fileHandle, ... }.
|
||
async function ensureZipLoaded(rootHandle, zipPath) {
|
||
var cached = zipCache.get(zipPath);
|
||
if (cached && cached.zip) return cached;
|
||
if (!rootHandle) throw new Error('source directory not connected');
|
||
var dir = await resolveDirHandle(rootHandle, relFromRoot(parentPath(zipPath)));
|
||
var fh = await dir.getFileHandle(baseName(zipPath));
|
||
var zip = await JSZip.loadAsync(await (await fh.getFile()).arrayBuffer());
|
||
var rec = { zip: zip, fileHandle: fh, folderHandle: dir };
|
||
zipCache.set(zipPath, rec);
|
||
return rec;
|
||
}
|
||
// Read a zip member's bytes as a Blob (lazily loading its archive).
|
||
async function extractZipMember(rootHandle, fileObj) {
|
||
var rec = await ensureZipLoaded(rootHandle, fileObj.zipPath);
|
||
var entry = rec.zip.file(fileObj.zipEntryPath);
|
||
if (!entry) throw new Error('zip member not found: ' + fileObj.zipEntryPath);
|
||
return await entry.async('blob');
|
||
}
|
||
|
||
// ── per-zip mode toggle (single file ⇄ expandable folder) ───────────────
|
||
function findNodeByPath(path) {
|
||
var hit = null;
|
||
(function walk(ns) { (ns || []).forEach(function (n) { if (hit) return; if (n.path === path) hit = n; else walk(n.children); }); })(window.app.folderTree || []);
|
||
return hit;
|
||
}
|
||
// Recompute subtree totals after a structural change (expand/collapse a zip).
|
||
function recomputeTotals() {
|
||
(function walk(ns) {
|
||
(ns || []).forEach(function (n) {
|
||
walk(n.children || []);
|
||
var rf = (n.files || []).length, rd = (n.children || []).length;
|
||
(n.children || []).forEach(function (c) { rf += c.runFiles || 0; rd += c.runDirs || 0; });
|
||
n.runFiles = rf; n.runDirs = rd;
|
||
n.fileCount = (n.files || []).length; n.subdirCount = (n.children || []).length;
|
||
});
|
||
})(window.app.folderTree || []);
|
||
}
|
||
// Turn a .zip FILE into an expandable archive folder in place: scan its
|
||
// members into the fileset and drop the now-meaningless single-file
|
||
// assignment. Members come from the live handle, or (snapshot-restored) are
|
||
// re-read from the workspace root via scanZipNode's fallback.
|
||
async function expandZipAsFolder(file) {
|
||
var parent = findNodeByPath(file.folderPath);
|
||
if (!parent) return null;
|
||
var zipName = zddc.joinExtension(file.originalFilename, file.extension);
|
||
var zipPath = parent.path + '/' + zipName;
|
||
var zipNode = makeNode({ name: zipName, kind: 'directory', isZipRoot: true, zipPath: zipPath }, zipPath, parent);
|
||
zipNode._zipFileObj = file.handle ? file : null; // null → scanZipNode resolves from root
|
||
zipNode.scanState = 'zip-pending';
|
||
zipNode.expanded = true;
|
||
parent.files = (parent.files || []).filter(function (f) { return f !== file; });
|
||
parent.children = parent.children || [];
|
||
parent.children.push(zipNode);
|
||
var c = window.app.modules.classify;
|
||
if (c && c.dropAssignments) c.dropAssignments([c.srcKeyForFile(file)]);
|
||
await scanZipNode(zipNode);
|
||
recomputeTotals();
|
||
return zipNode;
|
||
}
|
||
// Collapse an expanded archive folder back to a single .zip file, dropping
|
||
// every member's assignment.
|
||
function collapseZipToFile(zipNode) {
|
||
if (!zipNode || !zipNode.isZipRoot) return null;
|
||
var parent = zipNode.parent || findNodeByPath(zipNode.path.slice(0, zipNode.path.lastIndexOf('/')));
|
||
if (!parent) return null;
|
||
var c = window.app.modules.classify;
|
||
if (c && c.dropAssignments) {
|
||
var keys = [];
|
||
(function walk(n) { (n.files || []).forEach(function (f) { keys.push(c.srcKeyForFile(f)); }); (n.children || []).forEach(walk); })(zipNode);
|
||
c.dropAssignments(keys);
|
||
}
|
||
var split = zddc.splitExtension(zipNode.name);
|
||
var file = {
|
||
handle: (zipNode._zipFileObj && zipNode._zipFileObj.handle) || null,
|
||
folderHandle: (zipNode._zipFileObj && zipNode._zipFileObj.folderHandle) || null,
|
||
originalFilename: split.name, extension: split.extension,
|
||
size: null, lastModified: null,
|
||
trackingNumber: '', revision: '', status: '', title: '',
|
||
isDirty: false, error: false, errorMessage: '', validation: null, sha256: null,
|
||
folderPath: parent.path,
|
||
};
|
||
parent.children = (parent.children || []).filter(function (n) { return n !== zipNode; });
|
||
parent.files = parent.files || [];
|
||
parent.files.push(file);
|
||
zipCache.delete(zipNode.zipPath);
|
||
recomputeTotals();
|
||
return file;
|
||
}
|
||
async function resolveDirHandle(rootHandle, relPath) {
|
||
var cur = rootHandle;
|
||
var parts = (relPath || '').split('/').filter(Boolean);
|
||
for (var i = 0; i < parts.length; i++) { cur = await cur.getDirectoryHandle(parts[i]); }
|
||
return cur;
|
||
}
|
||
// Resolve (and cache) a file object's handle from the workspace root.
|
||
async function resolveFileHandle(rootHandle, fileObj) {
|
||
if (fileObj.handle) return fileObj.handle;
|
||
var dir = await resolveDirHandle(rootHandle, relFromRoot(fileObj.folderPath));
|
||
var name = zddc.joinExtension(fileObj.originalFilename, fileObj.extension);
|
||
var h = await dir.getFileHandle(name);
|
||
fileObj.handle = h;
|
||
fileObj.folderHandle = dir;
|
||
return h;
|
||
}
|
||
|
||
// Resume an interrupted scan: walk the loaded tree for 'pending' folders,
|
||
// resolve their handles from the (reconnected) root, and drain only those —
|
||
// already-scanned folders are left alone. Returns true if work was done.
|
||
async function resumeScan(rootHandle) {
|
||
if (!rootHandle) return false;
|
||
var pend = [];
|
||
(function walk(ns) {
|
||
(ns || []).forEach(function (n) {
|
||
if (n.scanState === 'pending') pend.push(n);
|
||
else walk(n.children);
|
||
});
|
||
})(window.app.folderTree || []);
|
||
if (!pend.length) return false;
|
||
|
||
var myGen = ++scanGen;
|
||
zipCache.clear();
|
||
scanStats = { folders: 0, files: 0, current: '', done: false, startedAt: Date.now() };
|
||
var ticker = setInterval(function () {
|
||
if (myGen !== scanGen || (scanStats && scanStats.done)) { clearInterval(ticker); return; }
|
||
updateScanStatus();
|
||
}, 1000);
|
||
|
||
for (var i = 0; i < pend.length; i++) {
|
||
try { pend[i].handle = await resolveDirHandle(rootHandle, relFromRoot(pend[i].path)); }
|
||
catch (e) { pend[i].scanState = 'done'; reportScanError(pend[i].path, e); }
|
||
}
|
||
await drainQueue(pend.filter(function (n) { return n.handle; }), myGen, SCAN_CONCURRENCY);
|
||
|
||
clearInterval(ticker);
|
||
if (myGen !== scanGen) return true;
|
||
scanStats.done = true;
|
||
scanStats.current = '';
|
||
flushRender();
|
||
if (window.zddc && typeof window.zddc.toast === 'function') {
|
||
window.zddc.toast('Resumed scan complete — ' + scanStats.folders + ' folders, '
|
||
+ scanStats.files + ' files added in ' + elapsedStr() + '.', 'success');
|
||
}
|
||
return true;
|
||
}
|
||
|
||
// Export module
|
||
window.app.modules.scanner = {
|
||
scanDirectory,
|
||
ensureScanned,
|
||
getZipCache,
|
||
extractZip,
|
||
snapshotTree,
|
||
loadSnapshot,
|
||
resolveFileHandle,
|
||
resolveDirHandle,
|
||
ensureZipLoaded,
|
||
extractZipMember,
|
||
expandZipAsFolder,
|
||
collapseZipToFile,
|
||
resumeScan
|
||
};
|
||
})();
|
||
|