ZDDC/archive/js/source.js

(function() {
    'use strict';
    // Source abstraction — local (File System Access API) and HTTP (Caddy JSON browse)
    //
    // Three scan modes, decided once at the entry point:
    //   1. Multi-project — ?projects=A,B URL param non-empty. The scan root holds project
    //      folders; for each in the filter, descend into its `archive/` subfolder (case-
    //      insensitive) and scan from there.
    //   2. Project-root — scan root has an `archive/` child (case-insensitive). Descend
    //      into it and scan from there. Other stage folders (reviewing/staging/mdl/working)
    //      are not entered.
    //   3. In-archive (default) — scan root's children are third-party (grouping) folders.
    //      Today's behavior, unchanged.
    //
    // The recursion below the entry point never re-applies the mode check: once we are
    // inside the archive folder for a given project, descent is uniform across modes.
    //
    // Listing skip: at any depth, a directory child whose lowercased name is a member of
    // FOLDER_TYPE_NAMES (issued/received/mdl/incoming) AND not currently in
    // enabledFolderTypes is skipped entirely — we do not even fetch its listing. Toggling
    // a type back on triggers a refresh in app.js.

    // Shared utility used by both source implementations
    function getDisplayPath(fullPath) {
        if (fullPath.length <= 100) {
            return fullPath;
        }
        const parts = fullPath.split('/');
        if (parts.length > 3) {
            return parts[0] + '/.../' + parts.slice(-2).join('/');
        }
        return '...' + fullPath.substring(fullPath.length - 80);
    }

    // True if a directory child should be skipped entirely (don't fetch its listing).
    function isHiddenFolderTypeName(rawName) {
        const lower = rawName.toLowerCase();
        return window.app.FOLDER_TYPE_NAMES.includes(lower)
            && !window.app.enabledFolderTypes.has(lower);
    }

    // createSource(type, options) returns a source object:
    //   source.type       — 'local' | 'http'
    //   source.canWrite   — boolean
    //   source.scan(rootIdentifier, callbacks) — Promise; walks tree calling:
    //       callbacks.onGroupingFolder(folder)     folder: { name, path, displayPath, handle? }
    //       callbacks.onTransmittalFolder(folder)  folder: { name, path, displayPath, handle?, url? }
    //       callbacks.onFile(file)                 file:   full file object (parsed + metadata)
    //       callbacks.onProgress(message)
    //   source.fetchFile(fileRef) — Promise<ArrayBuffer>

    function createSource(type, options) {
        if (type === 'local') {
            return createLocalSource();
        } else if (type === 'http') {
            return createHttpSource(options.baseUrl);
        }
        throw new Error('Unknown source type: ' + type);
    }

    // ---------------------------------------------------------------------------
    // Local source — wraps File System Access API
    // ---------------------------------------------------------------------------

    function createLocalSource() {
        return {
            type: 'local',
            canWrite: true,

            scan: function(dirHandle, callbacks) {
                return scanLocalRoot(dirHandle, dirHandle.name, callbacks);
            },

            fetchFile: function(fileRef) {
                return fileRef.handle.getFile().then(function(f) {
                    return f.arrayBuffer();
                });
            }
        };
    }

    async function listLocalEntries(dirHandle, currentPath) {
        const entries = [];
        try {
            for await (const entry of dirHandle.values()) {
                entries.push(entry);
            }
        } catch (err) {
            console.warn('Could not read directory ' + currentPath + ':', err);
            return null;
        }
        return entries;
    }

    function findArchiveInEntries(entries) {
        const stage = window.app.ARCHIVE_STAGE_NAME;
        for (const entry of entries) {
            if (entry.kind === 'directory' && entry.name.toLowerCase() === stage) {
                return entry;
            }
        }
        return null;
    }

    async function scanLocalRoot(dirHandle, rootPath, callbacks) {
        callbacks.onProgress('Scanning ' + rootPath + '...');

        const entries = await listLocalEntries(dirHandle, rootPath);
        if (!entries) return;

        // Mode 1 — multi-project (?projects= set)
        if (window.app.projectFilter && window.app.projectFilter.size > 0) {
            for (const entry of entries) {
                if (entry.kind !== 'directory') continue;
                if (!window.app.projectFilter.has(entry.name)) continue;
                const projPath = rootPath + '/' + entry.name;
                const projEntries = await listLocalEntries(entry, projPath);
                if (!projEntries) continue;
                const archive = findArchiveInEntries(projEntries);
                if (!archive) continue;
                const archivePath = projPath + '/' + archive.name;
                await scanLocalRecursive(archive, archivePath, 0, callbacks);
            }
            return;
        }

        // Mode 2 — project-root (scan root has an archive/ child)
        const archive = findArchiveInEntries(entries);
        if (archive) {
            const archivePath = rootPath + '/' + archive.name;
            await scanLocalRecursive(archive, archivePath, 0, callbacks);
            return;
        }

        // Mode 3 — in-archive (default)
        await processLocalEntries(entries, rootPath, 0, callbacks);
    }

    async function scanLocalRecursive(dirHandle, currentPath, depth, callbacks) {
        if (currentPath.length > 200) {
            console.warn('Path too long, skipping deeper scan: ' + currentPath);
            return;
        }
        if (depth > 10) {
            console.warn('Directory depth limit reached at: ' + currentPath);
            return;
        }

        callbacks.onProgress('Scanning ' + currentPath + '...');

        const entries = await listLocalEntries(dirHandle, currentPath);
        if (!entries) return;

        await processLocalEntries(entries, currentPath, depth, callbacks);
    }

    async function processLocalEntries(entries, currentPath, depth, callbacks) {
        for (const entry of entries) {
            if (entry.kind === 'directory') {
                if (isHiddenFolderTypeName(entry.name)) continue;

                const subPath = currentPath + '/' + entry.name;
                try {
                    if (window.app.modules.parser.isTransmittalFolder(entry.name)) {
                        const folder = {
                            name: entry.name,
                            path: subPath,
                            displayPath: getDisplayPath(subPath),
                            handle: entry
                        };
                        callbacks.onTransmittalFolder(folder);
                        await scanLocalTransmittalFolder(entry, subPath, 0, subPath, callbacks);
                    } else {
                        const folder = {
                            name: entry.name,
                            path: subPath,
                            displayPath: entry.name,
                            handle: entry
                        };
                        callbacks.onGroupingFolder(folder);
                        await scanLocalRecursive(entry, subPath, depth + 1, callbacks);
                    }
                } catch (err) {
                    console.warn('Could not process directory ' + entry.name + ':', err);
                }
            } else if (entry.kind === 'file') {
                // A zipped transmittal folder (e.g.
                // "2025-05-12_DOC-001 (IFI) - Title.zip") is treated as
                // that transmittal folder: open the zip in the browser
                // and scan its members like an uncompressed folder's
                // files. The .zip stays in the recorded path so it's
                // unambiguous; the displayed name drops it.
                if (window.app.modules.parser.isTransmittalFolderZip(entry.name)) {
                    const base = zddc.splitExtension(entry.name).name;
                    const zipPath = currentPath + '/' + entry.name;
                    try {
                        const zh = await window.zddc.zip.fromFileHandle(entry);
                        callbacks.onTransmittalFolder({
                            name: base,
                            path: zipPath,
                            displayPath: getDisplayPath(zipPath),
                            handle: zh
                        });
                        await scanLocalTransmittalFolder(zh, zipPath, 0, zipPath, callbacks);
                    } catch (zipErr) {
                        console.warn('Could not open zip transmittal ' + entry.name + ':', zipErr);
                    }
                    continue;
                }
                // File directly in a grouping folder — assign to the Outstanding virtual transmittal.
                // actualPath records the real containing folder for grouping-folder-scoped filtering.
                try {
                    const file = await entry.getFile();
                    const parsed = zddc.parseFilename(file.name) || {};
                    const fullPath = currentPath + '/' + file.name;
                    const displayPath = fullPath.length > 250
                        ? '...' + fullPath.substring(fullPath.length - 200)
                        : fullPath;

                    callbacks.onFile({
                        id: crypto.randomUUID(),
                        name: file.name,
                        path: fullPath,
                        displayPath: displayPath,
                        url: null,
                        size: file.size,
                        modified: file.lastModified,
                        handle: entry,
                        folderPath: '__outstanding__',
                        actualPath: currentPath,
                        hasPathError: false,
                        ...parsed
                    });
                } catch (fileErr) {
                    const fullPath = currentPath + '/' + entry.name;
                    const displayPath = fullPath.length > 250
                        ? '...' + fullPath.substring(fullPath.length - 200)
                        : fullPath;
                    const parsed = zddc.parseFilename(entry.name) || {};

                    callbacks.onFile({
                        id: crypto.randomUUID(),
                        name: entry.name,
                        path: fullPath,
                        displayPath: displayPath,
                        url: null,
                        size: null,
                        modified: null,
                        handle: null,
                        folderPath: '__outstanding__',
                        actualPath: currentPath,
                        hasPathError: true,
                        pathErrorMessage: fileErr.message || 'File access error',
                        ...parsed
                    });

                    console.warn('Could not access file ' + entry.name + ' (path error):', fileErr.message);
                }
            }
        }
    }

    async function scanLocalTransmittalFolder(dirHandle, folderPath, depth, transmittalPath, callbacks) {
        if (depth > 10) {
            console.warn('Directory depth limit reached in transmittal folder: ' + folderPath);
            return;
        }

        try {
            if (folderPath.length > 240) {
                console.warn('Path approaching Windows limit (' + folderPath.length + ' chars): ' + folderPath);
            }

            for await (const entry of dirHandle.values()) {
                if (entry.kind === 'file') {
                    try {
                        const file = await entry.getFile();
                        const parsed = zddc.parseFilename(file.name) || {};
                        const fullPath = folderPath + '/' + file.name;
                        const displayPath = fullPath.length > 250
                            ? '...' + fullPath.substring(fullPath.length - 200)
                            : fullPath;

                        callbacks.onFile({
                            id: crypto.randomUUID(),
                            name: file.name,
                            path: fullPath,
                            displayPath: displayPath,
                            url: null,
                            size: file.size,
                            modified: file.lastModified,
                            handle: entry,
                            folderPath: transmittalPath,
                            actualPath: folderPath,
                            hasPathError: false,
                            ...parsed
                        });
                    } catch (fileErr) {
                        const fullPath = folderPath + '/' + entry.name;
                        const displayPath = fullPath.length > 250
                            ? '...' + fullPath.substring(fullPath.length - 200)
                            : fullPath;
                        const parsed = zddc.parseFilename(entry.name) || {};

                        callbacks.onFile({
                            id: crypto.randomUUID(),
                            name: entry.name,
                            path: fullPath,
                            displayPath: displayPath,
                            url: null,
                            size: null,
                            modified: null,
                            handle: null,
                            folderPath: transmittalPath,
                            actualPath: folderPath,
                            hasPathError: true,
                            pathErrorMessage: fileErr.message || 'File access error',
                            ...parsed
                        });

                        console.warn('Could not access file ' + entry.name + ' (path error):', fileErr.message);
                    }
                } else if (entry.kind === 'directory') {
                    const subPath = folderPath + '/' + entry.name;
                    try {
                        await scanLocalTransmittalFolder(entry, subPath, depth + 1, transmittalPath, callbacks);
                    } catch (err) {
                        console.warn('Could not scan subdirectory ' + entry.name + ' in ' + folderPath + ':', err);
                    }
                }
            }
        } catch (err) {
            console.error('Error scanning folder ' + folderPath + ':', err);
        }
    }

    // ---------------------------------------------------------------------------
    // HTTP source — uses Caddy JSON browse (Accept: application/json)
    // ---------------------------------------------------------------------------

    function createHttpSource(baseUrl) {
        // Normalise: ensure baseUrl ends with /
        const root = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/';

        return {
            type: 'http',
            canWrite: false,

            scan: function(rootUrl, callbacks) {
                const scanRoot = (rootUrl && rootUrl !== root) ? rootUrl : root;
                return scanHttpRoot(scanRoot, root, callbacks);
            },

            fetchFile: function(fileRef) {
                return fetch(fileRef.url).then(function(r) {
                    if (!r.ok) throw new Error('HTTP ' + r.status + ' fetching ' + fileRef.url);
                    return r.arrayBuffer();
                });
            }
        };
    }

    async function fetchHttpListing(dirUrl) {
        try {
            const resp = await fetch(dirUrl, {
                headers: { 'Accept': 'application/json' }
            });
            if (!resp.ok) {
                // 403/404 on a sub-path is expected when ACLs deny access or a
                // listed dir doesn't exist — log at info level to avoid alarming
                // users in the console.
                console.info('skip ' + dirUrl + ' (' + resp.status + ')');
                return null;
            }
            const items = await resp.json();
            if (!Array.isArray(items)) {
                // Server returned 200 but the body wasn't a JSON array — most
                // commonly Caddy serving an HTML error page or an index.html
                // when file_browse isn't enabled at that path. Silent skip.
                return null;
            }
            return items;
        } catch (err) {
            // JSON parse failures, network errors, etc. — single concise line.
            console.info('skip ' + dirUrl + ': ' + (err.message || err));
            return null;
        }
    }

    function rawNameOf(item) {
        return item.name.endsWith('/') ? item.name.slice(0, -1) : item.name;
    }

    function findArchiveInItems(items) {
        const stage = window.app.ARCHIVE_STAGE_NAME;
        for (const item of items) {
            if (!item.is_dir) continue;
            const name = rawNameOf(item);
            if (name.toLowerCase() === stage) return { item: item, name: name };
        }
        return null;
    }

    async function scanHttpRoot(scanRootUrl, rootUrl, callbacks) {
        // Mode 1 — multi-project (?projects= set). Skip listing scanRootUrl
        // entirely: project URLs are deterministic, so go straight to each one
        // (the names in projectFilter, slash-normalised in app.js against the
        // server's root listing). Avoids depending on the root listing's shape.
        if (window.app.projectFilter && window.app.projectFilter.size > 0) {
            const tasks = [];
            for (const name of window.app.projectFilter) {
                if (!name || name.startsWith('.')) continue;
                const projUrl = resolveHttpUrl(scanRootUrl, name, true);
                tasks.push((async () => {
                    const projItems = await fetchHttpListing(projUrl);
                    if (!projItems) return;
                    const found = findArchiveInItems(projItems);
                    if (!found) return;
                    const archiveUrl = resolveHttpUrl(projUrl, found.name, true);
                    await scanHttpRecursive(archiveUrl, rootUrl, 0, null, callbacks);
                })());
            }
            await Promise.all(tasks);
            return;
        }

        const items = await fetchHttpListing(scanRootUrl);
        if (!items) return;

        // Mode 2 — project-root (scan root has archive/ child)
        const found = findArchiveInItems(items);
        if (found) {
            const archiveUrl = resolveHttpUrl(scanRootUrl, found.name, true);
            await scanHttpRecursive(archiveUrl, rootUrl, 0, null, callbacks);
            return;
        }

        // Mode 3 — in-archive (default)
        await processHttpItems(items, scanRootUrl, rootUrl, 0, null, callbacks);
    }

    async function scanHttpRecursive(dirUrl, rootUrl, depth, transmittalPath, callbacks) {
        if (depth > 10) {
            console.warn('HTTP directory depth limit reached at: ' + dirUrl);
            return;
        }

        const items = await fetchHttpListing(dirUrl);
        if (!items) return;

        await processHttpItems(items, dirUrl, rootUrl, depth, transmittalPath, callbacks);
    }

    async function processHttpItems(items, dirUrl, rootUrl, depth, transmittalPath, callbacks) {
        // Collect subdirectory scan promises so siblings run in parallel
        const subdirPromises = [];

        for (const item of items) {
            // Caddy appends "/" to directory names; strip it to get the bare name for matching
            const rawName = rawNameOf(item);

            // Skip hidden files
            if (rawName.startsWith('.')) continue;

            const isDir = item.is_dir === true;
            const itemUrl = resolveHttpUrl(dirUrl, rawName, isDir);
            const logicalPath = urlToLogicalPath(itemUrl, rootUrl);

            if (isDir) {
                // Skip listings for folder-types that are toggled off — applies at any depth.
                if (transmittalPath === null && isHiddenFolderTypeName(rawName)) continue;

                if (transmittalPath !== null) {
                    // Inside a transmittal folder — recurse into subdirectories
                    subdirPromises.push(
                        scanHttpRecursive(itemUrl, rootUrl, depth + 1, transmittalPath, callbacks)
                    );
                } else if (window.app.modules.parser.isTransmittalFolder(rawName)) {
                    const folder = {
                        name: rawName,
                        path: logicalPath,
                        displayPath: getDisplayPath(logicalPath),
                        handle: null,
                        url: itemUrl
                    };
                    callbacks.onTransmittalFolder(folder);
                    subdirPromises.push(
                        scanHttpRecursive(itemUrl, rootUrl, depth + 1, logicalPath, callbacks)
                    );
                } else {
                    const folder = {
                        name: rawName,
                        path: logicalPath,
                        displayPath: rawName,
                        handle: null,
                        url: itemUrl
                    };
                    callbacks.onGroupingFolder(folder);
                    subdirPromises.push(
                        scanHttpRecursive(itemUrl, rootUrl, depth + 1, null, callbacks)
                    );
                }
            } else {
                // It's a file
                // A zipped transmittal folder at the grouping level:
                // zddc-server serves "<…>.zip/" as a virtual directory
                // of the zip's members, so recurse into it like an
                // uncompressed transmittal folder. Members come back
                // with URLs like "<…>.zip/<member>" that the server
                // extracts on demand — no whole-zip download.
                if (transmittalPath === null && window.app.modules.parser.isTransmittalFolderZip(rawName)) {
                    const base = zddc.splitExtension(rawName).name;
                    const zipDirUrl = itemUrl + '/'; // itemUrl is the .zip file URL (no trailing slash)
                    callbacks.onTransmittalFolder({
                        name: base,
                        path: logicalPath,
                        displayPath: getDisplayPath(logicalPath),
                        handle: null,
                        url: zipDirUrl
                    });
                    subdirPromises.push(
                        scanHttpRecursive(zipDirUrl, rootUrl, depth + 1, logicalPath, callbacks)
                    );
                    continue;
                }
                if (transmittalPath === null) {
                    // File directly in a grouping folder — assign to Outstanding virtual transmittal.
                    // actualPath records the real containing folder for grouping-folder-scoped filtering.
                    const dirLogicalPath = urlToLogicalPath(dirUrl, rootUrl);
                    const parsed = zddc.parseFilename(rawName) || {};
                    const modified = item.mod_time ? new Date(item.mod_time).getTime() : null;

                    callbacks.onFile({
                        id: crypto.randomUUID(),
                        name: rawName,
                        path: logicalPath,
                        displayPath: logicalPath,
                        url: itemUrl,
                        size: item.size || null,
                        modified: modified,
                        handle: null,
                        folderPath: '__outstanding__',
                        actualPath: dirLogicalPath,
                        hasPathError: false,
                        ...parsed
                    });
                } else {
                    // Inside a transmittal folder
                    const parsed = zddc.parseFilename(rawName) || {};
                    // mod_time is an ISO 8601 string from Go's time.Time.UTC()
                    const modified = item.mod_time ? new Date(item.mod_time).getTime() : null;

                    callbacks.onFile({
                        id: crypto.randomUUID(),
                        name: rawName,
                        path: logicalPath,
                        displayPath: logicalPath,
                        url: itemUrl,
                        size: item.size || null,
                        modified: modified,
                        handle: null,
                        folderPath: transmittalPath,
                        actualPath: logicalPath.substring(0, logicalPath.lastIndexOf('/')),
                        hasPathError: false,
                        ...parsed
                    });
                }
            }
        }

        // Wait for all sibling subdirectory scans to complete in parallel
        if (subdirPromises.length > 0) {
            await Promise.all(subdirPromises);
        }
    }

    // Build an absolute URL for an item inside a directory listing URL
    function resolveHttpUrl(dirUrl, name, isDir) {
        const base = dirUrl.endsWith('/') ? dirUrl : dirUrl + '/';
        const encoded = encodeURIComponent(name);
        return base + encoded + (isDir ? '/' : '');
    }

    // Convert an absolute item URL to a logical relative path (for display / filtering)
    function urlToLogicalPath(itemUrl, rootUrl) {
        const root = rootUrl.endsWith('/') ? rootUrl : rootUrl + '/';
        let rel = itemUrl;
        if (rel.startsWith(root)) {
            rel = rel.substring(root.length);
        }
        // Decode percent-encoding for display
        try { rel = decodeURIComponent(rel); } catch (e) { /* leave encoded */ }
        // Strip trailing slash for directories
        if (rel.endsWith('/')) rel = rel.slice(0, -1);
        return rel;
    }

    window.app.modules.source = {
        getDisplayPath,
        createSource
    };

})();