diff --git a/crates/oxide/src/scanner/auto_source_detection.rs b/crates/oxide/src/scanner/auto_source_detection.rs index 0d723a1ab..c53756456 100644 --- a/crates/oxide/src/scanner/auto_source_detection.rs +++ b/crates/oxide/src/scanner/auto_source_detection.rs @@ -34,10 +34,10 @@ pub static IGNORED_CONTENT_DIRS: sync::LazyLock> = sync::LazyL .collect() }); -static IGNORED_CONTENT_DIRS_GLOB: sync::LazyLock = +pub static IGNORED_CONTENT_DIRS_GLOB: sync::LazyLock = sync::LazyLock::new(|| format!("{{{}}}/", IGNORED_CONTENT_DIRS.join(","))); -static IGNORED_EXTENSIONS_GLOB: sync::LazyLock = sync::LazyLock::new(|| { +pub static IGNORED_EXTENSIONS_GLOB: sync::LazyLock = sync::LazyLock::new(|| { format!( "*.{{{}}}", include_str!("fixtures/ignored-extensions.txt") @@ -59,7 +59,7 @@ pub static BINARY_EXTENSIONS_GLOB: sync::LazyLock = sync::LazyLock::new( ) }); -static IGNORED_FILES_GLOB: sync::LazyLock = sync::LazyLock::new(|| { +pub static IGNORED_FILES_GLOB: sync::LazyLock = sync::LazyLock::new(|| { format!( "{{{}}}", include_str!("fixtures/ignored-files.txt") diff --git a/crates/oxide/src/scanner/mod.rs b/crates/oxide/src/scanner/mod.rs index ab61b11b3..66696e7ec 100644 --- a/crates/oxide/src/scanner/mod.rs +++ b/crates/oxide/src/scanner/mod.rs @@ -22,16 +22,58 @@ use std::sync::{Arc, Mutex}; use std::time::SystemTime; use tracing::event; -// @source "some/folder"; // This is auto source detection -// @source "some/folder/**/*"; // This is auto source detection -// @source "some/folder/*.html"; // This is just a glob, but new files matching this should be included -// @source "node_modules/my-ui-lib"; // Auto source detection but since node_modules is explicit we allow it -// // Maybe could be considered `external(…)` automatically if: -// // 1. It's git ignored but listed explicitly -// // 2. It exists outside of the current working directory (do we know that?) +// # `@source` semantics // -// @source "do-include-me.bin"; // `.bin` is typically ignored, but now it's explicit so should be included -// @source "git-ignored.html"; // A git ignored file that is listed explicitly, should be scanned +// Every `@source` directive is classified as one of: +// +// - `Auto`: `@source "some/folder"` or `@source "some/folder/**/*"` — auto source detection. +// The folder is scanned recursively while respecting `.gitignore` files and the default rules +// (skip `node_modules`/`.git`/…, skip binary and irrelevant extensions, skip lock files, …). +// +// - `External`: an `Auto` source whose folder is itself ignored (by a `.gitignore` or because +// it's a default-ignored directory like `node_modules`), e.g. +// `@source "node_modules/my-ui-lib"`. Since the folder was listed explicitly, its ignoredness +// is bypassed: everything inside is scanned as if it were an `Auto` source, except that +// `.gitignore` files no longer apply inside (git ignores the whole tree anyway). The default +// rules still apply inside: nested `node_modules`, binary extensions, etc. stay ignored. +// +// - `Pattern`: `@source "some/folder/*.html"` — an explicit glob. Only files matching the glob +// are scanned. The *static* prefix of the glob (`some/folder`) is the explicit part: it is +// reached even when it is git ignored or hidden behind a default-ignored directory +// (`@source "node_modules/lib/dist/*.html"` works). The *wildcard* part is not explicit: +// while expanding it we still respect `.gitignore` files inside the walked subtree and the +// default-ignored directories (`@source "./**/*.html"` does not descend into `node_modules` +// or a git ignored `dist/`; `@source "./dist/**/*.html"` does descend into `dist/`). +// Individual *files* matching the glob are always included, even when git ignored — you were +// explicit about wanting files of that shape (`@source "git-ignored.html"` and +// `@source "*.styl"` work). Extensions that are ignored by default are only included when the +// glob pins an extension (`@source "logo.{jpg,png}"`, `@source "do-include-me.bin"`), an +// extension-less glob like `@source "some/folder/**/*"`… is `Auto`, and e.g. +// `@source "blog/*/post/**/*"` still applies the default extension rules. +// +// - `Ignored`: `@source not "…"` — excludes matching files/folders, even when a `.gitignore` +// or another `@source` allows them. +// +// Later directives win over earlier ones on conflict: `@source not "./x"` followed by +// `@source "./x/keep.html"` scans `keep.html`, and vice versa excludes it. +// +// # Implementation +// +// Scanning uses the vendored `ignore` crate for gitignore-aware, pruned directory walking (we +// never descend into directories that cannot contribute files). Sources are split over multiple +// walkers: +// +// - One walker for all `Auto` and `External` roots. Default rules and `@source not` rules are +// registered as explicit in-memory gitignores (they rank above `.gitignore` files found on +// disk, later directives above earlier ones). `External` roots additionally get a `!/**/*` +// whitelist (bypassing gitignore rules) plus a re-statement of the default rules. +// +// - One walker per `Pattern` base. It never looks at `.gitignore` files *above* its base +// (the static prefix is explicit), while `.gitignore` files inside the subtree still prune +// directories. Matching files are whitelisted (a glob match beats file-level ignores), and a +// filter closure makes the final per-file decision: the file must match a glob, honoring the +// relative order of `@source not` directives, and non-extension-pinning globs re-apply the +// default file rules. #[derive(Debug, Clone)] pub enum ChangedContent { @@ -60,8 +102,9 @@ pub struct Scanner { /// Content sources sources: Sources, - /// The walker to detect all files that we have to scan - walker: Option, + /// The walkers to detect all files that we have to scan: one for all auto/external source + /// roots, and one per pattern source base + walkers: Vec, /// All found extensions extensions: FxHashSet, @@ -111,11 +154,11 @@ impl Scanner { } } - let walker = create_walker(&sources); + let walkers = create_walkers(&sources); Self { sources, - walker, + walkers, ..Default::default() } } @@ -174,7 +217,7 @@ impl Scanner { // Figure out if the new unknown files are allowed to be scanned if !new_unknown_files.is_empty() { - if let Some(walk_builder) = &mut self.walker { + 'outer: for walk_builder in self.walkers.iter_mut() { for entry in walk_builder.build().filter_map(Result::ok) { let path = entry.path(); if !path.is_file() { @@ -219,7 +262,7 @@ impl Scanner { // We can stop walking the file system if all files we are interested in have // been found. if new_unknown_files.is_empty() { - break; + break 'outer; } } } @@ -379,17 +422,20 @@ impl Scanner { } self.sources_scanned = true; - let Some(walker) = &mut self.walker else { + if self.walkers.is_empty() { return (vec![], vec![], vec![]); - }; + } - // Use synchronous walk for the initial build (lower overhead) and parallel - // walk for subsequent calls (watch mode) where the overhead is amortised. - let all_entries = if self.has_scanned_once { - walk_parallel(walker) - } else { - walk_synchronous(walker) - }; + // Use synchronous walks for the initial build (lower overhead) and parallel + // walks for subsequent calls (watch mode) where the overhead is amortised. + let mut all_entries = vec![]; + for walker in self.walkers.iter_mut() { + if self.has_scanned_once { + all_entries.extend(walk_parallel(walker)); + } else { + all_entries.extend(walk_synchronous(walker)); + } + } let mut css_files: Vec = vec![]; let mut content_paths: Vec<(PathBuf, String)> = vec![]; @@ -707,75 +753,43 @@ fn walk_parallel(walker: &mut WalkBuilder) -> Vec { Arc::try_unwrap(collected).unwrap().into_inner().unwrap() } -/// Sets up a WalkBuilder with all source roots, gitignore rules, and source pattern matching. +/// Sets up all walkers for the given sources: one gitignore-aware walker for all `Auto` and +/// `External` roots, and one walker per `Pattern` base. +fn create_walkers(sources: &Sources) -> Vec { + let mut walkers = vec![]; + walkers.extend(create_auto_walker(sources)); + walkers.extend(create_pattern_walkers(sources)); + walkers +} + +/// A single walker for all `Auto` and `External` roots. /// -/// This is the common setup shared between the full walker (with mtime tracking for re-scans) -/// and the parallel walker (without mtime tracking for the initial scan). -fn create_walker(sources: &Sources) -> Option { - let mut other_roots: FxHashSet<&PathBuf> = FxHashSet::default(); - let mut first_root: Option<&PathBuf> = None; +/// The walker respects `.gitignore` files (also from parent directories, up to the git +/// repository root) plus a stack of explicit in-memory gitignores. Explicit gitignores rank +/// above the ignore files found on disk, and later-registered ones above earlier ones (see the +/// `CHANGED:` annotations in the vendored `ignore` crate). From low to high precedence: +/// +/// 1. The default auto source detection rules +/// 2. Per `@source` directive, in order: +/// - `External`: a `!/**/*` whitelist at its base (bypassing all gitignore rules), plus a +/// re-statement of the default rules so that e.g. nested `node_modules` stay ignored +/// - `Ignored` (`@source not`): its pattern at its base +fn create_auto_walker(sources: &Sources) -> Option { + let mut roots = sources.iter().filter_map(|source| match source { + SourceEntry::Auto { base } | SourceEntry::External { base } => Some(base), + _ => None, + }); - let mut ignores: Vec<(&PathBuf, Vec)> = Default::default(); - let mut emit = |base, pattern| match ignores.last_mut() { - Some((prev_base, patterns)) if *prev_base == base => { - patterns.push(pattern); - } - _ => { - ignores.push((base, vec![pattern])); - } - }; + let first_root = roots.next()?; - for source in sources.iter() { - match source { - SourceEntry::Auto { base } => { - if first_root.is_none() { - first_root = Some(base); - } else { - other_roots.insert(base); - } - } - SourceEntry::Pattern { base, pattern } => { - let pattern = pattern.to_owned(); - - if first_root.is_none() { - first_root = Some(base); - } else { - other_roots.insert(base); - } - - if !pattern.contains("**") { - // Specific patterns should take precedence even over git-ignored files: - emit(base, format!("!{}", pattern)); - } else { - // Assumption: the pattern we receive will already be brace expanded. So - // `*.{html,jsx}` will result in two separate patterns: `*.html` and `*.jsx`. - if let Some(extension) = Path::new(&pattern).extension() { - // Extend auto source detection to include the extension - emit(base, format!("!*.{}", extension.to_string_lossy())); - } - } - } - SourceEntry::Ignored { base, pattern } => { - emit(base, pattern.to_owned()); - } - SourceEntry::External { base } => { - if first_root.is_none() { - first_root = Some(base); - } else { - other_roots.insert(base); - } - - // External sources should take precedence even over git-ignored files: - emit(base, "!/**/*".to_owned()); - - // External sources should still disallow binary extensions: - emit(base, BINARY_EXTENSIONS_GLOB.clone()); - } + let mut builder = WalkBuilder::new(first_root); + let mut seen_roots = FxHashSet::from_iter([first_root]); + for root in roots { + if seen_roots.insert(root) { + builder.add(root); } } - let mut builder = WalkBuilder::new(first_root?); - // We have to follow symlinks builder.follow_links(true); @@ -820,92 +834,302 @@ fn create_walker(sources: &Sources) -> Option { // - my-project/apps/.gitignore // // Setting the require_git(true) flag conditionally allows us to do this. - for parent in first_root?.ancestors() { + for parent in first_root.ancestors() { if parent.join(".git").exists() { builder.require_git(true); break; } } - for root in other_roots { - builder.add(root); - } - // Setup auto source detection rules for ignore in auto_source_detection::RULES.iter() { builder.add_gitignore(ignore.clone()); } - // Setup ignores based on `@source` definitions - for (base, patterns) in ignores { - let mut ignore_builder = GitignoreBuilder::new(base); - for pattern in patterns { - ignore_builder.add_line(None, &pattern).unwrap(); + // Setup ignores based on `@source` definitions, in directive order so later directives win + for source in sources.iter() { + match source { + SourceEntry::External { base } => { + // External sources bypass all gitignore rules (the directory was explicitly + // listed even though it is ignored)… + let mut ignore_builder = GitignoreBuilder::new(base); + ignore_builder.add_line(None, "!/**/*").unwrap(); + + // …but the default auto source detection rules still apply inside of them, so + // nested `node_modules`, ignored extensions, and lock files stay ignored. + ignore_builder + .add_line(None, &auto_source_detection::IGNORED_CONTENT_DIRS_GLOB) + .unwrap(); + ignore_builder + .add_line(None, &auto_source_detection::IGNORED_EXTENSIONS_GLOB) + .unwrap(); + ignore_builder + .add_line(None, &auto_source_detection::IGNORED_FILES_GLOB) + .unwrap(); + builder.add_gitignore(ignore_builder.build().unwrap()); + + // Binary extensions are ignored as well, but only for files, so that a folder + // named e.g. `some.pages` is still scanned. + let mut ignore_builder = GitignoreBuilder::new(base); + ignore_builder + .only_on_files(true) + .add_line(None, &BINARY_EXTENSIONS_GLOB) + .unwrap(); + builder.add_gitignore(ignore_builder.build().unwrap()); + } + SourceEntry::Ignored { base, pattern } => { + let mut ignore_builder = GitignoreBuilder::new(base); + ignore_builder.add_line(None, pattern).unwrap(); + builder.add_gitignore(ignore_builder.build().unwrap()); + } + _ => {} } - let ignore = ignore_builder.build().unwrap(); - builder.add_gitignore(ignore); } - // Pre-compute source matching data to avoid allocations in the hot filter_entry path - let auto_bases: Vec = sources - .iter() - .filter_map(|source| match source { - SourceEntry::Auto { base } | SourceEntry::External { base } => Some(base.clone()), - _ => None, - }) - .collect(); - - let pattern_sources: Vec<(PathBuf, String)> = sources - .iter() - .filter_map(|source| match source { - SourceEntry::Pattern { base, pattern } => Some((base.into(), pattern.into())), - _ => None, - }) - .collect(); - - // Source pattern matching filter (lock-free, safe for parallel walking) - builder.filter_entry(move |entry| { - let path = entry.path(); - - // Ensure the entries are matching any of the provided source patterns (this is - // necessary for manual-patterns that can filter the file extension) - if path.is_file() { - let mut matches = false; - - for base in &auto_bases { - if path.starts_with(base) { - matches = true; - break; - } - } - - if !matches { - for (base, pattern) in &pattern_sources { - let remainder = path.strip_prefix(base); - if remainder.is_ok_and(|remainder| { - let mut path_str = remainder.to_string_lossy().to_string(); - if !path_str.starts_with("/") { - path_str = format!("/{path_str}"); - } - glob_match(pattern, path_str.as_bytes()) - }) { - matches = true; - break; - } - } - } - - if !matches { - return false; - } - } - - true - }); - Some(builder) } +/// A glob pattern of a `Pattern` source, together with the position of its `@source` directive. +#[derive(Debug, Clone)] +struct PatternRule { + /// Position of the `@source` directive, used to resolve conflicts with `@source not` + /// directives: the later directive wins. + idx: usize, + + /// The glob pattern, relative to the walker's base, e.g. `/ba*/*.html` + pattern: String, + + /// Whether the pattern pins a specific extension (e.g. `*.html` or `logo.png`). Patterns + /// that don't (e.g. `blog/*/**/*`) re-apply the default extension rules. + pins_extension: bool, +} + +/// An `@source not` directive, together with its position. +#[derive(Debug, Clone)] +struct NotRule { + /// Position of the `@source not` directive + idx: usize, + + base: PathBuf, + + /// The glob pattern, relative to `base`, e.g. `/ignored/**/*` + pattern: String, +} + +impl NotRule { + /// Whether this directive excludes the given file. + fn matches_file(&self, path: &Path) -> bool { + let Ok(remainder) = path.strip_prefix(&self.base) else { + return false; + }; + glob_match(&self.pattern, rooted_posix(remainder).as_bytes()) + } + + /// Whether this directive excludes the entire directory (and everything inside). + fn covers_dir(&self, path: &Path) -> bool { + // Directory-shaped `@source not "./some/dir"` directives are normalized to a `/**/*` + // pattern with the directory as its base. + self.pattern == "/**/*" && path.starts_with(&self.base) + } +} + +/// Serialize a path relative to some base as a `/`-rooted posix style string, e.g. +/// `/ba*/index.html`, matching how source patterns are stored. +fn rooted_posix(path: &Path) -> String { + let posix = crate::scanner::sources::path_to_posix_string(path); + if posix.starts_with('/') { + posix + } else { + format!("/{posix}") + } +} + +/// Whether a directory (relative to the pattern's base) can contain files matching the pattern. +/// Used to prune directories that can never contribute, e.g. for `/ba*/*.html` only `ba*` +/// directories are entered. +fn dir_could_contain_matches(pattern: &str, dir: &Path) -> bool { + let pattern_components: Vec<&str> = pattern + .trim_start_matches('/') + .split('/') + .filter(|c| !c.is_empty()) + .collect(); + + for (i, component) in dir.components().enumerate() { + let component = component.as_os_str().to_string_lossy(); + + // Once we see a `**` everything nested can contain matches + match pattern_components.get(i) { + Some(&"**") => return true, + // The last pattern component matches files, not directories. A directory nested + // deeper than the pattern's directory part can never contain matches. + Some(_) if i + 1 >= pattern_components.len() => return false, + Some(pattern_component) => { + if !glob_match(pattern_component, component.as_bytes()) { + return false; + } + } + None => return false, + } + } + + true +} + +/// One walker per `Pattern` base. +/// +/// The static base of a glob is the explicit part: it is used as the walk root, so `.gitignore` +/// files *above* it never apply (`parents(false)`), even when the base is hidden inside an +/// ignored directory. The wildcard part is not explicit: +/// +/// - `.gitignore` files inside the subtree still prune directories +/// - the default rules still prune directories (`node_modules` etc., unless the base itself +/// points inside of one) +/// +/// Files matching a glob are whitelisted explicitly (a glob match beats file-level gitignore +/// and default rules). The filter closure then makes the exact per-file decision. +fn create_pattern_walkers(sources: &Sources) -> Vec { + // Collect all `@source not` directives with their positions + let nots: Vec = sources + .iter() + .enumerate() + .filter_map(|(idx, source)| match source { + SourceEntry::Ignored { base, pattern } => Some(NotRule { + idx, + base: base.clone(), + pattern: pattern.clone(), + }), + _ => None, + }) + .collect(); + + // Group patterns by base, preserving directive order + let mut bases: Vec = vec![]; + let mut patterns_by_base: FxHashMap> = FxHashMap::default(); + for (idx, source) in sources.iter().enumerate() { + let SourceEntry::Pattern { base, pattern } = source else { + continue; + }; + + patterns_by_base + .entry(base.clone()) + .or_insert_with(|| { + bases.push(base.clone()); + vec![] + }) + .push(PatternRule { + idx, + pattern: pattern.clone(), + pins_extension: pattern_pins_extension(pattern), + }); + } + + bases + .into_iter() + .map(|base| { + let patterns = patterns_by_base.remove(&base).unwrap(); + + let mut builder = WalkBuilder::new(&base); + + // We have to follow symlinks + builder.follow_links(true); + + // Scan hidden files / directories + builder.hidden(false); + + // Don't respect global gitignore files + builder.git_global(false); + + // The static base is explicit: `.gitignore` files above it do not apply + builder.parents(false); + + // Apply `.gitignore` files inside the subtree regardless of whether a `.git` + // directory is present + builder.require_git(false); + + // The default rules prune directories (`node_modules`, …). Their file-level rules + // are rescued by the whitelists below when the glob matches. + for ignore in auto_source_detection::RULES.iter() { + builder.add_gitignore(ignore.clone()); + } + + // Whitelist the patterns themselves so that matching files win from file-level + // gitignore rules and the default rules. Restricted to files: the wildcard part of + // a pattern must not re-include ignored directories. + let mut ignore_builder = GitignoreBuilder::new(&base); + ignore_builder.only_on_files(true); + for pattern in &patterns { + ignore_builder + .add_line(None, &format!("!{}", pattern.pattern)) + .unwrap(); + } + builder.add_gitignore(ignore_builder.build().unwrap()); + + // The exact per-file decision (lock-free, safe for parallel walking) + let filter_base = base.clone(); + let filter_nots = nots.clone(); + builder.filter_entry(move |entry| { + // Always keep the walk root itself + if entry.depth() == 0 { + return true; + } + + let path = entry.path(); + let is_dir = entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false); + + let Ok(remainder) = path.strip_prefix(&filter_base) else { + return false; + }; + + if is_dir { + // Only descend when some pattern can match files inside this directory… + let relevant = patterns + .iter() + .filter(|p| dir_could_contain_matches(&p.pattern, remainder)) + .collect::>(); + if relevant.is_empty() { + return false; + } + + // …and no later `@source not` directive excludes the directory for all of + // those patterns. When a pattern comes after the `not`, keep walking; the + // file-level check below resolves the conflict exactly. + return !filter_nots.iter().any(|not| { + not.covers_dir(path) && relevant.iter().all(|p| p.idx < not.idx) + }); + } + + let rel = rooted_posix(remainder); + patterns.iter().any(|p| { + glob_match(&p.pattern, rel.as_bytes()) + && !filter_nots + .iter() + .any(|not| not.idx > p.idx && not.matches_file(path)) + && (p.pins_extension || !is_ignored_by_default_file_rules(path)) + }) + }); + + builder + }) + .collect() +} + +/// Whether a pattern pins a specific extension, e.g. `/*.html` or `/logo.png`. Patterns that +/// don't (e.g. `/blog/*/**/*`) keep the default extension rules applied. +fn pattern_pins_extension(pattern: &str) -> bool { + match Path::new(pattern).extension().and_then(|ext| ext.to_str()) { + Some(ext) => !ext.contains(['*', '?', '[']), + None => false, + } +} + +/// Whether a file is ignored by the default file-level rules (binary extensions, ignored +/// extensions, lock files, …). +fn is_ignored_by_default_file_rules(path: &Path) -> bool { + auto_source_detection::RULES + .iter() + .any(|ignore| ignore.matched(path, false).is_ignore()) +} + #[cfg(test)] mod tests { use super::{ChangedContent, Scanner}; diff --git a/crates/oxide/src/scanner/sources.rs b/crates/oxide/src/scanner/sources.rs index 99a4a5ec2..49619d41f 100644 --- a/crates/oxide/src/scanner/sources.rs +++ b/crates/oxide/src/scanner/sources.rs @@ -1,6 +1,6 @@ use crate::GlobEntry; use bexpand::Expression; -use fxhash::{FxHashMap, FxHashSet}; +use fxhash::FxHashMap; use ignore::gitignore::Gitignore; use std::path::{Component, Path, PathBuf}; use tracing::{event, Level}; @@ -77,164 +77,6 @@ impl Sources { } } -/// When dealing with a pattern, then it could be that we end up with: -/// -/// ```json -/// { base: '/some/folder', pattern: 'foo.ts' } -/// ``` -/// -/// If we just emit `!foo.ts` for the `/some/folder` path, then _everything_ else in that folder -/// would still be walked (but the result will be ignored). -/// -/// Instead, we have to ensure that we ignore everything in that folder _except_ for the `foo.ts` -/// pattern. -/// -/// This should be equivalent to: -/// ```gitignore -/// * -/// !foo.ts -/// ``` -/// -/// However, we have to be careful that we don't start ignoring files/folders that already exist. -/// ```css -/// @source "./some/folder/foo.ts"; -/// @source "./some/folder/bar.ts"; -/// ``` -/// Would result in: -/// ```json -/// { base: '/some/folder', pattern: 'foo.ts' } -/// { base: '/some/folder', pattern: 'bar.ts' } -/// ``` -/// -/// If we were to blindly emit `*` for each pattern, then the `.gitignore` equivalent would look like -/// this: -/// ```gitignore -/// * -/// !foo.ts -/// * -/// !bar.ts -/// ``` -/// -/// This would result in ignoring the `foo.ts` file as well. Therefore we only want to insert -/// this `*` pattern when nothing else exists yet. -/// -/// There is another problem that we need to solve. Let's say you have a pattern that contains a `*` -/// in the pattern: -/// ```css -/// @source './src/ba*/*.html'; -/// ``` -/// -/// This would result in -/// ```json -/// { base: '/src', pattern: '/ba*/*.html' } -/// ``` -/// -/// If we now inject the `*` pattern for the `/src` folder, then we wouldn't scan any `ba*` folders -/// (e.g. `bar` or `baz`). For this, we have to make sure that we add inverse patterns for these -/// folders. This would essentially result in: -/// ```gitignore -/// * ← ignore everything -/// !/ba*/ ← except for the `ba*/` pattern, so we scan these folders -/// !/ba*/*.html ← then ensure we scan the `*.html` files in it as well -/// ``` -/// -fn expand_restricted_patterns(sources: Vec) -> Vec { - let unrestricted_roots = sources - .iter() - .filter_map(|source| match source { - SourceEntry::Auto { base } | SourceEntry::External { base } => Some(base.clone()), - SourceEntry::Pattern { base, pattern } if pattern.contains("**") => Some(base.clone()), - _ => None, - }) - .collect::>(); - - // Bases of restricted patterns. Each of these becomes its own walk root with its own - // `*` + `!` rules, so an ancestor base must not ignore them recursively. - let pattern_roots = sources - .iter() - .filter_map(|source| match source { - SourceEntry::Pattern { base, .. } => Some(base.clone()), - _ => None, - }) - .collect::>(); - - let mut restricted_roots: FxHashSet = FxHashSet::default(); - let mut expanded = vec![]; - - for source in sources { - let SourceEntry::Pattern { base, pattern } = &source else { - expanded.push(source); - continue; - }; - - // `base` is already included by another `@source` that we know should be walked. This - // includes the case where `base` is _nested_ inside such a root, because everything under - // an unrestricted root is walked already. Restricting it would incorrectly hide siblings - // that the broader source is supposed to pick up. - if unrestricted_roots.iter().any(|root| base.starts_with(root)) { - expanded.push(source); - continue; - } - - // Ignore everything in the directory. We will later add the specific patterns we are - // interested in. - if restricted_roots.insert(base.clone()) { - // When another source root is nested inside this base — an unrestricted root, or the - // base of another restricted pattern (which is walked from its own root with its own - // rules) — only ignore direct children so the nested root can still be walked. - let has_nested_root = unrestricted_roots.iter().any(|root| root.starts_with(base)) - || pattern_roots - .iter() - .any(|root| root != base && root.starts_with(base)); - - let pattern = if has_nested_root { "/*" } else { "*" }; - - expanded.push(SourceEntry::Ignored { - base: base.clone(), - pattern: pattern.to_owned(), - }); - } - - // Ensure to _include_ parent paths, otherwise the `*` from above would block walking the - // folders that need to be walked. - // - // ```css - // @source './src/ba*/*.html'; - // ``` - // - // ```gitignore - // * ← added by the above rule - // !/ba*/ ← this is what we're focusing on in this block - // !/ba*/*.html ← this is added later - // ``` - { - let mut dir = PathBuf::new(); - let mut components = Path::new(pattern).components().peekable(); - - while let Some(component) = components.next() { - if components.peek().is_none() { - break; - } - - match component { - Component::Prefix(_) | Component::RootDir | Component::CurDir => continue, - Component::ParentDir | Component::Normal(_) => dir.push(component), - } - - expanded.push(SourceEntry::Ignored { - base: base.clone(), - pattern: format!("!/{}/", path_to_posix_string(&dir).trim_start_matches('/')), - }); - } - } - - // Track the original source - expanded.push(source); - } - - expanded -} - impl PublicSourceEntry { /// Optimize the PublicSourceEntry by trying to move all the static parts of the pattern to the /// base of the PublicSourceEntry. @@ -343,7 +185,7 @@ impl PublicSourceEntry { } } -fn path_to_posix_string(path: &Path) -> String { +pub(crate) fn path_to_posix_string(path: &Path) -> String { let mut parts = Vec::new(); let mut is_rooted = false; @@ -474,65 +316,7 @@ mod tests { } #[test] - fn concrete_patterns_are_expanded_to_restrict_their_base() { - let dir = tempdir().unwrap(); - fs::create_dir_all(dir.path().join("src")).unwrap(); - let base = dunce::canonicalize(dir.path().join("src")).unwrap(); - - let sources = public_source_entries_to_private_source_entries(vec![PublicSourceEntry { - base: dir.path().to_string_lossy().to_string(), - pattern: "src/foo.html".to_string(), - negated: false, - }]); - - assert_eq!( - sources, - vec![ - SourceEntry::Ignored { - base: base.clone(), - pattern: "*".to_string(), - }, - SourceEntry::Pattern { - base, - pattern: "/foo.html".to_string(), - }, - ] - ); - } - - #[test] - fn restricted_patterns_include_parent_directory_allow_rules() { - let dir = tempdir().unwrap(); - fs::create_dir_all(dir.path().join("src")).unwrap(); - let base = dunce::canonicalize(dir.path().join("src")).unwrap(); - - let sources = public_source_entries_to_private_source_entries(vec![PublicSourceEntry { - base: dir.path().to_string_lossy().to_string(), - pattern: "src/ef*/*.html".to_string(), - negated: false, - }]); - - assert_eq!( - sources, - vec![ - SourceEntry::Ignored { - base: base.clone(), - pattern: "*".to_string(), - }, - SourceEntry::Ignored { - base: base.clone(), - pattern: "!/ef*/".to_string(), - }, - SourceEntry::Pattern { - base, - pattern: "/ef*/*.html".to_string(), - }, - ] - ); - } - - #[test] - fn unrestricted_sources_do_not_expand_patterns_for_the_same_base() { + fn sources_are_converted_in_order() { let dir = tempdir().unwrap(); fs::create_dir_all(dir.path().join("src")).unwrap(); let base = dunce::canonicalize(dir.path().join("src")).unwrap(); @@ -548,72 +332,6 @@ mod tests { pattern: "src/foo.html".to_string(), negated: false, }, - ]); - - assert_eq!( - sources, - vec![ - SourceEntry::Auto { base: base.clone() }, - SourceEntry::Pattern { - base, - pattern: "/foo.html".to_string(), - }, - ] - ); - } - - #[test] - fn restricted_parent_bases_do_not_open_unrelated_siblings() { - let dir = tempdir().unwrap(); - let project = dir.path().join("Users").join("robin").join("docus-test"); - fs::create_dir_all(&project).unwrap(); - - let users = dunce::canonicalize(dir.path().join("Users")).unwrap(); - let project = dunce::canonicalize(project).unwrap(); - - let sources = public_source_entries_to_private_source_entries(vec![ - PublicSourceEntry { - base: project.to_string_lossy().to_string(), - pattern: "**/*".to_string(), - negated: false, - }, - PublicSourceEntry { - base: project.to_string_lossy().to_string(), - pattern: "../../app.config.ts".to_string(), - negated: false, - }, - ]); - - assert_eq!( - sources, - vec![ - SourceEntry::Auto { - base: project.clone(), - }, - SourceEntry::Ignored { - base: users.clone(), - pattern: "/*".to_string(), - }, - SourceEntry::Pattern { - base: users, - pattern: "/app.config.ts".to_string(), - }, - ] - ); - } - - #[test] - fn restricted_patterns_preserve_source_order() { - let dir = tempdir().unwrap(); - fs::create_dir_all(dir.path().join("src")).unwrap(); - let base = dunce::canonicalize(dir.path().join("src")).unwrap(); - - let sources = public_source_entries_to_private_source_entries(vec![ - PublicSourceEntry { - base: dir.path().to_string_lossy().to_string(), - pattern: "src/foo.html".to_string(), - negated: false, - }, PublicSourceEntry { base: dir.path().to_string_lossy().to_string(), pattern: "src/foo.html".to_string(), @@ -624,10 +342,7 @@ mod tests { assert_eq!( sources, vec![ - SourceEntry::Ignored { - base: base.clone(), - pattern: "*".to_string(), - }, + SourceEntry::Auto { base: base.clone() }, SourceEntry::Pattern { base: base.clone(), pattern: "/foo.html".to_string(), @@ -873,7 +588,7 @@ pub fn public_source_entries_to_private_source_entries( }) .collect::>(); - expand_restricted_patterns(sources) + sources } /// Convert a public source entry to a source entry diff --git a/crates/oxide/tests/scanner.rs b/crates/oxide/tests/scanner.rs index c30327bdd..764ad5a12 100644 --- a/crates/oxide/tests/scanner.rs +++ b/crates/oxide/tests/scanner.rs @@ -3947,6 +3947,340 @@ mod scanner { ]); } + // https://github.com/tailwindlabs/tailwindcss/issues/18870 + #[test] + fn glob_sources_can_descend_into_directories_ignored_from_within() { + // Laravel's `storage/` directories ship a `.gitignore` that ignores everything inside + // (`*` + `!.gitignore`). An explicit glob through such a directory should still find + // matching files, but nothing else. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + ("storage/.gitignore", "*\n!.gitignore"), + ( + "storage/cms/section.html", + "content-['storage/cms/section.html']", + ), + ( + "storage/cms/nested/deep.html", + "content-['storage/cms/nested/deep.html']", + ), + ("storage/cms/data.json", "content-['storage/cms/data.json']"), + ("storage/other/other.html", "content-['storage/other.html']"), + ], + vec!["@source './storage/cms/**/*.html'"], + ); + + assert_eq!( + candidates, + vec![ + "content-['storage/cms/nested/deep.html']", + "content-['storage/cms/section.html']", + ] + ); + assert_eq!( + files, + vec!["storage/cms/nested/deep.html", "storage/cms/section.html"] + ); + } + + // https://github.com/tailwindlabs/tailwindcss/issues/18870 + #[test] + fn glob_sources_can_descend_into_directories_ignored_by_an_ancestor() { + // Same as above, but the ignore comes from an ancestor `.gitignore` rather than one + // inside the ignored directory itself. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + (".gitignore", "/storage"), + ( + "storage/cms/section.html", + "content-['storage/cms/section.html']", + ), + ( + "storage/cms/nested/deep.html", + "content-['storage/cms/nested/deep.html']", + ), + ("storage/cms/script.js", "content-['storage/cms/script.js']"), + ], + vec!["@source './storage/cms/**/*.html'"], + ); + + assert_eq!( + candidates, + vec![ + "content-['storage/cms/nested/deep.html']", + "content-['storage/cms/section.html']", + ] + ); + assert_eq!( + files, + vec!["storage/cms/nested/deep.html", "storage/cms/section.html"] + ); + } + + #[test] + fn glob_sources_inside_ignored_directories_do_not_include_other_extensions() { + // `@source "./foo/*.html"` where `foo` is git ignored: only the `.html` files were asked + // for. Other files in `foo` must not be scanned, even when a broader auto source exists. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + (".gitignore", "/foo"), + ("foo/index.html", "content-['foo/index.html']"), + ("foo/script.js", "content-['foo/script.js']"), + ( + "foo/nested/nested.html", + "content-['foo/nested/nested.html']", + ), + ("index.html", "content-['index.html']"), + ], + vec!["@source '**/*'", "@source './foo/*.html'"], + ); + + assert_eq!( + candidates, + vec!["content-['foo/index.html']", "content-['index.html']"] + ); + assert_eq!(files, vec!["foo/index.html", "index.html"]); + } + + // https://github.com/tailwindlabs/tailwindcss/pull/20406 + #[test] + fn concrete_file_sources_behind_node_modules_do_not_include_siblings() { + // A concrete file `@source` pointing into a default-ignored directory (`node_modules`) + // must only include that file, not its siblings, even when the project root is an auto + // source. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + ( + "node_modules/.generated/ui/button.ts", + "content-['button.ts']", + ), + ("node_modules/.generated/ui/card.ts", "content-['card.ts']"), + ("index.html", "content-['index.html']"), + ], + vec![ + "@source '**/*'", + "@source './node_modules/.generated/ui/button.ts'", + ], + ); + + assert_eq!( + candidates, + vec!["content-['button.ts']", "content-['index.html']"] + ); + assert_eq!( + files, + vec!["index.html", "node_modules/.generated/ui/button.ts"] + ); + } + + // https://github.com/tailwindlabs/tailwindcss/pull/20406 + #[test] + fn concrete_file_sources_behind_gitignored_directories_do_not_include_siblings() { + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + (".gitignore", "/generated"), + ("generated/button.ts", "content-['button.ts']"), + ("generated/card.ts", "content-['card.ts']"), + ("index.html", "content-['index.html']"), + ], + vec!["@source '**/*'", "@source './generated/button.ts'"], + ); + + assert_eq!( + candidates, + vec!["content-['button.ts']", "content-['index.html']"] + ); + assert_eq!(files, vec!["generated/button.ts", "index.html"]); + } + + #[test] + fn glob_sources_do_not_reopen_node_modules() { + // A glob source respects the default auto source detection rules: `node_modules` inside + // the globbed tree stays pruned unless it is targeted explicitly. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + ("src/index.html", "content-['src/index.html']"), + ( + "src/node_modules/lib/index.html", + "content-['src/node_modules/lib/index.html']", + ), + ], + vec!["@source './src/**/*.html'"], + ); + + assert_eq!(candidates, vec!["content-['src/index.html']"]); + assert_eq!(files, vec!["src/index.html"]); + } + + #[test] + fn wildcards_in_glob_sources_do_not_descend_into_gitignored_directories() { + // The static part of a glob source is the "explicit" part: it bypasses ignore rules. + // The wildcard part does not: `@source "./**/*.html"` does not descend into a git + // ignored directory — you weren't explicit about it. To opt in, name the directory: + // `@source "./dist/**/*.html"`. + // + // Files are different: a git ignored *file* that matches the glob is still included + // (you asked for all `.html` files). + let paths_with_content = &[ + (".gitignore", "dist/\nignored.html"), + ("src/index.html", "content-['src/index.html']"), + ("dist/index.html", "content-['dist/index.html']"), + ("ignored.html", "content-['ignored.html']"), + ]; + + let ScanResult { + candidates, files, .. + } = scan_with_globs(paths_with_content, vec!["@source './**/*.html'"]); + + assert_eq!( + candidates, + vec!["content-['ignored.html']", "content-['src/index.html']"] + ); + assert_eq!(files, vec!["ignored.html", "src/index.html"]); + + // Being explicit about the ignored directory bypasses the ignore rule. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + paths_with_content, + vec!["@source './**/*.html'", "@source './dist/**/*.html'"], + ); + + assert_eq!( + candidates, + vec![ + "content-['dist/index.html']", + "content-['ignored.html']", + "content-['src/index.html']" + ] + ); + assert_eq!( + files, + vec!["dist/index.html", "ignored.html", "src/index.html"] + ); + } + + #[test] + fn glob_sources_only_rescue_matching_files_from_ignored_directories() { + // When a glob source forces the walker into a git ignored directory, files that do not + // match the glob must stay excluded, even though they are only ignored "via" their + // parent directory. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + (".gitignore", "gen/"), + ("gen/a.html", "content-['gen/a.html']"), + ("gen/b.js", "content-['gen/b.js']"), + ("index.html", "content-['index.html']"), + ], + vec!["@source '**/*'", "@source './gen/**/*.html'"], + ); + + assert_eq!( + candidates, + vec!["content-['gen/a.html']", "content-['index.html']"] + ); + assert_eq!(files, vec!["gen/a.html", "index.html"]); + } + + #[test] + fn unpinned_extension_globs_apply_default_extension_rules() { + // A glob that doesn't pin an extension (`**/*`-style tail after a wildcard directory) + // still applies the default extension rules, so binary files are not included. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + ("blog/2024/post/index.html", "content-['index.html']"), + ("blog/2024/post/image.png", "content-['image.png']"), + ("blog/2024/post/styles.scss", "content-['styles.scss']"), + ], + vec!["@source './blog/*/post/**/*'"], + ); + + assert_eq!(candidates, vec!["content-['index.html']"]); + assert_eq!(files, vec!["blog/2024/post/index.html"]); + } + + #[test] + fn external_sources_ignore_nested_default_ignored_directories() { + // An explicitly listed, git ignored directory (external source) still applies the + // default auto source detection rules inside of it: nested `node_modules` are not + // scanned. + let ScanResult { + candidates, files, .. + } = scan_with_globs( + &[ + (".gitignore", "node_modules"), + ( + "node_modules/my-lib/dist/index.html", + "content-['node_modules/my-lib/dist/index.html']", + ), + ( + "node_modules/my-lib/node_modules/dep/index.html", + "content-['node_modules/my-lib/node_modules/dep/index.html']", + ), + ( + "node_modules/my-lib/logo.png", + "content-['node_modules/my-lib/logo.png']", + ), + ], + vec!["@source './node_modules/my-lib'"], + ); + + assert_eq!( + candidates, + vec!["content-['node_modules/my-lib/dist/index.html']"] + ); + assert_eq!(files, vec!["node_modules/my-lib/dist/index.html"]); + } + + #[test] + fn later_pattern_sources_override_earlier_not_sources() { + // Order matters: a later positive `@source` wins from an earlier `@source not`, and + // vice versa. + let ScanResult { candidates, .. } = scan_with_globs( + &[ + ("src/keep.html", "content-['src/keep.html']"), + ("src/other.html", "content-['src/other.html']"), + ], + vec![ + "@source '**/*'", + "@source not './src'", + "@source './src/keep.html'", + ], + ); + + assert_eq!(candidates, vec!["content-['src/keep.html']"]); + + let ScanResult { candidates, .. } = scan_with_globs( + &[ + ("src/keep.html", "content-['src/keep.html']"), + ("src/other.html", "content-['src/other.html']"), + ], + vec![ + "@source '**/*'", + "@source './src/keep.html'", + "@source not './src'", + ], + ); + + assert!(candidates.is_empty()); + } + #[test] fn test_extract_used_css_variables_from_css() { let dir = tempdir().unwrap().into_path();